tests/transportpce_tests/common/test_utils.py

   1 #!/usr/bin/env python
   2 ##############################################################################
   3 # Copyright (c) 2020 Orange, Inc. and others.  All rights reserved.
   4 #
   5 # All rights reserved. This program and the accompanying materials
   6 # are made available under the terms of the Apache License, Version 2.0
   7 # which accompanies this distribution, and is available at
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 ##############################################################################
  10 import json
  11 import os
  12 import sys
  13 import re
  14 import signal
  15 import subprocess
  16 import time
  17
  18 import psutil
  19 import requests
  20
  21 import simulators
  22
  23 SIMS = simulators.SIMS
  24 HONEYNODE_EXECUTABLE = simulators.HONEYNODE_EXECUTABLE
  25 SAMPLES_DIRECTORY = simulators.SAMPLES_DIRECTORY
  26
  27 HONEYNODE_OK_START_MSG = "Netconf SSH endpoint started successfully at 0.0.0.0"
  28 KARAF_OK_START_MSG = re.escape(
  29     "Blueprint container for bundle org.opendaylight.netconf.restconf")+".* was successfully created"
  30
  31
  32 RESTCONF_BASE_URL = "http://localhost:8181/restconf"
  33 ODL_LOGIN = "admin"
  34 ODL_PWD = "admin"
  35 NODES_LOGIN = "admin"
  36 NODES_PWD = "admin"
  37 URL_CONFIG_NETCONF_TOPO = "{}/config/network-topology:network-topology/topology/topology-netconf/"
  38 URL_CONFIG_ORDM_TOPO = "{}/config/ietf-network:networks/network/openroadm-topology/"
  39 URL_CONFIG_OTN_TOPO = "{}/config/ietf-network:networks/network/otn-topology/"
  40 URL_CONFIG_CLLI_NET = "{}/config/ietf-network:networks/network/clli-network/"
  41 URL_CONFIG_ORDM_NET = "{}/config/ietf-network:networks/network/openroadm-network/"
  42 URL_PORTMAPPING = "{}/config/transportpce-portmapping:network/nodes/"
  43
  44 TYPE_APPLICATION_JSON = {'Content-Type': 'application/json', 'Accept': 'application/json'}
  45 TYPE_APPLICATION_XML = {'Content-Type': 'application/xml', 'Accept': 'application/xml'}
  46
  47 CODE_SHOULD_BE_200 = 'Http status code should be 200'
  48 CODE_SHOULD_BE_201 = 'Http status code should be 201'
  49
  50 LOG_DIRECTORY = os.path.dirname(os.path.realpath(__file__))
  51
  52 KARAF_LOG = os.path.join(
  53     os.path.dirname(os.path.realpath(__file__)),
  54     "..", "..", "..", "karaf", "target", "assembly", "data", "log", "karaf.log")
  55
  56 process_list = []
  57
  58 if "USE_LIGHTY" in os.environ and os.environ['USE_LIGHTY'] == 'True':
  59     TPCE_LOG = 'odl.log'
  60 else:
  61     TPCE_LOG = KARAF_LOG
  62
  63
  64 def start_sims(sims_list):
  65     for sim in sims_list:
  66         print("starting simulator for " + sim + "...")
  67         log_file = os.path.join(LOG_DIRECTORY, SIMS[sim]['logfile'])
  68         process = start_honeynode(log_file, SIMS[sim]['port'], SIMS[sim]['configfile'])
  69         if wait_until_log_contains(log_file, HONEYNODE_OK_START_MSG, 100):
  70             print("simulator for " + sim + " started")
  71         else:
  72             print("simulator for " + sim + " failed to start")
  73             shutdown_process(process)
  74             for pid in process_list:
  75                 shutdown_process(pid)
  76             sys.exit(3)
  77         process_list.append(process)
  78     return process_list
  79
  80
  81 def start_tpce():
  82     print("starting OpenDaylight...")
  83     if "USE_LIGHTY" in os.environ and os.environ['USE_LIGHTY'] == 'True':
  84         process = start_lighty()
  85         # TODO: add some sort of health check similar to Karaf below
  86     else:
  87         process = start_karaf()
  88         if wait_until_log_contains(KARAF_LOG, KARAF_OK_START_MSG, time_to_wait=60):
  89             print("OpenDaylight started !")
  90         else:
  91             print("OpenDaylight failed to start !")
  92             shutdown_process(process)
  93             for pid in process_list:
  94                 shutdown_process(pid)
  95             sys.exit(1)
  96     process_list.append(process)
  97     return process_list
  98
  99
 100 def start_karaf():
 101     print("starting KARAF TransportPCE build...")
 102     executable = os.path.join(
 103         os.path.dirname(os.path.realpath(__file__)),
 104         "..", "..", "..", "karaf", "target", "assembly", "bin", "karaf")
 105     with open('odl.log', 'w') as outfile:
 106         return subprocess.Popen(
 107             ["sh", executable, "server"], stdout=outfile, stderr=outfile, stdin=None)
 108
 109
 110 def start_lighty():
 111     print("starting LIGHTY.IO TransportPCE build...")
 112     executable = os.path.join(
 113         os.path.dirname(os.path.realpath(__file__)),
 114         "..", "..", "..", "lighty", "target", "tpce",
 115         "clean-start-controller.sh")
 116     with open('odl.log', 'w') as outfile:
 117         return subprocess.Popen(
 118             ["sh", executable], stdout=outfile, stderr=outfile, stdin=None)
 119
 120
 121 def install_karaf_feature(feature_name: str):
 122     print("installing feature " + feature_name)
 123     executable = os.path.join(
 124         os.path.dirname(os.path.realpath(__file__)),
 125         "..", "..", "..", "karaf", "target", "assembly", "bin", "client")
 126     return subprocess.run([executable],
 127                           input='feature:install ' + feature_name + '\n feature:list | grep tapi \n logout \n',
 128                           universal_newlines=True)
 129
 130
 131 def get_request(url):
 132     return requests.request(
 133         "GET", url.format(RESTCONF_BASE_URL),
 134         headers=TYPE_APPLICATION_JSON,
 135         auth=(ODL_LOGIN, ODL_PWD))
 136
 137
 138 def post_request(url, data):
 139     if data:
 140         return requests.request(
 141             "POST", url.format(RESTCONF_BASE_URL),
 142             data=json.dumps(data),
 143             headers=TYPE_APPLICATION_JSON,
 144             auth=(ODL_LOGIN, ODL_PWD))
 145     else:
 146         return requests.request(
 147             "POST", url.format(RESTCONF_BASE_URL),
 148             headers=TYPE_APPLICATION_JSON,
 149             auth=(ODL_LOGIN, ODL_PWD))
 150
 151
 152 def post_xmlrequest(url, data):
 153     if data:
 154         return requests.request(
 155             "POST", url.format(RESTCONF_BASE_URL),
 156             data=data,
 157             headers=TYPE_APPLICATION_XML,
 158             auth=(ODL_LOGIN, ODL_PWD))
 159
 160
 161 def put_request(url, data):
 162     return requests.request(
 163         "PUT", url.format(RESTCONF_BASE_URL),
 164         data=json.dumps(data),
 165         headers=TYPE_APPLICATION_JSON,
 166         auth=(ODL_LOGIN, ODL_PWD))
 167
 168
 169 def put_xmlrequest(url, data):
 170     return requests.request(
 171         "PUT", url.format(RESTCONF_BASE_URL),
 172         data=data,
 173         headers=TYPE_APPLICATION_XML,
 174         auth=(ODL_LOGIN, ODL_PWD))
 175
 176
 177 def rawput_request(url, data):
 178     return requests.request(
 179         "PUT", url.format(RESTCONF_BASE_URL),
 180         data=data,
 181         headers=TYPE_APPLICATION_JSON,
 182         auth=(ODL_LOGIN, ODL_PWD))
 183
 184
 185 def delete_request(url):
 186     return requests.request(
 187         "DELETE", url.format(RESTCONF_BASE_URL),
 188         headers=TYPE_APPLICATION_JSON,
 189         auth=(ODL_LOGIN, ODL_PWD))
 190
 191
 192 def mount_device(node_id, sim):
 193     url = URL_CONFIG_NETCONF_TOPO+"node/"+node_id
 194     body = {"node": [{
 195         "node-id": node_id,
 196         "netconf-node-topology:username": NODES_LOGIN,
 197         "netconf-node-topology:password": NODES_PWD,
 198         "netconf-node-topology:host": "127.0.0.1",
 199         "netconf-node-topology:port": SIMS[sim]['port'],
 200         "netconf-node-topology:tcp-only": "false",
 201         "netconf-node-topology:pass-through": {}}]}
 202     response = put_request(url, body)
 203     if wait_until_log_contains(TPCE_LOG, re.escape("Triggering notification stream NETCONF for node "+node_id), 60):
 204         print("Node "+node_id+" correctly added to tpce topology", end='... ', flush=True)
 205     else:
 206         print("Node "+node_id+" still not added to tpce topology", end='... ', flush=True)
 207         if response.status_code == requests.codes.ok:
 208             print("It was probably loaded at start-up", end='... ', flush=True)
 209         # TODO an else-clause to abort test would probably be nice here
 210     return response
 211
 212
 213 def unmount_device(node_id):
 214     url = URL_CONFIG_NETCONF_TOPO+"node/"+node_id
 215     response = delete_request(url)
 216     if wait_until_log_contains(TPCE_LOG, re.escape("onDeviceDisConnected: "+node_id), 60):
 217         print("Node "+node_id+" correctly deleted from tpce topology", end='... ', flush=True)
 218     else:
 219         print("Node "+node_id+" still not deleted from tpce topology", end='... ', flush=True)
 220     return response
 221
 222
 223 def connect_xpdr_to_rdm_request(xpdr_node: str, xpdr_num: str, network_num: str,
 224                                 rdm_node: str, srg_num: str, termination_num: str):
 225     url = "{}/operations/transportpce-networkutils:init-xpdr-rdm-links"
 226     data = {
 227         "networkutils:input": {
 228             "networkutils:links-input": {
 229                 "networkutils:xpdr-node": xpdr_node,
 230                 "networkutils:xpdr-num": xpdr_num,
 231                 "networkutils:network-num": network_num,
 232                 "networkutils:rdm-node": rdm_node,
 233                 "networkutils:srg-num": srg_num,
 234                 "networkutils:termination-point-num": termination_num
 235             }
 236         }
 237     }
 238     return post_request(url, data)
 239
 240
 241 def connect_rdm_to_xpdr_request(xpdr_node: str, xpdr_num: str, network_num: str,
 242                                 rdm_node: str, srg_num: str, termination_num: str):
 243     url = "{}/operations/transportpce-networkutils:init-rdm-xpdr-links"
 244     data = {
 245         "networkutils:input": {
 246             "networkutils:links-input": {
 247                 "networkutils:xpdr-node": xpdr_node,
 248                 "networkutils:xpdr-num": xpdr_num,
 249                 "networkutils:network-num": network_num,
 250                 "networkutils:rdm-node": rdm_node,
 251                 "networkutils:srg-num": srg_num,
 252                 "networkutils:termination-point-num": termination_num
 253             }
 254         }
 255     }
 256     return post_request(url, data)
 257
 258
 259 def check_netconf_node_request(node: str, suffix: str):
 260     url = URL_CONFIG_NETCONF_TOPO + (
 261         "node/" + node + "/yang-ext:mount/org-openroadm-device:org-openroadm-device/" + suffix
 262     )
 263     return get_request(url)
 264
 265
 266 def get_netconf_oper_request(node: str):
 267     url = "{}/operational/network-topology:network-topology/topology/topology-netconf/node/" + node
 268     return get_request(url)
 269
 270
 271 def get_ordm_topo_request(suffix: str):
 272     url = URL_CONFIG_ORDM_TOPO + suffix
 273     return get_request(url)
 274
 275
 276 def add_oms_attr_request(link: str, attr):
 277     url = URL_CONFIG_ORDM_TOPO + (
 278         "ietf-network-topology:link/" + link + "/org-openroadm-network-topology:OMS-attributes/span"
 279     )
 280     return put_request(url, attr)
 281
 282
 283 def del_oms_attr_request(link: str):
 284     url = URL_CONFIG_ORDM_TOPO + (
 285         "ietf-network-topology:link/" + link + "/org-openroadm-network-topology:OMS-attributes/span"
 286     )
 287     return delete_request(url)
 288
 289
 290 def get_clli_net_request():
 291     return get_request(URL_CONFIG_CLLI_NET)
 292
 293
 294 def get_ordm_net_request():
 295     return get_request(URL_CONFIG_ORDM_NET)
 296
 297
 298 def get_otn_topo_request():
 299     return get_request(URL_CONFIG_OTN_TOPO)
 300
 301
 302 def del_link_request(link: str):
 303     url = URL_CONFIG_ORDM_TOPO + ("ietf-network-topology:link/" + link)
 304     return delete_request(url)
 305
 306
 307 def del_node_request(node: str):
 308     url = URL_CONFIG_CLLI_NET + ("node/" + node)
 309     return delete_request(url)
 310
 311
 312 def portmapping_request(suffix: str):
 313     url = URL_PORTMAPPING + suffix
 314     return get_request(url)
 315
 316
 317 def shutdown_process(process):
 318     if process is not None:
 319         for child in psutil.Process(process.pid).children():
 320             child.send_signal(signal.SIGINT)
 321             child.wait()
 322         process.send_signal(signal.SIGINT)
 323
 324
 325 def start_honeynode(log_file: str, node_port: str, node_config_file_name: str):
 326     if os.path.isfile(HONEYNODE_EXECUTABLE):
 327         with open(log_file, 'w') as outfile:
 328             return subprocess.Popen(
 329                 [HONEYNODE_EXECUTABLE, node_port, os.path.join(SAMPLES_DIRECTORY, node_config_file_name)],
 330                 stdout=outfile, stderr=outfile)
 331
 332
 333 def wait_until_log_contains(log_file, regexp, time_to_wait=20):
 334     stringfound = False
 335     filefound = False
 336     line = None
 337     try:
 338         with TimeOut(seconds=time_to_wait):
 339             while not os.path.exists(log_file):
 340                 time.sleep(0.2)
 341             filelogs = open(log_file, 'r')
 342             filelogs.seek(0, 2)
 343             filefound = True
 344             print("Searching for pattern '"+regexp+"' in "+os.path.basename(log_file), end='... ', flush=True)
 345             compiled_regexp = re.compile(regexp)
 346             while True:
 347                 line = filelogs.readline()
 348                 if compiled_regexp.search(line):
 349                     print("Pattern found!", end=' ')
 350                     stringfound = True
 351                     break
 352                 if not line:
 353                     time.sleep(0.1)
 354     except TimeoutError:
 355         print("Pattern not found after "+str(time_to_wait), end=" seconds! ", flush=True)
 356     except PermissionError:
 357         print("Permission Error when trying to access the log file", end=" ... ", flush=True)
 358     finally:
 359         if filefound:
 360             filelogs.close()
 361         else:
 362             print("log file does not exist or is not accessible... ", flush=True)
 363         return stringfound
 364
 365
 366 class TimeOut:
 367     def __init__(self, seconds=1, error_message='Timeout'):
 368         self.seconds = seconds
 369         self.error_message = error_message
 370
 371     def handle_timeout(self, signum, frame):
 372         raise TimeoutError(self.error_message)
 373
 374     def __enter__(self):
 375         signal.signal(signal.SIGALRM, self.handle_timeout)
 376         signal.alarm(self.seconds)
 377
 378     def __exit__(self, type, value, traceback):
 379         signal.alarm(0)