tests/transportpce_tests/common/test_utils.py

   1 #!/usr/bin/env python
   2
   3 ##############################################################################
   4 # Copyright (c) 2020 Orange, Inc. and others.  All rights reserved.
   5 #
   6 # All rights reserved. This program and the accompanying materials
   7 # are made available under the terms of the Apache License, Version 2.0
   8 # which accompanies this distribution, and is available at
   9 # http://www.apache.org/licenses/LICENSE-2.0
  10 ##############################################################################
  11
  12 # pylint: disable=no-member
  13
  14 import json
  15 import os
  16 import sys
  17 import re
  18 import signal
  19 import subprocess
  20 import time
  21
  22 import psutil
  23 import requests
  24
  25 import simulators
  26
  27 SIMS = simulators.SIMS
  28
  29 HONEYNODE_OK_START_MSG = "Netconf SSH endpoint started successfully at 0.0.0.0"
  30 KARAF_OK_START_MSG = re.escape(
  31     "Blueprint container for bundle org.opendaylight.netconf.restconf")+".* was successfully created"
  32 LIGHTY_OK_START_MSG = re.escape("lighty.io and RESTCONF-NETCONF started")
  33
  34 ODL_LOGIN = "admin"
  35 ODL_PWD = "admin"
  36 NODES_LOGIN = "admin"
  37 NODES_PWD = "admin"
  38 URL_CONFIG_NETCONF_TOPO = "{}/config/network-topology:network-topology/topology/topology-netconf/"
  39 URL_CONFIG_ORDM_TOPO = "{}/config/ietf-network:networks/network/openroadm-topology/"
  40 URL_CONFIG_OTN_TOPO = "{}/config/ietf-network:networks/network/otn-topology/"
  41 URL_CONFIG_CLLI_NET = "{}/config/ietf-network:networks/network/clli-network/"
  42 URL_CONFIG_ORDM_NET = "{}/config/ietf-network:networks/network/openroadm-network/"
  43 URL_PORTMAPPING = "{}/config/transportpce-portmapping:network/nodes/"
  44 URL_OPER_SERV_LIST = "{}/operational/org-openroadm-service:service-list/"
  45 URL_GET_NBINOTIFICATIONS_PROCESS_SERV = "{}/operations/nbi-notifications:get-notifications-process-service/"
  46 URL_GET_NBINOTIFICATIONS_ALARM_SERV = "{}/operations/nbi-notifications:get-notifications-alarm-service/"
  47 URL_SERV_CREATE = "{}/operations/org-openroadm-service:service-create"
  48 URL_SERV_DELETE = "{}/operations/org-openroadm-service:service-delete"
  49 URL_SERVICE_PATH = "{}/operations/transportpce-device-renderer:service-path"
  50 URL_OTN_SERVICE_PATH = "{}/operations/transportpce-device-renderer:otn-service-path"
  51 URL_CREATE_OTS_OMS = "{}/operations/transportpce-device-renderer:create-ots-oms"
  52 URL_PATH_COMPUTATION_REQUEST = "{}/operations/transportpce-pce:path-computation-request"
  53 URL_FULL_PORTMAPPING = "{}/config/transportpce-portmapping:network"
  54
  55 TYPE_APPLICATION_JSON = {'Content-Type': 'application/json', 'Accept': 'application/json'}
  56 TYPE_APPLICATION_XML = {'Content-Type': 'application/xml', 'Accept': 'application/xml'}
  57
  58 CODE_SHOULD_BE_200 = 'Http status code should be 200'
  59 CODE_SHOULD_BE_201 = 'Http status code should be 201'
  60
  61 SIM_LOG_DIRECTORY = os.path.join(os.path.dirname(os.path.realpath(__file__)), "log")
  62
  63 process_list = []
  64
  65
  66 if "USE_ODL_ALT_RESTCONF_PORT" in os.environ:
  67     RESTCONF_BASE_URL = "http://localhost:" + os.environ['USE_ODL_ALT_RESTCONF_PORT'] + "/restconf"
  68 else:
  69     RESTCONF_BASE_URL = "http://localhost:8181/restconf"
  70
  71 if "USE_ODL_ALT_KARAF_INSTALL_DIR" in os.environ:
  72     KARAF_INSTALLDIR = os.environ['USE_ODL_ALT_KARAF_INSTALL_DIR']
  73 else:
  74     KARAF_INSTALLDIR = "karaf"
  75
  76 KARAF_LOG = os.path.join(
  77     os.path.dirname(os.path.realpath(__file__)),
  78     "..", "..", "..", KARAF_INSTALLDIR, "target", "assembly", "data", "log", "karaf.log")
  79
  80 if "USE_LIGHTY" in os.environ and os.environ['USE_LIGHTY'] == 'True':
  81     TPCE_LOG = 'odl-' + str(os.getpid()) + '.log'
  82 else:
  83     TPCE_LOG = KARAF_LOG
  84
  85
  86 def start_sims(sims_list):
  87     for sim in sims_list:
  88         print("starting simulator " + sim[0] + " in OpenROADM device version " + sim[1] + "...")
  89         log_file = os.path.join(SIM_LOG_DIRECTORY, SIMS[sim]['logfile'])
  90         process = start_honeynode(log_file, sim)
  91         if wait_until_log_contains(log_file, HONEYNODE_OK_START_MSG, 100):
  92             print("simulator for " + sim[0] + " started")
  93         else:
  94             print("simulator for " + sim[0] + " failed to start")
  95             shutdown_process(process)
  96             for pid in process_list:
  97                 shutdown_process(pid)
  98             sys.exit(3)
  99         process_list.append(process)
 100     return process_list
 101
 102
 103 def start_tpce():
 104     print("starting OpenDaylight...")
 105     if "USE_LIGHTY" in os.environ and os.environ['USE_LIGHTY'] == 'True':
 106         process = start_lighty()
 107         start_msg = LIGHTY_OK_START_MSG
 108     else:
 109         process = start_karaf()
 110         start_msg = KARAF_OK_START_MSG
 111     if wait_until_log_contains(TPCE_LOG, start_msg, time_to_wait=300):
 112         print("OpenDaylight started !")
 113     else:
 114         print("OpenDaylight failed to start !")
 115         shutdown_process(process)
 116         for pid in process_list:
 117             shutdown_process(pid)
 118         sys.exit(1)
 119     process_list.append(process)
 120     return process_list
 121
 122
 123 def start_karaf():
 124     print("starting KARAF TransportPCE build...")
 125     executable = os.path.join(
 126         os.path.dirname(os.path.realpath(__file__)),
 127         "..", "..", "..", KARAF_INSTALLDIR, "target", "assembly", "bin", "karaf")
 128     with open('odl.log', 'w') as outfile:
 129         return subprocess.Popen(
 130             ["sh", executable, "server"], stdout=outfile, stderr=outfile, stdin=None)
 131
 132
 133 def start_lighty():
 134     print("starting LIGHTY.IO TransportPCE build...")
 135     executable = os.path.join(
 136         os.path.dirname(os.path.realpath(__file__)),
 137         "..", "..", "..", "lighty", "target", "tpce",
 138         "clean-start-controller.sh")
 139     with open(TPCE_LOG, 'w') as outfile:
 140         return subprocess.Popen(
 141             ["sh", executable], stdout=outfile, stderr=outfile, stdin=None)
 142
 143
 144 def install_karaf_feature(feature_name: str):
 145     print("installing feature " + feature_name)
 146     executable = os.path.join(
 147         os.path.dirname(os.path.realpath(__file__)),
 148         "..", "..", "..", KARAF_INSTALLDIR, "target", "assembly", "bin", "client")
 149     return subprocess.run([executable],
 150                           input='feature:install ' + feature_name + '\n feature:list | grep '
 151                           + feature_name + ' \n logout \n',
 152                           universal_newlines=True, check=False)
 153
 154
 155 def get_request(url):
 156     return requests.request(
 157         "GET", url.format(RESTCONF_BASE_URL),
 158         headers=TYPE_APPLICATION_JSON,
 159         auth=(ODL_LOGIN, ODL_PWD))
 160
 161
 162 def post_request(url, data):
 163     if data:
 164         print(json.dumps(data))
 165         return requests.request(
 166             "POST", url.format(RESTCONF_BASE_URL),
 167             data=json.dumps(data),
 168             headers=TYPE_APPLICATION_JSON,
 169             auth=(ODL_LOGIN, ODL_PWD))
 170
 171     return requests.request(
 172         "POST", url.format(RESTCONF_BASE_URL),
 173         headers=TYPE_APPLICATION_JSON,
 174         auth=(ODL_LOGIN, ODL_PWD))
 175
 176
 177 def post_xmlrequest(url, data):
 178     if data:
 179         return requests.request(
 180             "POST", url.format(RESTCONF_BASE_URL),
 181             data=data,
 182             headers=TYPE_APPLICATION_XML,
 183             auth=(ODL_LOGIN, ODL_PWD))
 184     return None
 185
 186
 187 def put_request(url, data):
 188     return requests.request(
 189         "PUT", url.format(RESTCONF_BASE_URL),
 190         data=json.dumps(data),
 191         headers=TYPE_APPLICATION_JSON,
 192         auth=(ODL_LOGIN, ODL_PWD))
 193
 194
 195 def put_xmlrequest(url, data):
 196     return requests.request(
 197         "PUT", url.format(RESTCONF_BASE_URL),
 198         data=data,
 199         headers=TYPE_APPLICATION_XML,
 200         auth=(ODL_LOGIN, ODL_PWD))
 201
 202
 203 def put_jsonrequest(url, data):
 204     return requests.request(
 205         "PUT", url.format(RESTCONF_BASE_URL),
 206         data=data,
 207         headers=TYPE_APPLICATION_JSON,
 208         auth=(ODL_LOGIN, ODL_PWD))
 209
 210
 211 def rawput_request(url, data):
 212     return requests.request(
 213         "PUT", url.format(RESTCONF_BASE_URL),
 214         data=data,
 215         headers=TYPE_APPLICATION_JSON,
 216         auth=(ODL_LOGIN, ODL_PWD))
 217
 218
 219 def rawpost_request(url, data):
 220     return requests.request(
 221         "POST", url.format(RESTCONF_BASE_URL),
 222         data=data,
 223         headers=TYPE_APPLICATION_JSON,
 224         auth=(ODL_LOGIN, ODL_PWD))
 225
 226
 227 def delete_request(url):
 228     return requests.request(
 229         "DELETE", url.format(RESTCONF_BASE_URL),
 230         headers=TYPE_APPLICATION_JSON,
 231         auth=(ODL_LOGIN, ODL_PWD))
 232
 233
 234 def mount_device(node_id, sim):
 235     url = URL_CONFIG_NETCONF_TOPO + "node/" + node_id
 236     body = {"node": [{
 237         "node-id": node_id,
 238         "netconf-node-topology:username": NODES_LOGIN,
 239         "netconf-node-topology:password": NODES_PWD,
 240         "netconf-node-topology:host": "127.0.0.1",
 241         "netconf-node-topology:port": SIMS[sim]['port'],
 242         "netconf-node-topology:tcp-only": "false",
 243         "netconf-node-topology:pass-through": {}}]}
 244     response = put_request(url, body)
 245     if wait_until_log_contains(TPCE_LOG, re.escape("Triggering notification stream NETCONF for node " + node_id), 180):
 246         print("Node " + node_id + " correctly added to tpce topology", end='... ', flush=True)
 247     else:
 248         print("Node " + node_id + " still not added to tpce topology", end='... ', flush=True)
 249         if response.status_code == requests.codes.ok:
 250             print("It was probably loaded at start-up", end='... ', flush=True)
 251         # TODO an else-clause to abort test would probably be nice here
 252     return response
 253
 254
 255 def unmount_device(node_id):
 256     url = URL_CONFIG_NETCONF_TOPO + "node/" + node_id
 257     response = delete_request(url)
 258     if wait_until_log_contains(TPCE_LOG, re.escape("onDeviceDisConnected: " + node_id), 180):
 259         print("Node " + node_id + " correctly deleted from tpce topology", end='... ', flush=True)
 260     else:
 261         print("Node " + node_id + " still not deleted from tpce topology", end='... ', flush=True)
 262     return response
 263
 264
 265 def connect_xpdr_to_rdm_request(xpdr_node: str, xpdr_num: str, network_num: str,
 266                                 rdm_node: str, srg_num: str, termination_num: str):
 267     url = "{}/operations/transportpce-networkutils:init-xpdr-rdm-links"
 268     data = {
 269         "networkutils:input": {
 270             "networkutils:links-input": {
 271                 "networkutils:xpdr-node": xpdr_node,
 272                 "networkutils:xpdr-num": xpdr_num,
 273                 "networkutils:network-num": network_num,
 274                 "networkutils:rdm-node": rdm_node,
 275                 "networkutils:srg-num": srg_num,
 276                 "networkutils:termination-point-num": termination_num
 277             }
 278         }
 279     }
 280     return post_request(url, data)
 281
 282
 283 def connect_rdm_to_xpdr_request(xpdr_node: str, xpdr_num: str, network_num: str,
 284                                 rdm_node: str, srg_num: str, termination_num: str):
 285     url = "{}/operations/transportpce-networkutils:init-rdm-xpdr-links"
 286     data = {
 287         "networkutils:input": {
 288             "networkutils:links-input": {
 289                 "networkutils:xpdr-node": xpdr_node,
 290                 "networkutils:xpdr-num": xpdr_num,
 291                 "networkutils:network-num": network_num,
 292                 "networkutils:rdm-node": rdm_node,
 293                 "networkutils:srg-num": srg_num,
 294                 "networkutils:termination-point-num": termination_num
 295             }
 296         }
 297     }
 298     return post_request(url, data)
 299
 300
 301 def check_netconf_node_request(node: str, suffix: str):
 302     url = URL_CONFIG_NETCONF_TOPO + (
 303         "node/" + node + "/yang-ext:mount/org-openroadm-device:org-openroadm-device/" + suffix
 304     )
 305     return get_request(url)
 306
 307
 308 def get_netconf_oper_request(node: str):
 309     url = "{}/operational/network-topology:network-topology/topology/topology-netconf/node/" + node
 310     return get_request(url)
 311
 312
 313 def get_ordm_topo_request(suffix: str):
 314     url = URL_CONFIG_ORDM_TOPO + suffix
 315     return get_request(url)
 316
 317
 318 def add_oms_attr_request(link: str, attr):
 319     url = URL_CONFIG_ORDM_TOPO + (
 320         "ietf-network-topology:link/" + link + "/org-openroadm-network-topology:OMS-attributes/span"
 321     )
 322     return put_request(url, attr)
 323
 324
 325 def del_oms_attr_request(link: str):
 326     url = URL_CONFIG_ORDM_TOPO + (
 327         "ietf-network-topology:link/" + link + "/org-openroadm-network-topology:OMS-attributes/span"
 328     )
 329     return delete_request(url)
 330
 331
 332 def get_clli_net_request():
 333     return get_request(URL_CONFIG_CLLI_NET)
 334
 335
 336 def get_ordm_net_request():
 337     return get_request(URL_CONFIG_ORDM_NET)
 338
 339
 340 def get_otn_topo_request():
 341     return get_request(URL_CONFIG_OTN_TOPO)
 342
 343
 344 def del_link_request(link: str):
 345     url = URL_CONFIG_ORDM_TOPO + ("ietf-network-topology:link/" + link)
 346     return delete_request(url)
 347
 348
 349 def del_node_request(node: str):
 350     url = URL_CONFIG_CLLI_NET + ("node/" + node)
 351     return delete_request(url)
 352
 353
 354 def portmapping_request(suffix: str):
 355     url = URL_PORTMAPPING + suffix
 356     return get_request(url)
 357
 358
 359 def get_notifications_process_service_request(attr):
 360     return post_request(URL_GET_NBINOTIFICATIONS_PROCESS_SERV, attr)
 361
 362
 363 def get_notifications_alarm_service_request(attr):
 364     return post_request(URL_GET_NBINOTIFICATIONS_ALARM_SERV, attr)
 365
 366
 367 def get_service_list_request(suffix: str):
 368     url = URL_OPER_SERV_LIST + suffix
 369     return get_request(url)
 370
 371
 372 def service_create_request(attr):
 373     return post_request(URL_SERV_CREATE, attr)
 374
 375
 376 def service_delete_request(servicename: str,
 377                            requestid="e3028bae-a90f-4ddd-a83f-cf224eba0e58",
 378                            notificationurl="http://localhost:8585/NotificationServer/notify"):
 379     attr = {"input": {
 380         "sdnc-request-header": {
 381             "request-id": requestid,
 382             "rpc-action": "service-delete",
 383             "request-system-id": "appname",
 384             "notification-url": notificationurl},
 385         "service-delete-req-info": {
 386             "service-name": servicename,
 387             "tail-retention": "no"}}}
 388     return post_request(URL_SERV_DELETE, attr)
 389
 390
 391 def service_path_request(operation: str, servicename: str, wavenumber: str, nodes, centerfreq: str,
 392                          slotwidth: int, minfreq: float, maxfreq: float, lowerslotnumber: int,
 393                          higherslotnumber: int):
 394     attr = {"renderer:input": {
 395         "renderer:service-name": servicename,
 396         "renderer:wave-number": wavenumber,
 397         "renderer:modulation-format": "dp-qpsk",
 398         "renderer:operation": operation,
 399         "renderer:nodes": nodes,
 400         "renderer:center-freq": centerfreq,
 401         "renderer:width": slotwidth,
 402         "renderer:min-freq": minfreq,
 403         "renderer:max-freq": maxfreq,
 404         "renderer:lower-spectral-slot-number": lowerslotnumber,
 405         "renderer:higher-spectral-slot-number": higherslotnumber}}
 406     return post_request(URL_SERVICE_PATH, attr)
 407
 408
 409 def otn_service_path_request(operation: str, servicename: str, servicerate: str, serviceformat: str, nodes,
 410                              eth_attr=None):
 411     attr = {"service-name": servicename,
 412             "operation": operation,
 413             "service-rate": servicerate,
 414             "service-format": serviceformat,
 415             "nodes": nodes}
 416     if eth_attr:
 417         attr.update(eth_attr)
 418     return post_request(URL_OTN_SERVICE_PATH, {"renderer:input": attr})
 419
 420
 421 def create_ots_oms_request(nodeid: str, lcp: str):
 422     attr = {"input": {
 423         "node-id": nodeid,
 424         "logical-connection-point": lcp}}
 425     return post_request(URL_CREATE_OTS_OMS, attr)
 426
 427
 428 def path_computation_request(requestid: str, servicename: str, serviceaend, servicezend,
 429                              hardconstraints=None, softconstraints=None, metric="hop-count", other_attr=None):
 430     attr = {"service-name": servicename,
 431             "resource-reserve": "true",
 432             "service-handler-header": {"request-id": requestid},
 433             "service-a-end": serviceaend,
 434             "service-z-end": servicezend,
 435             "pce-metric": metric}
 436     if hardconstraints:
 437         attr.update({"hard-constraints": hardconstraints})
 438     if softconstraints:
 439         attr.update({"soft-constraints": softconstraints})
 440     if other_attr:
 441         attr.update(other_attr)
 442     return post_request(URL_PATH_COMPUTATION_REQUEST, {"input": attr})
 443
 444
 445 def shutdown_process(process):
 446     if process is not None:
 447         for child in psutil.Process(process.pid).children():
 448             child.send_signal(signal.SIGINT)
 449             child.wait()
 450         process.send_signal(signal.SIGINT)
 451
 452
 453 def start_honeynode(log_file: str, sim):
 454     executable = os.path.join(os.path.dirname(os.path.realpath(__file__)),
 455                               "..", "..", "honeynode", sim[1], "honeynode-simulator", "honeycomb-tpce")
 456     sample_directory = os.path.join(os.path.dirname(os.path.realpath(__file__)),
 457                                     "..", "..", "sample_configs", "openroadm", sim[1])
 458     if os.path.isfile(executable):
 459         with open(log_file, 'w') as outfile:
 460             return subprocess.Popen(
 461                 [executable, SIMS[sim]['port'], os.path.join(sample_directory, SIMS[sim]['configfile'])],
 462                 stdout=outfile, stderr=outfile)
 463     return None
 464
 465
 466 def wait_until_log_contains(log_file, regexp, time_to_wait=60):
 467     # pylint: disable=lost-exception
 468     stringfound = False
 469     filefound = False
 470     line = None
 471     try:
 472         with TimeOut(seconds=time_to_wait):
 473             while not os.path.exists(log_file):
 474                 time.sleep(0.2)
 475             filelogs = open(log_file, 'r')
 476             filelogs.seek(0, 2)
 477             filefound = True
 478             print("Searching for pattern '" + regexp + "' in " + os.path.basename(log_file), end='... ', flush=True)
 479             compiled_regexp = re.compile(regexp)
 480             while True:
 481                 line = filelogs.readline()
 482                 if compiled_regexp.search(line):
 483                     print("Pattern found!", end=' ')
 484                     stringfound = True
 485                     break
 486                 if not line:
 487                     time.sleep(0.1)
 488     except TimeoutError:
 489         print("Pattern not found after " + str(time_to_wait), end=" seconds! ", flush=True)
 490     except PermissionError:
 491         print("Permission Error when trying to access the log file", end=" ... ", flush=True)
 492     finally:
 493         if filefound:
 494             filelogs.close()
 495         else:
 496             print("log file does not exist or is not accessible... ", flush=True)
 497         return stringfound
 498
 499
 500 class TimeOut:
 501     def __init__(self, seconds=1, error_message='Timeout'):
 502         self.seconds = seconds
 503         self.error_message = error_message
 504
 505     def handle_timeout(self, signum, frame):
 506         raise TimeoutError(self.error_message)
 507
 508     def __enter__(self):
 509         signal.signal(signal.SIGALRM, self.handle_timeout)
 510         signal.alarm(self.seconds)
 511
 512     def __exit__(self, type, value, traceback):
 513         # pylint: disable=W0622
 514         signal.alarm(0)