tests/transportpce_tests/common/test_utils.py

   1 #!/usr/bin/env python
   2
   3 ##############################################################################
   4 # Copyright (c) 2020 Orange, Inc. and others.  All rights reserved.
   5 #
   6 # All rights reserved. This program and the accompanying materials
   7 # are made available under the terms of the Apache License, Version 2.0
   8 # which accompanies this distribution, and is available at
   9 # http://www.apache.org/licenses/LICENSE-2.0
  10 ##############################################################################
  11
  12 # pylint: disable=no-member
  13
  14 import json
  15 import os
  16 # pylint: disable=wrong-import-order
  17 import sys
  18 import re
  19 import signal
  20 import subprocess
  21 import time
  22
  23 import psutil
  24 import requests
  25
  26 # pylint: disable=import-error
  27 import simulators
  28
  29 SIMS = simulators.SIMS
  30
  31 HONEYNODE_OK_START_MSG = "Netconf SSH endpoint started successfully at 0.0.0.0"
  32 KARAF_OK_START_MSG = re.escape(
  33     "Blueprint container for bundle org.opendaylight.netconf.restconf")+".* was successfully created"
  34 LIGHTY_OK_START_MSG = re.escape("lighty.io and RESTCONF-NETCONF started")
  35
  36 ODL_LOGIN = "admin"
  37 ODL_PWD = "admin"
  38 NODES_LOGIN = "admin"
  39 NODES_PWD = "admin"
  40 URL_CONFIG_NETCONF_TOPO = "{}/config/network-topology:network-topology/topology/topology-netconf/"
  41 URL_CONFIG_ORDM_TOPO = "{}/config/ietf-network:networks/network/openroadm-topology/"
  42 URL_CONFIG_OTN_TOPO = "{}/config/ietf-network:networks/network/otn-topology/"
  43 URL_CONFIG_CLLI_NET = "{}/config/ietf-network:networks/network/clli-network/"
  44 URL_CONFIG_ORDM_NET = "{}/config/ietf-network:networks/network/openroadm-network/"
  45 URL_PORTMAPPING = "{}/config/transportpce-portmapping:network/nodes/"
  46 URL_OPER_SERV_LIST = "{}/operational/org-openroadm-service:service-list/"
  47 URL_GET_NBINOTIFICATIONS_PROCESS_SERV = "{}/operations/nbi-notifications:get-notifications-process-service/"
  48 URL_GET_NBINOTIFICATIONS_ALARM_SERV = "{}/operations/nbi-notifications:get-notifications-alarm-service/"
  49 URL_SERV_CREATE = "{}/operations/org-openroadm-service:service-create"
  50 URL_SERV_DELETE = "{}/operations/org-openroadm-service:service-delete"
  51 URL_SERVICE_PATH = "{}/operations/transportpce-device-renderer:service-path"
  52 URL_OTN_SERVICE_PATH = "{}/operations/transportpce-device-renderer:otn-service-path"
  53 URL_TAPI_CREATE_CONNECTIVITY = "{}/operations/tapi-connectivity:create-connectivity-service"
  54 URL_TAPI_DELETE_CONNECTIVITY = "{}/operations/tapi-connectivity:delete-connectivity-service"
  55 URL_CREATE_OTS_OMS = "{}/operations/transportpce-device-renderer:create-ots-oms"
  56 URL_PATH_COMPUTATION_REQUEST = "{}/operations/transportpce-pce:path-computation-request"
  57 URL_FULL_PORTMAPPING = "{}/config/transportpce-portmapping:network"
  58 URL_TAPI_TOPOLOGY_DETAILS = "{}/operations/tapi-topology:get-topology-details"
  59 URL_TAPI_NODE_DETAILS = "{}/operations/tapi-topology:get-node-details"
  60 URL_TAPI_SIP_LIST = "{}/operations/tapi-common:get-service-interface-point-list"
  61 URL_TAPI_SERVICE_LIST = "{}/operations/tapi-connectivity:get-connectivity-service-list"
  62
  63 TYPE_APPLICATION_JSON = {'Content-Type': 'application/json', 'Accept': 'application/json'}
  64 TYPE_APPLICATION_XML = {'Content-Type': 'application/xml', 'Accept': 'application/xml'}
  65
  66 CODE_SHOULD_BE_200 = 'Http status code should be 200'
  67 CODE_SHOULD_BE_201 = 'Http status code should be 201'
  68
  69 SIM_LOG_DIRECTORY = os.path.join(os.path.dirname(os.path.realpath(__file__)), "log")
  70
  71 process_list = []
  72
  73
  74 if "USE_ODL_ALT_RESTCONF_PORT" in os.environ:
  75     RESTCONF_BASE_URL = "http://localhost:" + os.environ['USE_ODL_ALT_RESTCONF_PORT'] + "/restconf"
  76 else:
  77     RESTCONF_BASE_URL = "http://localhost:8181/restconf"
  78
  79 if "USE_ODL_ALT_KARAF_INSTALL_DIR" in os.environ:
  80     KARAF_INSTALLDIR = os.environ['USE_ODL_ALT_KARAF_INSTALL_DIR']
  81 else:
  82     KARAF_INSTALLDIR = "karaf"
  83
  84 KARAF_LOG = os.path.join(
  85     os.path.dirname(os.path.realpath(__file__)),
  86     "..", "..", "..", KARAF_INSTALLDIR, "target", "assembly", "data", "log", "karaf.log")
  87
  88 if "USE_LIGHTY" in os.environ and os.environ['USE_LIGHTY'] == 'True':
  89     TPCE_LOG = 'odl-' + str(os.getpid()) + '.log'
  90 else:
  91     TPCE_LOG = KARAF_LOG
  92
  93
  94 def start_sims(sims_list):
  95     for sim in sims_list:
  96         print("starting simulator " + sim[0] + " in OpenROADM device version " + sim[1] + "...")
  97         log_file = os.path.join(SIM_LOG_DIRECTORY, SIMS[sim]['logfile'])
  98         process = start_honeynode(log_file, sim)
  99         if wait_until_log_contains(log_file, HONEYNODE_OK_START_MSG, 100):
 100             print("simulator for " + sim[0] + " started")
 101         else:
 102             print("simulator for " + sim[0] + " failed to start")
 103             shutdown_process(process)
 104             for pid in process_list:
 105                 shutdown_process(pid)
 106             sys.exit(3)
 107         process_list.append(process)
 108     return process_list
 109
 110
 111 def start_tpce():
 112     print("starting OpenDaylight...")
 113     if "USE_LIGHTY" in os.environ and os.environ['USE_LIGHTY'] == 'True':
 114         process = start_lighty()
 115         start_msg = LIGHTY_OK_START_MSG
 116     else:
 117         process = start_karaf()
 118         start_msg = KARAF_OK_START_MSG
 119     if wait_until_log_contains(TPCE_LOG, start_msg, time_to_wait=300):
 120         print("OpenDaylight started !")
 121     else:
 122         print("OpenDaylight failed to start !")
 123         shutdown_process(process)
 124         for pid in process_list:
 125             shutdown_process(pid)
 126         sys.exit(1)
 127     process_list.append(process)
 128     return process_list
 129
 130
 131 def start_karaf():
 132     print("starting KARAF TransportPCE build...")
 133     executable = os.path.join(
 134         os.path.dirname(os.path.realpath(__file__)),
 135         "..", "..", "..", KARAF_INSTALLDIR, "target", "assembly", "bin", "karaf")
 136     with open('odl.log', 'w', encoding='utf-8') as outfile:
 137         return subprocess.Popen(
 138             ["sh", executable, "server"], stdout=outfile, stderr=outfile, stdin=None)
 139
 140
 141 def start_lighty():
 142     print("starting LIGHTY.IO TransportPCE build...")
 143     executable = os.path.join(
 144         os.path.dirname(os.path.realpath(__file__)),
 145         "..", "..", "..", "lighty", "target", "tpce",
 146         "clean-start-controller.sh")
 147     with open(TPCE_LOG, 'w', encoding='utf-8') as outfile:
 148         return subprocess.Popen(
 149             ["sh", executable], stdout=outfile, stderr=outfile, stdin=None)
 150
 151
 152 def install_karaf_feature(feature_name: str):
 153     print("installing feature " + feature_name)
 154     executable = os.path.join(
 155         os.path.dirname(os.path.realpath(__file__)),
 156         "..", "..", "..", KARAF_INSTALLDIR, "target", "assembly", "bin", "client")
 157     return subprocess.run([executable],
 158                           input='feature:install ' + feature_name + '\n feature:list | grep '
 159                           + feature_name + ' \n logout \n',
 160                           universal_newlines=True, check=False)
 161
 162
 163 def get_request(url):
 164     return requests.request(
 165         "GET", url.format(RESTCONF_BASE_URL),
 166         headers=TYPE_APPLICATION_JSON,
 167         auth=(ODL_LOGIN, ODL_PWD))
 168
 169
 170 def post_request(url, data):
 171     if data:
 172         print(json.dumps(data))
 173         return requests.request(
 174             "POST", url.format(RESTCONF_BASE_URL),
 175             data=json.dumps(data),
 176             headers=TYPE_APPLICATION_JSON,
 177             auth=(ODL_LOGIN, ODL_PWD))
 178
 179     return requests.request(
 180         "POST", url.format(RESTCONF_BASE_URL),
 181         headers=TYPE_APPLICATION_JSON,
 182         auth=(ODL_LOGIN, ODL_PWD))
 183
 184
 185 def post_xmlrequest(url, data):
 186     if data:
 187         return requests.request(
 188             "POST", url.format(RESTCONF_BASE_URL),
 189             data=data,
 190             headers=TYPE_APPLICATION_XML,
 191             auth=(ODL_LOGIN, ODL_PWD))
 192     return None
 193
 194
 195 def put_request(url, data):
 196     return requests.request(
 197         "PUT", url.format(RESTCONF_BASE_URL),
 198         data=json.dumps(data),
 199         headers=TYPE_APPLICATION_JSON,
 200         auth=(ODL_LOGIN, ODL_PWD))
 201
 202
 203 def put_xmlrequest(url, data):
 204     return requests.request(
 205         "PUT", url.format(RESTCONF_BASE_URL),
 206         data=data,
 207         headers=TYPE_APPLICATION_XML,
 208         auth=(ODL_LOGIN, ODL_PWD))
 209
 210
 211 def put_jsonrequest(url, data):
 212     return requests.request(
 213         "PUT", url.format(RESTCONF_BASE_URL),
 214         data=data,
 215         headers=TYPE_APPLICATION_JSON,
 216         auth=(ODL_LOGIN, ODL_PWD))
 217
 218
 219 def rawput_request(url, data):
 220     return requests.request(
 221         "PUT", url.format(RESTCONF_BASE_URL),
 222         data=data,
 223         headers=TYPE_APPLICATION_JSON,
 224         auth=(ODL_LOGIN, ODL_PWD))
 225
 226
 227 def rawpost_request(url, data):
 228     return requests.request(
 229         "POST", url.format(RESTCONF_BASE_URL),
 230         data=data,
 231         headers=TYPE_APPLICATION_JSON,
 232         auth=(ODL_LOGIN, ODL_PWD))
 233
 234
 235 def delete_request(url):
 236     return requests.request(
 237         "DELETE", url.format(RESTCONF_BASE_URL),
 238         headers=TYPE_APPLICATION_JSON,
 239         auth=(ODL_LOGIN, ODL_PWD))
 240
 241
 242 def mount_device(node_id, sim):
 243     url = URL_CONFIG_NETCONF_TOPO + "node/" + node_id
 244     body = {"node": [{
 245         "node-id": node_id,
 246         "netconf-node-topology:username": NODES_LOGIN,
 247         "netconf-node-topology:password": NODES_PWD,
 248         "netconf-node-topology:host": "127.0.0.1",
 249         "netconf-node-topology:port": SIMS[sim]['port'],
 250         "netconf-node-topology:tcp-only": "false",
 251         "netconf-node-topology:pass-through": {}}]}
 252     response = put_request(url, body)
 253     if wait_until_log_contains(TPCE_LOG, re.escape("Triggering notification stream NETCONF for node " + node_id), 180):
 254         print("Node " + node_id + " correctly added to tpce topology", end='... ', flush=True)
 255     else:
 256         print("Node " + node_id + " still not added to tpce topology", end='... ', flush=True)
 257         if response.status_code == requests.codes.ok:
 258             print("It was probably loaded at start-up", end='... ', flush=True)
 259         # TODO an else-clause to abort test would probably be nice here
 260     return response
 261
 262
 263 def unmount_device(node_id):
 264     url = URL_CONFIG_NETCONF_TOPO + "node/" + node_id
 265     response = delete_request(url)
 266     if wait_until_log_contains(TPCE_LOG, re.escape("onDeviceDisConnected: " + node_id), 180):
 267         print("Node " + node_id + " correctly deleted from tpce topology", end='... ', flush=True)
 268     else:
 269         print("Node " + node_id + " still not deleted from tpce topology", end='... ', flush=True)
 270     return response
 271
 272
 273 def connect_xpdr_to_rdm_request(xpdr_node: str, xpdr_num: str, network_num: str,
 274                                 rdm_node: str, srg_num: str, termination_num: str):
 275     url = "{}/operations/transportpce-networkutils:init-xpdr-rdm-links"
 276     data = {
 277         "networkutils:input": {
 278             "networkutils:links-input": {
 279                 "networkutils:xpdr-node": xpdr_node,
 280                 "networkutils:xpdr-num": xpdr_num,
 281                 "networkutils:network-num": network_num,
 282                 "networkutils:rdm-node": rdm_node,
 283                 "networkutils:srg-num": srg_num,
 284                 "networkutils:termination-point-num": termination_num
 285             }
 286         }
 287     }
 288     return post_request(url, data)
 289
 290
 291 def connect_rdm_to_xpdr_request(xpdr_node: str, xpdr_num: str, network_num: str,
 292                                 rdm_node: str, srg_num: str, termination_num: str):
 293     url = "{}/operations/transportpce-networkutils:init-rdm-xpdr-links"
 294     data = {
 295         "networkutils:input": {
 296             "networkutils:links-input": {
 297                 "networkutils:xpdr-node": xpdr_node,
 298                 "networkutils:xpdr-num": xpdr_num,
 299                 "networkutils:network-num": network_num,
 300                 "networkutils:rdm-node": rdm_node,
 301                 "networkutils:srg-num": srg_num,
 302                 "networkutils:termination-point-num": termination_num
 303             }
 304         }
 305     }
 306     return post_request(url, data)
 307
 308
 309 def check_netconf_node_request(node: str, suffix: str):
 310     url = URL_CONFIG_NETCONF_TOPO + (
 311         "node/" + node + "/yang-ext:mount/org-openroadm-device:org-openroadm-device/" + suffix
 312     )
 313     return get_request(url)
 314
 315
 316 def get_netconf_oper_request(node: str):
 317     url = "{}/operational/network-topology:network-topology/topology/topology-netconf/node/" + node
 318     return get_request(url)
 319
 320
 321 def get_ordm_topo_request(suffix: str):
 322     url = URL_CONFIG_ORDM_TOPO + suffix
 323     return get_request(url)
 324
 325
 326 def add_oms_attr_request(link: str, attr):
 327     url = URL_CONFIG_ORDM_TOPO + (
 328         "ietf-network-topology:link/" + link + "/org-openroadm-network-topology:OMS-attributes/span"
 329     )
 330     return put_request(url, attr)
 331
 332
 333 def del_oms_attr_request(link: str):
 334     url = URL_CONFIG_ORDM_TOPO + (
 335         "ietf-network-topology:link/" + link + "/org-openroadm-network-topology:OMS-attributes/span"
 336     )
 337     return delete_request(url)
 338
 339
 340 def get_clli_net_request():
 341     return get_request(URL_CONFIG_CLLI_NET)
 342
 343
 344 def get_ordm_net_request():
 345     return get_request(URL_CONFIG_ORDM_NET)
 346
 347
 348 def get_otn_topo_request():
 349     return get_request(URL_CONFIG_OTN_TOPO)
 350
 351
 352 def del_link_request(link: str):
 353     url = URL_CONFIG_ORDM_TOPO + ("ietf-network-topology:link/" + link)
 354     return delete_request(url)
 355
 356
 357 def del_node_request(node: str):
 358     url = URL_CONFIG_CLLI_NET + ("node/" + node)
 359     return delete_request(url)
 360
 361
 362 def portmapping_request(suffix: str):
 363     url = URL_PORTMAPPING + suffix
 364     return get_request(url)
 365
 366
 367 def get_notifications_process_service_request(attr):
 368     return post_request(URL_GET_NBINOTIFICATIONS_PROCESS_SERV, attr)
 369
 370
 371 def get_notifications_alarm_service_request(attr):
 372     return post_request(URL_GET_NBINOTIFICATIONS_ALARM_SERV, attr)
 373
 374
 375 def get_service_list_request(suffix: str):
 376     url = URL_OPER_SERV_LIST + suffix
 377     return get_request(url)
 378
 379
 380 def service_create_request(attr):
 381     return post_request(URL_SERV_CREATE, attr)
 382
 383
 384 def service_delete_request(servicename: str,
 385                            requestid="e3028bae-a90f-4ddd-a83f-cf224eba0e58",
 386                            notificationurl="http://localhost:8585/NotificationServer/notify"):
 387     attr = {"input": {
 388         "sdnc-request-header": {
 389             "request-id": requestid,
 390             "rpc-action": "service-delete",
 391             "request-system-id": "appname",
 392             "notification-url": notificationurl},
 393         "service-delete-req-info": {
 394             "service-name": servicename,
 395             "tail-retention": "no"}}}
 396     return post_request(URL_SERV_DELETE, attr)
 397
 398
 399 def service_path_request(operation: str, servicename: str, wavenumber: str, nodes, centerfreq: str,
 400                          slotwidth: int, minfreq: float, maxfreq: float, lowerslotnumber: int,
 401                          higherslotnumber: int):
 402     attr = {"renderer:input": {
 403         "renderer:service-name": servicename,
 404         "renderer:wave-number": wavenumber,
 405         "renderer:modulation-format": "dp-qpsk",
 406         "renderer:operation": operation,
 407         "renderer:nodes": nodes,
 408         "renderer:center-freq": centerfreq,
 409         "renderer:width": slotwidth,
 410         "renderer:min-freq": minfreq,
 411         "renderer:max-freq": maxfreq,
 412         "renderer:lower-spectral-slot-number": lowerslotnumber,
 413         "renderer:higher-spectral-slot-number": higherslotnumber}}
 414     return post_request(URL_SERVICE_PATH, attr)
 415
 416
 417 def otn_service_path_request(operation: str, servicename: str, servicerate: str, serviceformat: str, nodes,
 418                              eth_attr=None):
 419     attr = {"service-name": servicename,
 420             "operation": operation,
 421             "service-rate": servicerate,
 422             "service-format": serviceformat,
 423             "nodes": nodes}
 424     if eth_attr:
 425         attr.update(eth_attr)
 426     return post_request(URL_OTN_SERVICE_PATH, {"renderer:input": attr})
 427
 428
 429 def create_ots_oms_request(nodeid: str, lcp: str):
 430     attr = {"input": {
 431         "node-id": nodeid,
 432         "logical-connection-point": lcp}}
 433     return post_request(URL_CREATE_OTS_OMS, attr)
 434
 435
 436 def path_computation_request(requestid: str, servicename: str, serviceaend, servicezend,
 437                              hardconstraints=None, softconstraints=None, metric="hop-count", other_attr=None):
 438     attr = {"service-name": servicename,
 439             "resource-reserve": "true",
 440             "service-handler-header": {"request-id": requestid},
 441             "service-a-end": serviceaend,
 442             "service-z-end": servicezend,
 443             "pce-metric": metric}
 444     if hardconstraints:
 445         attr.update({"hard-constraints": hardconstraints})
 446     if softconstraints:
 447         attr.update({"soft-constraints": softconstraints})
 448     if other_attr:
 449         attr.update(other_attr)
 450     return post_request(URL_PATH_COMPUTATION_REQUEST, {"input": attr})
 451
 452
 453 def tapi_create_connectivity_request(topologyidorname):
 454     return post_request(URL_TAPI_CREATE_CONNECTIVITY, topologyidorname)
 455
 456
 457 def tapi_delete_connectivity_request(serviceidorname):
 458     attr = {
 459         "input": {
 460             "service-id-or-name": serviceidorname}}
 461     return post_request(URL_TAPI_DELETE_CONNECTIVITY, attr)
 462
 463
 464 def tapi_get_topology_details_request(topologyidorname):
 465     attr = {
 466         "input": {
 467             "topology-id-or-name": topologyidorname}}
 468     return post_request(URL_TAPI_TOPOLOGY_DETAILS, attr)
 469
 470
 471 def tapi_get_node_details_request(topologyidorname, nodeidorname):
 472     attr = {
 473         "input": {
 474             "topology-id-or-name": topologyidorname,
 475             "node-id-or-name": nodeidorname}}
 476     return post_request(URL_TAPI_NODE_DETAILS, attr)
 477
 478
 479 def tapi_get_sip_details_request():
 480     return post_request(URL_TAPI_SIP_LIST, "")
 481
 482
 483 def tapi_get_service_list_request():
 484     return post_request(URL_TAPI_SERVICE_LIST, "")
 485
 486
 487 def shutdown_process(process):
 488     if process is not None:
 489         for child in psutil.Process(process.pid).children():
 490             child.send_signal(signal.SIGINT)
 491             child.wait()
 492         process.send_signal(signal.SIGINT)
 493
 494
 495 def start_honeynode(log_file: str, sim):
 496     executable = os.path.join(os.path.dirname(os.path.realpath(__file__)),
 497                               "..", "..", "honeynode", sim[1], "honeynode-simulator", "honeycomb-tpce")
 498     sample_directory = os.path.join(os.path.dirname(os.path.realpath(__file__)),
 499                                     "..", "..", "sample_configs", "openroadm", sim[1])
 500     if os.path.isfile(executable):
 501         with open(log_file, 'w', encoding='utf-8') as outfile:
 502             return subprocess.Popen(
 503                 [executable, SIMS[sim]['port'], os.path.join(sample_directory, SIMS[sim]['configfile'])],
 504                 stdout=outfile, stderr=outfile)
 505     return None
 506
 507
 508 def wait_until_log_contains(log_file, regexp, time_to_wait=60):
 509     # pylint: disable=lost-exception
 510     stringfound = False
 511     filefound = False
 512     line = None
 513     try:
 514         with TimeOut(seconds=time_to_wait):
 515             while not os.path.exists(log_file):
 516                 time.sleep(0.2)
 517             filelogs = open(log_file, 'r', encoding='utf-8')
 518             filelogs.seek(0, 2)
 519             filefound = True
 520             print("Searching for pattern '" + regexp + "' in " + os.path.basename(log_file), end='... ', flush=True)
 521             compiled_regexp = re.compile(regexp)
 522             while True:
 523                 line = filelogs.readline()
 524                 if compiled_regexp.search(line):
 525                     print("Pattern found!", end=' ')
 526                     stringfound = True
 527                     break
 528                 if not line:
 529                     time.sleep(0.1)
 530     except TimeoutError:
 531         print("Pattern not found after " + str(time_to_wait), end=" seconds! ", flush=True)
 532     except PermissionError:
 533         print("Permission Error when trying to access the log file", end=" ... ", flush=True)
 534     finally:
 535         if filefound:
 536             filelogs.close()
 537         else:
 538             print("log file does not exist or is not accessible... ", flush=True)
 539         return stringfound
 540
 541
 542 class TimeOut:
 543     def __init__(self, seconds=1, error_message='Timeout'):
 544         self.seconds = seconds
 545         self.error_message = error_message
 546
 547     def handle_timeout(self, signum, frame):
 548         raise TimeoutError(self.error_message)
 549
 550     def __enter__(self):
 551         signal.signal(signal.SIGALRM, self.handle_timeout)
 552         signal.alarm(self.seconds)
 553
 554     def __exit__(self, type, value, traceback):
 555         # pylint: disable=W0622
 556         signal.alarm(0)