tests/transportpce_tests/common/test_utils.py

   1 #!/usr/bin/env python
   2
   3 ##############################################################################
   4 # Copyright (c) 2020 Orange, Inc. and others.  All rights reserved.
   5 #
   6 # All rights reserved. This program and the accompanying materials
   7 # are made available under the terms of the Apache License, Version 2.0
   8 # which accompanies this distribution, and is available at
   9 # http://www.apache.org/licenses/LICENSE-2.0
  10 ##############################################################################
  11
  12 # pylint: disable=no-member
  13 # pylint: disable=too-many-arguments
  14
  15 import json
  16 import os
  17 # pylint: disable=wrong-import-order
  18 import sys
  19 import re
  20 import signal
  21 import subprocess
  22 import time
  23
  24 import psutil
  25 import requests
  26
  27 # pylint: disable=import-error
  28 import simulators
  29
  30 SIMS = simulators.SIMS
  31
  32 HONEYNODE_OK_START_MSG = "Netconf SSH endpoint started successfully at 0.0.0.0"
  33 KARAF_OK_START_MSG = re.escape(
  34     "Blueprint container for bundle org.opendaylight.netconf.restconf")+".* was successfully created"
  35 LIGHTY_OK_START_MSG = re.escape("lighty.io and RESTCONF-NETCONF started")
  36
  37 ODL_LOGIN = "admin"
  38 ODL_PWD = "admin"
  39 NODES_LOGIN = "admin"
  40 NODES_PWD = "admin"
  41 URL_CONFIG_NETCONF_TOPO = "{}/config/network-topology:network-topology/topology/topology-netconf/"
  42 URL_CONFIG_ORDM_TOPO = "{}/config/ietf-network:networks/network/openroadm-topology/"
  43 URL_CONFIG_OTN_TOPO = "{}/config/ietf-network:networks/network/otn-topology/"
  44 URL_CONFIG_CLLI_NET = "{}/config/ietf-network:networks/network/clli-network/"
  45 URL_CONFIG_ORDM_NET = "{}/config/ietf-network:networks/network/openroadm-network/"
  46 URL_PORTMAPPING = "{}/config/transportpce-portmapping:network/nodes/"
  47 URL_OPER_SERV_LIST = "{}/operational/org-openroadm-service:service-list/"
  48 URL_GET_NBINOTIFICATIONS_PROCESS_SERV = "{}/operations/nbi-notifications:get-notifications-process-service/"
  49 URL_GET_NBINOTIFICATIONS_ALARM_SERV = "{}/operations/nbi-notifications:get-notifications-alarm-service/"
  50 URL_SERV_CREATE = "{}/operations/org-openroadm-service:service-create"
  51 URL_SERV_DELETE = "{}/operations/org-openroadm-service:service-delete"
  52 URL_SERVICE_PATH = "{}/operations/transportpce-device-renderer:service-path"
  53 URL_OTN_SERVICE_PATH = "{}/operations/transportpce-device-renderer:otn-service-path"
  54 URL_TAPI_CREATE_CONNECTIVITY = "{}/operations/tapi-connectivity:create-connectivity-service"
  55 URL_TAPI_DELETE_CONNECTIVITY = "{}/operations/tapi-connectivity:delete-connectivity-service"
  56 URL_CREATE_OTS_OMS = "{}/operations/transportpce-device-renderer:create-ots-oms"
  57 URL_PATH_COMPUTATION_REQUEST = "{}/operations/transportpce-pce:path-computation-request"
  58 URL_FULL_PORTMAPPING = "{}/config/transportpce-portmapping:network"
  59 URL_TAPI_TOPOLOGY_DETAILS = "{}/operations/tapi-topology:get-topology-details"
  60 URL_TAPI_NODE_DETAILS = "{}/operations/tapi-topology:get-node-details"
  61 URL_TAPI_SIP_LIST = "{}/operations/tapi-common:get-service-interface-point-list"
  62 URL_TAPI_SERVICE_LIST = "{}/operations/tapi-connectivity:get-connectivity-service-list"
  63
  64 TYPE_APPLICATION_JSON = {'Content-Type': 'application/json', 'Accept': 'application/json'}
  65 TYPE_APPLICATION_XML = {'Content-Type': 'application/xml', 'Accept': 'application/xml'}
  66
  67 CODE_SHOULD_BE_200 = 'Http status code should be 200'
  68 CODE_SHOULD_BE_201 = 'Http status code should be 201'
  69
  70 SIM_LOG_DIRECTORY = os.path.join(os.path.dirname(os.path.realpath(__file__)), "log")
  71
  72 process_list = []
  73
  74
  75 if "USE_ODL_ALT_RESTCONF_PORT" in os.environ:
  76     RESTCONF_BASE_URL = "http://localhost:" + os.environ['USE_ODL_ALT_RESTCONF_PORT'] + "/restconf"
  77 else:
  78     RESTCONF_BASE_URL = "http://localhost:8181/restconf"
  79
  80 if "USE_ODL_ALT_KARAF_INSTALL_DIR" in os.environ:
  81     KARAF_INSTALLDIR = os.environ['USE_ODL_ALT_KARAF_INSTALL_DIR']
  82 else:
  83     KARAF_INSTALLDIR = "karaf"
  84
  85 KARAF_LOG = os.path.join(
  86     os.path.dirname(os.path.realpath(__file__)),
  87     "..", "..", "..", KARAF_INSTALLDIR, "target", "assembly", "data", "log", "karaf.log")
  88
  89 if "USE_LIGHTY" in os.environ and os.environ['USE_LIGHTY'] == 'True':
  90     TPCE_LOG = 'odl-' + str(os.getpid()) + '.log'
  91 else:
  92     TPCE_LOG = KARAF_LOG
  93
  94
  95 def start_sims(sims_list):
  96     for sim in sims_list:
  97         print("starting simulator " + sim[0] + " in OpenROADM device version " + sim[1] + "...")
  98         log_file = os.path.join(SIM_LOG_DIRECTORY, SIMS[sim]['logfile'])
  99         process = start_honeynode(log_file, sim)
 100         if wait_until_log_contains(log_file, HONEYNODE_OK_START_MSG, 100):
 101             print("simulator for " + sim[0] + " started")
 102         else:
 103             print("simulator for " + sim[0] + " failed to start")
 104             shutdown_process(process)
 105             for pid in process_list:
 106                 shutdown_process(pid)
 107             sys.exit(3)
 108         process_list.append(process)
 109     return process_list
 110
 111
 112 def start_tpce():
 113     print("starting OpenDaylight...")
 114     if "USE_LIGHTY" in os.environ and os.environ['USE_LIGHTY'] == 'True':
 115         process = start_lighty()
 116         start_msg = LIGHTY_OK_START_MSG
 117     else:
 118         process = start_karaf()
 119         start_msg = KARAF_OK_START_MSG
 120     if wait_until_log_contains(TPCE_LOG, start_msg, time_to_wait=300):
 121         print("OpenDaylight started !")
 122     else:
 123         print("OpenDaylight failed to start !")
 124         shutdown_process(process)
 125         for pid in process_list:
 126             shutdown_process(pid)
 127         sys.exit(1)
 128     process_list.append(process)
 129     return process_list
 130
 131
 132 def start_karaf():
 133     print("starting KARAF TransportPCE build...")
 134     executable = os.path.join(
 135         os.path.dirname(os.path.realpath(__file__)),
 136         "..", "..", "..", KARAF_INSTALLDIR, "target", "assembly", "bin", "karaf")
 137     with open('odl.log', 'w', encoding='utf-8') as outfile:
 138         return subprocess.Popen(
 139             ["sh", executable, "server"], stdout=outfile, stderr=outfile, stdin=None)
 140
 141
 142 def start_lighty():
 143     print("starting LIGHTY.IO TransportPCE build...")
 144     executable = os.path.join(
 145         os.path.dirname(os.path.realpath(__file__)),
 146         "..", "..", "..", "lighty", "target", "tpce",
 147         "clean-start-controller.sh")
 148     with open(TPCE_LOG, 'w', encoding='utf-8') as outfile:
 149         return subprocess.Popen(
 150             ["sh", executable], stdout=outfile, stderr=outfile, stdin=None)
 151
 152
 153 def install_karaf_feature(feature_name: str):
 154     print("installing feature " + feature_name)
 155     executable = os.path.join(
 156         os.path.dirname(os.path.realpath(__file__)),
 157         "..", "..", "..", KARAF_INSTALLDIR, "target", "assembly", "bin", "client")
 158     return subprocess.run([executable],
 159                           input='feature:install ' + feature_name + '\n feature:list | grep '
 160                           + feature_name + ' \n logout \n',
 161                           universal_newlines=True, check=False)
 162
 163
 164 def get_request(url):
 165     return requests.request(
 166         "GET", url.format(RESTCONF_BASE_URL),
 167         headers=TYPE_APPLICATION_JSON,
 168         auth=(ODL_LOGIN, ODL_PWD))
 169
 170
 171 def post_request(url, data):
 172     if data:
 173         print(json.dumps(data))
 174         return requests.request(
 175             "POST", url.format(RESTCONF_BASE_URL),
 176             data=json.dumps(data),
 177             headers=TYPE_APPLICATION_JSON,
 178             auth=(ODL_LOGIN, ODL_PWD))
 179
 180     return requests.request(
 181         "POST", url.format(RESTCONF_BASE_URL),
 182         headers=TYPE_APPLICATION_JSON,
 183         auth=(ODL_LOGIN, ODL_PWD))
 184
 185
 186 def post_xmlrequest(url, data):
 187     if data:
 188         return requests.request(
 189             "POST", url.format(RESTCONF_BASE_URL),
 190             data=data,
 191             headers=TYPE_APPLICATION_XML,
 192             auth=(ODL_LOGIN, ODL_PWD))
 193     return None
 194
 195
 196 def put_request(url, data):
 197     return requests.request(
 198         "PUT", url.format(RESTCONF_BASE_URL),
 199         data=json.dumps(data),
 200         headers=TYPE_APPLICATION_JSON,
 201         auth=(ODL_LOGIN, ODL_PWD))
 202
 203
 204 def put_xmlrequest(url, data):
 205     return requests.request(
 206         "PUT", url.format(RESTCONF_BASE_URL),
 207         data=data,
 208         headers=TYPE_APPLICATION_XML,
 209         auth=(ODL_LOGIN, ODL_PWD))
 210
 211
 212 def put_jsonrequest(url, data):
 213     return requests.request(
 214         "PUT", url.format(RESTCONF_BASE_URL),
 215         data=data,
 216         headers=TYPE_APPLICATION_JSON,
 217         auth=(ODL_LOGIN, ODL_PWD))
 218
 219
 220 def rawput_request(url, data):
 221     return requests.request(
 222         "PUT", url.format(RESTCONF_BASE_URL),
 223         data=data,
 224         headers=TYPE_APPLICATION_JSON,
 225         auth=(ODL_LOGIN, ODL_PWD))
 226
 227
 228 def rawpost_request(url, data):
 229     return requests.request(
 230         "POST", url.format(RESTCONF_BASE_URL),
 231         data=data,
 232         headers=TYPE_APPLICATION_JSON,
 233         auth=(ODL_LOGIN, ODL_PWD))
 234
 235
 236 def delete_request(url):
 237     return requests.request(
 238         "DELETE", url.format(RESTCONF_BASE_URL),
 239         headers=TYPE_APPLICATION_JSON,
 240         auth=(ODL_LOGIN, ODL_PWD))
 241
 242
 243 def mount_device(node_id, sim):
 244     url = URL_CONFIG_NETCONF_TOPO + "node/" + node_id
 245     body = {"node": [{
 246         "node-id": node_id,
 247         "netconf-node-topology:username": NODES_LOGIN,
 248         "netconf-node-topology:password": NODES_PWD,
 249         "netconf-node-topology:host": "127.0.0.1",
 250         "netconf-node-topology:port": SIMS[sim]['port'],
 251         "netconf-node-topology:tcp-only": "false",
 252         "netconf-node-topology:pass-through": {}}]}
 253     response = put_request(url, body)
 254     if wait_until_log_contains(TPCE_LOG, re.escape("Triggering notification stream NETCONF for node " + node_id), 180):
 255         print("Node " + node_id + " correctly added to tpce topology", end='... ', flush=True)
 256     else:
 257         print("Node " + node_id + " still not added to tpce topology", end='... ', flush=True)
 258         if response.status_code == requests.codes.ok:
 259             print("It was probably loaded at start-up", end='... ', flush=True)
 260         # TODO an else-clause to abort test would probably be nice here
 261     return response
 262
 263
 264 def unmount_device(node_id):
 265     url = URL_CONFIG_NETCONF_TOPO + "node/" + node_id
 266     response = delete_request(url)
 267     if wait_until_log_contains(TPCE_LOG, re.escape("onDeviceDisConnected: " + node_id), 180):
 268         print("Node " + node_id + " correctly deleted from tpce topology", end='... ', flush=True)
 269     else:
 270         print("Node " + node_id + " still not deleted from tpce topology", end='... ', flush=True)
 271     return response
 272
 273
 274 def connect_xpdr_to_rdm_request(xpdr_node: str, xpdr_num: str, network_num: str,
 275                                 rdm_node: str, srg_num: str, termination_num: str):
 276     url = "{}/operations/transportpce-networkutils:init-xpdr-rdm-links"
 277     data = {
 278         "networkutils:input": {
 279             "networkutils:links-input": {
 280                 "networkutils:xpdr-node": xpdr_node,
 281                 "networkutils:xpdr-num": xpdr_num,
 282                 "networkutils:network-num": network_num,
 283                 "networkutils:rdm-node": rdm_node,
 284                 "networkutils:srg-num": srg_num,
 285                 "networkutils:termination-point-num": termination_num
 286             }
 287         }
 288     }
 289     return post_request(url, data)
 290
 291
 292 def connect_rdm_to_xpdr_request(xpdr_node: str, xpdr_num: str, network_num: str,
 293                                 rdm_node: str, srg_num: str, termination_num: str):
 294     url = "{}/operations/transportpce-networkutils:init-rdm-xpdr-links"
 295     data = {
 296         "networkutils:input": {
 297             "networkutils:links-input": {
 298                 "networkutils:xpdr-node": xpdr_node,
 299                 "networkutils:xpdr-num": xpdr_num,
 300                 "networkutils:network-num": network_num,
 301                 "networkutils:rdm-node": rdm_node,
 302                 "networkutils:srg-num": srg_num,
 303                 "networkutils:termination-point-num": termination_num
 304             }
 305         }
 306     }
 307     return post_request(url, data)
 308
 309
 310 def check_netconf_node_request(node: str, suffix: str):
 311     url = URL_CONFIG_NETCONF_TOPO + (
 312         "node/" + node + "/yang-ext:mount/org-openroadm-device:org-openroadm-device/" + suffix
 313     )
 314     return get_request(url)
 315
 316
 317 def get_netconf_oper_request(node: str):
 318     url = "{}/operational/network-topology:network-topology/topology/topology-netconf/node/" + node
 319     return get_request(url)
 320
 321
 322 def get_ordm_topo_request(suffix: str):
 323     url = URL_CONFIG_ORDM_TOPO + suffix
 324     return get_request(url)
 325
 326
 327 def add_oms_attr_request(link: str, attr):
 328     url = URL_CONFIG_ORDM_TOPO + (
 329         "ietf-network-topology:link/" + link + "/org-openroadm-network-topology:OMS-attributes/span"
 330     )
 331     return put_request(url, attr)
 332
 333
 334 def del_oms_attr_request(link: str):
 335     url = URL_CONFIG_ORDM_TOPO + (
 336         "ietf-network-topology:link/" + link + "/org-openroadm-network-topology:OMS-attributes/span"
 337     )
 338     return delete_request(url)
 339
 340
 341 def get_clli_net_request():
 342     return get_request(URL_CONFIG_CLLI_NET)
 343
 344
 345 def get_ordm_net_request():
 346     return get_request(URL_CONFIG_ORDM_NET)
 347
 348
 349 def get_otn_topo_request():
 350     return get_request(URL_CONFIG_OTN_TOPO)
 351
 352
 353 def del_link_request(link: str):
 354     url = URL_CONFIG_ORDM_TOPO + ("ietf-network-topology:link/" + link)
 355     return delete_request(url)
 356
 357
 358 def del_node_request(node: str):
 359     url = URL_CONFIG_CLLI_NET + ("node/" + node)
 360     return delete_request(url)
 361
 362
 363 def portmapping_request(suffix: str):
 364     url = URL_PORTMAPPING + suffix
 365     return get_request(url)
 366
 367
 368 def get_notifications_process_service_request(attr):
 369     return post_request(URL_GET_NBINOTIFICATIONS_PROCESS_SERV, attr)
 370
 371
 372 def get_notifications_alarm_service_request(attr):
 373     return post_request(URL_GET_NBINOTIFICATIONS_ALARM_SERV, attr)
 374
 375
 376 def get_service_list_request(suffix: str):
 377     url = URL_OPER_SERV_LIST + suffix
 378     return get_request(url)
 379
 380
 381 def service_create_request(attr):
 382     return post_request(URL_SERV_CREATE, attr)
 383
 384
 385 def service_delete_request(servicename: str,
 386                            requestid="e3028bae-a90f-4ddd-a83f-cf224eba0e58",
 387                            notificationurl="http://localhost:8585/NotificationServer/notify"):
 388     attr = {"input": {
 389         "sdnc-request-header": {
 390             "request-id": requestid,
 391             "rpc-action": "service-delete",
 392             "request-system-id": "appname",
 393             "notification-url": notificationurl},
 394         "service-delete-req-info": {
 395             "service-name": servicename,
 396             "tail-retention": "no"}}}
 397     return post_request(URL_SERV_DELETE, attr)
 398
 399
 400 def service_path_request(operation: str, servicename: str, wavenumber: str, nodes, centerfreq: str,
 401                          slotwidth: int, minfreq: float, maxfreq: float, lowerslotnumber: int,
 402                          higherslotnumber: int):
 403     attr = {"renderer:input": {
 404         "renderer:service-name": servicename,
 405         "renderer:wave-number": wavenumber,
 406         "renderer:modulation-format": "dp-qpsk",
 407         "renderer:operation": operation,
 408         "renderer:nodes": nodes,
 409         "renderer:center-freq": centerfreq,
 410         "renderer:width": slotwidth,
 411         "renderer:min-freq": minfreq,
 412         "renderer:max-freq": maxfreq,
 413         "renderer:lower-spectral-slot-number": lowerslotnumber,
 414         "renderer:higher-spectral-slot-number": higherslotnumber}}
 415     return post_request(URL_SERVICE_PATH, attr)
 416
 417
 418 def otn_service_path_request(operation: str, servicename: str, servicerate: str, serviceformat: str, nodes,
 419                              eth_attr=None):
 420     attr = {"service-name": servicename,
 421             "operation": operation,
 422             "service-rate": servicerate,
 423             "service-format": serviceformat,
 424             "nodes": nodes}
 425     if eth_attr:
 426         attr.update(eth_attr)
 427     return post_request(URL_OTN_SERVICE_PATH, {"renderer:input": attr})
 428
 429
 430 def create_ots_oms_request(nodeid: str, lcp: str):
 431     attr = {"input": {
 432         "node-id": nodeid,
 433         "logical-connection-point": lcp}}
 434     return post_request(URL_CREATE_OTS_OMS, attr)
 435
 436
 437 def path_computation_request(requestid: str, servicename: str, serviceaend, servicezend,
 438                              hardconstraints=None, softconstraints=None, metric="hop-count", other_attr=None):
 439     attr = {"service-name": servicename,
 440             "resource-reserve": "true",
 441             "service-handler-header": {"request-id": requestid},
 442             "service-a-end": serviceaend,
 443             "service-z-end": servicezend,
 444             "pce-metric": metric}
 445     if hardconstraints:
 446         attr.update({"hard-constraints": hardconstraints})
 447     if softconstraints:
 448         attr.update({"soft-constraints": softconstraints})
 449     if other_attr:
 450         attr.update(other_attr)
 451     return post_request(URL_PATH_COMPUTATION_REQUEST, {"input": attr})
 452
 453
 454 def tapi_create_connectivity_request(topologyidorname):
 455     return post_request(URL_TAPI_CREATE_CONNECTIVITY, topologyidorname)
 456
 457
 458 def tapi_delete_connectivity_request(serviceidorname):
 459     attr = {
 460         "input": {
 461             "service-id-or-name": serviceidorname}}
 462     return post_request(URL_TAPI_DELETE_CONNECTIVITY, attr)
 463
 464
 465 def tapi_get_topology_details_request(topologyidorname):
 466     attr = {
 467         "input": {
 468             "topology-id-or-name": topologyidorname}}
 469     return post_request(URL_TAPI_TOPOLOGY_DETAILS, attr)
 470
 471
 472 def tapi_get_node_details_request(topologyidorname, nodeidorname):
 473     attr = {
 474         "input": {
 475             "topology-id-or-name": topologyidorname,
 476             "node-id-or-name": nodeidorname}}
 477     return post_request(URL_TAPI_NODE_DETAILS, attr)
 478
 479
 480 def tapi_get_sip_details_request():
 481     return post_request(URL_TAPI_SIP_LIST, "")
 482
 483
 484 def tapi_get_service_list_request():
 485     return post_request(URL_TAPI_SERVICE_LIST, "")
 486
 487
 488 def shutdown_process(process):
 489     if process is not None:
 490         for child in psutil.Process(process.pid).children():
 491             child.send_signal(signal.SIGINT)
 492             child.wait()
 493         process.send_signal(signal.SIGINT)
 494
 495
 496 def start_honeynode(log_file: str, sim):
 497     executable = os.path.join(os.path.dirname(os.path.realpath(__file__)),
 498                               "..", "..", "honeynode", sim[1], "honeynode-simulator", "honeycomb-tpce")
 499     sample_directory = os.path.join(os.path.dirname(os.path.realpath(__file__)),
 500                                     "..", "..", "sample_configs", "openroadm", sim[1])
 501     if os.path.isfile(executable):
 502         with open(log_file, 'w', encoding='utf-8') as outfile:
 503             return subprocess.Popen(
 504                 [executable, SIMS[sim]['port'], os.path.join(sample_directory, SIMS[sim]['configfile'])],
 505                 stdout=outfile, stderr=outfile)
 506     return None
 507
 508
 509 def wait_until_log_contains(log_file, regexp, time_to_wait=60):
 510     # pylint: disable=lost-exception
 511     # pylint: disable=consider-using-with
 512     stringfound = False
 513     filefound = False
 514     line = None
 515     try:
 516         with TimeOut(seconds=time_to_wait):
 517             while not os.path.exists(log_file):
 518                 time.sleep(0.2)
 519             filelogs = open(log_file, 'r', encoding='utf-8')
 520             filelogs.seek(0, 2)
 521             filefound = True
 522             print("Searching for pattern '" + regexp + "' in " + os.path.basename(log_file), end='... ', flush=True)
 523             compiled_regexp = re.compile(regexp)
 524             while True:
 525                 line = filelogs.readline()
 526                 if compiled_regexp.search(line):
 527                     print("Pattern found!", end=' ')
 528                     stringfound = True
 529                     break
 530                 if not line:
 531                     time.sleep(0.1)
 532     except TimeoutError:
 533         print("Pattern not found after " + str(time_to_wait), end=" seconds! ", flush=True)
 534     except PermissionError:
 535         print("Permission Error when trying to access the log file", end=" ... ", flush=True)
 536     finally:
 537         if filefound:
 538             filelogs.close()
 539         else:
 540             print("log file does not exist or is not accessible... ", flush=True)
 541         return stringfound
 542
 543
 544 class TimeOut:
 545     def __init__(self, seconds=1, error_message='Timeout'):
 546         self.seconds = seconds
 547         self.error_message = error_message
 548
 549     def handle_timeout(self, signum, frame):
 550         raise TimeoutError(self.error_message)
 551
 552     def __enter__(self):
 553         signal.signal(signal.SIGALRM, self.handle_timeout)
 554         signal.alarm(self.seconds)
 555
 556     def __exit__(self, type, value, traceback):
 557         # pylint: disable=W0622
 558         signal.alarm(0)