tests/transportpce_tests/common/test_utils.py

   1 #!/usr/bin/env python
   2
   3 ##############################################################################
   4 # Copyright (c) 2020 Orange, Inc. and others.  All rights reserved.
   5 #
   6 # All rights reserved. This program and the accompanying materials
   7 # are made available under the terms of the Apache License, Version 2.0
   8 # which accompanies this distribution, and is available at
   9 # http://www.apache.org/licenses/LICENSE-2.0
  10 ##############################################################################
  11
  12 # pylint: disable=no-member
  13
  14 import json
  15 import os
  16 import sys
  17 import re
  18 import signal
  19 import subprocess
  20 import time
  21
  22 import psutil
  23 import requests
  24
  25 import simulators
  26
  27 SIMS = simulators.SIMS
  28
  29 HONEYNODE_OK_START_MSG = "Netconf SSH endpoint started successfully at 0.0.0.0"
  30 KARAF_OK_START_MSG = re.escape(
  31     "Blueprint container for bundle org.opendaylight.netconf.restconf")+".* was successfully created"
  32 LIGHTY_OK_START_MSG = re.escape("lighty.io and RESTCONF-NETCONF started")
  33
  34 RESTCONF_BASE_URL = "http://localhost:8181/restconf"
  35 ODL_LOGIN = "admin"
  36 ODL_PWD = "admin"
  37 NODES_LOGIN = "admin"
  38 NODES_PWD = "admin"
  39 URL_CONFIG_NETCONF_TOPO = "{}/config/network-topology:network-topology/topology/topology-netconf/"
  40 URL_CONFIG_ORDM_TOPO = "{}/config/ietf-network:networks/network/openroadm-topology/"
  41 URL_CONFIG_OTN_TOPO = "{}/config/ietf-network:networks/network/otn-topology/"
  42 URL_CONFIG_CLLI_NET = "{}/config/ietf-network:networks/network/clli-network/"
  43 URL_CONFIG_ORDM_NET = "{}/config/ietf-network:networks/network/openroadm-network/"
  44 URL_PORTMAPPING = "{}/config/transportpce-portmapping:network/nodes/"
  45 URL_OPER_SERV_LIST = "{}/operational/org-openroadm-service:service-list/"
  46 URL_GET_NBINOTIFICATIONS_SERV = "{}/operations/nbi-notifications:get-notifications-service/"
  47 URL_SERV_CREATE = "{}/operations/org-openroadm-service:service-create"
  48 URL_SERV_DELETE = "{}/operations/org-openroadm-service:service-delete"
  49 URL_SERVICE_PATH = "{}/operations/transportpce-device-renderer:service-path"
  50 URL_OTN_SERVICE_PATH = "{}/operations/transportpce-device-renderer:otn-service-path"
  51 URL_CREATE_OTS_OMS = "{}/operations/transportpce-device-renderer:create-ots-oms"
  52 URL_PATH_COMPUTATION_REQUEST = "{}/operations/transportpce-pce:path-computation-request"
  53 URL_FULL_PORTMAPPING = "{}/config/transportpce-portmapping:network"
  54
  55 TYPE_APPLICATION_JSON = {'Content-Type': 'application/json', 'Accept': 'application/json'}
  56 TYPE_APPLICATION_XML = {'Content-Type': 'application/xml', 'Accept': 'application/xml'}
  57
  58 CODE_SHOULD_BE_200 = 'Http status code should be 200'
  59 CODE_SHOULD_BE_201 = 'Http status code should be 201'
  60
  61 SIM_LOG_DIRECTORY = os.path.join(os.path.dirname(os.path.realpath(__file__)), "log")
  62 KARAF_LOG = os.path.join(
  63     os.path.dirname(os.path.realpath(__file__)),
  64     "..", "..", "..", "karaf", "target", "assembly", "data", "log", "karaf.log")
  65
  66 process_list = []
  67
  68 if "USE_LIGHTY" in os.environ and os.environ['USE_LIGHTY'] == 'True':
  69     TPCE_LOG = 'odl.log'
  70 else:
  71     TPCE_LOG = KARAF_LOG
  72
  73
  74 def start_sims(sims_list):
  75     for sim in sims_list:
  76         print("starting simulator " + sim[0] + " in OpenROADM device version " + sim[1] + "...")
  77         log_file = os.path.join(SIM_LOG_DIRECTORY, SIMS[sim]['logfile'])
  78         process = start_honeynode(log_file, sim)
  79         if wait_until_log_contains(log_file, HONEYNODE_OK_START_MSG, 100):
  80             print("simulator for " + sim[0] + " started")
  81         else:
  82             print("simulator for " + sim[0] + " failed to start")
  83             shutdown_process(process)
  84             for pid in process_list:
  85                 shutdown_process(pid)
  86             sys.exit(3)
  87         process_list.append(process)
  88     return process_list
  89
  90
  91 def start_tpce():
  92     print("starting OpenDaylight...")
  93     if "USE_LIGHTY" in os.environ and os.environ['USE_LIGHTY'] == 'True':
  94         process = start_lighty()
  95         start_msg = LIGHTY_OK_START_MSG
  96     else:
  97         process = start_karaf()
  98         start_msg = KARAF_OK_START_MSG
  99     if wait_until_log_contains(TPCE_LOG, start_msg, time_to_wait=120):
 100         print("OpenDaylight started !")
 101     else:
 102         print("OpenDaylight failed to start !")
 103         shutdown_process(process)
 104         for pid in process_list:
 105             shutdown_process(pid)
 106         sys.exit(1)
 107     process_list.append(process)
 108     return process_list
 109
 110
 111 def start_karaf():
 112     print("starting KARAF TransportPCE build...")
 113     executable = os.path.join(
 114         os.path.dirname(os.path.realpath(__file__)),
 115         "..", "..", "..", "karaf", "target", "assembly", "bin", "karaf")
 116     with open('odl.log', 'w') as outfile:
 117         return subprocess.Popen(
 118             ["sh", executable, "server"], stdout=outfile, stderr=outfile, stdin=None)
 119
 120
 121 def start_lighty():
 122     print("starting LIGHTY.IO TransportPCE build...")
 123     executable = os.path.join(
 124         os.path.dirname(os.path.realpath(__file__)),
 125         "..", "..", "..", "lighty", "target", "tpce",
 126         "clean-start-controller.sh")
 127     with open('odl.log', 'w') as outfile:
 128         return subprocess.Popen(
 129             ["sh", executable], stdout=outfile, stderr=outfile, stdin=None)
 130
 131
 132 def install_karaf_feature(feature_name: str):
 133     print("installing feature " + feature_name)
 134     executable = os.path.join(
 135         os.path.dirname(os.path.realpath(__file__)),
 136         "..", "..", "..", "karaf", "target", "assembly", "bin", "client")
 137     return subprocess.run([executable],
 138                           input='feature:install ' + feature_name + '\n feature:list | grep '
 139                           + feature_name + ' \n logout \n',
 140                           universal_newlines=True, check=False)
 141
 142
 143 def get_request(url):
 144     return requests.request(
 145         "GET", url.format(RESTCONF_BASE_URL),
 146         headers=TYPE_APPLICATION_JSON,
 147         auth=(ODL_LOGIN, ODL_PWD))
 148
 149
 150 def post_request(url, data):
 151     if data:
 152         print(json.dumps(data))
 153         return requests.request(
 154             "POST", url.format(RESTCONF_BASE_URL),
 155             data=json.dumps(data),
 156             headers=TYPE_APPLICATION_JSON,
 157             auth=(ODL_LOGIN, ODL_PWD))
 158
 159     return requests.request(
 160         "POST", url.format(RESTCONF_BASE_URL),
 161         headers=TYPE_APPLICATION_JSON,
 162         auth=(ODL_LOGIN, ODL_PWD))
 163
 164
 165 def post_xmlrequest(url, data):
 166     if data:
 167         return requests.request(
 168             "POST", url.format(RESTCONF_BASE_URL),
 169             data=data,
 170             headers=TYPE_APPLICATION_XML,
 171             auth=(ODL_LOGIN, ODL_PWD))
 172     return None
 173
 174
 175 def put_request(url, data):
 176     return requests.request(
 177         "PUT", url.format(RESTCONF_BASE_URL),
 178         data=json.dumps(data),
 179         headers=TYPE_APPLICATION_JSON,
 180         auth=(ODL_LOGIN, ODL_PWD))
 181
 182
 183 def put_xmlrequest(url, data):
 184     return requests.request(
 185         "PUT", url.format(RESTCONF_BASE_URL),
 186         data=data,
 187         headers=TYPE_APPLICATION_XML,
 188         auth=(ODL_LOGIN, ODL_PWD))
 189
 190 def put_jsonrequest(url, data):
 191     return requests.request(
 192         "PUT", url.format(RESTCONF_BASE_URL),
 193         data=data,
 194         headers=TYPE_APPLICATION_JSON,
 195         auth=(ODL_LOGIN, ODL_PWD))
 196
 197 def rawput_request(url, data):
 198     return requests.request(
 199         "PUT", url.format(RESTCONF_BASE_URL),
 200         data=data,
 201         headers=TYPE_APPLICATION_JSON,
 202         auth=(ODL_LOGIN, ODL_PWD))
 203
 204 def rawpost_request(url, data):
 205     return requests.request(
 206         "POST", url.format(RESTCONF_BASE_URL),
 207         data=data,
 208         headers=TYPE_APPLICATION_JSON,
 209         auth=(ODL_LOGIN, ODL_PWD))
 210
 211
 212 def delete_request(url):
 213     return requests.request(
 214         "DELETE", url.format(RESTCONF_BASE_URL),
 215         headers=TYPE_APPLICATION_JSON,
 216         auth=(ODL_LOGIN, ODL_PWD))
 217
 218
 219 def mount_device(node_id, sim):
 220     url = URL_CONFIG_NETCONF_TOPO + "node/" + node_id
 221     body = {"node": [{
 222         "node-id": node_id,
 223         "netconf-node-topology:username": NODES_LOGIN,
 224         "netconf-node-topology:password": NODES_PWD,
 225         "netconf-node-topology:host": "127.0.0.1",
 226         "netconf-node-topology:port": SIMS[sim]['port'],
 227         "netconf-node-topology:tcp-only": "false",
 228         "netconf-node-topology:pass-through": {}}]}
 229     response = put_request(url, body)
 230     if wait_until_log_contains(TPCE_LOG, re.escape("Triggering notification stream NETCONF for node " + node_id), 60):
 231         print("Node " + node_id + " correctly added to tpce topology", end='... ', flush=True)
 232     else:
 233         print("Node " + node_id + " still not added to tpce topology", end='... ', flush=True)
 234         if response.status_code == requests.codes.ok:
 235             print("It was probably loaded at start-up", end='... ', flush=True)
 236         # TODO an else-clause to abort test would probably be nice here
 237     return response
 238
 239
 240 def unmount_device(node_id):
 241     url = URL_CONFIG_NETCONF_TOPO + "node/" + node_id
 242     response = delete_request(url)
 243     if wait_until_log_contains(TPCE_LOG, re.escape("onDeviceDisConnected: " + node_id), 60):
 244         print("Node " + node_id + " correctly deleted from tpce topology", end='... ', flush=True)
 245     else:
 246         print("Node " + node_id + " still not deleted from tpce topology", end='... ', flush=True)
 247     return response
 248
 249
 250 def connect_xpdr_to_rdm_request(xpdr_node: str, xpdr_num: str, network_num: str,
 251                                 rdm_node: str, srg_num: str, termination_num: str):
 252     url = "{}/operations/transportpce-networkutils:init-xpdr-rdm-links"
 253     data = {
 254         "networkutils:input": {
 255             "networkutils:links-input": {
 256                 "networkutils:xpdr-node": xpdr_node,
 257                 "networkutils:xpdr-num": xpdr_num,
 258                 "networkutils:network-num": network_num,
 259                 "networkutils:rdm-node": rdm_node,
 260                 "networkutils:srg-num": srg_num,
 261                 "networkutils:termination-point-num": termination_num
 262             }
 263         }
 264     }
 265     return post_request(url, data)
 266
 267
 268 def connect_rdm_to_xpdr_request(xpdr_node: str, xpdr_num: str, network_num: str,
 269                                 rdm_node: str, srg_num: str, termination_num: str):
 270     url = "{}/operations/transportpce-networkutils:init-rdm-xpdr-links"
 271     data = {
 272         "networkutils:input": {
 273             "networkutils:links-input": {
 274                 "networkutils:xpdr-node": xpdr_node,
 275                 "networkutils:xpdr-num": xpdr_num,
 276                 "networkutils:network-num": network_num,
 277                 "networkutils:rdm-node": rdm_node,
 278                 "networkutils:srg-num": srg_num,
 279                 "networkutils:termination-point-num": termination_num
 280             }
 281         }
 282     }
 283     return post_request(url, data)
 284
 285
 286 def check_netconf_node_request(node: str, suffix: str):
 287     url = URL_CONFIG_NETCONF_TOPO + (
 288         "node/" + node + "/yang-ext:mount/org-openroadm-device:org-openroadm-device/" + suffix
 289     )
 290     return get_request(url)
 291
 292
 293 def get_netconf_oper_request(node: str):
 294     url = "{}/operational/network-topology:network-topology/topology/topology-netconf/node/" + node
 295     return get_request(url)
 296
 297
 298 def get_ordm_topo_request(suffix: str):
 299     url = URL_CONFIG_ORDM_TOPO + suffix
 300     return get_request(url)
 301
 302
 303 def add_oms_attr_request(link: str, attr):
 304     url = URL_CONFIG_ORDM_TOPO + (
 305         "ietf-network-topology:link/" + link + "/org-openroadm-network-topology:OMS-attributes/span"
 306     )
 307     return put_request(url, attr)
 308
 309
 310 def del_oms_attr_request(link: str):
 311     url = URL_CONFIG_ORDM_TOPO + (
 312         "ietf-network-topology:link/" + link + "/org-openroadm-network-topology:OMS-attributes/span"
 313     )
 314     return delete_request(url)
 315
 316
 317 def get_clli_net_request():
 318     return get_request(URL_CONFIG_CLLI_NET)
 319
 320
 321 def get_ordm_net_request():
 322     return get_request(URL_CONFIG_ORDM_NET)
 323
 324
 325 def get_otn_topo_request():
 326     return get_request(URL_CONFIG_OTN_TOPO)
 327
 328
 329 def del_link_request(link: str):
 330     url = URL_CONFIG_ORDM_TOPO + ("ietf-network-topology:link/" + link)
 331     return delete_request(url)
 332
 333
 334 def del_node_request(node: str):
 335     url = URL_CONFIG_CLLI_NET + ("node/" + node)
 336     return delete_request(url)
 337
 338
 339 def portmapping_request(suffix: str):
 340     url = URL_PORTMAPPING + suffix
 341     return get_request(url)
 342
 343
 344 def get_notifications_service_request(attr):
 345     return post_request(URL_GET_NBINOTIFICATIONS_SERV, attr)
 346
 347
 348 def get_service_list_request(suffix: str):
 349     url = URL_OPER_SERV_LIST + suffix
 350     return get_request(url)
 351
 352
 353 def service_create_request(attr):
 354     return post_request(URL_SERV_CREATE, attr)
 355
 356
 357 def service_delete_request(servicename: str,
 358                            requestid="e3028bae-a90f-4ddd-a83f-cf224eba0e58",
 359                            notificationurl="http://localhost:8585/NotificationServer/notify"):
 360     attr = {"input": {
 361         "sdnc-request-header": {
 362             "request-id": requestid,
 363             "rpc-action": "service-delete",
 364             "request-system-id": "appname",
 365             "notification-url": notificationurl},
 366         "service-delete-req-info": {
 367             "service-name": servicename,
 368             "tail-retention": "no"}}}
 369     return post_request(URL_SERV_DELETE, attr)
 370
 371
 372 def service_path_request(operation: str, servicename: str, wavenumber: str, nodes, centerfreq: str,
 373                          slotwidth: int, minfreq: float, maxfreq: float, lowerslotnumber: int,
 374                          higherslotnumber: int):
 375     attr = {"renderer:input": {
 376         "renderer:service-name": servicename,
 377         "renderer:wave-number": wavenumber,
 378         "renderer:modulation-format": "dp-qpsk",
 379         "renderer:operation": operation,
 380         "renderer:nodes": nodes,
 381         "renderer:center-freq": centerfreq,
 382         "renderer:width": slotwidth,
 383         "renderer:min-freq": minfreq,
 384         "renderer:max-freq": maxfreq,
 385         "renderer:lower-spectral-slot-number": lowerslotnumber,
 386         "renderer:higher-spectral-slot-number": higherslotnumber}}
 387     return post_request(URL_SERVICE_PATH, attr)
 388
 389
 390 def otn_service_path_request(operation: str, servicename: str, servicerate: str, servicetype: str, nodes,
 391                              eth_attr=None):
 392     attr = {"service-name": servicename,
 393             "operation": operation,
 394             "service-rate": servicerate,
 395             "service-type": servicetype,
 396             "nodes": nodes}
 397     if eth_attr:
 398         attr.update(eth_attr)
 399     return post_request(URL_OTN_SERVICE_PATH, {"renderer:input": attr})
 400
 401
 402 def create_ots_oms_request(nodeid: str, lcp: str):
 403     attr = {"input": {
 404         "node-id": nodeid,
 405         "logical-connection-point": lcp}}
 406     return post_request(URL_CREATE_OTS_OMS, attr)
 407
 408
 409 def path_computation_request(requestid: str, servicename: str, serviceaend, servicezend,
 410                              hardconstraints=None, softconstraints=None, metric="hop-count", other_attr=None):
 411     attr = {"service-name": servicename,
 412             "resource-reserve": "true",
 413             "service-handler-header": {"request-id": requestid},
 414             "service-a-end": serviceaend,
 415             "service-z-end": servicezend,
 416             "pce-metric": metric}
 417     if hardconstraints:
 418         attr.update({"hard-constraints": hardconstraints})
 419     if softconstraints:
 420         attr.update({"soft-constraints": softconstraints})
 421     if other_attr:
 422         attr.update(other_attr)
 423     return post_request(URL_PATH_COMPUTATION_REQUEST, {"input": attr})
 424
 425
 426 def shutdown_process(process):
 427     if process is not None:
 428         for child in psutil.Process(process.pid).children():
 429             child.send_signal(signal.SIGINT)
 430             child.wait()
 431         process.send_signal(signal.SIGINT)
 432
 433
 434 def start_honeynode(log_file: str, sim):
 435     executable = os.path.join(os.path.dirname(os.path.realpath(__file__)),
 436                               "..", "..", "honeynode", sim[1], "honeynode-simulator", "honeycomb-tpce")
 437     sample_directory = os.path.join(os.path.dirname(os.path.realpath(__file__)),
 438                                     "..", "..", "sample_configs", "openroadm", sim[1])
 439     if os.path.isfile(executable):
 440         with open(log_file, 'w') as outfile:
 441             return subprocess.Popen(
 442                 [executable, SIMS[sim]['port'], os.path.join(sample_directory, SIMS[sim]['configfile'])],
 443                 stdout=outfile, stderr=outfile)
 444     return None
 445
 446
 447 def wait_until_log_contains(log_file, regexp, time_to_wait=20):
 448     # pylint: disable=lost-exception
 449     stringfound = False
 450     filefound = False
 451     line = None
 452     try:
 453         with TimeOut(seconds=time_to_wait):
 454             while not os.path.exists(log_file):
 455                 time.sleep(0.2)
 456             filelogs = open(log_file, 'r')
 457             filelogs.seek(0, 2)
 458             filefound = True
 459             print("Searching for pattern '" + regexp + "' in " + os.path.basename(log_file), end='... ', flush=True)
 460             compiled_regexp = re.compile(regexp)
 461             while True:
 462                 line = filelogs.readline()
 463                 if compiled_regexp.search(line):
 464                     print("Pattern found!", end=' ')
 465                     stringfound = True
 466                     break
 467                 if not line:
 468                     time.sleep(0.1)
 469     except TimeoutError:
 470         print("Pattern not found after " + str(time_to_wait), end=" seconds! ", flush=True)
 471     except PermissionError:
 472         print("Permission Error when trying to access the log file", end=" ... ", flush=True)
 473     finally:
 474         if filefound:
 475             filelogs.close()
 476         else:
 477             print("log file does not exist or is not accessible... ", flush=True)
 478         return stringfound
 479
 480
 481 class TimeOut:
 482     def __init__(self, seconds=1, error_message='Timeout'):
 483         self.seconds = seconds
 484         self.error_message = error_message
 485
 486     def handle_timeout(self, signum, frame):
 487         raise TimeoutError(self.error_message)
 488
 489     def __enter__(self):
 490         signal.signal(signal.SIGALRM, self.handle_timeout)
 491         signal.alarm(self.seconds)
 492
 493     def __exit__(self, type, value, traceback):
 494         # pylint: disable=W0622
 495         signal.alarm(0)