tests/transportpce_tests/common/test_utils.py

   1 #!/usr/bin/env python
   2
   3 ##############################################################################
   4 # Copyright (c) 2020 Orange, Inc. and others.  All rights reserved.
   5 #
   6 # All rights reserved. This program and the accompanying materials
   7 # are made available under the terms of the Apache License, Version 2.0
   8 # which accompanies this distribution, and is available at
   9 # http://www.apache.org/licenses/LICENSE-2.0
  10 ##############################################################################
  11
  12 # pylint: disable=no-member
  13
  14 import json
  15 import os
  16 import sys
  17 import re
  18 import signal
  19 import subprocess
  20 import time
  21
  22 import psutil
  23 import requests
  24
  25 import simulators
  26
  27 SIMS = simulators.SIMS
  28
  29 HONEYNODE_OK_START_MSG = "Netconf SSH endpoint started successfully at 0.0.0.0"
  30 KARAF_OK_START_MSG = re.escape(
  31     "Blueprint container for bundle org.opendaylight.netconf.restconf")+".* was successfully created"
  32 LIGHTY_OK_START_MSG = re.escape("lighty.io and RESTCONF-NETCONF started")
  33
  34 ODL_LOGIN = "admin"
  35 ODL_PWD = "admin"
  36 NODES_LOGIN = "admin"
  37 NODES_PWD = "admin"
  38 URL_CONFIG_NETCONF_TOPO = "{}/config/network-topology:network-topology/topology/topology-netconf/"
  39 URL_CONFIG_ORDM_TOPO = "{}/config/ietf-network:networks/network/openroadm-topology/"
  40 URL_CONFIG_OTN_TOPO = "{}/config/ietf-network:networks/network/otn-topology/"
  41 URL_CONFIG_CLLI_NET = "{}/config/ietf-network:networks/network/clli-network/"
  42 URL_CONFIG_ORDM_NET = "{}/config/ietf-network:networks/network/openroadm-network/"
  43 URL_PORTMAPPING = "{}/config/transportpce-portmapping:network/nodes/"
  44 URL_OPER_SERV_LIST = "{}/operational/org-openroadm-service:service-list/"
  45 URL_GET_NBINOTIFICATIONS_PROCESS_SERV = "{}/operations/nbi-notifications:get-notifications-process-service/"
  46 URL_GET_NBINOTIFICATIONS_ALARM_SERV = "{}/operations/nbi-notifications:get-notifications-alarm-service/"
  47 URL_SERV_CREATE = "{}/operations/org-openroadm-service:service-create"
  48 URL_SERV_DELETE = "{}/operations/org-openroadm-service:service-delete"
  49 URL_SERVICE_PATH = "{}/operations/transportpce-device-renderer:service-path"
  50 URL_OTN_SERVICE_PATH = "{}/operations/transportpce-device-renderer:otn-service-path"
  51 URL_CREATE_OTS_OMS = "{}/operations/transportpce-device-renderer:create-ots-oms"
  52 URL_PATH_COMPUTATION_REQUEST = "{}/operations/transportpce-pce:path-computation-request"
  53 URL_FULL_PORTMAPPING = "{}/config/transportpce-portmapping:network"
  54
  55 TYPE_APPLICATION_JSON = {'Content-Type': 'application/json', 'Accept': 'application/json'}
  56 TYPE_APPLICATION_XML = {'Content-Type': 'application/xml', 'Accept': 'application/xml'}
  57
  58 CODE_SHOULD_BE_200 = 'Http status code should be 200'
  59 CODE_SHOULD_BE_201 = 'Http status code should be 201'
  60
  61 SIM_LOG_DIRECTORY = os.path.join(os.path.dirname(os.path.realpath(__file__)), "log")
  62
  63 process_list = []
  64
  65
  66 if "USE_ODL_ALT_RESTCONF_PORT" in os.environ:
  67     RESTCONF_BASE_URL = "http://localhost:" + os.environ['USE_ODL_ALT_RESTCONF_PORT'] + "/restconf"
  68 else:
  69     RESTCONF_BASE_URL = "http://localhost:8181/restconf"
  70
  71 if "USE_ODL_ALT_KARAF_INSTALL_DIR" in os.environ:
  72     KARAF_INSTALLDIR = os.environ['USE_ODL_ALT_KARAF_INSTALL_DIR']
  73 else:
  74     KARAF_INSTALLDIR = "karaf"
  75
  76 KARAF_LOG = os.path.join(
  77     os.path.dirname(os.path.realpath(__file__)),
  78     "..", "..", "..", KARAF_INSTALLDIR, "target", "assembly", "data", "log", "karaf.log")
  79
  80 if "USE_LIGHTY" in os.environ and os.environ['USE_LIGHTY'] == 'True':
  81     TPCE_LOG = 'odl-' + str(os.getpid()) + '.log'
  82 else:
  83     TPCE_LOG = KARAF_LOG
  84
  85
  86 def start_sims(sims_list):
  87     for sim in sims_list:
  88         print("starting simulator " + sim[0] + " in OpenROADM device version " + sim[1] + "...")
  89         log_file = os.path.join(SIM_LOG_DIRECTORY, SIMS[sim]['logfile'])
  90         process = start_honeynode(log_file, sim)
  91         if wait_until_log_contains(log_file, HONEYNODE_OK_START_MSG, 100):
  92             print("simulator for " + sim[0] + " started")
  93         else:
  94             print("simulator for " + sim[0] + " failed to start")
  95             shutdown_process(process)
  96             for pid in process_list:
  97                 shutdown_process(pid)
  98             sys.exit(3)
  99         process_list.append(process)
 100     return process_list
 101
 102
 103 def start_tpce():
 104     print("starting OpenDaylight...")
 105     if "USE_LIGHTY" in os.environ and os.environ['USE_LIGHTY'] == 'True':
 106         process = start_lighty()
 107         start_msg = LIGHTY_OK_START_MSG
 108     else:
 109         process = start_karaf()
 110         start_msg = KARAF_OK_START_MSG
 111     if wait_until_log_contains(TPCE_LOG, start_msg, time_to_wait=300):
 112         print("OpenDaylight started !")
 113     else:
 114         print("OpenDaylight failed to start !")
 115         shutdown_process(process)
 116         for pid in process_list:
 117             shutdown_process(pid)
 118         sys.exit(1)
 119     process_list.append(process)
 120     return process_list
 121
 122
 123 def start_karaf():
 124     print("starting KARAF TransportPCE build...")
 125     executable = os.path.join(
 126         os.path.dirname(os.path.realpath(__file__)),
 127         "..", "..", "..", KARAF_INSTALLDIR, "target", "assembly", "bin", "karaf")
 128     with open('odl.log', 'w') as outfile:
 129         return subprocess.Popen(
 130             ["sh", executable, "server"], stdout=outfile, stderr=outfile, stdin=None)
 131
 132
 133 def start_lighty():
 134     print("starting LIGHTY.IO TransportPCE build...")
 135     executable = os.path.join(
 136         os.path.dirname(os.path.realpath(__file__)),
 137         "..", "..", "..", "lighty", "target", "tpce",
 138         "clean-start-controller.sh")
 139     with open(TPCE_LOG, 'w') as outfile:
 140         return subprocess.Popen(
 141             ["sh", executable], stdout=outfile, stderr=outfile, stdin=None)
 142
 143
 144 def install_karaf_feature(feature_name: str):
 145     print("installing feature " + feature_name)
 146     executable = os.path.join(
 147         os.path.dirname(os.path.realpath(__file__)),
 148         "..", "..", "..", KARAF_INSTALLDIR, "target", "assembly", "bin", "client")
 149     return subprocess.run([executable],
 150                           input='feature:install ' + feature_name + '\n feature:list | grep '
 151                           + feature_name + ' \n logout \n',
 152                           universal_newlines=True, check=False)
 153
 154
 155 def get_request(url):
 156     return requests.request(
 157         "GET", url.format(RESTCONF_BASE_URL),
 158         headers=TYPE_APPLICATION_JSON,
 159         auth=(ODL_LOGIN, ODL_PWD))
 160
 161
 162 def post_request(url, data):
 163     if data:
 164         print(json.dumps(data))
 165         return requests.request(
 166             "POST", url.format(RESTCONF_BASE_URL),
 167             data=json.dumps(data),
 168             headers=TYPE_APPLICATION_JSON,
 169             auth=(ODL_LOGIN, ODL_PWD))
 170
 171     return requests.request(
 172         "POST", url.format(RESTCONF_BASE_URL),
 173         headers=TYPE_APPLICATION_JSON,
 174         auth=(ODL_LOGIN, ODL_PWD))
 175
 176
 177 def post_xmlrequest(url, data):
 178     if data:
 179         return requests.request(
 180             "POST", url.format(RESTCONF_BASE_URL),
 181             data=data,
 182             headers=TYPE_APPLICATION_XML,
 183             auth=(ODL_LOGIN, ODL_PWD))
 184     return None
 185
 186
 187 def put_request(url, data):
 188     return requests.request(
 189         "PUT", url.format(RESTCONF_BASE_URL),
 190         data=json.dumps(data),
 191         headers=TYPE_APPLICATION_JSON,
 192         auth=(ODL_LOGIN, ODL_PWD))
 193
 194
 195 def put_xmlrequest(url, data):
 196     return requests.request(
 197         "PUT", url.format(RESTCONF_BASE_URL),
 198         data=data,
 199         headers=TYPE_APPLICATION_XML,
 200         auth=(ODL_LOGIN, ODL_PWD))
 201
 202 def put_jsonrequest(url, data):
 203     return requests.request(
 204         "PUT", url.format(RESTCONF_BASE_URL),
 205         data=data,
 206         headers=TYPE_APPLICATION_JSON,
 207         auth=(ODL_LOGIN, ODL_PWD))
 208
 209 def rawput_request(url, data):
 210     return requests.request(
 211         "PUT", url.format(RESTCONF_BASE_URL),
 212         data=data,
 213         headers=TYPE_APPLICATION_JSON,
 214         auth=(ODL_LOGIN, ODL_PWD))
 215
 216 def rawpost_request(url, data):
 217     return requests.request(
 218         "POST", url.format(RESTCONF_BASE_URL),
 219         data=data,
 220         headers=TYPE_APPLICATION_JSON,
 221         auth=(ODL_LOGIN, ODL_PWD))
 222
 223
 224 def delete_request(url):
 225     return requests.request(
 226         "DELETE", url.format(RESTCONF_BASE_URL),
 227         headers=TYPE_APPLICATION_JSON,
 228         auth=(ODL_LOGIN, ODL_PWD))
 229
 230
 231 def mount_device(node_id, sim):
 232     url = URL_CONFIG_NETCONF_TOPO + "node/" + node_id
 233     body = {"node": [{
 234         "node-id": node_id,
 235         "netconf-node-topology:username": NODES_LOGIN,
 236         "netconf-node-topology:password": NODES_PWD,
 237         "netconf-node-topology:host": "127.0.0.1",
 238         "netconf-node-topology:port": SIMS[sim]['port'],
 239         "netconf-node-topology:tcp-only": "false",
 240         "netconf-node-topology:pass-through": {}}]}
 241     response = put_request(url, body)
 242     if wait_until_log_contains(TPCE_LOG, re.escape("Triggering notification stream NETCONF for node " + node_id), 180):
 243         print("Node " + node_id + " correctly added to tpce topology", end='... ', flush=True)
 244     else:
 245         print("Node " + node_id + " still not added to tpce topology", end='... ', flush=True)
 246         if response.status_code == requests.codes.ok:
 247             print("It was probably loaded at start-up", end='... ', flush=True)
 248         # TODO an else-clause to abort test would probably be nice here
 249     return response
 250
 251
 252 def unmount_device(node_id):
 253     url = URL_CONFIG_NETCONF_TOPO + "node/" + node_id
 254     response = delete_request(url)
 255     if wait_until_log_contains(TPCE_LOG, re.escape("onDeviceDisConnected: " + node_id), 180):
 256         print("Node " + node_id + " correctly deleted from tpce topology", end='... ', flush=True)
 257     else:
 258         print("Node " + node_id + " still not deleted from tpce topology", end='... ', flush=True)
 259     return response
 260
 261
 262 def connect_xpdr_to_rdm_request(xpdr_node: str, xpdr_num: str, network_num: str,
 263                                 rdm_node: str, srg_num: str, termination_num: str):
 264     url = "{}/operations/transportpce-networkutils:init-xpdr-rdm-links"
 265     data = {
 266         "networkutils:input": {
 267             "networkutils:links-input": {
 268                 "networkutils:xpdr-node": xpdr_node,
 269                 "networkutils:xpdr-num": xpdr_num,
 270                 "networkutils:network-num": network_num,
 271                 "networkutils:rdm-node": rdm_node,
 272                 "networkutils:srg-num": srg_num,
 273                 "networkutils:termination-point-num": termination_num
 274             }
 275         }
 276     }
 277     return post_request(url, data)
 278
 279
 280 def connect_rdm_to_xpdr_request(xpdr_node: str, xpdr_num: str, network_num: str,
 281                                 rdm_node: str, srg_num: str, termination_num: str):
 282     url = "{}/operations/transportpce-networkutils:init-rdm-xpdr-links"
 283     data = {
 284         "networkutils:input": {
 285             "networkutils:links-input": {
 286                 "networkutils:xpdr-node": xpdr_node,
 287                 "networkutils:xpdr-num": xpdr_num,
 288                 "networkutils:network-num": network_num,
 289                 "networkutils:rdm-node": rdm_node,
 290                 "networkutils:srg-num": srg_num,
 291                 "networkutils:termination-point-num": termination_num
 292             }
 293         }
 294     }
 295     return post_request(url, data)
 296
 297
 298 def check_netconf_node_request(node: str, suffix: str):
 299     url = URL_CONFIG_NETCONF_TOPO + (
 300         "node/" + node + "/yang-ext:mount/org-openroadm-device:org-openroadm-device/" + suffix
 301     )
 302     return get_request(url)
 303
 304
 305 def get_netconf_oper_request(node: str):
 306     url = "{}/operational/network-topology:network-topology/topology/topology-netconf/node/" + node
 307     return get_request(url)
 308
 309
 310 def get_ordm_topo_request(suffix: str):
 311     url = URL_CONFIG_ORDM_TOPO + suffix
 312     return get_request(url)
 313
 314
 315 def add_oms_attr_request(link: str, attr):
 316     url = URL_CONFIG_ORDM_TOPO + (
 317         "ietf-network-topology:link/" + link + "/org-openroadm-network-topology:OMS-attributes/span"
 318     )
 319     return put_request(url, attr)
 320
 321
 322 def del_oms_attr_request(link: str):
 323     url = URL_CONFIG_ORDM_TOPO + (
 324         "ietf-network-topology:link/" + link + "/org-openroadm-network-topology:OMS-attributes/span"
 325     )
 326     return delete_request(url)
 327
 328
 329 def get_clli_net_request():
 330     return get_request(URL_CONFIG_CLLI_NET)
 331
 332
 333 def get_ordm_net_request():
 334     return get_request(URL_CONFIG_ORDM_NET)
 335
 336
 337 def get_otn_topo_request():
 338     return get_request(URL_CONFIG_OTN_TOPO)
 339
 340
 341 def del_link_request(link: str):
 342     url = URL_CONFIG_ORDM_TOPO + ("ietf-network-topology:link/" + link)
 343     return delete_request(url)
 344
 345
 346 def del_node_request(node: str):
 347     url = URL_CONFIG_CLLI_NET + ("node/" + node)
 348     return delete_request(url)
 349
 350
 351 def portmapping_request(suffix: str):
 352     url = URL_PORTMAPPING + suffix
 353     return get_request(url)
 354
 355
 356 def get_notifications_process_service_request(attr):
 357     return post_request(URL_GET_NBINOTIFICATIONS_PROCESS_SERV, attr)
 358
 359
 360 def get_notifications_alarm_service_request(attr):
 361     return post_request(URL_GET_NBINOTIFICATIONS_ALARM_SERV, attr)
 362
 363
 364 def get_service_list_request(suffix: str):
 365     url = URL_OPER_SERV_LIST + suffix
 366     return get_request(url)
 367
 368
 369 def service_create_request(attr):
 370     return post_request(URL_SERV_CREATE, attr)
 371
 372
 373 def service_delete_request(servicename: str,
 374                            requestid="e3028bae-a90f-4ddd-a83f-cf224eba0e58",
 375                            notificationurl="http://localhost:8585/NotificationServer/notify"):
 376     attr = {"input": {
 377         "sdnc-request-header": {
 378             "request-id": requestid,
 379             "rpc-action": "service-delete",
 380             "request-system-id": "appname",
 381             "notification-url": notificationurl},
 382         "service-delete-req-info": {
 383             "service-name": servicename,
 384             "tail-retention": "no"}}}
 385     return post_request(URL_SERV_DELETE, attr)
 386
 387
 388 def service_path_request(operation: str, servicename: str, wavenumber: str, nodes, centerfreq: str,
 389                          slotwidth: int, minfreq: float, maxfreq: float, lowerslotnumber: int,
 390                          higherslotnumber: int):
 391     attr = {"renderer:input": {
 392         "renderer:service-name": servicename,
 393         "renderer:wave-number": wavenumber,
 394         "renderer:modulation-format": "dp-qpsk",
 395         "renderer:operation": operation,
 396         "renderer:nodes": nodes,
 397         "renderer:center-freq": centerfreq,
 398         "renderer:width": slotwidth,
 399         "renderer:min-freq": minfreq,
 400         "renderer:max-freq": maxfreq,
 401         "renderer:lower-spectral-slot-number": lowerslotnumber,
 402         "renderer:higher-spectral-slot-number": higherslotnumber}}
 403     return post_request(URL_SERVICE_PATH, attr)
 404
 405
 406 def otn_service_path_request(operation: str, servicename: str, servicerate: str, serviceformat: str, nodes,
 407                              eth_attr=None):
 408     attr = {"service-name": servicename,
 409             "operation": operation,
 410             "service-rate": servicerate,
 411             "service-format": serviceformat,
 412             "nodes": nodes}
 413     if eth_attr:
 414         attr.update(eth_attr)
 415     return post_request(URL_OTN_SERVICE_PATH, {"renderer:input": attr})
 416
 417
 418 def create_ots_oms_request(nodeid: str, lcp: str):
 419     attr = {"input": {
 420         "node-id": nodeid,
 421         "logical-connection-point": lcp}}
 422     return post_request(URL_CREATE_OTS_OMS, attr)
 423
 424
 425 def path_computation_request(requestid: str, servicename: str, serviceaend, servicezend,
 426                              hardconstraints=None, softconstraints=None, metric="hop-count", other_attr=None):
 427     attr = {"service-name": servicename,
 428             "resource-reserve": "true",
 429             "service-handler-header": {"request-id": requestid},
 430             "service-a-end": serviceaend,
 431             "service-z-end": servicezend,
 432             "pce-metric": metric}
 433     if hardconstraints:
 434         attr.update({"hard-constraints": hardconstraints})
 435     if softconstraints:
 436         attr.update({"soft-constraints": softconstraints})
 437     if other_attr:
 438         attr.update(other_attr)
 439     return post_request(URL_PATH_COMPUTATION_REQUEST, {"input": attr})
 440
 441
 442 def shutdown_process(process):
 443     if process is not None:
 444         for child in psutil.Process(process.pid).children():
 445             child.send_signal(signal.SIGINT)
 446             child.wait()
 447         process.send_signal(signal.SIGINT)
 448
 449
 450 def start_honeynode(log_file: str, sim):
 451     executable = os.path.join(os.path.dirname(os.path.realpath(__file__)),
 452                               "..", "..", "honeynode", sim[1], "honeynode-simulator", "honeycomb-tpce")
 453     sample_directory = os.path.join(os.path.dirname(os.path.realpath(__file__)),
 454                                     "..", "..", "sample_configs", "openroadm", sim[1])
 455     if os.path.isfile(executable):
 456         with open(log_file, 'w') as outfile:
 457             return subprocess.Popen(
 458                 [executable, SIMS[sim]['port'], os.path.join(sample_directory, SIMS[sim]['configfile'])],
 459                 stdout=outfile, stderr=outfile)
 460     return None
 461
 462
 463 def wait_until_log_contains(log_file, regexp, time_to_wait=60):
 464     # pylint: disable=lost-exception
 465     stringfound = False
 466     filefound = False
 467     line = None
 468     try:
 469         with TimeOut(seconds=time_to_wait):
 470             while not os.path.exists(log_file):
 471                 time.sleep(0.2)
 472             filelogs = open(log_file, 'r')
 473             filelogs.seek(0, 2)
 474             filefound = True
 475             print("Searching for pattern '" + regexp + "' in " + os.path.basename(log_file), end='... ', flush=True)
 476             compiled_regexp = re.compile(regexp)
 477             while True:
 478                 line = filelogs.readline()
 479                 if compiled_regexp.search(line):
 480                     print("Pattern found!", end=' ')
 481                     stringfound = True
 482                     break
 483                 if not line:
 484                     time.sleep(0.1)
 485     except TimeoutError:
 486         print("Pattern not found after " + str(time_to_wait), end=" seconds! ", flush=True)
 487     except PermissionError:
 488         print("Permission Error when trying to access the log file", end=" ... ", flush=True)
 489     finally:
 490         if filefound:
 491             filelogs.close()
 492         else:
 493             print("log file does not exist or is not accessible... ", flush=True)
 494         return stringfound
 495
 496
 497 class TimeOut:
 498     def __init__(self, seconds=1, error_message='Timeout'):
 499         self.seconds = seconds
 500         self.error_message = error_message
 501
 502     def handle_timeout(self, signum, frame):
 503         raise TimeoutError(self.error_message)
 504
 505     def __enter__(self):
 506         signal.signal(signal.SIGALRM, self.handle_timeout)
 507         signal.alarm(self.seconds)
 508
 509     def __exit__(self, type, value, traceback):
 510         # pylint: disable=W0622
 511         signal.alarm(0)