tests/transportpce_tests/common/test_utils.py

   1 #!/usr/bin/env python
   2
   3 ##############################################################################
   4 # Copyright (c) 2020 Orange, Inc. and others.  All rights reserved.
   5 #
   6 # All rights reserved. This program and the accompanying materials
   7 # are made available under the terms of the Apache License, Version 2.0
   8 # which accompanies this distribution, and is available at
   9 # http://www.apache.org/licenses/LICENSE-2.0
  10 ##############################################################################
  11
  12 # pylint: disable=no-member
  13
  14 import json
  15 import os
  16 import sys
  17 import re
  18 import signal
  19 import subprocess
  20 import time
  21
  22 import psutil
  23 import requests
  24
  25 import simulators
  26
  27 SIMS = simulators.SIMS
  28
  29 HONEYNODE_OK_START_MSG = "Netconf SSH endpoint started successfully at 0.0.0.0"
  30 KARAF_OK_START_MSG = re.escape(
  31     "Blueprint container for bundle org.opendaylight.netconf.restconf")+".* was successfully created"
  32 LIGHTY_OK_START_MSG = re.escape("lighty.io and RESTCONF-NETCONF started")
  33
  34 ODL_LOGIN = "admin"
  35 ODL_PWD = "admin"
  36 NODES_LOGIN = "admin"
  37 NODES_PWD = "admin"
  38 URL_CONFIG_NETCONF_TOPO = "{}/config/network-topology:network-topology/topology/topology-netconf/"
  39 URL_CONFIG_ORDM_TOPO = "{}/config/ietf-network:networks/network/openroadm-topology/"
  40 URL_CONFIG_OTN_TOPO = "{}/config/ietf-network:networks/network/otn-topology/"
  41 URL_CONFIG_CLLI_NET = "{}/config/ietf-network:networks/network/clli-network/"
  42 URL_CONFIG_ORDM_NET = "{}/config/ietf-network:networks/network/openroadm-network/"
  43 URL_PORTMAPPING = "{}/config/transportpce-portmapping:network/nodes/"
  44 URL_OPER_SERV_LIST = "{}/operational/org-openroadm-service:service-list/"
  45 URL_GET_NBINOTIFICATIONS_SERV = "{}/operations/nbi-notifications:get-notifications-service/"
  46 URL_GET_NBINOTIFICATIONS_ALARM_SERV = "{}/operations/nbi-notifications:get-notifications-alarm-service/"
  47 URL_SERV_CREATE = "{}/operations/org-openroadm-service:service-create"
  48 URL_SERV_DELETE = "{}/operations/org-openroadm-service:service-delete"
  49 URL_SERVICE_PATH = "{}/operations/transportpce-device-renderer:service-path"
  50 URL_OTN_SERVICE_PATH = "{}/operations/transportpce-device-renderer:otn-service-path"
  51 URL_CREATE_OTS_OMS = "{}/operations/transportpce-device-renderer:create-ots-oms"
  52 URL_PATH_COMPUTATION_REQUEST = "{}/operations/transportpce-pce:path-computation-request"
  53 URL_FULL_PORTMAPPING = "{}/config/transportpce-portmapping:network"
  54
  55 TYPE_APPLICATION_JSON = {'Content-Type': 'application/json', 'Accept': 'application/json'}
  56 TYPE_APPLICATION_XML = {'Content-Type': 'application/xml', 'Accept': 'application/xml'}
  57
  58 CODE_SHOULD_BE_200 = 'Http status code should be 200'
  59 CODE_SHOULD_BE_201 = 'Http status code should be 201'
  60
  61 SIM_LOG_DIRECTORY = os.path.join(os.path.dirname(os.path.realpath(__file__)), "log")
  62 KARAF_LOG = os.path.join(
  63     os.path.dirname(os.path.realpath(__file__)),
  64     "..", "..", "..", "karaf", "target", "assembly", "data", "log", "karaf.log")
  65
  66 process_list = []
  67
  68
  69 if "USE_ODL_ALT_RESTCONF_PORT" in os.environ:
  70     RESTCONF_BASE_URL = "http://localhost:" + os.environ['USE_ODL_ALT_RESTCONF_PORT'] + "/restconf"
  71 else:
  72     RESTCONF_BASE_URL = "http://localhost:8181/restconf"
  73
  74 if "USE_LIGHTY" in os.environ and os.environ['USE_LIGHTY'] == 'True':
  75     TPCE_LOG = 'odl.log'
  76 else:
  77     TPCE_LOG = KARAF_LOG
  78
  79
  80 def start_sims(sims_list):
  81     for sim in sims_list:
  82         print("starting simulator " + sim[0] + " in OpenROADM device version " + sim[1] + "...")
  83         log_file = os.path.join(SIM_LOG_DIRECTORY, SIMS[sim]['logfile'])
  84         process = start_honeynode(log_file, sim)
  85         if wait_until_log_contains(log_file, HONEYNODE_OK_START_MSG, 100):
  86             print("simulator for " + sim[0] + " started")
  87         else:
  88             print("simulator for " + sim[0] + " failed to start")
  89             shutdown_process(process)
  90             for pid in process_list:
  91                 shutdown_process(pid)
  92             sys.exit(3)
  93         process_list.append(process)
  94     return process_list
  95
  96
  97 def start_tpce():
  98     print("starting OpenDaylight...")
  99     if "USE_LIGHTY" in os.environ and os.environ['USE_LIGHTY'] == 'True':
 100         process = start_lighty()
 101         start_msg = LIGHTY_OK_START_MSG
 102     else:
 103         process = start_karaf()
 104         start_msg = KARAF_OK_START_MSG
 105     if wait_until_log_contains(TPCE_LOG, start_msg, time_to_wait=120):
 106         print("OpenDaylight started !")
 107     else:
 108         print("OpenDaylight failed to start !")
 109         shutdown_process(process)
 110         for pid in process_list:
 111             shutdown_process(pid)
 112         sys.exit(1)
 113     process_list.append(process)
 114     return process_list
 115
 116
 117 def start_karaf():
 118     print("starting KARAF TransportPCE build...")
 119     executable = os.path.join(
 120         os.path.dirname(os.path.realpath(__file__)),
 121         "..", "..", "..", "karaf", "target", "assembly", "bin", "karaf")
 122     with open('odl.log', 'w') as outfile:
 123         return subprocess.Popen(
 124             ["sh", executable, "server"], stdout=outfile, stderr=outfile, stdin=None)
 125
 126
 127 def start_lighty():
 128     print("starting LIGHTY.IO TransportPCE build...")
 129     executable = os.path.join(
 130         os.path.dirname(os.path.realpath(__file__)),
 131         "..", "..", "..", "lighty", "target", "tpce",
 132         "clean-start-controller.sh")
 133     with open('odl.log', 'w') as outfile:
 134         return subprocess.Popen(
 135             ["sh", executable], stdout=outfile, stderr=outfile, stdin=None)
 136
 137
 138 def install_karaf_feature(feature_name: str):
 139     print("installing feature " + feature_name)
 140     executable = os.path.join(
 141         os.path.dirname(os.path.realpath(__file__)),
 142         "..", "..", "..", "karaf", "target", "assembly", "bin", "client")
 143     return subprocess.run([executable],
 144                           input='feature:install ' + feature_name + '\n feature:list | grep '
 145                           + feature_name + ' \n logout \n',
 146                           universal_newlines=True, check=False)
 147
 148
 149 def get_request(url):
 150     return requests.request(
 151         "GET", url.format(RESTCONF_BASE_URL),
 152         headers=TYPE_APPLICATION_JSON,
 153         auth=(ODL_LOGIN, ODL_PWD))
 154
 155
 156 def post_request(url, data):
 157     if data:
 158         print(json.dumps(data))
 159         return requests.request(
 160             "POST", url.format(RESTCONF_BASE_URL),
 161             data=json.dumps(data),
 162             headers=TYPE_APPLICATION_JSON,
 163             auth=(ODL_LOGIN, ODL_PWD))
 164
 165     return requests.request(
 166         "POST", url.format(RESTCONF_BASE_URL),
 167         headers=TYPE_APPLICATION_JSON,
 168         auth=(ODL_LOGIN, ODL_PWD))
 169
 170
 171 def post_xmlrequest(url, data):
 172     if data:
 173         return requests.request(
 174             "POST", url.format(RESTCONF_BASE_URL),
 175             data=data,
 176             headers=TYPE_APPLICATION_XML,
 177             auth=(ODL_LOGIN, ODL_PWD))
 178     return None
 179
 180
 181 def put_request(url, data):
 182     return requests.request(
 183         "PUT", url.format(RESTCONF_BASE_URL),
 184         data=json.dumps(data),
 185         headers=TYPE_APPLICATION_JSON,
 186         auth=(ODL_LOGIN, ODL_PWD))
 187
 188
 189 def put_xmlrequest(url, data):
 190     return requests.request(
 191         "PUT", url.format(RESTCONF_BASE_URL),
 192         data=data,
 193         headers=TYPE_APPLICATION_XML,
 194         auth=(ODL_LOGIN, ODL_PWD))
 195
 196 def put_jsonrequest(url, data):
 197     return requests.request(
 198         "PUT", url.format(RESTCONF_BASE_URL),
 199         data=data,
 200         headers=TYPE_APPLICATION_JSON,
 201         auth=(ODL_LOGIN, ODL_PWD))
 202
 203 def rawput_request(url, data):
 204     return requests.request(
 205         "PUT", url.format(RESTCONF_BASE_URL),
 206         data=data,
 207         headers=TYPE_APPLICATION_JSON,
 208         auth=(ODL_LOGIN, ODL_PWD))
 209
 210 def rawpost_request(url, data):
 211     return requests.request(
 212         "POST", url.format(RESTCONF_BASE_URL),
 213         data=data,
 214         headers=TYPE_APPLICATION_JSON,
 215         auth=(ODL_LOGIN, ODL_PWD))
 216
 217
 218 def delete_request(url):
 219     return requests.request(
 220         "DELETE", url.format(RESTCONF_BASE_URL),
 221         headers=TYPE_APPLICATION_JSON,
 222         auth=(ODL_LOGIN, ODL_PWD))
 223
 224
 225 def mount_device(node_id, sim):
 226     url = URL_CONFIG_NETCONF_TOPO + "node/" + node_id
 227     body = {"node": [{
 228         "node-id": node_id,
 229         "netconf-node-topology:username": NODES_LOGIN,
 230         "netconf-node-topology:password": NODES_PWD,
 231         "netconf-node-topology:host": "127.0.0.1",
 232         "netconf-node-topology:port": SIMS[sim]['port'],
 233         "netconf-node-topology:tcp-only": "false",
 234         "netconf-node-topology:pass-through": {}}]}
 235     response = put_request(url, body)
 236     if wait_until_log_contains(TPCE_LOG, re.escape("Triggering notification stream NETCONF for node " + node_id), 60):
 237         print("Node " + node_id + " correctly added to tpce topology", end='... ', flush=True)
 238     else:
 239         print("Node " + node_id + " still not added to tpce topology", end='... ', flush=True)
 240         if response.status_code == requests.codes.ok:
 241             print("It was probably loaded at start-up", end='... ', flush=True)
 242         # TODO an else-clause to abort test would probably be nice here
 243     return response
 244
 245
 246 def unmount_device(node_id):
 247     url = URL_CONFIG_NETCONF_TOPO + "node/" + node_id
 248     response = delete_request(url)
 249     if wait_until_log_contains(TPCE_LOG, re.escape("onDeviceDisConnected: " + node_id), 60):
 250         print("Node " + node_id + " correctly deleted from tpce topology", end='... ', flush=True)
 251     else:
 252         print("Node " + node_id + " still not deleted from tpce topology", end='... ', flush=True)
 253     return response
 254
 255
 256 def connect_xpdr_to_rdm_request(xpdr_node: str, xpdr_num: str, network_num: str,
 257                                 rdm_node: str, srg_num: str, termination_num: str):
 258     url = "{}/operations/transportpce-networkutils:init-xpdr-rdm-links"
 259     data = {
 260         "networkutils:input": {
 261             "networkutils:links-input": {
 262                 "networkutils:xpdr-node": xpdr_node,
 263                 "networkutils:xpdr-num": xpdr_num,
 264                 "networkutils:network-num": network_num,
 265                 "networkutils:rdm-node": rdm_node,
 266                 "networkutils:srg-num": srg_num,
 267                 "networkutils:termination-point-num": termination_num
 268             }
 269         }
 270     }
 271     return post_request(url, data)
 272
 273
 274 def connect_rdm_to_xpdr_request(xpdr_node: str, xpdr_num: str, network_num: str,
 275                                 rdm_node: str, srg_num: str, termination_num: str):
 276     url = "{}/operations/transportpce-networkutils:init-rdm-xpdr-links"
 277     data = {
 278         "networkutils:input": {
 279             "networkutils:links-input": {
 280                 "networkutils:xpdr-node": xpdr_node,
 281                 "networkutils:xpdr-num": xpdr_num,
 282                 "networkutils:network-num": network_num,
 283                 "networkutils:rdm-node": rdm_node,
 284                 "networkutils:srg-num": srg_num,
 285                 "networkutils:termination-point-num": termination_num
 286             }
 287         }
 288     }
 289     return post_request(url, data)
 290
 291
 292 def check_netconf_node_request(node: str, suffix: str):
 293     url = URL_CONFIG_NETCONF_TOPO + (
 294         "node/" + node + "/yang-ext:mount/org-openroadm-device:org-openroadm-device/" + suffix
 295     )
 296     return get_request(url)
 297
 298
 299 def get_netconf_oper_request(node: str):
 300     url = "{}/operational/network-topology:network-topology/topology/topology-netconf/node/" + node
 301     return get_request(url)
 302
 303
 304 def get_ordm_topo_request(suffix: str):
 305     url = URL_CONFIG_ORDM_TOPO + suffix
 306     return get_request(url)
 307
 308
 309 def add_oms_attr_request(link: str, attr):
 310     url = URL_CONFIG_ORDM_TOPO + (
 311         "ietf-network-topology:link/" + link + "/org-openroadm-network-topology:OMS-attributes/span"
 312     )
 313     return put_request(url, attr)
 314
 315
 316 def del_oms_attr_request(link: str):
 317     url = URL_CONFIG_ORDM_TOPO + (
 318         "ietf-network-topology:link/" + link + "/org-openroadm-network-topology:OMS-attributes/span"
 319     )
 320     return delete_request(url)
 321
 322
 323 def get_clli_net_request():
 324     return get_request(URL_CONFIG_CLLI_NET)
 325
 326
 327 def get_ordm_net_request():
 328     return get_request(URL_CONFIG_ORDM_NET)
 329
 330
 331 def get_otn_topo_request():
 332     return get_request(URL_CONFIG_OTN_TOPO)
 333
 334
 335 def del_link_request(link: str):
 336     url = URL_CONFIG_ORDM_TOPO + ("ietf-network-topology:link/" + link)
 337     return delete_request(url)
 338
 339
 340 def del_node_request(node: str):
 341     url = URL_CONFIG_CLLI_NET + ("node/" + node)
 342     return delete_request(url)
 343
 344
 345 def portmapping_request(suffix: str):
 346     url = URL_PORTMAPPING + suffix
 347     return get_request(url)
 348
 349
 350 def get_notifications_service_request(attr):
 351     return post_request(URL_GET_NBINOTIFICATIONS_SERV, attr)
 352
 353
 354 def get_notifications_alarm_service_request(attr):
 355     return post_request(URL_GET_NBINOTIFICATIONS_ALARM_SERV, attr)
 356
 357
 358 def get_service_list_request(suffix: str):
 359     url = URL_OPER_SERV_LIST + suffix
 360     return get_request(url)
 361
 362
 363 def service_create_request(attr):
 364     return post_request(URL_SERV_CREATE, attr)
 365
 366
 367 def service_delete_request(servicename: str,
 368                            requestid="e3028bae-a90f-4ddd-a83f-cf224eba0e58",
 369                            notificationurl="http://localhost:8585/NotificationServer/notify"):
 370     attr = {"input": {
 371         "sdnc-request-header": {
 372             "request-id": requestid,
 373             "rpc-action": "service-delete",
 374             "request-system-id": "appname",
 375             "notification-url": notificationurl},
 376         "service-delete-req-info": {
 377             "service-name": servicename,
 378             "tail-retention": "no"}}}
 379     return post_request(URL_SERV_DELETE, attr)
 380
 381
 382 def service_path_request(operation: str, servicename: str, wavenumber: str, nodes, centerfreq: str,
 383                          slotwidth: int, minfreq: float, maxfreq: float, lowerslotnumber: int,
 384                          higherslotnumber: int):
 385     attr = {"renderer:input": {
 386         "renderer:service-name": servicename,
 387         "renderer:wave-number": wavenumber,
 388         "renderer:modulation-format": "dp-qpsk",
 389         "renderer:operation": operation,
 390         "renderer:nodes": nodes,
 391         "renderer:center-freq": centerfreq,
 392         "renderer:width": slotwidth,
 393         "renderer:min-freq": minfreq,
 394         "renderer:max-freq": maxfreq,
 395         "renderer:lower-spectral-slot-number": lowerslotnumber,
 396         "renderer:higher-spectral-slot-number": higherslotnumber}}
 397     return post_request(URL_SERVICE_PATH, attr)
 398
 399
 400 def otn_service_path_request(operation: str, servicename: str, servicerate: str, serviceformat: str, nodes,
 401                              eth_attr=None):
 402     attr = {"service-name": servicename,
 403             "operation": operation,
 404             "service-rate": servicerate,
 405             "service-format": serviceformat,
 406             "nodes": nodes}
 407     if eth_attr:
 408         attr.update(eth_attr)
 409     return post_request(URL_OTN_SERVICE_PATH, {"renderer:input": attr})
 410
 411
 412 def create_ots_oms_request(nodeid: str, lcp: str):
 413     attr = {"input": {
 414         "node-id": nodeid,
 415         "logical-connection-point": lcp}}
 416     return post_request(URL_CREATE_OTS_OMS, attr)
 417
 418
 419 def path_computation_request(requestid: str, servicename: str, serviceaend, servicezend,
 420                              hardconstraints=None, softconstraints=None, metric="hop-count", other_attr=None):
 421     attr = {"service-name": servicename,
 422             "resource-reserve": "true",
 423             "service-handler-header": {"request-id": requestid},
 424             "service-a-end": serviceaend,
 425             "service-z-end": servicezend,
 426             "pce-metric": metric}
 427     if hardconstraints:
 428         attr.update({"hard-constraints": hardconstraints})
 429     if softconstraints:
 430         attr.update({"soft-constraints": softconstraints})
 431     if other_attr:
 432         attr.update(other_attr)
 433     return post_request(URL_PATH_COMPUTATION_REQUEST, {"input": attr})
 434
 435
 436 def shutdown_process(process):
 437     if process is not None:
 438         for child in psutil.Process(process.pid).children():
 439             child.send_signal(signal.SIGINT)
 440             child.wait()
 441         process.send_signal(signal.SIGINT)
 442
 443
 444 def start_honeynode(log_file: str, sim):
 445     executable = os.path.join(os.path.dirname(os.path.realpath(__file__)),
 446                               "..", "..", "honeynode", sim[1], "honeynode-simulator", "honeycomb-tpce")
 447     sample_directory = os.path.join(os.path.dirname(os.path.realpath(__file__)),
 448                                     "..", "..", "sample_configs", "openroadm", sim[1])
 449     if os.path.isfile(executable):
 450         with open(log_file, 'w') as outfile:
 451             return subprocess.Popen(
 452                 [executable, SIMS[sim]['port'], os.path.join(sample_directory, SIMS[sim]['configfile'])],
 453                 stdout=outfile, stderr=outfile)
 454     return None
 455
 456
 457 def wait_until_log_contains(log_file, regexp, time_to_wait=20):
 458     # pylint: disable=lost-exception
 459     stringfound = False
 460     filefound = False
 461     line = None
 462     try:
 463         with TimeOut(seconds=time_to_wait):
 464             while not os.path.exists(log_file):
 465                 time.sleep(0.2)
 466             filelogs = open(log_file, 'r')
 467             filelogs.seek(0, 2)
 468             filefound = True
 469             print("Searching for pattern '" + regexp + "' in " + os.path.basename(log_file), end='... ', flush=True)
 470             compiled_regexp = re.compile(regexp)
 471             while True:
 472                 line = filelogs.readline()
 473                 if compiled_regexp.search(line):
 474                     print("Pattern found!", end=' ')
 475                     stringfound = True
 476                     break
 477                 if not line:
 478                     time.sleep(0.1)
 479     except TimeoutError:
 480         print("Pattern not found after " + str(time_to_wait), end=" seconds! ", flush=True)
 481     except PermissionError:
 482         print("Permission Error when trying to access the log file", end=" ... ", flush=True)
 483     finally:
 484         if filefound:
 485             filelogs.close()
 486         else:
 487             print("log file does not exist or is not accessible... ", flush=True)
 488         return stringfound
 489
 490
 491 class TimeOut:
 492     def __init__(self, seconds=1, error_message='Timeout'):
 493         self.seconds = seconds
 494         self.error_message = error_message
 495
 496     def handle_timeout(self, signum, frame):
 497         raise TimeoutError(self.error_message)
 498
 499     def __enter__(self):
 500         signal.signal(signal.SIGALRM, self.handle_timeout)
 501         signal.alarm(self.seconds)
 502
 503     def __exit__(self, type, value, traceback):
 504         # pylint: disable=W0622
 505         signal.alarm(0)