tests/transportpce_tests/common/test_utils.py

   1 #!/usr/bin/env python
   2
   3 ##############################################################################
   4 # Copyright (c) 2020 Orange, Inc. and others.  All rights reserved.
   5 #
   6 # All rights reserved. This program and the accompanying materials
   7 # are made available under the terms of the Apache License, Version 2.0
   8 # which accompanies this distribution, and is available at
   9 # http://www.apache.org/licenses/LICENSE-2.0
  10 ##############################################################################
  11
  12 # pylint: disable=no-member
  13
  14 import json
  15 import os
  16 import sys
  17 import re
  18 import signal
  19 import subprocess
  20 import time
  21
  22 import psutil
  23 import requests
  24
  25 import simulators
  26
  27 SIMS = simulators.SIMS
  28
  29 HONEYNODE_OK_START_MSG = "Netconf SSH endpoint started successfully at 0.0.0.0"
  30 KARAF_OK_START_MSG = re.escape(
  31     "Blueprint container for bundle org.opendaylight.netconf.restconf")+".* was successfully created"
  32 LIGHTY_OK_START_MSG = re.escape("lighty.io and RESTCONF-NETCONF started")
  33
  34 ODL_LOGIN = "admin"
  35 ODL_PWD = "admin"
  36 NODES_LOGIN = "admin"
  37 NODES_PWD = "admin"
  38 URL_CONFIG_NETCONF_TOPO = "{}/config/network-topology:network-topology/topology/topology-netconf/"
  39 URL_CONFIG_ORDM_TOPO = "{}/config/ietf-network:networks/network/openroadm-topology/"
  40 URL_CONFIG_OTN_TOPO = "{}/config/ietf-network:networks/network/otn-topology/"
  41 URL_CONFIG_CLLI_NET = "{}/config/ietf-network:networks/network/clli-network/"
  42 URL_CONFIG_ORDM_NET = "{}/config/ietf-network:networks/network/openroadm-network/"
  43 URL_PORTMAPPING = "{}/config/transportpce-portmapping:network/nodes/"
  44 URL_OPER_SERV_LIST = "{}/operational/org-openroadm-service:service-list/"
  45 URL_GET_NBINOTIFICATIONS_PROCESS_SERV = "{}/operations/nbi-notifications:get-notifications-process-service/"
  46 URL_GET_NBINOTIFICATIONS_ALARM_SERV = "{}/operations/nbi-notifications:get-notifications-alarm-service/"
  47 URL_SERV_CREATE = "{}/operations/org-openroadm-service:service-create"
  48 URL_SERV_DELETE = "{}/operations/org-openroadm-service:service-delete"
  49 URL_SERVICE_PATH = "{}/operations/transportpce-device-renderer:service-path"
  50 URL_OTN_SERVICE_PATH = "{}/operations/transportpce-device-renderer:otn-service-path"
  51 URL_TAPI_CREATE_CONNECTIVITY = "{}/operations/tapi-connectivity:create-connectivity-service"
  52 URL_TAPI_DELETE_CONNECTIVITY = "{}/operations/tapi-connectivity:delete-connectivity-service"
  53 URL_CREATE_OTS_OMS = "{}/operations/transportpce-device-renderer:create-ots-oms"
  54 URL_PATH_COMPUTATION_REQUEST = "{}/operations/transportpce-pce:path-computation-request"
  55 URL_FULL_PORTMAPPING = "{}/config/transportpce-portmapping:network"
  56 URL_TAPI_TOPOLOGY_DETAILS = "{}/operations/tapi-topology:get-topology-details"
  57 URL_TAPI_NODE_DETAILS = "{}/operations/tapi-topology:get-node-details"
  58 URL_TAPI_SIP_LIST = "{}/operations/tapi-common:get-service-interface-point-list"
  59 URL_TAPI_SERVICE_LIST = "{}/operations/tapi-connectivity:get-connectivity-service-list"
  60
  61 TYPE_APPLICATION_JSON = {'Content-Type': 'application/json', 'Accept': 'application/json'}
  62 TYPE_APPLICATION_XML = {'Content-Type': 'application/xml', 'Accept': 'application/xml'}
  63
  64 CODE_SHOULD_BE_200 = 'Http status code should be 200'
  65 CODE_SHOULD_BE_201 = 'Http status code should be 201'
  66
  67 SIM_LOG_DIRECTORY = os.path.join(os.path.dirname(os.path.realpath(__file__)), "log")
  68
  69 process_list = []
  70
  71
  72 if "USE_ODL_ALT_RESTCONF_PORT" in os.environ:
  73     RESTCONF_BASE_URL = "http://localhost:" + os.environ['USE_ODL_ALT_RESTCONF_PORT'] + "/restconf"
  74 else:
  75     RESTCONF_BASE_URL = "http://localhost:8181/restconf"
  76
  77 if "USE_ODL_ALT_KARAF_INSTALL_DIR" in os.environ:
  78     KARAF_INSTALLDIR = os.environ['USE_ODL_ALT_KARAF_INSTALL_DIR']
  79 else:
  80     KARAF_INSTALLDIR = "karaf"
  81
  82 KARAF_LOG = os.path.join(
  83     os.path.dirname(os.path.realpath(__file__)),
  84     "..", "..", "..", KARAF_INSTALLDIR, "target", "assembly", "data", "log", "karaf.log")
  85
  86 if "USE_LIGHTY" in os.environ and os.environ['USE_LIGHTY'] == 'True':
  87     TPCE_LOG = 'odl-' + str(os.getpid()) + '.log'
  88 else:
  89     TPCE_LOG = KARAF_LOG
  90
  91
  92 def start_sims(sims_list):
  93     for sim in sims_list:
  94         print("starting simulator " + sim[0] + " in OpenROADM device version " + sim[1] + "...")
  95         log_file = os.path.join(SIM_LOG_DIRECTORY, SIMS[sim]['logfile'])
  96         process = start_honeynode(log_file, sim)
  97         if wait_until_log_contains(log_file, HONEYNODE_OK_START_MSG, 100):
  98             print("simulator for " + sim[0] + " started")
  99         else:
 100             print("simulator for " + sim[0] + " failed to start")
 101             shutdown_process(process)
 102             for pid in process_list:
 103                 shutdown_process(pid)
 104             sys.exit(3)
 105         process_list.append(process)
 106     return process_list
 107
 108
 109 def start_tpce():
 110     print("starting OpenDaylight...")
 111     if "USE_LIGHTY" in os.environ and os.environ['USE_LIGHTY'] == 'True':
 112         process = start_lighty()
 113         start_msg = LIGHTY_OK_START_MSG
 114     else:
 115         process = start_karaf()
 116         start_msg = KARAF_OK_START_MSG
 117     if wait_until_log_contains(TPCE_LOG, start_msg, time_to_wait=300):
 118         print("OpenDaylight started !")
 119     else:
 120         print("OpenDaylight failed to start !")
 121         shutdown_process(process)
 122         for pid in process_list:
 123             shutdown_process(pid)
 124         sys.exit(1)
 125     process_list.append(process)
 126     return process_list
 127
 128
 129 def start_karaf():
 130     print("starting KARAF TransportPCE build...")
 131     executable = os.path.join(
 132         os.path.dirname(os.path.realpath(__file__)),
 133         "..", "..", "..", KARAF_INSTALLDIR, "target", "assembly", "bin", "karaf")
 134     with open('odl.log', 'w') as outfile:
 135         return subprocess.Popen(
 136             ["sh", executable, "server"], stdout=outfile, stderr=outfile, stdin=None)
 137
 138
 139 def start_lighty():
 140     print("starting LIGHTY.IO TransportPCE build...")
 141     executable = os.path.join(
 142         os.path.dirname(os.path.realpath(__file__)),
 143         "..", "..", "..", "lighty", "target", "tpce",
 144         "clean-start-controller.sh")
 145     with open(TPCE_LOG, 'w') as outfile:
 146         return subprocess.Popen(
 147             ["sh", executable], stdout=outfile, stderr=outfile, stdin=None)
 148
 149
 150 def install_karaf_feature(feature_name: str):
 151     print("installing feature " + feature_name)
 152     executable = os.path.join(
 153         os.path.dirname(os.path.realpath(__file__)),
 154         "..", "..", "..", KARAF_INSTALLDIR, "target", "assembly", "bin", "client")
 155     return subprocess.run([executable],
 156                           input='feature:install ' + feature_name + '\n feature:list | grep '
 157                           + feature_name + ' \n logout \n',
 158                           universal_newlines=True, check=False)
 159
 160
 161 def get_request(url):
 162     return requests.request(
 163         "GET", url.format(RESTCONF_BASE_URL),
 164         headers=TYPE_APPLICATION_JSON,
 165         auth=(ODL_LOGIN, ODL_PWD))
 166
 167
 168 def post_request(url, data):
 169     if data:
 170         print(json.dumps(data))
 171         return requests.request(
 172             "POST", url.format(RESTCONF_BASE_URL),
 173             data=json.dumps(data),
 174             headers=TYPE_APPLICATION_JSON,
 175             auth=(ODL_LOGIN, ODL_PWD))
 176
 177     return requests.request(
 178         "POST", url.format(RESTCONF_BASE_URL),
 179         headers=TYPE_APPLICATION_JSON,
 180         auth=(ODL_LOGIN, ODL_PWD))
 181
 182
 183 def post_xmlrequest(url, data):
 184     if data:
 185         return requests.request(
 186             "POST", url.format(RESTCONF_BASE_URL),
 187             data=data,
 188             headers=TYPE_APPLICATION_XML,
 189             auth=(ODL_LOGIN, ODL_PWD))
 190     return None
 191
 192
 193 def put_request(url, data):
 194     return requests.request(
 195         "PUT", url.format(RESTCONF_BASE_URL),
 196         data=json.dumps(data),
 197         headers=TYPE_APPLICATION_JSON,
 198         auth=(ODL_LOGIN, ODL_PWD))
 199
 200
 201 def put_xmlrequest(url, data):
 202     return requests.request(
 203         "PUT", url.format(RESTCONF_BASE_URL),
 204         data=data,
 205         headers=TYPE_APPLICATION_XML,
 206         auth=(ODL_LOGIN, ODL_PWD))
 207
 208
 209 def put_jsonrequest(url, data):
 210     return requests.request(
 211         "PUT", url.format(RESTCONF_BASE_URL),
 212         data=data,
 213         headers=TYPE_APPLICATION_JSON,
 214         auth=(ODL_LOGIN, ODL_PWD))
 215
 216
 217 def rawput_request(url, data):
 218     return requests.request(
 219         "PUT", url.format(RESTCONF_BASE_URL),
 220         data=data,
 221         headers=TYPE_APPLICATION_JSON,
 222         auth=(ODL_LOGIN, ODL_PWD))
 223
 224
 225 def rawpost_request(url, data):
 226     return requests.request(
 227         "POST", url.format(RESTCONF_BASE_URL),
 228         data=data,
 229         headers=TYPE_APPLICATION_JSON,
 230         auth=(ODL_LOGIN, ODL_PWD))
 231
 232
 233 def delete_request(url):
 234     return requests.request(
 235         "DELETE", url.format(RESTCONF_BASE_URL),
 236         headers=TYPE_APPLICATION_JSON,
 237         auth=(ODL_LOGIN, ODL_PWD))
 238
 239
 240 def mount_device(node_id, sim):
 241     url = URL_CONFIG_NETCONF_TOPO + "node/" + node_id
 242     body = {"node": [{
 243         "node-id": node_id,
 244         "netconf-node-topology:username": NODES_LOGIN,
 245         "netconf-node-topology:password": NODES_PWD,
 246         "netconf-node-topology:host": "127.0.0.1",
 247         "netconf-node-topology:port": SIMS[sim]['port'],
 248         "netconf-node-topology:tcp-only": "false",
 249         "netconf-node-topology:pass-through": {}}]}
 250     response = put_request(url, body)
 251     if wait_until_log_contains(TPCE_LOG, re.escape("Triggering notification stream NETCONF for node " + node_id), 180):
 252         print("Node " + node_id + " correctly added to tpce topology", end='... ', flush=True)
 253     else:
 254         print("Node " + node_id + " still not added to tpce topology", end='... ', flush=True)
 255         if response.status_code == requests.codes.ok:
 256             print("It was probably loaded at start-up", end='... ', flush=True)
 257         # TODO an else-clause to abort test would probably be nice here
 258     return response
 259
 260
 261 def unmount_device(node_id):
 262     url = URL_CONFIG_NETCONF_TOPO + "node/" + node_id
 263     response = delete_request(url)
 264     if wait_until_log_contains(TPCE_LOG, re.escape("onDeviceDisConnected: " + node_id), 180):
 265         print("Node " + node_id + " correctly deleted from tpce topology", end='... ', flush=True)
 266     else:
 267         print("Node " + node_id + " still not deleted from tpce topology", end='... ', flush=True)
 268     return response
 269
 270
 271 def connect_xpdr_to_rdm_request(xpdr_node: str, xpdr_num: str, network_num: str,
 272                                 rdm_node: str, srg_num: str, termination_num: str):
 273     url = "{}/operations/transportpce-networkutils:init-xpdr-rdm-links"
 274     data = {
 275         "networkutils:input": {
 276             "networkutils:links-input": {
 277                 "networkutils:xpdr-node": xpdr_node,
 278                 "networkutils:xpdr-num": xpdr_num,
 279                 "networkutils:network-num": network_num,
 280                 "networkutils:rdm-node": rdm_node,
 281                 "networkutils:srg-num": srg_num,
 282                 "networkutils:termination-point-num": termination_num
 283             }
 284         }
 285     }
 286     return post_request(url, data)
 287
 288
 289 def connect_rdm_to_xpdr_request(xpdr_node: str, xpdr_num: str, network_num: str,
 290                                 rdm_node: str, srg_num: str, termination_num: str):
 291     url = "{}/operations/transportpce-networkutils:init-rdm-xpdr-links"
 292     data = {
 293         "networkutils:input": {
 294             "networkutils:links-input": {
 295                 "networkutils:xpdr-node": xpdr_node,
 296                 "networkutils:xpdr-num": xpdr_num,
 297                 "networkutils:network-num": network_num,
 298                 "networkutils:rdm-node": rdm_node,
 299                 "networkutils:srg-num": srg_num,
 300                 "networkutils:termination-point-num": termination_num
 301             }
 302         }
 303     }
 304     return post_request(url, data)
 305
 306
 307 def check_netconf_node_request(node: str, suffix: str):
 308     url = URL_CONFIG_NETCONF_TOPO + (
 309         "node/" + node + "/yang-ext:mount/org-openroadm-device:org-openroadm-device/" + suffix
 310     )
 311     return get_request(url)
 312
 313
 314 def get_netconf_oper_request(node: str):
 315     url = "{}/operational/network-topology:network-topology/topology/topology-netconf/node/" + node
 316     return get_request(url)
 317
 318
 319 def get_ordm_topo_request(suffix: str):
 320     url = URL_CONFIG_ORDM_TOPO + suffix
 321     return get_request(url)
 322
 323
 324 def add_oms_attr_request(link: str, attr):
 325     url = URL_CONFIG_ORDM_TOPO + (
 326         "ietf-network-topology:link/" + link + "/org-openroadm-network-topology:OMS-attributes/span"
 327     )
 328     return put_request(url, attr)
 329
 330
 331 def del_oms_attr_request(link: str):
 332     url = URL_CONFIG_ORDM_TOPO + (
 333         "ietf-network-topology:link/" + link + "/org-openroadm-network-topology:OMS-attributes/span"
 334     )
 335     return delete_request(url)
 336
 337
 338 def get_clli_net_request():
 339     return get_request(URL_CONFIG_CLLI_NET)
 340
 341
 342 def get_ordm_net_request():
 343     return get_request(URL_CONFIG_ORDM_NET)
 344
 345
 346 def get_otn_topo_request():
 347     return get_request(URL_CONFIG_OTN_TOPO)
 348
 349
 350 def del_link_request(link: str):
 351     url = URL_CONFIG_ORDM_TOPO + ("ietf-network-topology:link/" + link)
 352     return delete_request(url)
 353
 354
 355 def del_node_request(node: str):
 356     url = URL_CONFIG_CLLI_NET + ("node/" + node)
 357     return delete_request(url)
 358
 359
 360 def portmapping_request(suffix: str):
 361     url = URL_PORTMAPPING + suffix
 362     return get_request(url)
 363
 364
 365 def get_notifications_process_service_request(attr):
 366     return post_request(URL_GET_NBINOTIFICATIONS_PROCESS_SERV, attr)
 367
 368
 369 def get_notifications_alarm_service_request(attr):
 370     return post_request(URL_GET_NBINOTIFICATIONS_ALARM_SERV, attr)
 371
 372
 373 def get_service_list_request(suffix: str):
 374     url = URL_OPER_SERV_LIST + suffix
 375     return get_request(url)
 376
 377
 378 def service_create_request(attr):
 379     return post_request(URL_SERV_CREATE, attr)
 380
 381
 382 def service_delete_request(servicename: str,
 383                            requestid="e3028bae-a90f-4ddd-a83f-cf224eba0e58",
 384                            notificationurl="http://localhost:8585/NotificationServer/notify"):
 385     attr = {"input": {
 386         "sdnc-request-header": {
 387             "request-id": requestid,
 388             "rpc-action": "service-delete",
 389             "request-system-id": "appname",
 390             "notification-url": notificationurl},
 391         "service-delete-req-info": {
 392             "service-name": servicename,
 393             "tail-retention": "no"}}}
 394     return post_request(URL_SERV_DELETE, attr)
 395
 396 def service_path_request(operation: str, servicename: str, wavenumber: str, nodes, centerfreq: str,
 397                          slotwidth: int, minfreq: float, maxfreq: float, lowerslotnumber: int,
 398                          higherslotnumber: int):
 399     attr = {"renderer:input": {
 400         "renderer:service-name": servicename,
 401         "renderer:wave-number": wavenumber,
 402         "renderer:modulation-format": "dp-qpsk",
 403         "renderer:operation": operation,
 404         "renderer:nodes": nodes,
 405         "renderer:center-freq": centerfreq,
 406         "renderer:width": slotwidth,
 407         "renderer:min-freq": minfreq,
 408         "renderer:max-freq": maxfreq,
 409         "renderer:lower-spectral-slot-number": lowerslotnumber,
 410         "renderer:higher-spectral-slot-number": higherslotnumber}}
 411     return post_request(URL_SERVICE_PATH, attr)
 412
 413
 414 def otn_service_path_request(operation: str, servicename: str, servicerate: str, serviceformat: str, nodes,
 415                              eth_attr=None):
 416     attr = {"service-name": servicename,
 417             "operation": operation,
 418             "service-rate": servicerate,
 419             "service-format": serviceformat,
 420             "nodes": nodes}
 421     if eth_attr:
 422         attr.update(eth_attr)
 423     return post_request(URL_OTN_SERVICE_PATH, {"renderer:input": attr})
 424
 425
 426 def create_ots_oms_request(nodeid: str, lcp: str):
 427     attr = {"input": {
 428         "node-id": nodeid,
 429         "logical-connection-point": lcp}}
 430     return post_request(URL_CREATE_OTS_OMS, attr)
 431
 432
 433 def path_computation_request(requestid: str, servicename: str, serviceaend, servicezend,
 434                              hardconstraints=None, softconstraints=None, metric="hop-count", other_attr=None):
 435     attr = {"service-name": servicename,
 436             "resource-reserve": "true",
 437             "service-handler-header": {"request-id": requestid},
 438             "service-a-end": serviceaend,
 439             "service-z-end": servicezend,
 440             "pce-metric": metric}
 441     if hardconstraints:
 442         attr.update({"hard-constraints": hardconstraints})
 443     if softconstraints:
 444         attr.update({"soft-constraints": softconstraints})
 445     if other_attr:
 446         attr.update(other_attr)
 447     return post_request(URL_PATH_COMPUTATION_REQUEST, {"input": attr})
 448
 449 def tapi_create_connectivity_request(topologyidorname):
 450     return post_request(URL_TAPI_CREATE_CONNECTIVITY, topologyidorname)
 451
 452 def tapi_delete_connectivity_request(serviceidorname):
 453     attr = {
 454         "input": {
 455             "service-id-or-name": serviceidorname}}
 456     return post_request(URL_TAPI_DELETE_CONNECTIVITY , attr)
 457
 458 def tapi_get_topology_details_request(topologyidorname):
 459     attr = {
 460         "input": {
 461             "topology-id-or-name": topologyidorname}}
 462     return post_request(URL_TAPI_TOPOLOGY_DETAILS , attr)
 463
 464 def tapi_get_node_details_request(topologyidorname, nodeidorname):
 465     attr = {
 466         "input": {
 467             "topology-id-or-name": topologyidorname,
 468             "node-id-or-name": nodeidorname}}
 469     return post_request(URL_TAPI_NODE_DETAILS, attr)
 470
 471 def tapi_get_sip_details_request():
 472     return post_request(URL_TAPI_SIP_LIST, "")
 473
 474 def tapi_get_service_list_request():
 475     return post_request(URL_TAPI_SERVICE_LIST, "")
 476
 477 def shutdown_process(process):
 478     if process is not None:
 479         for child in psutil.Process(process.pid).children():
 480             child.send_signal(signal.SIGINT)
 481             child.wait()
 482         process.send_signal(signal.SIGINT)
 483
 484
 485 def start_honeynode(log_file: str, sim):
 486     executable = os.path.join(os.path.dirname(os.path.realpath(__file__)),
 487                               "..", "..", "honeynode", sim[1], "honeynode-simulator", "honeycomb-tpce")
 488     sample_directory = os.path.join(os.path.dirname(os.path.realpath(__file__)),
 489                                     "..", "..", "sample_configs", "openroadm", sim[1])
 490     if os.path.isfile(executable):
 491         with open(log_file, 'w') as outfile:
 492             return subprocess.Popen(
 493                 [executable, SIMS[sim]['port'], os.path.join(sample_directory, SIMS[sim]['configfile'])],
 494                 stdout=outfile, stderr=outfile)
 495     return None
 496
 497
 498 def wait_until_log_contains(log_file, regexp, time_to_wait=60):
 499     # pylint: disable=lost-exception
 500     stringfound = False
 501     filefound = False
 502     line = None
 503     try:
 504         with TimeOut(seconds=time_to_wait):
 505             while not os.path.exists(log_file):
 506                 time.sleep(0.2)
 507             filelogs = open(log_file, 'r')
 508             filelogs.seek(0, 2)
 509             filefound = True
 510             print("Searching for pattern '" + regexp + "' in " + os.path.basename(log_file), end='... ', flush=True)
 511             compiled_regexp = re.compile(regexp)
 512             while True:
 513                 line = filelogs.readline()
 514                 if compiled_regexp.search(line):
 515                     print("Pattern found!", end=' ')
 516                     stringfound = True
 517                     break
 518                 if not line:
 519                     time.sleep(0.1)
 520     except TimeoutError:
 521         print("Pattern not found after " + str(time_to_wait), end=" seconds! ", flush=True)
 522     except PermissionError:
 523         print("Permission Error when trying to access the log file", end=" ... ", flush=True)
 524     finally:
 525         if filefound:
 526             filelogs.close()
 527         else:
 528             print("log file does not exist or is not accessible... ", flush=True)
 529         return stringfound
 530
 531
 532 class TimeOut:
 533     def __init__(self, seconds=1, error_message='Timeout'):
 534         self.seconds = seconds
 535         self.error_message = error_message
 536
 537     def handle_timeout(self, signum, frame):
 538         raise TimeoutError(self.error_message)
 539
 540     def __enter__(self):
 541         signal.signal(signal.SIGALRM, self.handle_timeout)
 542         signal.alarm(self.seconds)
 543
 544     def __exit__(self, type, value, traceback):
 545         # pylint: disable=W0622
 546         signal.alarm(0)