tests/transportpce_tests/common/test_utils.py

   1 #!/usr/bin/env python
   2
   3 ##############################################################################
   4 # Copyright (c) 2020 Orange, Inc. and others.  All rights reserved.
   5 #
   6 # All rights reserved. This program and the accompanying materials
   7 # are made available under the terms of the Apache License, Version 2.0
   8 # which accompanies this distribution, and is available at
   9 # http://www.apache.org/licenses/LICENSE-2.0
  10 ##############################################################################
  11
  12 # pylint: disable=no-member
  13
  14 import json
  15 import os
  16 import sys
  17 import re
  18 import signal
  19 import subprocess
  20 import time
  21
  22 import psutil
  23 import requests
  24
  25 import simulators
  26
  27 SIMS = simulators.SIMS
  28
  29 HONEYNODE_OK_START_MSG = "Netconf SSH endpoint started successfully at 0.0.0.0"
  30 KARAF_OK_START_MSG = re.escape(
  31     "Blueprint container for bundle org.opendaylight.netconf.restconf")+".* was successfully created"
  32 LIGHTY_OK_START_MSG = re.escape("lighty.io and RESTCONF-NETCONF started")
  33
  34 ODL_LOGIN = "admin"
  35 ODL_PWD = "admin"
  36 NODES_LOGIN = "admin"
  37 NODES_PWD = "admin"
  38 URL_CONFIG_NETCONF_TOPO = "{}/config/network-topology:network-topology/topology/topology-netconf/"
  39 URL_CONFIG_ORDM_TOPO = "{}/config/ietf-network:networks/network/openroadm-topology/"
  40 URL_CONFIG_OTN_TOPO = "{}/config/ietf-network:networks/network/otn-topology/"
  41 URL_CONFIG_CLLI_NET = "{}/config/ietf-network:networks/network/clli-network/"
  42 URL_CONFIG_ORDM_NET = "{}/config/ietf-network:networks/network/openroadm-network/"
  43 URL_PORTMAPPING = "{}/config/transportpce-portmapping:network/nodes/"
  44 URL_OPER_SERV_LIST = "{}/operational/org-openroadm-service:service-list/"
  45 URL_GET_NBINOTIFICATIONS_SERV = "{}/operations/nbi-notifications:get-notifications-service/"
  46 URL_SERV_CREATE = "{}/operations/org-openroadm-service:service-create"
  47 URL_SERV_DELETE = "{}/operations/org-openroadm-service:service-delete"
  48 URL_SERVICE_PATH = "{}/operations/transportpce-device-renderer:service-path"
  49 URL_OTN_SERVICE_PATH = "{}/operations/transportpce-device-renderer:otn-service-path"
  50 URL_CREATE_OTS_OMS = "{}/operations/transportpce-device-renderer:create-ots-oms"
  51 URL_PATH_COMPUTATION_REQUEST = "{}/operations/transportpce-pce:path-computation-request"
  52 URL_FULL_PORTMAPPING = "{}/config/transportpce-portmapping:network"
  53
  54 TYPE_APPLICATION_JSON = {'Content-Type': 'application/json', 'Accept': 'application/json'}
  55 TYPE_APPLICATION_XML = {'Content-Type': 'application/xml', 'Accept': 'application/xml'}
  56
  57 CODE_SHOULD_BE_200 = 'Http status code should be 200'
  58 CODE_SHOULD_BE_201 = 'Http status code should be 201'
  59
  60 SIM_LOG_DIRECTORY = os.path.join(os.path.dirname(os.path.realpath(__file__)), "log")
  61 KARAF_LOG = os.path.join(
  62     os.path.dirname(os.path.realpath(__file__)),
  63     "..", "..", "..", "karaf", "target", "assembly", "data", "log", "karaf.log")
  64
  65 process_list = []
  66
  67
  68 if "USE_ODL_ALT_RESTCONF_PORT" in os.environ:
  69     RESTCONF_BASE_URL = "http://localhost:" + os.environ['USE_ODL_ALT_RESTCONF_PORT'] + "/restconf"
  70 else:
  71     RESTCONF_BASE_URL = "http://localhost:8181/restconf"
  72
  73 if "USE_LIGHTY" in os.environ and os.environ['USE_LIGHTY'] == 'True':
  74     TPCE_LOG = 'odl.log'
  75 else:
  76     TPCE_LOG = KARAF_LOG
  77
  78
  79 def start_sims(sims_list):
  80     for sim in sims_list:
  81         print("starting simulator " + sim[0] + " in OpenROADM device version " + sim[1] + "...")
  82         log_file = os.path.join(SIM_LOG_DIRECTORY, SIMS[sim]['logfile'])
  83         process = start_honeynode(log_file, sim)
  84         if wait_until_log_contains(log_file, HONEYNODE_OK_START_MSG, 100):
  85             print("simulator for " + sim[0] + " started")
  86         else:
  87             print("simulator for " + sim[0] + " failed to start")
  88             shutdown_process(process)
  89             for pid in process_list:
  90                 shutdown_process(pid)
  91             sys.exit(3)
  92         process_list.append(process)
  93     return process_list
  94
  95
  96 def start_tpce():
  97     print("starting OpenDaylight...")
  98     if "USE_LIGHTY" in os.environ and os.environ['USE_LIGHTY'] == 'True':
  99         process = start_lighty()
 100         start_msg = LIGHTY_OK_START_MSG
 101     else:
 102         process = start_karaf()
 103         start_msg = KARAF_OK_START_MSG
 104     if wait_until_log_contains(TPCE_LOG, start_msg, time_to_wait=120):
 105         print("OpenDaylight started !")
 106     else:
 107         print("OpenDaylight failed to start !")
 108         shutdown_process(process)
 109         for pid in process_list:
 110             shutdown_process(pid)
 111         sys.exit(1)
 112     process_list.append(process)
 113     return process_list
 114
 115
 116 def start_karaf():
 117     print("starting KARAF TransportPCE build...")
 118     executable = os.path.join(
 119         os.path.dirname(os.path.realpath(__file__)),
 120         "..", "..", "..", "karaf", "target", "assembly", "bin", "karaf")
 121     with open('odl.log', 'w') as outfile:
 122         return subprocess.Popen(
 123             ["sh", executable, "server"], stdout=outfile, stderr=outfile, stdin=None)
 124
 125
 126 def start_lighty():
 127     print("starting LIGHTY.IO TransportPCE build...")
 128     executable = os.path.join(
 129         os.path.dirname(os.path.realpath(__file__)),
 130         "..", "..", "..", "lighty", "target", "tpce",
 131         "clean-start-controller.sh")
 132     with open('odl.log', 'w') as outfile:
 133         return subprocess.Popen(
 134             ["sh", executable], stdout=outfile, stderr=outfile, stdin=None)
 135
 136
 137 def install_karaf_feature(feature_name: str):
 138     print("installing feature " + feature_name)
 139     executable = os.path.join(
 140         os.path.dirname(os.path.realpath(__file__)),
 141         "..", "..", "..", "karaf", "target", "assembly", "bin", "client")
 142     return subprocess.run([executable],
 143                           input='feature:install ' + feature_name + '\n feature:list | grep '
 144                           + feature_name + ' \n logout \n',
 145                           universal_newlines=True, check=False)
 146
 147
 148 def get_request(url):
 149     return requests.request(
 150         "GET", url.format(RESTCONF_BASE_URL),
 151         headers=TYPE_APPLICATION_JSON,
 152         auth=(ODL_LOGIN, ODL_PWD))
 153
 154
 155 def post_request(url, data):
 156     if data:
 157         print(json.dumps(data))
 158         return requests.request(
 159             "POST", url.format(RESTCONF_BASE_URL),
 160             data=json.dumps(data),
 161             headers=TYPE_APPLICATION_JSON,
 162             auth=(ODL_LOGIN, ODL_PWD))
 163
 164     return requests.request(
 165         "POST", url.format(RESTCONF_BASE_URL),
 166         headers=TYPE_APPLICATION_JSON,
 167         auth=(ODL_LOGIN, ODL_PWD))
 168
 169
 170 def post_xmlrequest(url, data):
 171     if data:
 172         return requests.request(
 173             "POST", url.format(RESTCONF_BASE_URL),
 174             data=data,
 175             headers=TYPE_APPLICATION_XML,
 176             auth=(ODL_LOGIN, ODL_PWD))
 177     return None
 178
 179
 180 def put_request(url, data):
 181     return requests.request(
 182         "PUT", url.format(RESTCONF_BASE_URL),
 183         data=json.dumps(data),
 184         headers=TYPE_APPLICATION_JSON,
 185         auth=(ODL_LOGIN, ODL_PWD))
 186
 187
 188 def put_xmlrequest(url, data):
 189     return requests.request(
 190         "PUT", url.format(RESTCONF_BASE_URL),
 191         data=data,
 192         headers=TYPE_APPLICATION_XML,
 193         auth=(ODL_LOGIN, ODL_PWD))
 194
 195 def put_jsonrequest(url, data):
 196     return requests.request(
 197         "PUT", url.format(RESTCONF_BASE_URL),
 198         data=data,
 199         headers=TYPE_APPLICATION_JSON,
 200         auth=(ODL_LOGIN, ODL_PWD))
 201
 202 def rawput_request(url, data):
 203     return requests.request(
 204         "PUT", url.format(RESTCONF_BASE_URL),
 205         data=data,
 206         headers=TYPE_APPLICATION_JSON,
 207         auth=(ODL_LOGIN, ODL_PWD))
 208
 209 def rawpost_request(url, data):
 210     return requests.request(
 211         "POST", url.format(RESTCONF_BASE_URL),
 212         data=data,
 213         headers=TYPE_APPLICATION_JSON,
 214         auth=(ODL_LOGIN, ODL_PWD))
 215
 216
 217 def delete_request(url):
 218     return requests.request(
 219         "DELETE", url.format(RESTCONF_BASE_URL),
 220         headers=TYPE_APPLICATION_JSON,
 221         auth=(ODL_LOGIN, ODL_PWD))
 222
 223
 224 def mount_device(node_id, sim):
 225     url = URL_CONFIG_NETCONF_TOPO + "node/" + node_id
 226     body = {"node": [{
 227         "node-id": node_id,
 228         "netconf-node-topology:username": NODES_LOGIN,
 229         "netconf-node-topology:password": NODES_PWD,
 230         "netconf-node-topology:host": "127.0.0.1",
 231         "netconf-node-topology:port": SIMS[sim]['port'],
 232         "netconf-node-topology:tcp-only": "false",
 233         "netconf-node-topology:pass-through": {}}]}
 234     response = put_request(url, body)
 235     if wait_until_log_contains(TPCE_LOG, re.escape("Triggering notification stream NETCONF for node " + node_id), 60):
 236         print("Node " + node_id + " correctly added to tpce topology", end='... ', flush=True)
 237     else:
 238         print("Node " + node_id + " still not added to tpce topology", end='... ', flush=True)
 239         if response.status_code == requests.codes.ok:
 240             print("It was probably loaded at start-up", end='... ', flush=True)
 241         # TODO an else-clause to abort test would probably be nice here
 242     return response
 243
 244
 245 def unmount_device(node_id):
 246     url = URL_CONFIG_NETCONF_TOPO + "node/" + node_id
 247     response = delete_request(url)
 248     if wait_until_log_contains(TPCE_LOG, re.escape("onDeviceDisConnected: " + node_id), 60):
 249         print("Node " + node_id + " correctly deleted from tpce topology", end='... ', flush=True)
 250     else:
 251         print("Node " + node_id + " still not deleted from tpce topology", end='... ', flush=True)
 252     return response
 253
 254
 255 def connect_xpdr_to_rdm_request(xpdr_node: str, xpdr_num: str, network_num: str,
 256                                 rdm_node: str, srg_num: str, termination_num: str):
 257     url = "{}/operations/transportpce-networkutils:init-xpdr-rdm-links"
 258     data = {
 259         "networkutils:input": {
 260             "networkutils:links-input": {
 261                 "networkutils:xpdr-node": xpdr_node,
 262                 "networkutils:xpdr-num": xpdr_num,
 263                 "networkutils:network-num": network_num,
 264                 "networkutils:rdm-node": rdm_node,
 265                 "networkutils:srg-num": srg_num,
 266                 "networkutils:termination-point-num": termination_num
 267             }
 268         }
 269     }
 270     return post_request(url, data)
 271
 272
 273 def connect_rdm_to_xpdr_request(xpdr_node: str, xpdr_num: str, network_num: str,
 274                                 rdm_node: str, srg_num: str, termination_num: str):
 275     url = "{}/operations/transportpce-networkutils:init-rdm-xpdr-links"
 276     data = {
 277         "networkutils:input": {
 278             "networkutils:links-input": {
 279                 "networkutils:xpdr-node": xpdr_node,
 280                 "networkutils:xpdr-num": xpdr_num,
 281                 "networkutils:network-num": network_num,
 282                 "networkutils:rdm-node": rdm_node,
 283                 "networkutils:srg-num": srg_num,
 284                 "networkutils:termination-point-num": termination_num
 285             }
 286         }
 287     }
 288     return post_request(url, data)
 289
 290
 291 def check_netconf_node_request(node: str, suffix: str):
 292     url = URL_CONFIG_NETCONF_TOPO + (
 293         "node/" + node + "/yang-ext:mount/org-openroadm-device:org-openroadm-device/" + suffix
 294     )
 295     return get_request(url)
 296
 297
 298 def get_netconf_oper_request(node: str):
 299     url = "{}/operational/network-topology:network-topology/topology/topology-netconf/node/" + node
 300     return get_request(url)
 301
 302
 303 def get_ordm_topo_request(suffix: str):
 304     url = URL_CONFIG_ORDM_TOPO + suffix
 305     return get_request(url)
 306
 307
 308 def add_oms_attr_request(link: str, attr):
 309     url = URL_CONFIG_ORDM_TOPO + (
 310         "ietf-network-topology:link/" + link + "/org-openroadm-network-topology:OMS-attributes/span"
 311     )
 312     return put_request(url, attr)
 313
 314
 315 def del_oms_attr_request(link: str):
 316     url = URL_CONFIG_ORDM_TOPO + (
 317         "ietf-network-topology:link/" + link + "/org-openroadm-network-topology:OMS-attributes/span"
 318     )
 319     return delete_request(url)
 320
 321
 322 def get_clli_net_request():
 323     return get_request(URL_CONFIG_CLLI_NET)
 324
 325
 326 def get_ordm_net_request():
 327     return get_request(URL_CONFIG_ORDM_NET)
 328
 329
 330 def get_otn_topo_request():
 331     return get_request(URL_CONFIG_OTN_TOPO)
 332
 333
 334 def del_link_request(link: str):
 335     url = URL_CONFIG_ORDM_TOPO + ("ietf-network-topology:link/" + link)
 336     return delete_request(url)
 337
 338
 339 def del_node_request(node: str):
 340     url = URL_CONFIG_CLLI_NET + ("node/" + node)
 341     return delete_request(url)
 342
 343
 344 def portmapping_request(suffix: str):
 345     url = URL_PORTMAPPING + suffix
 346     return get_request(url)
 347
 348
 349 def get_notifications_service_request(attr):
 350     return post_request(URL_GET_NBINOTIFICATIONS_SERV, attr)
 351
 352
 353 def get_service_list_request(suffix: str):
 354     url = URL_OPER_SERV_LIST + suffix
 355     return get_request(url)
 356
 357
 358 def service_create_request(attr):
 359     return post_request(URL_SERV_CREATE, attr)
 360
 361
 362 def service_delete_request(servicename: str,
 363                            requestid="e3028bae-a90f-4ddd-a83f-cf224eba0e58",
 364                            notificationurl="http://localhost:8585/NotificationServer/notify"):
 365     attr = {"input": {
 366         "sdnc-request-header": {
 367             "request-id": requestid,
 368             "rpc-action": "service-delete",
 369             "request-system-id": "appname",
 370             "notification-url": notificationurl},
 371         "service-delete-req-info": {
 372             "service-name": servicename,
 373             "tail-retention": "no"}}}
 374     return post_request(URL_SERV_DELETE, attr)
 375
 376
 377 def service_path_request(operation: str, servicename: str, wavenumber: str, nodes, centerfreq: str,
 378                          slotwidth: int, minfreq: float, maxfreq: float, lowerslotnumber: int,
 379                          higherslotnumber: int):
 380     attr = {"renderer:input": {
 381         "renderer:service-name": servicename,
 382         "renderer:wave-number": wavenumber,
 383         "renderer:modulation-format": "dp-qpsk",
 384         "renderer:operation": operation,
 385         "renderer:nodes": nodes,
 386         "renderer:center-freq": centerfreq,
 387         "renderer:width": slotwidth,
 388         "renderer:min-freq": minfreq,
 389         "renderer:max-freq": maxfreq,
 390         "renderer:lower-spectral-slot-number": lowerslotnumber,
 391         "renderer:higher-spectral-slot-number": higherslotnumber}}
 392     return post_request(URL_SERVICE_PATH, attr)
 393
 394
 395 def otn_service_path_request(operation: str, servicename: str, servicerate: str, servicetype: str, nodes,
 396                              eth_attr=None):
 397     attr = {"service-name": servicename,
 398             "operation": operation,
 399             "service-rate": servicerate,
 400             "service-type": servicetype,
 401             "nodes": nodes}
 402     if eth_attr:
 403         attr.update(eth_attr)
 404     return post_request(URL_OTN_SERVICE_PATH, {"renderer:input": attr})
 405
 406
 407 def create_ots_oms_request(nodeid: str, lcp: str):
 408     attr = {"input": {
 409         "node-id": nodeid,
 410         "logical-connection-point": lcp}}
 411     return post_request(URL_CREATE_OTS_OMS, attr)
 412
 413
 414 def path_computation_request(requestid: str, servicename: str, serviceaend, servicezend,
 415                              hardconstraints=None, softconstraints=None, metric="hop-count", other_attr=None):
 416     attr = {"service-name": servicename,
 417             "resource-reserve": "true",
 418             "service-handler-header": {"request-id": requestid},
 419             "service-a-end": serviceaend,
 420             "service-z-end": servicezend,
 421             "pce-metric": metric}
 422     if hardconstraints:
 423         attr.update({"hard-constraints": hardconstraints})
 424     if softconstraints:
 425         attr.update({"soft-constraints": softconstraints})
 426     if other_attr:
 427         attr.update(other_attr)
 428     return post_request(URL_PATH_COMPUTATION_REQUEST, {"input": attr})
 429
 430
 431 def shutdown_process(process):
 432     if process is not None:
 433         for child in psutil.Process(process.pid).children():
 434             child.send_signal(signal.SIGINT)
 435             child.wait()
 436         process.send_signal(signal.SIGINT)
 437
 438
 439 def start_honeynode(log_file: str, sim):
 440     executable = os.path.join(os.path.dirname(os.path.realpath(__file__)),
 441                               "..", "..", "honeynode", sim[1], "honeynode-simulator", "honeycomb-tpce")
 442     sample_directory = os.path.join(os.path.dirname(os.path.realpath(__file__)),
 443                                     "..", "..", "sample_configs", "openroadm", sim[1])
 444     if os.path.isfile(executable):
 445         with open(log_file, 'w') as outfile:
 446             return subprocess.Popen(
 447                 [executable, SIMS[sim]['port'], os.path.join(sample_directory, SIMS[sim]['configfile'])],
 448                 stdout=outfile, stderr=outfile)
 449     return None
 450
 451
 452 def wait_until_log_contains(log_file, regexp, time_to_wait=20):
 453     # pylint: disable=lost-exception
 454     stringfound = False
 455     filefound = False
 456     line = None
 457     try:
 458         with TimeOut(seconds=time_to_wait):
 459             while not os.path.exists(log_file):
 460                 time.sleep(0.2)
 461             filelogs = open(log_file, 'r')
 462             filelogs.seek(0, 2)
 463             filefound = True
 464             print("Searching for pattern '" + regexp + "' in " + os.path.basename(log_file), end='... ', flush=True)
 465             compiled_regexp = re.compile(regexp)
 466             while True:
 467                 line = filelogs.readline()
 468                 if compiled_regexp.search(line):
 469                     print("Pattern found!", end=' ')
 470                     stringfound = True
 471                     break
 472                 if not line:
 473                     time.sleep(0.1)
 474     except TimeoutError:
 475         print("Pattern not found after " + str(time_to_wait), end=" seconds! ", flush=True)
 476     except PermissionError:
 477         print("Permission Error when trying to access the log file", end=" ... ", flush=True)
 478     finally:
 479         if filefound:
 480             filelogs.close()
 481         else:
 482             print("log file does not exist or is not accessible... ", flush=True)
 483         return stringfound
 484
 485
 486 class TimeOut:
 487     def __init__(self, seconds=1, error_message='Timeout'):
 488         self.seconds = seconds
 489         self.error_message = error_message
 490
 491     def handle_timeout(self, signum, frame):
 492         raise TimeoutError(self.error_message)
 493
 494     def __enter__(self):
 495         signal.signal(signal.SIGALRM, self.handle_timeout)
 496         signal.alarm(self.seconds)
 497
 498     def __exit__(self, type, value, traceback):
 499         # pylint: disable=W0622
 500         signal.alarm(0)