tests/transportpce_tests/common/test_utils.py

   1 #!/usr/bin/env python
   2
   3 ##############################################################################
   4 # Copyright (c) 2020 Orange, Inc. and others.  All rights reserved.
   5 #
   6 # All rights reserved. This program and the accompanying materials
   7 # are made available under the terms of the Apache License, Version 2.0
   8 # which accompanies this distribution, and is available at
   9 # http://www.apache.org/licenses/LICENSE-2.0
  10 ##############################################################################
  11
  12 # pylint: disable=no-member
  13
  14 import json
  15 import os
  16 import sys
  17 import re
  18 import signal
  19 import subprocess
  20 import time
  21
  22 import psutil
  23 import requests
  24
  25 import simulators
  26
  27 SIMS = simulators.SIMS
  28 HONEYNODE_EXECUTABLE = simulators.HONEYNODE_EXECUTABLE
  29 SAMPLES_DIRECTORY = simulators.SAMPLES_DIRECTORY
  30
  31 HONEYNODE_OK_START_MSG = "Netconf SSH endpoint started successfully at 0.0.0.0"
  32 KARAF_OK_START_MSG = re.escape(
  33     "Blueprint container for bundle org.opendaylight.netconf.restconf")+".* was successfully created"
  34 LIGHTY_OK_START_MSG = re.escape("lighty.io and RESTCONF-NETCONF started")
  35
  36 RESTCONF_BASE_URL = "http://localhost:8181/restconf"
  37 ODL_LOGIN = "admin"
  38 ODL_PWD = "admin"
  39 NODES_LOGIN = "admin"
  40 NODES_PWD = "admin"
  41 URL_CONFIG_NETCONF_TOPO = "{}/config/network-topology:network-topology/topology/topology-netconf/"
  42 URL_CONFIG_ORDM_TOPO = "{}/config/ietf-network:networks/network/openroadm-topology/"
  43 URL_CONFIG_OTN_TOPO = "{}/config/ietf-network:networks/network/otn-topology/"
  44 URL_CONFIG_CLLI_NET = "{}/config/ietf-network:networks/network/clli-network/"
  45 URL_CONFIG_ORDM_NET = "{}/config/ietf-network:networks/network/openroadm-network/"
  46 URL_PORTMAPPING = "{}/config/transportpce-portmapping:network/nodes/"
  47 URL_OPER_SERV_LIST = "{}/operational/org-openroadm-service:service-list/"
  48 URL_SERV_CREATE = "{}/operations/org-openroadm-service:service-create"
  49 URL_SERV_DELETE = "{}/operations/org-openroadm-service:service-delete"
  50 URL_SERVICE_PATH = "{}/operations/transportpce-device-renderer:service-path"
  51 URL_OTN_SERVICE_PATH = "{}/operations/transportpce-device-renderer:otn-service-path"
  52 URL_CREATE_OTS_OMS = "{}/operations/transportpce-device-renderer:create-ots-oms"
  53 URL_PATH_COMPUTATION_REQUEST = "{}/operations/transportpce-pce:path-computation-request"
  54 URL_FULL_PORTMAPPING = "{}/config/transportpce-portmapping:network"
  55
  56 TYPE_APPLICATION_JSON = {'Content-Type': 'application/json', 'Accept': 'application/json'}
  57 TYPE_APPLICATION_XML = {'Content-Type': 'application/xml', 'Accept': 'application/xml'}
  58
  59 CODE_SHOULD_BE_200 = 'Http status code should be 200'
  60 CODE_SHOULD_BE_201 = 'Http status code should be 201'
  61
  62 LOG_DIRECTORY = os.path.dirname(os.path.realpath(__file__))
  63
  64 KARAF_LOG = os.path.join(
  65     os.path.dirname(os.path.realpath(__file__)),
  66     "..", "..", "..", "karaf", "target", "assembly", "data", "log", "karaf.log")
  67
  68 process_list = []
  69
  70 if "USE_LIGHTY" in os.environ and os.environ['USE_LIGHTY'] == 'True':
  71     TPCE_LOG = 'odl.log'
  72 else:
  73     TPCE_LOG = KARAF_LOG
  74
  75
  76 def start_sims(sims_list):
  77     for sim in sims_list:
  78         print("starting simulator for " + sim + "...")
  79         log_file = os.path.join(LOG_DIRECTORY, SIMS[sim]['logfile'])
  80         process = start_honeynode(log_file, SIMS[sim]['port'], SIMS[sim]['configfile'])
  81         if wait_until_log_contains(log_file, HONEYNODE_OK_START_MSG, 100):
  82             print("simulator for " + sim + " started")
  83         else:
  84             print("simulator for " + sim + " failed to start")
  85             shutdown_process(process)
  86             for pid in process_list:
  87                 shutdown_process(pid)
  88             sys.exit(3)
  89         process_list.append(process)
  90     return process_list
  91
  92
  93 def start_tpce():
  94     print("starting OpenDaylight...")
  95     if "USE_LIGHTY" in os.environ and os.environ['USE_LIGHTY'] == 'True':
  96         process = start_lighty()
  97         start_msg = LIGHTY_OK_START_MSG
  98     else:
  99         process = start_karaf()
 100         start_msg = KARAF_OK_START_MSG
 101     if wait_until_log_contains(TPCE_LOG, start_msg, time_to_wait=60):
 102         print("OpenDaylight started !")
 103     else:
 104         print("OpenDaylight failed to start !")
 105         shutdown_process(process)
 106         for pid in process_list:
 107             shutdown_process(pid)
 108         sys.exit(1)
 109     process_list.append(process)
 110     return process_list
 111
 112
 113 def start_karaf():
 114     print("starting KARAF TransportPCE build...")
 115     executable = os.path.join(
 116         os.path.dirname(os.path.realpath(__file__)),
 117         "..", "..", "..", "karaf", "target", "assembly", "bin", "karaf")
 118     with open('odl.log', 'w') as outfile:
 119         return subprocess.Popen(
 120             ["sh", executable, "server"], stdout=outfile, stderr=outfile, stdin=None)
 121
 122
 123 def start_lighty():
 124     print("starting LIGHTY.IO TransportPCE build...")
 125     executable = os.path.join(
 126         os.path.dirname(os.path.realpath(__file__)),
 127         "..", "..", "..", "lighty", "target", "tpce",
 128         "clean-start-controller.sh")
 129     with open('odl.log', 'w') as outfile:
 130         return subprocess.Popen(
 131             ["sh", executable], stdout=outfile, stderr=outfile, stdin=None)
 132
 133
 134 def install_karaf_feature(feature_name: str):
 135     print("installing feature " + feature_name)
 136     executable = os.path.join(
 137         os.path.dirname(os.path.realpath(__file__)),
 138         "..", "..", "..", "karaf", "target", "assembly", "bin", "client")
 139     return subprocess.run([executable],
 140                           input='feature:install ' + feature_name + '\n feature:list | grep '
 141                           + feature_name + ' \n logout \n',
 142                           universal_newlines=True, check=False)
 143
 144
 145 def get_request(url):
 146     return requests.request(
 147         "GET", url.format(RESTCONF_BASE_URL),
 148         headers=TYPE_APPLICATION_JSON,
 149         auth=(ODL_LOGIN, ODL_PWD))
 150
 151
 152 def post_request(url, data):
 153     if data:
 154         print(json.dumps(data))
 155         return requests.request(
 156             "POST", url.format(RESTCONF_BASE_URL),
 157             data=json.dumps(data),
 158             headers=TYPE_APPLICATION_JSON,
 159             auth=(ODL_LOGIN, ODL_PWD))
 160
 161     return requests.request(
 162         "POST", url.format(RESTCONF_BASE_URL),
 163         headers=TYPE_APPLICATION_JSON,
 164         auth=(ODL_LOGIN, ODL_PWD))
 165
 166
 167 def post_xmlrequest(url, data):
 168     if data:
 169         return requests.request(
 170             "POST", url.format(RESTCONF_BASE_URL),
 171             data=data,
 172             headers=TYPE_APPLICATION_XML,
 173             auth=(ODL_LOGIN, ODL_PWD))
 174     return None
 175
 176
 177 def put_request(url, data):
 178     return requests.request(
 179         "PUT", url.format(RESTCONF_BASE_URL),
 180         data=json.dumps(data),
 181         headers=TYPE_APPLICATION_JSON,
 182         auth=(ODL_LOGIN, ODL_PWD))
 183
 184
 185 def put_xmlrequest(url, data):
 186     return requests.request(
 187         "PUT", url.format(RESTCONF_BASE_URL),
 188         data=data,
 189         headers=TYPE_APPLICATION_XML,
 190         auth=(ODL_LOGIN, ODL_PWD))
 191
 192
 193 def rawput_request(url, data):
 194     return requests.request(
 195         "PUT", url.format(RESTCONF_BASE_URL),
 196         data=data,
 197         headers=TYPE_APPLICATION_JSON,
 198         auth=(ODL_LOGIN, ODL_PWD))
 199
 200 def rawpost_request(url, data):
 201     return requests.request(
 202         "POST", url.format(RESTCONF_BASE_URL),
 203         data=data,
 204         headers=TYPE_APPLICATION_JSON,
 205         auth=(ODL_LOGIN, ODL_PWD))
 206
 207
 208 def delete_request(url):
 209     return requests.request(
 210         "DELETE", url.format(RESTCONF_BASE_URL),
 211         headers=TYPE_APPLICATION_JSON,
 212         auth=(ODL_LOGIN, ODL_PWD))
 213
 214
 215 def mount_device(node_id, sim):
 216     url = URL_CONFIG_NETCONF_TOPO + "node/" + node_id
 217     body = {"node": [{
 218         "node-id": node_id,
 219         "netconf-node-topology:username": NODES_LOGIN,
 220         "netconf-node-topology:password": NODES_PWD,
 221         "netconf-node-topology:host": "127.0.0.1",
 222         "netconf-node-topology:port": SIMS[sim]['port'],
 223         "netconf-node-topology:tcp-only": "false",
 224         "netconf-node-topology:pass-through": {}}]}
 225     response = put_request(url, body)
 226     if wait_until_log_contains(TPCE_LOG, re.escape("Triggering notification stream NETCONF for node " + node_id), 60):
 227         print("Node " + node_id + " correctly added to tpce topology", end='... ', flush=True)
 228     else:
 229         print("Node " + node_id + " still not added to tpce topology", end='... ', flush=True)
 230         if response.status_code == requests.codes.ok:
 231             print("It was probably loaded at start-up", end='... ', flush=True)
 232         # TODO an else-clause to abort test would probably be nice here
 233     return response
 234
 235
 236 def unmount_device(node_id):
 237     url = URL_CONFIG_NETCONF_TOPO + "node/" + node_id
 238     response = delete_request(url)
 239     if wait_until_log_contains(TPCE_LOG, re.escape("onDeviceDisConnected: " + node_id), 60):
 240         print("Node " + node_id + " correctly deleted from tpce topology", end='... ', flush=True)
 241     else:
 242         print("Node " + node_id + " still not deleted from tpce topology", end='... ', flush=True)
 243     return response
 244
 245
 246 def connect_xpdr_to_rdm_request(xpdr_node: str, xpdr_num: str, network_num: str,
 247                                 rdm_node: str, srg_num: str, termination_num: str):
 248     url = "{}/operations/transportpce-networkutils:init-xpdr-rdm-links"
 249     data = {
 250         "networkutils:input": {
 251             "networkutils:links-input": {
 252                 "networkutils:xpdr-node": xpdr_node,
 253                 "networkutils:xpdr-num": xpdr_num,
 254                 "networkutils:network-num": network_num,
 255                 "networkutils:rdm-node": rdm_node,
 256                 "networkutils:srg-num": srg_num,
 257                 "networkutils:termination-point-num": termination_num
 258             }
 259         }
 260     }
 261     return post_request(url, data)
 262
 263
 264 def connect_rdm_to_xpdr_request(xpdr_node: str, xpdr_num: str, network_num: str,
 265                                 rdm_node: str, srg_num: str, termination_num: str):
 266     url = "{}/operations/transportpce-networkutils:init-rdm-xpdr-links"
 267     data = {
 268         "networkutils:input": {
 269             "networkutils:links-input": {
 270                 "networkutils:xpdr-node": xpdr_node,
 271                 "networkutils:xpdr-num": xpdr_num,
 272                 "networkutils:network-num": network_num,
 273                 "networkutils:rdm-node": rdm_node,
 274                 "networkutils:srg-num": srg_num,
 275                 "networkutils:termination-point-num": termination_num
 276             }
 277         }
 278     }
 279     return post_request(url, data)
 280
 281
 282 def check_netconf_node_request(node: str, suffix: str):
 283     url = URL_CONFIG_NETCONF_TOPO + (
 284         "node/" + node + "/yang-ext:mount/org-openroadm-device:org-openroadm-device/" + suffix
 285     )
 286     return get_request(url)
 287
 288
 289 def get_netconf_oper_request(node: str):
 290     url = "{}/operational/network-topology:network-topology/topology/topology-netconf/node/" + node
 291     return get_request(url)
 292
 293
 294 def get_ordm_topo_request(suffix: str):
 295     url = URL_CONFIG_ORDM_TOPO + suffix
 296     return get_request(url)
 297
 298
 299 def add_oms_attr_request(link: str, attr):
 300     url = URL_CONFIG_ORDM_TOPO + (
 301         "ietf-network-topology:link/" + link + "/org-openroadm-network-topology:OMS-attributes/span"
 302     )
 303     return put_request(url, attr)
 304
 305
 306 def del_oms_attr_request(link: str):
 307     url = URL_CONFIG_ORDM_TOPO + (
 308         "ietf-network-topology:link/" + link + "/org-openroadm-network-topology:OMS-attributes/span"
 309     )
 310     return delete_request(url)
 311
 312
 313 def get_clli_net_request():
 314     return get_request(URL_CONFIG_CLLI_NET)
 315
 316
 317 def get_ordm_net_request():
 318     return get_request(URL_CONFIG_ORDM_NET)
 319
 320
 321 def get_otn_topo_request():
 322     return get_request(URL_CONFIG_OTN_TOPO)
 323
 324
 325 def del_link_request(link: str):
 326     url = URL_CONFIG_ORDM_TOPO + ("ietf-network-topology:link/" + link)
 327     return delete_request(url)
 328
 329
 330 def del_node_request(node: str):
 331     url = URL_CONFIG_CLLI_NET + ("node/" + node)
 332     return delete_request(url)
 333
 334
 335 def portmapping_request(suffix: str):
 336     url = URL_PORTMAPPING + suffix
 337     return get_request(url)
 338
 339
 340 def get_service_list_request(suffix: str):
 341     url = URL_OPER_SERV_LIST + suffix
 342     return get_request(url)
 343
 344
 345 def service_create_request(attr):
 346     return post_request(URL_SERV_CREATE, attr)
 347
 348
 349 def service_delete_request(servicename: str,
 350                            requestid="e3028bae-a90f-4ddd-a83f-cf224eba0e58",
 351                            notificationurl="http://localhost:8585/NotificationServer/notify"):
 352     attr = {"input": {
 353         "sdnc-request-header": {
 354             "request-id": requestid,
 355             "rpc-action": "service-delete",
 356             "request-system-id": "appname",
 357             "notification-url": notificationurl},
 358         "service-delete-req-info": {
 359             "service-name": servicename,
 360             "tail-retention": "no"}}}
 361     return post_request(URL_SERV_DELETE, attr)
 362
 363
 364 def service_path_request(operation: str, servicename: str, wavenumber: str, nodes, centerfreq: str,
 365                          slotwidth: int, minfreq: float, maxfreq: float, lowerslotnumber: int,
 366                          higherslotnumber: int):
 367     attr = {"renderer:input": {
 368         "renderer:service-name": servicename,
 369         "renderer:wave-number": wavenumber,
 370         "renderer:modulation-format": "dp-qpsk",
 371         "renderer:operation": operation,
 372         "renderer:nodes": nodes,
 373         "renderer:center-freq": centerfreq,
 374         "renderer:width": slotwidth,
 375         "renderer:min-freq": minfreq,
 376         "renderer:max-freq": maxfreq,
 377         "renderer:lower-spectral-slot-number": lowerslotnumber,
 378         "renderer:higher-spectral-slot-number": higherslotnumber}}
 379     return post_request(URL_SERVICE_PATH, attr)
 380
 381
 382 def otn_service_path_request(operation: str, servicename: str, servicerate: str, servicetype: str, nodes,
 383                              eth_attr=None):
 384     attr = {"service-name": servicename,
 385             "operation": operation,
 386             "service-rate": servicerate,
 387             "service-type": servicetype,
 388             "nodes": nodes}
 389     if eth_attr:
 390         attr.update(eth_attr)
 391     return post_request(URL_OTN_SERVICE_PATH, {"renderer:input": attr})
 392
 393
 394 def create_ots_oms_request(nodeid: str, lcp: str):
 395     attr = {"input": {
 396         "node-id": nodeid,
 397         "logical-connection-point": lcp}}
 398     return post_request(URL_CREATE_OTS_OMS, attr)
 399
 400
 401 def path_computation_request(requestid: str, servicename: str, serviceaend, servicezend,
 402                              hardconstraints=None, softconstraints=None, metric="hop-count", other_attr=None):
 403     attr = {"service-name": servicename,
 404             "resource-reserve": "true",
 405             "service-handler-header": {"request-id": requestid},
 406             "service-a-end": serviceaend,
 407             "service-z-end": servicezend,
 408             "pce-metric": metric}
 409     if hardconstraints:
 410         attr.update({"hard-constraints": hardconstraints})
 411     if softconstraints:
 412         attr.update({"soft-constraints": softconstraints})
 413     if other_attr:
 414         attr.update(other_attr)
 415     return post_request(URL_PATH_COMPUTATION_REQUEST, {"input": attr})
 416
 417
 418 def shutdown_process(process):
 419     if process is not None:
 420         for child in psutil.Process(process.pid).children():
 421             child.send_signal(signal.SIGINT)
 422             child.wait()
 423         process.send_signal(signal.SIGINT)
 424
 425
 426 def start_honeynode(log_file: str, node_port: str, node_config_file_name: str):
 427     if os.path.isfile(HONEYNODE_EXECUTABLE):
 428         with open(log_file, 'w') as outfile:
 429             return subprocess.Popen(
 430                 [HONEYNODE_EXECUTABLE, node_port, os.path.join(SAMPLES_DIRECTORY, node_config_file_name)],
 431                 stdout=outfile, stderr=outfile)
 432     return None
 433
 434
 435 def wait_until_log_contains(log_file, regexp, time_to_wait=20):
 436     # pylint: disable=lost-exception
 437     stringfound = False
 438     filefound = False
 439     line = None
 440     try:
 441         with TimeOut(seconds=time_to_wait):
 442             while not os.path.exists(log_file):
 443                 time.sleep(0.2)
 444             filelogs = open(log_file, 'r')
 445             filelogs.seek(0, 2)
 446             filefound = True
 447             print("Searching for pattern '" + regexp + "' in " + os.path.basename(log_file), end='... ', flush=True)
 448             compiled_regexp = re.compile(regexp)
 449             while True:
 450                 line = filelogs.readline()
 451                 if compiled_regexp.search(line):
 452                     print("Pattern found!", end=' ')
 453                     stringfound = True
 454                     break
 455                 if not line:
 456                     time.sleep(0.1)
 457     except TimeoutError:
 458         print("Pattern not found after " + str(time_to_wait), end=" seconds! ", flush=True)
 459     except PermissionError:
 460         print("Permission Error when trying to access the log file", end=" ... ", flush=True)
 461     finally:
 462         if filefound:
 463             filelogs.close()
 464         else:
 465             print("log file does not exist or is not accessible... ", flush=True)
 466         return stringfound
 467
 468
 469 class TimeOut:
 470     def __init__(self, seconds=1, error_message='Timeout'):
 471         self.seconds = seconds
 472         self.error_message = error_message
 473
 474     def handle_timeout(self, signum, frame):
 475         raise TimeoutError(self.error_message)
 476
 477     def __enter__(self):
 478         signal.signal(signal.SIGALRM, self.handle_timeout)
 479         signal.alarm(self.seconds)
 480
 481     def __exit__(self, type, value, traceback):
 482         # pylint: disable=W0622
 483         signal.alarm(0)
 484