tests/transportpce_tests/common/test_utils.py

   1 #!/usr/bin/env python
   2
   3 ##############################################################################
   4 # Copyright (c) 2020 Orange, Inc. and others.  All rights reserved.
   5 #
   6 # All rights reserved. This program and the accompanying materials
   7 # are made available under the terms of the Apache License, Version 2.0
   8 # which accompanies this distribution, and is available at
   9 # http://www.apache.org/licenses/LICENSE-2.0
  10 ##############################################################################
  11
  12 # pylint: disable=no-member
  13
  14 import json
  15 import os
  16 import sys
  17 import re
  18 import signal
  19 import subprocess
  20 import time
  21
  22 import psutil
  23 import requests
  24
  25 import simulators
  26
  27 SIMS = simulators.SIMS
  28 HONEYNODE_EXECUTABLE = simulators.HONEYNODE_EXECUTABLE
  29 SAMPLES_DIRECTORY = simulators.SAMPLES_DIRECTORY
  30
  31 HONEYNODE_OK_START_MSG = "Netconf SSH endpoint started successfully at 0.0.0.0"
  32 KARAF_OK_START_MSG = re.escape(
  33     "Blueprint container for bundle org.opendaylight.netconf.restconf")+".* was successfully created"
  34 LIGHTY_OK_START_MSG = re.escape("lighty.io and RESTCONF-NETCONF started")
  35
  36 RESTCONF_BASE_URL = "http://localhost:8181/restconf"
  37 ODL_LOGIN = "admin"
  38 ODL_PWD = "admin"
  39 NODES_LOGIN = "admin"
  40 NODES_PWD = "admin"
  41 URL_CONFIG_NETCONF_TOPO = "{}/config/network-topology:network-topology/topology/topology-netconf/"
  42 URL_CONFIG_ORDM_TOPO = "{}/config/ietf-network:networks/network/openroadm-topology/"
  43 URL_CONFIG_OTN_TOPO = "{}/config/ietf-network:networks/network/otn-topology/"
  44 URL_CONFIG_CLLI_NET = "{}/config/ietf-network:networks/network/clli-network/"
  45 URL_CONFIG_ORDM_NET = "{}/config/ietf-network:networks/network/openroadm-network/"
  46 URL_PORTMAPPING = "{}/config/transportpce-portmapping:network/nodes/"
  47 URL_OPER_SERV_LIST = "{}/operational/org-openroadm-service:service-list/"
  48 URL_SERV_CREATE = "{}/operations/org-openroadm-service:service-create"
  49 URL_SERV_DELETE = "{}/operations/org-openroadm-service:service-delete"
  50 URL_SERVICE_PATH = "{}/operations/transportpce-device-renderer:service-path"
  51 URL_OTN_SERVICE_PATH = "{}/operations/transportpce-device-renderer:otn-service-path"
  52 URL_CREATE_OTS_OMS = "{}/operations/transportpce-device-renderer:create-ots-oms"
  53 URL_PATH_COMPUTATION_REQUEST = "{}/operations/transportpce-pce:path-computation-request"
  54 URL_FULL_PORTMAPPING = "{}/config/transportpce-portmapping:network"
  55
  56 TYPE_APPLICATION_JSON = {'Content-Type': 'application/json', 'Accept': 'application/json'}
  57 TYPE_APPLICATION_XML = {'Content-Type': 'application/xml', 'Accept': 'application/xml'}
  58
  59 CODE_SHOULD_BE_200 = 'Http status code should be 200'
  60 CODE_SHOULD_BE_201 = 'Http status code should be 201'
  61
  62 LOG_DIRECTORY = os.path.dirname(os.path.realpath(__file__))
  63
  64 KARAF_LOG = os.path.join(
  65     os.path.dirname(os.path.realpath(__file__)),
  66     "..", "..", "..", "karaf", "target", "assembly", "data", "log", "karaf.log")
  67
  68 process_list = []
  69
  70 if "USE_LIGHTY" in os.environ and os.environ['USE_LIGHTY'] == 'True':
  71     TPCE_LOG = 'odl.log'
  72 else:
  73     TPCE_LOG = KARAF_LOG
  74
  75
  76 def start_sims(sims_list):
  77     for sim in sims_list:
  78         print("starting simulator for " + sim + "...")
  79         log_file = os.path.join(LOG_DIRECTORY, SIMS[sim]['logfile'])
  80         process = start_honeynode(log_file, SIMS[sim]['port'], SIMS[sim]['configfile'])
  81         if wait_until_log_contains(log_file, HONEYNODE_OK_START_MSG, 100):
  82             print("simulator for " + sim + " started")
  83         else:
  84             print("simulator for " + sim + " failed to start")
  85             shutdown_process(process)
  86             for pid in process_list:
  87                 shutdown_process(pid)
  88             sys.exit(3)
  89         process_list.append(process)
  90     return process_list
  91
  92
  93 def start_tpce():
  94     print("starting OpenDaylight...")
  95     if "USE_LIGHTY" in os.environ and os.environ['USE_LIGHTY'] == 'True':
  96         process = start_lighty()
  97         start_msg = LIGHTY_OK_START_MSG
  98     else:
  99         process = start_karaf()
 100         start_msg = KARAF_OK_START_MSG
 101     if wait_until_log_contains(TPCE_LOG, start_msg, time_to_wait=60):
 102         print("OpenDaylight started !")
 103     else:
 104         print("OpenDaylight failed to start !")
 105         shutdown_process(process)
 106         for pid in process_list:
 107             shutdown_process(pid)
 108         sys.exit(1)
 109     process_list.append(process)
 110     return process_list
 111
 112
 113 def start_karaf():
 114     print("starting KARAF TransportPCE build...")
 115     executable = os.path.join(
 116         os.path.dirname(os.path.realpath(__file__)),
 117         "..", "..", "..", "karaf", "target", "assembly", "bin", "karaf")
 118     with open('odl.log', 'w') as outfile:
 119         return subprocess.Popen(
 120             ["sh", executable, "server"], stdout=outfile, stderr=outfile, stdin=None)
 121
 122
 123 def start_lighty():
 124     print("starting LIGHTY.IO TransportPCE build...")
 125     executable = os.path.join(
 126         os.path.dirname(os.path.realpath(__file__)),
 127         "..", "..", "..", "lighty", "target", "tpce",
 128         "clean-start-controller.sh")
 129     with open('odl.log', 'w') as outfile:
 130         return subprocess.Popen(
 131             ["sh", executable], stdout=outfile, stderr=outfile, stdin=None)
 132
 133
 134 def install_karaf_feature(feature_name: str):
 135     print("installing feature " + feature_name)
 136     executable = os.path.join(
 137         os.path.dirname(os.path.realpath(__file__)),
 138         "..", "..", "..", "karaf", "target", "assembly", "bin", "client")
 139     return subprocess.run([executable],
 140                           input='feature:install ' + feature_name + '\n feature:list | grep tapi \n logout \n',
 141                           universal_newlines=True, check=False)
 142
 143
 144 def get_request(url):
 145     return requests.request(
 146         "GET", url.format(RESTCONF_BASE_URL),
 147         headers=TYPE_APPLICATION_JSON,
 148         auth=(ODL_LOGIN, ODL_PWD))
 149
 150
 151 def post_request(url, data):
 152     if data:
 153         print(json.dumps(data))
 154         return requests.request(
 155             "POST", url.format(RESTCONF_BASE_URL),
 156             data=json.dumps(data),
 157             headers=TYPE_APPLICATION_JSON,
 158             auth=(ODL_LOGIN, ODL_PWD))
 159
 160     return requests.request(
 161         "POST", url.format(RESTCONF_BASE_URL),
 162         headers=TYPE_APPLICATION_JSON,
 163         auth=(ODL_LOGIN, ODL_PWD))
 164
 165
 166 def post_xmlrequest(url, data):
 167     if data:
 168         return requests.request(
 169             "POST", url.format(RESTCONF_BASE_URL),
 170             data=data,
 171             headers=TYPE_APPLICATION_XML,
 172             auth=(ODL_LOGIN, ODL_PWD))
 173     return None
 174
 175
 176 def put_request(url, data):
 177     return requests.request(
 178         "PUT", url.format(RESTCONF_BASE_URL),
 179         data=json.dumps(data),
 180         headers=TYPE_APPLICATION_JSON,
 181         auth=(ODL_LOGIN, ODL_PWD))
 182
 183
 184 def put_xmlrequest(url, data):
 185     return requests.request(
 186         "PUT", url.format(RESTCONF_BASE_URL),
 187         data=data,
 188         headers=TYPE_APPLICATION_XML,
 189         auth=(ODL_LOGIN, ODL_PWD))
 190
 191
 192 def rawput_request(url, data):
 193     return requests.request(
 194         "PUT", url.format(RESTCONF_BASE_URL),
 195         data=data,
 196         headers=TYPE_APPLICATION_JSON,
 197         auth=(ODL_LOGIN, ODL_PWD))
 198
 199 def rawpost_request(url, data):
 200     return requests.request(
 201         "POST", url.format(RESTCONF_BASE_URL),
 202         data=data,
 203         headers=TYPE_APPLICATION_JSON,
 204         auth=(ODL_LOGIN, ODL_PWD))
 205
 206
 207 def delete_request(url):
 208     return requests.request(
 209         "DELETE", url.format(RESTCONF_BASE_URL),
 210         headers=TYPE_APPLICATION_JSON,
 211         auth=(ODL_LOGIN, ODL_PWD))
 212
 213
 214 def mount_device(node_id, sim):
 215     url = URL_CONFIG_NETCONF_TOPO + "node/" + node_id
 216     body = {"node": [{
 217         "node-id": node_id,
 218         "netconf-node-topology:username": NODES_LOGIN,
 219         "netconf-node-topology:password": NODES_PWD,
 220         "netconf-node-topology:host": "127.0.0.1",
 221         "netconf-node-topology:port": SIMS[sim]['port'],
 222         "netconf-node-topology:tcp-only": "false",
 223         "netconf-node-topology:pass-through": {}}]}
 224     response = put_request(url, body)
 225     if wait_until_log_contains(TPCE_LOG, re.escape("Triggering notification stream NETCONF for node " + node_id), 60):
 226         print("Node " + node_id + " correctly added to tpce topology", end='... ', flush=True)
 227     else:
 228         print("Node " + node_id + " still not added to tpce topology", end='... ', flush=True)
 229         if response.status_code == requests.codes.ok:
 230             print("It was probably loaded at start-up", end='... ', flush=True)
 231         # TODO an else-clause to abort test would probably be nice here
 232     return response
 233
 234
 235 def unmount_device(node_id):
 236     url = URL_CONFIG_NETCONF_TOPO + "node/" + node_id
 237     response = delete_request(url)
 238     if wait_until_log_contains(TPCE_LOG, re.escape("onDeviceDisConnected: " + node_id), 60):
 239         print("Node " + node_id + " correctly deleted from tpce topology", end='... ', flush=True)
 240     else:
 241         print("Node " + node_id + " still not deleted from tpce topology", end='... ', flush=True)
 242     return response
 243
 244
 245 def connect_xpdr_to_rdm_request(xpdr_node: str, xpdr_num: str, network_num: str,
 246                                 rdm_node: str, srg_num: str, termination_num: str):
 247     url = "{}/operations/transportpce-networkutils:init-xpdr-rdm-links"
 248     data = {
 249         "networkutils:input": {
 250             "networkutils:links-input": {
 251                 "networkutils:xpdr-node": xpdr_node,
 252                 "networkutils:xpdr-num": xpdr_num,
 253                 "networkutils:network-num": network_num,
 254                 "networkutils:rdm-node": rdm_node,
 255                 "networkutils:srg-num": srg_num,
 256                 "networkutils:termination-point-num": termination_num
 257             }
 258         }
 259     }
 260     return post_request(url, data)
 261
 262
 263 def connect_rdm_to_xpdr_request(xpdr_node: str, xpdr_num: str, network_num: str,
 264                                 rdm_node: str, srg_num: str, termination_num: str):
 265     url = "{}/operations/transportpce-networkutils:init-rdm-xpdr-links"
 266     data = {
 267         "networkutils:input": {
 268             "networkutils:links-input": {
 269                 "networkutils:xpdr-node": xpdr_node,
 270                 "networkutils:xpdr-num": xpdr_num,
 271                 "networkutils:network-num": network_num,
 272                 "networkutils:rdm-node": rdm_node,
 273                 "networkutils:srg-num": srg_num,
 274                 "networkutils:termination-point-num": termination_num
 275             }
 276         }
 277     }
 278     return post_request(url, data)
 279
 280
 281 def check_netconf_node_request(node: str, suffix: str):
 282     url = URL_CONFIG_NETCONF_TOPO + (
 283         "node/" + node + "/yang-ext:mount/org-openroadm-device:org-openroadm-device/" + suffix
 284     )
 285     return get_request(url)
 286
 287
 288 def get_netconf_oper_request(node: str):
 289     url = "{}/operational/network-topology:network-topology/topology/topology-netconf/node/" + node
 290     return get_request(url)
 291
 292
 293 def get_ordm_topo_request(suffix: str):
 294     url = URL_CONFIG_ORDM_TOPO + suffix
 295     return get_request(url)
 296
 297
 298 def add_oms_attr_request(link: str, attr):
 299     url = URL_CONFIG_ORDM_TOPO + (
 300         "ietf-network-topology:link/" + link + "/org-openroadm-network-topology:OMS-attributes/span"
 301     )
 302     return put_request(url, attr)
 303
 304
 305 def del_oms_attr_request(link: str):
 306     url = URL_CONFIG_ORDM_TOPO + (
 307         "ietf-network-topology:link/" + link + "/org-openroadm-network-topology:OMS-attributes/span"
 308     )
 309     return delete_request(url)
 310
 311
 312 def get_clli_net_request():
 313     return get_request(URL_CONFIG_CLLI_NET)
 314
 315
 316 def get_ordm_net_request():
 317     return get_request(URL_CONFIG_ORDM_NET)
 318
 319
 320 def get_otn_topo_request():
 321     return get_request(URL_CONFIG_OTN_TOPO)
 322
 323
 324 def del_link_request(link: str):
 325     url = URL_CONFIG_ORDM_TOPO + ("ietf-network-topology:link/" + link)
 326     return delete_request(url)
 327
 328
 329 def del_node_request(node: str):
 330     url = URL_CONFIG_CLLI_NET + ("node/" + node)
 331     return delete_request(url)
 332
 333
 334 def portmapping_request(suffix: str):
 335     url = URL_PORTMAPPING + suffix
 336     return get_request(url)
 337
 338
 339 def get_service_list_request(suffix: str):
 340     url = URL_OPER_SERV_LIST + suffix
 341     return get_request(url)
 342
 343
 344 def service_create_request(attr):
 345     return post_request(URL_SERV_CREATE, attr)
 346
 347
 348 def service_delete_request(servicename: str,
 349                            requestid="e3028bae-a90f-4ddd-a83f-cf224eba0e58",
 350                            notificationurl="http://localhost:8585/NotificationServer/notify"):
 351     attr = {"input": {
 352         "sdnc-request-header": {
 353             "request-id": requestid,
 354             "rpc-action": "service-delete",
 355             "request-system-id": "appname",
 356             "notification-url": notificationurl},
 357         "service-delete-req-info": {
 358             "service-name": servicename,
 359             "tail-retention": "no"}}}
 360     return post_request(URL_SERV_DELETE, attr)
 361
 362
 363 def service_path_request(operation: str, servicename: str, wavenumber: str, nodes, centerfreq: str,
 364                          slotwidth: int, minfreq: float, maxfreq: float, lowerslotnumber: int,
 365                          higherslotnumber: int):
 366     attr = {"renderer:input": {
 367         "renderer:service-name": servicename,
 368         "renderer:wave-number": wavenumber,
 369         "renderer:modulation-format": "dp-qpsk",
 370         "renderer:operation": operation,
 371         "renderer:nodes": nodes,
 372         "renderer:center-freq": centerfreq,
 373         "renderer:width": slotwidth,
 374         "renderer:min-freq": minfreq,
 375         "renderer:max-freq": maxfreq,
 376         "renderer:lower-spectral-slot-number": lowerslotnumber,
 377         "renderer:higher-spectral-slot-number": higherslotnumber}}
 378     return post_request(URL_SERVICE_PATH, attr)
 379
 380
 381 def otn_service_path_request(operation: str, servicename: str, servicerate: str, servicetype: str, nodes,
 382                              eth_attr=None):
 383     attr = {"service-name": servicename,
 384             "operation": operation,
 385             "service-rate": servicerate,
 386             "service-type": servicetype,
 387             "nodes": nodes}
 388     if eth_attr:
 389         attr.update(eth_attr)
 390     return post_request(URL_OTN_SERVICE_PATH, {"renderer:input": attr})
 391
 392
 393 def create_ots_oms_request(nodeid: str, lcp: str):
 394     attr = {"input": {
 395         "node-id": nodeid,
 396         "logical-connection-point": lcp}}
 397     return post_request(URL_CREATE_OTS_OMS, attr)
 398
 399
 400 def path_computation_request(requestid: str, servicename: str, serviceaend, servicezend,
 401                              hardconstraints=None, softconstraints=None, metric="hop-count", other_attr=None):
 402     attr = {"service-name": servicename,
 403             "resource-reserve": "true",
 404             "service-handler-header": {"request-id": requestid},
 405             "service-a-end": serviceaend,
 406             "service-z-end": servicezend,
 407             "pce-metric": metric}
 408     if hardconstraints:
 409         attr.update({"hard-constraints": hardconstraints})
 410     if softconstraints:
 411         attr.update({"soft-constraints": softconstraints})
 412     if other_attr:
 413         attr.update(other_attr)
 414     return post_request(URL_PATH_COMPUTATION_REQUEST, {"input": attr})
 415
 416
 417 def shutdown_process(process):
 418     if process is not None:
 419         for child in psutil.Process(process.pid).children():
 420             child.send_signal(signal.SIGINT)
 421             child.wait()
 422         process.send_signal(signal.SIGINT)
 423
 424
 425 def start_honeynode(log_file: str, node_port: str, node_config_file_name: str):
 426     if os.path.isfile(HONEYNODE_EXECUTABLE):
 427         with open(log_file, 'w') as outfile:
 428             return subprocess.Popen(
 429                 [HONEYNODE_EXECUTABLE, node_port, os.path.join(SAMPLES_DIRECTORY, node_config_file_name)],
 430                 stdout=outfile, stderr=outfile)
 431     return None
 432
 433
 434 def wait_until_log_contains(log_file, regexp, time_to_wait=20):
 435     # pylint: disable=lost-exception
 436     stringfound = False
 437     filefound = False
 438     line = None
 439     try:
 440         with TimeOut(seconds=time_to_wait):
 441             while not os.path.exists(log_file):
 442                 time.sleep(0.2)
 443             filelogs = open(log_file, 'r')
 444             filelogs.seek(0, 2)
 445             filefound = True
 446             print("Searching for pattern '" + regexp + "' in " + os.path.basename(log_file), end='... ', flush=True)
 447             compiled_regexp = re.compile(regexp)
 448             while True:
 449                 line = filelogs.readline()
 450                 if compiled_regexp.search(line):
 451                     print("Pattern found!", end=' ')
 452                     stringfound = True
 453                     break
 454                 if not line:
 455                     time.sleep(0.1)
 456     except TimeoutError:
 457         print("Pattern not found after " + str(time_to_wait), end=" seconds! ", flush=True)
 458     except PermissionError:
 459         print("Permission Error when trying to access the log file", end=" ... ", flush=True)
 460     finally:
 461         if filefound:
 462             filelogs.close()
 463         else:
 464             print("log file does not exist or is not accessible... ", flush=True)
 465         return stringfound
 466
 467
 468 class TimeOut:
 469     def __init__(self, seconds=1, error_message='Timeout'):
 470         self.seconds = seconds
 471         self.error_message = error_message
 472
 473     def handle_timeout(self, signum, frame):
 474         raise TimeoutError(self.error_message)
 475
 476     def __enter__(self):
 477         signal.signal(signal.SIGALRM, self.handle_timeout)
 478         signal.alarm(self.seconds)
 479
 480     def __exit__(self, type, value, traceback):
 481         # pylint: disable=W0622
 482         signal.alarm(0)
 483