tests/transportpce_tests/common/test_utils.py

   1 #!/usr/bin/env python
   2
   3 ##############################################################################
   4 # Copyright (c) 2020 Orange, Inc. and others.  All rights reserved.
   5 #
   6 # All rights reserved. This program and the accompanying materials
   7 # are made available under the terms of the Apache License, Version 2.0
   8 # which accompanies this distribution, and is available at
   9 # http://www.apache.org/licenses/LICENSE-2.0
  10 ##############################################################################
  11
  12 # pylint: disable=no-member
  13
  14 import json
  15 import os
  16 import sys
  17 import re
  18 import signal
  19 import subprocess
  20 import time
  21
  22 import psutil
  23 import requests
  24
  25 import simulators
  26
  27 SIMS = simulators.SIMS
  28 HONEYNODE_EXECUTABLE = simulators.HONEYNODE_EXECUTABLE
  29 SAMPLES_DIRECTORY = simulators.SAMPLES_DIRECTORY
  30
  31 HONEYNODE_OK_START_MSG = "Netconf SSH endpoint started successfully at 0.0.0.0"
  32 KARAF_OK_START_MSG = re.escape(
  33     "Blueprint container for bundle org.opendaylight.netconf.restconf")+".* was successfully created"
  34 LIGHTY_OK_START_MSG = re.escape("lighty.io and RESTCONF-NETCONF started")
  35
  36 RESTCONF_BASE_URL = "http://localhost:8181/restconf"
  37 ODL_LOGIN = "admin"
  38 ODL_PWD = "admin"
  39 NODES_LOGIN = "admin"
  40 NODES_PWD = "admin"
  41 URL_CONFIG_NETCONF_TOPO = "{}/config/network-topology:network-topology/topology/topology-netconf/"
  42 URL_CONFIG_ORDM_TOPO = "{}/config/ietf-network:networks/network/openroadm-topology/"
  43 URL_CONFIG_OTN_TOPO = "{}/config/ietf-network:networks/network/otn-topology/"
  44 URL_CONFIG_CLLI_NET = "{}/config/ietf-network:networks/network/clli-network/"
  45 URL_CONFIG_ORDM_NET = "{}/config/ietf-network:networks/network/openroadm-network/"
  46 URL_PORTMAPPING = "{}/config/transportpce-portmapping:network/nodes/"
  47 URL_OPER_SERV_LIST = "{}/operational/org-openroadm-service:service-list/"
  48 URL_GET_NBINOTIFICATIONS_SERV = "{}/operations/nbi-notifications:get-notifications-service/"
  49 URL_SERV_CREATE = "{}/operations/org-openroadm-service:service-create"
  50 URL_SERV_DELETE = "{}/operations/org-openroadm-service:service-delete"
  51 URL_SERVICE_PATH = "{}/operations/transportpce-device-renderer:service-path"
  52 URL_OTN_SERVICE_PATH = "{}/operations/transportpce-device-renderer:otn-service-path"
  53 URL_CREATE_OTS_OMS = "{}/operations/transportpce-device-renderer:create-ots-oms"
  54 URL_PATH_COMPUTATION_REQUEST = "{}/operations/transportpce-pce:path-computation-request"
  55 URL_FULL_PORTMAPPING = "{}/config/transportpce-portmapping:network"
  56
  57 TYPE_APPLICATION_JSON = {'Content-Type': 'application/json', 'Accept': 'application/json'}
  58 TYPE_APPLICATION_XML = {'Content-Type': 'application/xml', 'Accept': 'application/xml'}
  59
  60 CODE_SHOULD_BE_200 = 'Http status code should be 200'
  61 CODE_SHOULD_BE_201 = 'Http status code should be 201'
  62
  63 LOG_DIRECTORY = os.path.dirname(os.path.realpath(__file__))
  64
  65 KARAF_LOG = os.path.join(
  66     os.path.dirname(os.path.realpath(__file__)),
  67     "..", "..", "..", "karaf", "target", "assembly", "data", "log", "karaf.log")
  68
  69 process_list = []
  70
  71 if "USE_LIGHTY" in os.environ and os.environ['USE_LIGHTY'] == 'True':
  72     TPCE_LOG = 'odl.log'
  73 else:
  74     TPCE_LOG = KARAF_LOG
  75
  76
  77 def start_sims(sims_list):
  78     for sim in sims_list:
  79         print("starting simulator for " + sim + "...")
  80         log_file = os.path.join(LOG_DIRECTORY, SIMS[sim]['logfile'])
  81         process = start_honeynode(log_file, SIMS[sim]['port'], SIMS[sim]['configfile'])
  82         if wait_until_log_contains(log_file, HONEYNODE_OK_START_MSG, 200):
  83             print("simulator for " + sim + " started")
  84         else:
  85             print("simulator for " + sim + " failed to start")
  86             shutdown_process(process)
  87             for pid in process_list:
  88                 shutdown_process(pid)
  89             sys.exit(3)
  90         process_list.append(process)
  91     return process_list
  92
  93
  94 def start_tpce():
  95     print("starting OpenDaylight...")
  96     if "USE_LIGHTY" in os.environ and os.environ['USE_LIGHTY'] == 'True':
  97         process = start_lighty()
  98         start_msg = LIGHTY_OK_START_MSG
  99     else:
 100         process = start_karaf()
 101         start_msg = KARAF_OK_START_MSG
 102     if wait_until_log_contains(TPCE_LOG, start_msg, time_to_wait=120):
 103         print("OpenDaylight started !")
 104     else:
 105         print("OpenDaylight failed to start !")
 106         shutdown_process(process)
 107         for pid in process_list:
 108             shutdown_process(pid)
 109         sys.exit(1)
 110     process_list.append(process)
 111     return process_list
 112
 113
 114 def start_karaf():
 115     print("starting KARAF TransportPCE build...")
 116     executable = os.path.join(
 117         os.path.dirname(os.path.realpath(__file__)),
 118         "..", "..", "..", "karaf", "target", "assembly", "bin", "karaf")
 119     with open('odl.log', 'w') as outfile:
 120         return subprocess.Popen(
 121             ["sh", executable, "server"], stdout=outfile, stderr=outfile, stdin=None)
 122
 123
 124 def start_lighty():
 125     print("starting LIGHTY.IO TransportPCE build...")
 126     executable = os.path.join(
 127         os.path.dirname(os.path.realpath(__file__)),
 128         "..", "..", "..", "lighty", "target", "tpce",
 129         "clean-start-controller.sh")
 130     with open('odl.log', 'w') as outfile:
 131         return subprocess.Popen(
 132             ["sh", executable], stdout=outfile, stderr=outfile, stdin=None)
 133
 134
 135 def install_karaf_feature(feature_name: str):
 136     print("installing feature " + feature_name)
 137     executable = os.path.join(
 138         os.path.dirname(os.path.realpath(__file__)),
 139         "..", "..", "..", "karaf", "target", "assembly", "bin", "client")
 140     return subprocess.run([executable],
 141                           input='feature:install ' + feature_name + '\n feature:list | grep '
 142                           + feature_name + ' \n logout \n',
 143                           universal_newlines=True, check=False)
 144
 145
 146 def get_request(url):
 147     return requests.request(
 148         "GET", url.format(RESTCONF_BASE_URL),
 149         headers=TYPE_APPLICATION_JSON,
 150         auth=(ODL_LOGIN, ODL_PWD))
 151
 152
 153 def post_request(url, data):
 154     if data:
 155         print(json.dumps(data))
 156         return requests.request(
 157             "POST", url.format(RESTCONF_BASE_URL),
 158             data=json.dumps(data),
 159             headers=TYPE_APPLICATION_JSON,
 160             auth=(ODL_LOGIN, ODL_PWD))
 161
 162     return requests.request(
 163         "POST", url.format(RESTCONF_BASE_URL),
 164         headers=TYPE_APPLICATION_JSON,
 165         auth=(ODL_LOGIN, ODL_PWD))
 166
 167
 168 def post_xmlrequest(url, data):
 169     if data:
 170         return requests.request(
 171             "POST", url.format(RESTCONF_BASE_URL),
 172             data=data,
 173             headers=TYPE_APPLICATION_XML,
 174             auth=(ODL_LOGIN, ODL_PWD))
 175     return None
 176
 177
 178 def put_request(url, data):
 179     return requests.request(
 180         "PUT", url.format(RESTCONF_BASE_URL),
 181         data=json.dumps(data),
 182         headers=TYPE_APPLICATION_JSON,
 183         auth=(ODL_LOGIN, ODL_PWD))
 184
 185
 186 def put_xmlrequest(url, data):
 187     return requests.request(
 188         "PUT", url.format(RESTCONF_BASE_URL),
 189         data=data,
 190         headers=TYPE_APPLICATION_XML,
 191         auth=(ODL_LOGIN, ODL_PWD))
 192
 193 def put_jsonrequest(url, data):
 194     return requests.request(
 195         "PUT", url.format(RESTCONF_BASE_URL),
 196         data=data,
 197         headers=TYPE_APPLICATION_JSON,
 198         auth=(ODL_LOGIN, ODL_PWD))
 199
 200 def rawput_request(url, data):
 201     return requests.request(
 202         "PUT", url.format(RESTCONF_BASE_URL),
 203         data=data,
 204         headers=TYPE_APPLICATION_JSON,
 205         auth=(ODL_LOGIN, ODL_PWD))
 206
 207 def rawpost_request(url, data):
 208     return requests.request(
 209         "POST", url.format(RESTCONF_BASE_URL),
 210         data=data,
 211         headers=TYPE_APPLICATION_JSON,
 212         auth=(ODL_LOGIN, ODL_PWD))
 213
 214
 215 def delete_request(url):
 216     return requests.request(
 217         "DELETE", url.format(RESTCONF_BASE_URL),
 218         headers=TYPE_APPLICATION_JSON,
 219         auth=(ODL_LOGIN, ODL_PWD))
 220
 221
 222 def mount_device(node_id, sim):
 223     url = URL_CONFIG_NETCONF_TOPO + "node/" + node_id
 224     body = {"node": [{
 225         "node-id": node_id,
 226         "netconf-node-topology:username": NODES_LOGIN,
 227         "netconf-node-topology:password": NODES_PWD,
 228         "netconf-node-topology:host": "127.0.0.1",
 229         "netconf-node-topology:port": SIMS[sim]['port'],
 230         "netconf-node-topology:tcp-only": "false",
 231         "netconf-node-topology:pass-through": {}}]}
 232     response = put_request(url, body)
 233     if wait_until_log_contains(TPCE_LOG, re.escape("Triggering notification stream NETCONF for node " + node_id), 60):
 234         print("Node " + node_id + " correctly added to tpce topology", end='... ', flush=True)
 235     else:
 236         print("Node " + node_id + " still not added to tpce topology", end='... ', flush=True)
 237         if response.status_code == requests.codes.ok:
 238             print("It was probably loaded at start-up", end='... ', flush=True)
 239         # TODO an else-clause to abort test would probably be nice here
 240     return response
 241
 242
 243 def unmount_device(node_id):
 244     url = URL_CONFIG_NETCONF_TOPO + "node/" + node_id
 245     response = delete_request(url)
 246     if wait_until_log_contains(TPCE_LOG, re.escape("onDeviceDisConnected: " + node_id), 60):
 247         print("Node " + node_id + " correctly deleted from tpce topology", end='... ', flush=True)
 248     else:
 249         print("Node " + node_id + " still not deleted from tpce topology", end='... ', flush=True)
 250     return response
 251
 252
 253 def connect_xpdr_to_rdm_request(xpdr_node: str, xpdr_num: str, network_num: str,
 254                                 rdm_node: str, srg_num: str, termination_num: str):
 255     url = "{}/operations/transportpce-networkutils:init-xpdr-rdm-links"
 256     data = {
 257         "networkutils:input": {
 258             "networkutils:links-input": {
 259                 "networkutils:xpdr-node": xpdr_node,
 260                 "networkutils:xpdr-num": xpdr_num,
 261                 "networkutils:network-num": network_num,
 262                 "networkutils:rdm-node": rdm_node,
 263                 "networkutils:srg-num": srg_num,
 264                 "networkutils:termination-point-num": termination_num
 265             }
 266         }
 267     }
 268     return post_request(url, data)
 269
 270
 271 def connect_rdm_to_xpdr_request(xpdr_node: str, xpdr_num: str, network_num: str,
 272                                 rdm_node: str, srg_num: str, termination_num: str):
 273     url = "{}/operations/transportpce-networkutils:init-rdm-xpdr-links"
 274     data = {
 275         "networkutils:input": {
 276             "networkutils:links-input": {
 277                 "networkutils:xpdr-node": xpdr_node,
 278                 "networkutils:xpdr-num": xpdr_num,
 279                 "networkutils:network-num": network_num,
 280                 "networkutils:rdm-node": rdm_node,
 281                 "networkutils:srg-num": srg_num,
 282                 "networkutils:termination-point-num": termination_num
 283             }
 284         }
 285     }
 286     return post_request(url, data)
 287
 288
 289 def check_netconf_node_request(node: str, suffix: str):
 290     url = URL_CONFIG_NETCONF_TOPO + (
 291         "node/" + node + "/yang-ext:mount/org-openroadm-device:org-openroadm-device/" + suffix
 292     )
 293     return get_request(url)
 294
 295
 296 def get_netconf_oper_request(node: str):
 297     url = "{}/operational/network-topology:network-topology/topology/topology-netconf/node/" + node
 298     return get_request(url)
 299
 300
 301 def get_ordm_topo_request(suffix: str):
 302     url = URL_CONFIG_ORDM_TOPO + suffix
 303     return get_request(url)
 304
 305
 306 def add_oms_attr_request(link: str, attr):
 307     url = URL_CONFIG_ORDM_TOPO + (
 308         "ietf-network-topology:link/" + link + "/org-openroadm-network-topology:OMS-attributes/span"
 309     )
 310     return put_request(url, attr)
 311
 312
 313 def del_oms_attr_request(link: str):
 314     url = URL_CONFIG_ORDM_TOPO + (
 315         "ietf-network-topology:link/" + link + "/org-openroadm-network-topology:OMS-attributes/span"
 316     )
 317     return delete_request(url)
 318
 319
 320 def get_clli_net_request():
 321     return get_request(URL_CONFIG_CLLI_NET)
 322
 323
 324 def get_ordm_net_request():
 325     return get_request(URL_CONFIG_ORDM_NET)
 326
 327
 328 def get_otn_topo_request():
 329     return get_request(URL_CONFIG_OTN_TOPO)
 330
 331
 332 def del_link_request(link: str):
 333     url = URL_CONFIG_ORDM_TOPO + ("ietf-network-topology:link/" + link)
 334     return delete_request(url)
 335
 336
 337 def del_node_request(node: str):
 338     url = URL_CONFIG_CLLI_NET + ("node/" + node)
 339     return delete_request(url)
 340
 341
 342 def portmapping_request(suffix: str):
 343     url = URL_PORTMAPPING + suffix
 344     return get_request(url)
 345
 346
 347 def get_notifications_service_request(attr):
 348     return post_request(URL_GET_NBINOTIFICATIONS_SERV, attr)
 349
 350
 351 def get_service_list_request(suffix: str):
 352     url = URL_OPER_SERV_LIST + suffix
 353     return get_request(url)
 354
 355
 356 def service_create_request(attr):
 357     return post_request(URL_SERV_CREATE, attr)
 358
 359
 360 def service_delete_request(servicename: str,
 361                            requestid="e3028bae-a90f-4ddd-a83f-cf224eba0e58",
 362                            notificationurl="http://localhost:8585/NotificationServer/notify"):
 363     attr = {"input": {
 364         "sdnc-request-header": {
 365             "request-id": requestid,
 366             "rpc-action": "service-delete",
 367             "request-system-id": "appname",
 368             "notification-url": notificationurl},
 369         "service-delete-req-info": {
 370             "service-name": servicename,
 371             "tail-retention": "no"}}}
 372     return post_request(URL_SERV_DELETE, attr)
 373
 374
 375 def service_path_request(operation: str, servicename: str, wavenumber: str, nodes, centerfreq: str,
 376                          slotwidth: int, minfreq: float, maxfreq: float, lowerslotnumber: int,
 377                          higherslotnumber: int):
 378     attr = {"renderer:input": {
 379         "renderer:service-name": servicename,
 380         "renderer:wave-number": wavenumber,
 381         "renderer:modulation-format": "dp-qpsk",
 382         "renderer:operation": operation,
 383         "renderer:nodes": nodes,
 384         "renderer:center-freq": centerfreq,
 385         "renderer:width": slotwidth,
 386         "renderer:min-freq": minfreq,
 387         "renderer:max-freq": maxfreq,
 388         "renderer:lower-spectral-slot-number": lowerslotnumber,
 389         "renderer:higher-spectral-slot-number": higherslotnumber}}
 390     return post_request(URL_SERVICE_PATH, attr)
 391
 392
 393 def otn_service_path_request(operation: str, servicename: str, servicerate: str, servicetype: str, nodes,
 394                              eth_attr=None):
 395     attr = {"service-name": servicename,
 396             "operation": operation,
 397             "service-rate": servicerate,
 398             "service-type": servicetype,
 399             "nodes": nodes}
 400     if eth_attr:
 401         attr.update(eth_attr)
 402     return post_request(URL_OTN_SERVICE_PATH, {"renderer:input": attr})
 403
 404
 405 def create_ots_oms_request(nodeid: str, lcp: str):
 406     attr = {"input": {
 407         "node-id": nodeid,
 408         "logical-connection-point": lcp}}
 409     return post_request(URL_CREATE_OTS_OMS, attr)
 410
 411
 412 def path_computation_request(requestid: str, servicename: str, serviceaend, servicezend,
 413                              hardconstraints=None, softconstraints=None, metric="hop-count", other_attr=None):
 414     attr = {"service-name": servicename,
 415             "resource-reserve": "true",
 416             "service-handler-header": {"request-id": requestid},
 417             "service-a-end": serviceaend,
 418             "service-z-end": servicezend,
 419             "pce-metric": metric}
 420     if hardconstraints:
 421         attr.update({"hard-constraints": hardconstraints})
 422     if softconstraints:
 423         attr.update({"soft-constraints": softconstraints})
 424     if other_attr:
 425         attr.update(other_attr)
 426     return post_request(URL_PATH_COMPUTATION_REQUEST, {"input": attr})
 427
 428
 429 def shutdown_process(process):
 430     if process is not None:
 431         for child in psutil.Process(process.pid).children():
 432             child.send_signal(signal.SIGINT)
 433             child.wait()
 434         process.send_signal(signal.SIGINT)
 435
 436
 437 def start_honeynode(log_file: str, node_port: str, node_config_file_name: str):
 438     if os.path.isfile(HONEYNODE_EXECUTABLE):
 439         with open(log_file, 'w') as outfile:
 440             return subprocess.Popen(
 441                 [HONEYNODE_EXECUTABLE, node_port, os.path.join(SAMPLES_DIRECTORY, node_config_file_name)],
 442                 stdout=outfile, stderr=outfile)
 443     return None
 444
 445
 446 def wait_until_log_contains(log_file, regexp, time_to_wait=20):
 447     # pylint: disable=lost-exception
 448     stringfound = False
 449     filefound = False
 450     line = None
 451     try:
 452         with TimeOut(seconds=time_to_wait):
 453             while not os.path.exists(log_file):
 454                 time.sleep(0.2)
 455             filelogs = open(log_file, 'r')
 456             filelogs.seek(0, 2)
 457             filefound = True
 458             print("Searching for pattern '" + regexp + "' in " + os.path.basename(log_file), end='... ', flush=True)
 459             compiled_regexp = re.compile(regexp)
 460             while True:
 461                 line = filelogs.readline()
 462                 if compiled_regexp.search(line):
 463                     print("Pattern found!", end=' ')
 464                     stringfound = True
 465                     break
 466                 if not line:
 467                     time.sleep(0.1)
 468     except TimeoutError:
 469         print("Pattern not found after " + str(time_to_wait), end=" seconds! ", flush=True)
 470     except PermissionError:
 471         print("Permission Error when trying to access the log file", end=" ... ", flush=True)
 472     finally:
 473         if filefound:
 474             filelogs.close()
 475         else:
 476             print("log file does not exist or is not accessible... ", flush=True)
 477         return stringfound
 478
 479
 480 class TimeOut:
 481     def __init__(self, seconds=1, error_message='Timeout'):
 482         self.seconds = seconds
 483         self.error_message = error_message
 484
 485     def handle_timeout(self, signum, frame):
 486         raise TimeoutError(self.error_message)
 487
 488     def __enter__(self):
 489         signal.signal(signal.SIGALRM, self.handle_timeout)
 490         signal.alarm(self.seconds)
 491
 492     def __exit__(self, type, value, traceback):
 493         # pylint: disable=W0622
 494         signal.alarm(0)
 495