tests/transportpce_tests/common/test_utils.py

   1 #!/usr/bin/env python
   2
   3 ##############################################################################
   4 # Copyright (c) 2020 Orange, Inc. and others.  All rights reserved.
   5 #
   6 # All rights reserved. This program and the accompanying materials
   7 # are made available under the terms of the Apache License, Version 2.0
   8 # which accompanies this distribution, and is available at
   9 # http://www.apache.org/licenses/LICENSE-2.0
  10 ##############################################################################
  11
  12 # pylint: disable=no-member
  13
  14 import json
  15 import os
  16 import sys
  17 import re
  18 import signal
  19 import subprocess
  20 import time
  21
  22 import psutil
  23 import requests
  24
  25 import simulators
  26
  27 SIMS = simulators.SIMS
  28 HONEYNODE_EXECUTABLE = simulators.HONEYNODE_EXECUTABLE
  29 SAMPLES_DIRECTORY = simulators.SAMPLES_DIRECTORY
  30
  31 HONEYNODE_OK_START_MSG = "Netconf SSH endpoint started successfully at 0.0.0.0"
  32 KARAF_OK_START_MSG = re.escape(
  33     "Blueprint container for bundle org.opendaylight.netconf.restconf")+".* was successfully created"
  34 LIGHTY_OK_START_MSG = re.escape("lighty.io and RESTCONF-NETCONF started")
  35
  36 RESTCONF_BASE_URL = "http://localhost:8181/restconf"
  37 ODL_LOGIN = "admin"
  38 ODL_PWD = "admin"
  39 NODES_LOGIN = "admin"
  40 NODES_PWD = "admin"
  41 URL_CONFIG_NETCONF_TOPO = "{}/config/network-topology:network-topology/topology/topology-netconf/"
  42 URL_CONFIG_ORDM_TOPO = "{}/config/ietf-network:networks/network/openroadm-topology/"
  43 URL_CONFIG_OTN_TOPO = "{}/config/ietf-network:networks/network/otn-topology/"
  44 URL_CONFIG_CLLI_NET = "{}/config/ietf-network:networks/network/clli-network/"
  45 URL_CONFIG_ORDM_NET = "{}/config/ietf-network:networks/network/openroadm-network/"
  46 URL_PORTMAPPING = "{}/config/transportpce-portmapping:network/nodes/"
  47 URL_OPER_SERV_LIST = "{}/operational/org-openroadm-service:service-list/"
  48 URL_SERV_CREATE = "{}/operations/org-openroadm-service:service-create"
  49 URL_SERV_DELETE = "{}/operations/org-openroadm-service:service-delete"
  50 URL_SERVICE_PATH = "{}/operations/transportpce-device-renderer:service-path"
  51 URL_OTN_SERVICE_PATH = "{}/operations/transportpce-device-renderer:otn-service-path"
  52 URL_CREATE_OTS_OMS = "{}/operations/transportpce-device-renderer:create-ots-oms"
  53 URL_PATH_COMPUTATION_REQUEST = "{}/operations/transportpce-pce:path-computation-request"
  54 URL_FULL_PORTMAPPING = "{}/config/transportpce-portmapping:network"
  55
  56 TYPE_APPLICATION_JSON = {'Content-Type': 'application/json', 'Accept': 'application/json'}
  57 TYPE_APPLICATION_XML = {'Content-Type': 'application/xml', 'Accept': 'application/xml'}
  58
  59 CODE_SHOULD_BE_200 = 'Http status code should be 200'
  60 CODE_SHOULD_BE_201 = 'Http status code should be 201'
  61
  62 LOG_DIRECTORY = os.path.dirname(os.path.realpath(__file__))
  63
  64 KARAF_LOG = os.path.join(
  65     os.path.dirname(os.path.realpath(__file__)),
  66     "..", "..", "..", "karaf", "target", "assembly", "data", "log", "karaf.log")
  67
  68 process_list = []
  69
  70 if "USE_LIGHTY" in os.environ and os.environ['USE_LIGHTY'] == 'True':
  71     TPCE_LOG = 'odl.log'
  72 else:
  73     TPCE_LOG = KARAF_LOG
  74
  75
  76 def start_sims(sims_list):
  77     for sim in sims_list:
  78         print("starting simulator for " + sim + "...")
  79         log_file = os.path.join(LOG_DIRECTORY, SIMS[sim]['logfile'])
  80         process = start_honeynode(log_file, SIMS[sim]['port'], SIMS[sim]['configfile'])
  81         if wait_until_log_contains(log_file, HONEYNODE_OK_START_MSG, 100):
  82             print("simulator for " + sim + " started")
  83         else:
  84             print("simulator for " + sim + " failed to start")
  85             shutdown_process(process)
  86             for pid in process_list:
  87                 shutdown_process(pid)
  88             sys.exit(3)
  89         process_list.append(process)
  90     return process_list
  91
  92
  93 def start_tpce():
  94     print("starting OpenDaylight...")
  95     if "USE_LIGHTY" in os.environ and os.environ['USE_LIGHTY'] == 'True':
  96         process = start_lighty()
  97         start_msg = LIGHTY_OK_START_MSG
  98     else:
  99         process = start_karaf()
 100         start_msg = KARAF_OK_START_MSG
 101     if wait_until_log_contains(TPCE_LOG, start_msg, time_to_wait=60):
 102         print("OpenDaylight started !")
 103     else:
 104         print("OpenDaylight failed to start !")
 105         shutdown_process(process)
 106         for pid in process_list:
 107             shutdown_process(pid)
 108         sys.exit(1)
 109     process_list.append(process)
 110     return process_list
 111
 112
 113 def start_karaf():
 114     print("starting KARAF TransportPCE build...")
 115     executable = os.path.join(
 116         os.path.dirname(os.path.realpath(__file__)),
 117         "..", "..", "..", "karaf", "target", "assembly", "bin", "karaf")
 118     with open('odl.log', 'w') as outfile:
 119         return subprocess.Popen(
 120             ["sh", executable, "server"], stdout=outfile, stderr=outfile, stdin=None)
 121
 122
 123 def start_lighty():
 124     print("starting LIGHTY.IO TransportPCE build...")
 125     executable = os.path.join(
 126         os.path.dirname(os.path.realpath(__file__)),
 127         "..", "..", "..", "lighty", "target", "tpce",
 128         "clean-start-controller.sh")
 129     with open('odl.log', 'w') as outfile:
 130         return subprocess.Popen(
 131             ["sh", executable], stdout=outfile, stderr=outfile, stdin=None)
 132
 133
 134 def install_karaf_feature(feature_name: str):
 135     print("installing feature " + feature_name)
 136     executable = os.path.join(
 137         os.path.dirname(os.path.realpath(__file__)),
 138         "..", "..", "..", "karaf", "target", "assembly", "bin", "client")
 139     return subprocess.run([executable],
 140                           input='feature:install ' + feature_name + '\n feature:list | grep tapi \n logout \n',
 141                           universal_newlines=True, check=False)
 142
 143
 144 def get_request(url):
 145     return requests.request(
 146         "GET", url.format(RESTCONF_BASE_URL),
 147         headers=TYPE_APPLICATION_JSON,
 148         auth=(ODL_LOGIN, ODL_PWD))
 149
 150
 151 def post_request(url, data):
 152     if data:
 153         return requests.request(
 154             "POST", url.format(RESTCONF_BASE_URL),
 155             data=json.dumps(data),
 156             headers=TYPE_APPLICATION_JSON,
 157             auth=(ODL_LOGIN, ODL_PWD))
 158
 159     return requests.request(
 160         "POST", url.format(RESTCONF_BASE_URL),
 161         headers=TYPE_APPLICATION_JSON,
 162         auth=(ODL_LOGIN, ODL_PWD))
 163
 164
 165 def post_xmlrequest(url, data):
 166     if data:
 167         return requests.request(
 168             "POST", url.format(RESTCONF_BASE_URL),
 169             data=data,
 170             headers=TYPE_APPLICATION_XML,
 171             auth=(ODL_LOGIN, ODL_PWD))
 172     return None
 173
 174
 175 def put_request(url, data):
 176     return requests.request(
 177         "PUT", url.format(RESTCONF_BASE_URL),
 178         data=json.dumps(data),
 179         headers=TYPE_APPLICATION_JSON,
 180         auth=(ODL_LOGIN, ODL_PWD))
 181
 182
 183 def put_xmlrequest(url, data):
 184     return requests.request(
 185         "PUT", url.format(RESTCONF_BASE_URL),
 186         data=data,
 187         headers=TYPE_APPLICATION_XML,
 188         auth=(ODL_LOGIN, ODL_PWD))
 189
 190
 191 def rawput_request(url, data):
 192     return requests.request(
 193         "PUT", url.format(RESTCONF_BASE_URL),
 194         data=data,
 195         headers=TYPE_APPLICATION_JSON,
 196         auth=(ODL_LOGIN, ODL_PWD))
 197
 198 def rawpost_request(url, data):
 199     return requests.request(
 200         "POST", url.format(RESTCONF_BASE_URL),
 201         data=data,
 202         headers=TYPE_APPLICATION_JSON,
 203         auth=(ODL_LOGIN, ODL_PWD))
 204
 205
 206 def delete_request(url):
 207     return requests.request(
 208         "DELETE", url.format(RESTCONF_BASE_URL),
 209         headers=TYPE_APPLICATION_JSON,
 210         auth=(ODL_LOGIN, ODL_PWD))
 211
 212
 213 def mount_device(node_id, sim):
 214     url = URL_CONFIG_NETCONF_TOPO + "node/" + node_id
 215     body = {"node": [{
 216         "node-id": node_id,
 217         "netconf-node-topology:username": NODES_LOGIN,
 218         "netconf-node-topology:password": NODES_PWD,
 219         "netconf-node-topology:host": "127.0.0.1",
 220         "netconf-node-topology:port": SIMS[sim]['port'],
 221         "netconf-node-topology:tcp-only": "false",
 222         "netconf-node-topology:pass-through": {}}]}
 223     response = put_request(url, body)
 224     if wait_until_log_contains(TPCE_LOG, re.escape("Triggering notification stream NETCONF for node " + node_id), 60):
 225         print("Node " + node_id + " correctly added to tpce topology", end='... ', flush=True)
 226     else:
 227         print("Node " + node_id + " still not added to tpce topology", end='... ', flush=True)
 228         if response.status_code == requests.codes.ok:
 229             print("It was probably loaded at start-up", end='... ', flush=True)
 230         # TODO an else-clause to abort test would probably be nice here
 231     return response
 232
 233
 234 def unmount_device(node_id):
 235     url = URL_CONFIG_NETCONF_TOPO + "node/" + node_id
 236     response = delete_request(url)
 237     if wait_until_log_contains(TPCE_LOG, re.escape("onDeviceDisConnected: " + node_id), 60):
 238         print("Node " + node_id + " correctly deleted from tpce topology", end='... ', flush=True)
 239     else:
 240         print("Node " + node_id + " still not deleted from tpce topology", end='... ', flush=True)
 241     return response
 242
 243
 244 def connect_xpdr_to_rdm_request(xpdr_node: str, xpdr_num: str, network_num: str,
 245                                 rdm_node: str, srg_num: str, termination_num: str):
 246     url = "{}/operations/transportpce-networkutils:init-xpdr-rdm-links"
 247     data = {
 248         "networkutils:input": {
 249             "networkutils:links-input": {
 250                 "networkutils:xpdr-node": xpdr_node,
 251                 "networkutils:xpdr-num": xpdr_num,
 252                 "networkutils:network-num": network_num,
 253                 "networkutils:rdm-node": rdm_node,
 254                 "networkutils:srg-num": srg_num,
 255                 "networkutils:termination-point-num": termination_num
 256             }
 257         }
 258     }
 259     return post_request(url, data)
 260
 261
 262 def connect_rdm_to_xpdr_request(xpdr_node: str, xpdr_num: str, network_num: str,
 263                                 rdm_node: str, srg_num: str, termination_num: str):
 264     url = "{}/operations/transportpce-networkutils:init-rdm-xpdr-links"
 265     data = {
 266         "networkutils:input": {
 267             "networkutils:links-input": {
 268                 "networkutils:xpdr-node": xpdr_node,
 269                 "networkutils:xpdr-num": xpdr_num,
 270                 "networkutils:network-num": network_num,
 271                 "networkutils:rdm-node": rdm_node,
 272                 "networkutils:srg-num": srg_num,
 273                 "networkutils:termination-point-num": termination_num
 274             }
 275         }
 276     }
 277     return post_request(url, data)
 278
 279
 280 def check_netconf_node_request(node: str, suffix: str):
 281     url = URL_CONFIG_NETCONF_TOPO + (
 282         "node/" + node + "/yang-ext:mount/org-openroadm-device:org-openroadm-device/" + suffix
 283     )
 284     return get_request(url)
 285
 286
 287 def get_netconf_oper_request(node: str):
 288     url = "{}/operational/network-topology:network-topology/topology/topology-netconf/node/" + node
 289     return get_request(url)
 290
 291
 292 def get_ordm_topo_request(suffix: str):
 293     url = URL_CONFIG_ORDM_TOPO + suffix
 294     return get_request(url)
 295
 296
 297 def add_oms_attr_request(link: str, attr):
 298     url = URL_CONFIG_ORDM_TOPO + (
 299         "ietf-network-topology:link/" + link + "/org-openroadm-network-topology:OMS-attributes/span"
 300     )
 301     return put_request(url, attr)
 302
 303
 304 def del_oms_attr_request(link: str):
 305     url = URL_CONFIG_ORDM_TOPO + (
 306         "ietf-network-topology:link/" + link + "/org-openroadm-network-topology:OMS-attributes/span"
 307     )
 308     return delete_request(url)
 309
 310
 311 def get_clli_net_request():
 312     return get_request(URL_CONFIG_CLLI_NET)
 313
 314
 315 def get_ordm_net_request():
 316     return get_request(URL_CONFIG_ORDM_NET)
 317
 318
 319 def get_otn_topo_request():
 320     return get_request(URL_CONFIG_OTN_TOPO)
 321
 322
 323 def del_link_request(link: str):
 324     url = URL_CONFIG_ORDM_TOPO + ("ietf-network-topology:link/" + link)
 325     return delete_request(url)
 326
 327
 328 def del_node_request(node: str):
 329     url = URL_CONFIG_CLLI_NET + ("node/" + node)
 330     return delete_request(url)
 331
 332
 333 def portmapping_request(suffix: str):
 334     url = URL_PORTMAPPING + suffix
 335     return get_request(url)
 336
 337
 338 def get_service_list_request(suffix: str):
 339     url = URL_OPER_SERV_LIST + suffix
 340     return get_request(url)
 341
 342
 343 def service_create_request(attr):
 344     return post_request(URL_SERV_CREATE, attr)
 345
 346
 347 def service_delete_request(servicename: str,
 348                            requestid="e3028bae-a90f-4ddd-a83f-cf224eba0e58",
 349                            notificationurl="http://localhost:8585/NotificationServer/notify"):
 350     attr = {"input": {
 351         "sdnc-request-header": {
 352             "request-id": requestid,
 353             "rpc-action": "service-delete",
 354             "request-system-id": "appname",
 355             "notification-url": notificationurl},
 356         "service-delete-req-info": {
 357             "service-name": servicename,
 358             "tail-retention": "no"}}}
 359     return post_request(URL_SERV_DELETE, attr)
 360
 361
 362 def service_path_request(operation: str, servicename: str, wavenumber: str, nodes):
 363     attr = {"renderer:input": {
 364         "renderer:service-name": servicename,
 365         "renderer:wave-number": wavenumber,
 366         "renderer:modulation-format": "qpsk",
 367         "renderer:operation": operation,
 368         "renderer:nodes": nodes}}
 369     return post_request(URL_SERVICE_PATH, attr)
 370
 371
 372 def otn_service_path_request(operation: str, servicename: str, servicerate: str, servicetype: str, nodes,
 373                              eth_attr=None):
 374     attr = {"service-name": servicename,
 375             "operation": operation,
 376             "service-rate": servicerate,
 377             "service-type": servicetype,
 378             "nodes": nodes}
 379     if eth_attr:
 380         attr.update(eth_attr)
 381     return post_request(URL_OTN_SERVICE_PATH, {"renderer:input": attr})
 382
 383
 384 def create_ots_oms_request(nodeid: str, lcp: str):
 385     attr = {"input": {
 386         "node-id": nodeid,
 387         "logical-connection-point": lcp}}
 388     return post_request(URL_CREATE_OTS_OMS, attr)
 389
 390
 391 def path_computation_request(requestid: str, servicename: str, serviceaend, servicezend,
 392                              hardconstraints=None, softconstraints=None, metric="hop-count", other_attr=None):
 393     attr = {"service-name": servicename,
 394             "resource-reserve": "true",
 395             "service-handler-header": {"request-id": requestid},
 396             "service-a-end": serviceaend,
 397             "service-z-end": servicezend,
 398             "pce-metric": metric}
 399     if hardconstraints:
 400         attr.update({"hard-constraints": hardconstraints})
 401     if softconstraints:
 402         attr.update({"soft-constraints": softconstraints})
 403     if other_attr:
 404         attr.update(other_attr)
 405     return post_request(URL_PATH_COMPUTATION_REQUEST, {"input": attr})
 406
 407
 408 def shutdown_process(process):
 409     if process is not None:
 410         for child in psutil.Process(process.pid).children():
 411             child.send_signal(signal.SIGINT)
 412             child.wait()
 413         process.send_signal(signal.SIGINT)
 414
 415
 416 def start_honeynode(log_file: str, node_port: str, node_config_file_name: str):
 417     if os.path.isfile(HONEYNODE_EXECUTABLE):
 418         with open(log_file, 'w') as outfile:
 419             return subprocess.Popen(
 420                 [HONEYNODE_EXECUTABLE, node_port, os.path.join(SAMPLES_DIRECTORY, node_config_file_name)],
 421                 stdout=outfile, stderr=outfile)
 422     return None
 423
 424
 425 def wait_until_log_contains(log_file, regexp, time_to_wait=20):
 426     # pylint: disable=lost-exception
 427     stringfound = False
 428     filefound = False
 429     line = None
 430     try:
 431         with TimeOut(seconds=time_to_wait):
 432             while not os.path.exists(log_file):
 433                 time.sleep(0.2)
 434             filelogs = open(log_file, 'r')
 435             filelogs.seek(0, 2)
 436             filefound = True
 437             print("Searching for pattern '" + regexp + "' in " + os.path.basename(log_file), end='... ', flush=True)
 438             compiled_regexp = re.compile(regexp)
 439             while True:
 440                 line = filelogs.readline()
 441                 if compiled_regexp.search(line):
 442                     print("Pattern found!", end=' ')
 443                     stringfound = True
 444                     break
 445                 if not line:
 446                     time.sleep(0.1)
 447     except TimeoutError:
 448         print("Pattern not found after " + str(time_to_wait), end=" seconds! ", flush=True)
 449     except PermissionError:
 450         print("Permission Error when trying to access the log file", end=" ... ", flush=True)
 451     finally:
 452         if filefound:
 453             filelogs.close()
 454         else:
 455             print("log file does not exist or is not accessible... ", flush=True)
 456         return stringfound
 457
 458
 459 class TimeOut:
 460     def __init__(self, seconds=1, error_message='Timeout'):
 461         self.seconds = seconds
 462         self.error_message = error_message
 463
 464     def handle_timeout(self, signum, frame):
 465         raise TimeoutError(self.error_message)
 466
 467     def __enter__(self):
 468         signal.signal(signal.SIGALRM, self.handle_timeout)
 469         signal.alarm(self.seconds)
 470
 471     def __exit__(self, type, value, traceback):
 472         # pylint: disable=W0622
 473         signal.alarm(0)
 474