tests/transportpce_tests/common/test_utils.py

   1 #!/usr/bin/env python
   2
   3 ##############################################################################
   4 # Copyright (c) 2020 Orange, Inc. and others.  All rights reserved.
   5 #
   6 # All rights reserved. This program and the accompanying materials
   7 # are made available under the terms of the Apache License, Version 2.0
   8 # which accompanies this distribution, and is available at
   9 # http://www.apache.org/licenses/LICENSE-2.0
  10 ##############################################################################
  11
  12 # pylint: disable=no-member
  13
  14 import json
  15 import os
  16 import sys
  17 import re
  18 import signal
  19 import subprocess
  20 import time
  21
  22 import psutil
  23 import requests
  24
  25 import simulators
  26
  27 SIMS = simulators.SIMS
  28 HONEYNODE_EXECUTABLE = simulators.HONEYNODE_EXECUTABLE
  29 SAMPLES_DIRECTORY = simulators.SAMPLES_DIRECTORY
  30
  31 HONEYNODE_OK_START_MSG = "Netconf SSH endpoint started successfully at 0.0.0.0"
  32 KARAF_OK_START_MSG = re.escape(
  33     "Blueprint container for bundle org.opendaylight.netconf.restconf")+".* was successfully created"
  34 LIGHTY_OK_START_MSG = re.escape("lighty.io and RESTCONF-NETCONF started")
  35
  36 RESTCONF_BASE_URL = "http://localhost:8181/restconf"
  37 ODL_LOGIN = "admin"
  38 ODL_PWD = "admin"
  39 NODES_LOGIN = "admin"
  40 NODES_PWD = "admin"
  41 URL_CONFIG_NETCONF_TOPO = "{}/config/network-topology:network-topology/topology/topology-netconf/"
  42 URL_CONFIG_ORDM_TOPO = "{}/config/ietf-network:networks/network/openroadm-topology/"
  43 URL_CONFIG_OTN_TOPO = "{}/config/ietf-network:networks/network/otn-topology/"
  44 URL_CONFIG_CLLI_NET = "{}/config/ietf-network:networks/network/clli-network/"
  45 URL_CONFIG_ORDM_NET = "{}/config/ietf-network:networks/network/openroadm-network/"
  46 URL_PORTMAPPING = "{}/config/transportpce-portmapping:network/nodes/"
  47 URL_OPER_SERV_LIST = "{}/operational/org-openroadm-service:service-list/"
  48 URL_GET_NBINOTIFICATIONS_SERV = "{}/operations/nbi-notifications:get-notifications-service/"
  49 URL_SERV_CREATE = "{}/operations/org-openroadm-service:service-create"
  50 URL_SERV_DELETE = "{}/operations/org-openroadm-service:service-delete"
  51 URL_SERVICE_PATH = "{}/operations/transportpce-device-renderer:service-path"
  52 URL_OTN_SERVICE_PATH = "{}/operations/transportpce-device-renderer:otn-service-path"
  53 URL_CREATE_OTS_OMS = "{}/operations/transportpce-device-renderer:create-ots-oms"
  54 URL_PATH_COMPUTATION_REQUEST = "{}/operations/transportpce-pce:path-computation-request"
  55 URL_FULL_PORTMAPPING = "{}/config/transportpce-portmapping:network"
  56
  57 TYPE_APPLICATION_JSON = {'Content-Type': 'application/json', 'Accept': 'application/json'}
  58 TYPE_APPLICATION_XML = {'Content-Type': 'application/xml', 'Accept': 'application/xml'}
  59
  60 CODE_SHOULD_BE_200 = 'Http status code should be 200'
  61 CODE_SHOULD_BE_201 = 'Http status code should be 201'
  62
  63 LOG_DIRECTORY = os.path.dirname(os.path.realpath(__file__))
  64
  65 KARAF_LOG = os.path.join(
  66     os.path.dirname(os.path.realpath(__file__)),
  67     "..", "..", "..", "karaf", "target", "assembly", "data", "log", "karaf.log")
  68
  69 process_list = []
  70
  71 if "USE_LIGHTY" in os.environ and os.environ['USE_LIGHTY'] == 'True':
  72     TPCE_LOG = 'odl.log'
  73 else:
  74     TPCE_LOG = KARAF_LOG
  75
  76
  77 def start_sims(sims_list):
  78     for sim in sims_list:
  79         print("starting simulator for " + sim + "...")
  80         log_file = os.path.join(LOG_DIRECTORY, SIMS[sim]['logfile'])
  81         process = start_honeynode(log_file, SIMS[sim]['port'], SIMS[sim]['configfile'])
  82         if wait_until_log_contains(log_file, HONEYNODE_OK_START_MSG, 100):
  83             print("simulator for " + sim + " started")
  84         else:
  85             print("simulator for " + sim + " failed to start")
  86             shutdown_process(process)
  87             for pid in process_list:
  88                 shutdown_process(pid)
  89             sys.exit(3)
  90         process_list.append(process)
  91     return process_list
  92
  93
  94 def start_tpce():
  95     print("starting OpenDaylight...")
  96     if "USE_LIGHTY" in os.environ and os.environ['USE_LIGHTY'] == 'True':
  97         process = start_lighty()
  98         start_msg = LIGHTY_OK_START_MSG
  99     else:
 100         process = start_karaf()
 101         start_msg = KARAF_OK_START_MSG
 102     if wait_until_log_contains(TPCE_LOG, start_msg, time_to_wait=60):
 103         print("OpenDaylight started !")
 104     else:
 105         print("OpenDaylight failed to start !")
 106         shutdown_process(process)
 107         for pid in process_list:
 108             shutdown_process(pid)
 109         sys.exit(1)
 110     process_list.append(process)
 111     return process_list
 112
 113
 114 def start_karaf():
 115     print("starting KARAF TransportPCE build...")
 116     executable = os.path.join(
 117         os.path.dirname(os.path.realpath(__file__)),
 118         "..", "..", "..", "karaf", "target", "assembly", "bin", "karaf")
 119     with open('odl.log', 'w') as outfile:
 120         return subprocess.Popen(
 121             ["sh", executable, "server"], stdout=outfile, stderr=outfile, stdin=None)
 122
 123
 124 def start_lighty():
 125     print("starting LIGHTY.IO TransportPCE build...")
 126     executable = os.path.join(
 127         os.path.dirname(os.path.realpath(__file__)),
 128         "..", "..", "..", "lighty", "target", "tpce",
 129         "clean-start-controller.sh")
 130     with open('odl.log', 'w') as outfile:
 131         return subprocess.Popen(
 132             ["sh", executable], stdout=outfile, stderr=outfile, stdin=None)
 133
 134
 135 def install_karaf_feature(feature_name: str):
 136     print("installing feature " + feature_name)
 137     executable = os.path.join(
 138         os.path.dirname(os.path.realpath(__file__)),
 139         "..", "..", "..", "karaf", "target", "assembly", "bin", "client")
 140     return subprocess.run([executable],
 141                           input='feature:install ' + feature_name + '\n feature:list | grep '
 142                           + feature_name + ' \n logout \n',
 143                           universal_newlines=True, check=False)
 144
 145
 146 def get_request(url):
 147     return requests.request(
 148         "GET", url.format(RESTCONF_BASE_URL),
 149         headers=TYPE_APPLICATION_JSON,
 150         auth=(ODL_LOGIN, ODL_PWD))
 151
 152
 153 def post_request(url, data):
 154     if data:
 155         print(json.dumps(data))
 156         return requests.request(
 157             "POST", url.format(RESTCONF_BASE_URL),
 158             data=json.dumps(data),
 159             headers=TYPE_APPLICATION_JSON,
 160             auth=(ODL_LOGIN, ODL_PWD))
 161
 162     return requests.request(
 163         "POST", url.format(RESTCONF_BASE_URL),
 164         headers=TYPE_APPLICATION_JSON,
 165         auth=(ODL_LOGIN, ODL_PWD))
 166
 167
 168 def post_xmlrequest(url, data):
 169     if data:
 170         return requests.request(
 171             "POST", url.format(RESTCONF_BASE_URL),
 172             data=data,
 173             headers=TYPE_APPLICATION_XML,
 174             auth=(ODL_LOGIN, ODL_PWD))
 175     return None
 176
 177
 178 def put_request(url, data):
 179     return requests.request(
 180         "PUT", url.format(RESTCONF_BASE_URL),
 181         data=json.dumps(data),
 182         headers=TYPE_APPLICATION_JSON,
 183         auth=(ODL_LOGIN, ODL_PWD))
 184
 185
 186 def put_xmlrequest(url, data):
 187     return requests.request(
 188         "PUT", url.format(RESTCONF_BASE_URL),
 189         data=data,
 190         headers=TYPE_APPLICATION_XML,
 191         auth=(ODL_LOGIN, ODL_PWD))
 192
 193
 194 def rawput_request(url, data):
 195     return requests.request(
 196         "PUT", url.format(RESTCONF_BASE_URL),
 197         data=data,
 198         headers=TYPE_APPLICATION_JSON,
 199         auth=(ODL_LOGIN, ODL_PWD))
 200
 201 def rawpost_request(url, data):
 202     return requests.request(
 203         "POST", url.format(RESTCONF_BASE_URL),
 204         data=data,
 205         headers=TYPE_APPLICATION_JSON,
 206         auth=(ODL_LOGIN, ODL_PWD))
 207
 208
 209 def delete_request(url):
 210     return requests.request(
 211         "DELETE", url.format(RESTCONF_BASE_URL),
 212         headers=TYPE_APPLICATION_JSON,
 213         auth=(ODL_LOGIN, ODL_PWD))
 214
 215
 216 def mount_device(node_id, sim):
 217     url = URL_CONFIG_NETCONF_TOPO + "node/" + node_id
 218     body = {"node": [{
 219         "node-id": node_id,
 220         "netconf-node-topology:username": NODES_LOGIN,
 221         "netconf-node-topology:password": NODES_PWD,
 222         "netconf-node-topology:host": "127.0.0.1",
 223         "netconf-node-topology:port": SIMS[sim]['port'],
 224         "netconf-node-topology:tcp-only": "false",
 225         "netconf-node-topology:pass-through": {}}]}
 226     response = put_request(url, body)
 227     if wait_until_log_contains(TPCE_LOG, re.escape("Triggering notification stream NETCONF for node " + node_id), 60):
 228         print("Node " + node_id + " correctly added to tpce topology", end='... ', flush=True)
 229     else:
 230         print("Node " + node_id + " still not added to tpce topology", end='... ', flush=True)
 231         if response.status_code == requests.codes.ok:
 232             print("It was probably loaded at start-up", end='... ', flush=True)
 233         # TODO an else-clause to abort test would probably be nice here
 234     return response
 235
 236
 237 def unmount_device(node_id):
 238     url = URL_CONFIG_NETCONF_TOPO + "node/" + node_id
 239     response = delete_request(url)
 240     if wait_until_log_contains(TPCE_LOG, re.escape("onDeviceDisConnected: " + node_id), 60):
 241         print("Node " + node_id + " correctly deleted from tpce topology", end='... ', flush=True)
 242     else:
 243         print("Node " + node_id + " still not deleted from tpce topology", end='... ', flush=True)
 244     return response
 245
 246
 247 def connect_xpdr_to_rdm_request(xpdr_node: str, xpdr_num: str, network_num: str,
 248                                 rdm_node: str, srg_num: str, termination_num: str):
 249     url = "{}/operations/transportpce-networkutils:init-xpdr-rdm-links"
 250     data = {
 251         "networkutils:input": {
 252             "networkutils:links-input": {
 253                 "networkutils:xpdr-node": xpdr_node,
 254                 "networkutils:xpdr-num": xpdr_num,
 255                 "networkutils:network-num": network_num,
 256                 "networkutils:rdm-node": rdm_node,
 257                 "networkutils:srg-num": srg_num,
 258                 "networkutils:termination-point-num": termination_num
 259             }
 260         }
 261     }
 262     return post_request(url, data)
 263
 264
 265 def connect_rdm_to_xpdr_request(xpdr_node: str, xpdr_num: str, network_num: str,
 266                                 rdm_node: str, srg_num: str, termination_num: str):
 267     url = "{}/operations/transportpce-networkutils:init-rdm-xpdr-links"
 268     data = {
 269         "networkutils:input": {
 270             "networkutils:links-input": {
 271                 "networkutils:xpdr-node": xpdr_node,
 272                 "networkutils:xpdr-num": xpdr_num,
 273                 "networkutils:network-num": network_num,
 274                 "networkutils:rdm-node": rdm_node,
 275                 "networkutils:srg-num": srg_num,
 276                 "networkutils:termination-point-num": termination_num
 277             }
 278         }
 279     }
 280     return post_request(url, data)
 281
 282
 283 def check_netconf_node_request(node: str, suffix: str):
 284     url = URL_CONFIG_NETCONF_TOPO + (
 285         "node/" + node + "/yang-ext:mount/org-openroadm-device:org-openroadm-device/" + suffix
 286     )
 287     return get_request(url)
 288
 289
 290 def get_netconf_oper_request(node: str):
 291     url = "{}/operational/network-topology:network-topology/topology/topology-netconf/node/" + node
 292     return get_request(url)
 293
 294
 295 def get_ordm_topo_request(suffix: str):
 296     url = URL_CONFIG_ORDM_TOPO + suffix
 297     return get_request(url)
 298
 299
 300 def add_oms_attr_request(link: str, attr):
 301     url = URL_CONFIG_ORDM_TOPO + (
 302         "ietf-network-topology:link/" + link + "/org-openroadm-network-topology:OMS-attributes/span"
 303     )
 304     return put_request(url, attr)
 305
 306
 307 def del_oms_attr_request(link: str):
 308     url = URL_CONFIG_ORDM_TOPO + (
 309         "ietf-network-topology:link/" + link + "/org-openroadm-network-topology:OMS-attributes/span"
 310     )
 311     return delete_request(url)
 312
 313
 314 def get_clli_net_request():
 315     return get_request(URL_CONFIG_CLLI_NET)
 316
 317
 318 def get_ordm_net_request():
 319     return get_request(URL_CONFIG_ORDM_NET)
 320
 321
 322 def get_otn_topo_request():
 323     return get_request(URL_CONFIG_OTN_TOPO)
 324
 325
 326 def del_link_request(link: str):
 327     url = URL_CONFIG_ORDM_TOPO + ("ietf-network-topology:link/" + link)
 328     return delete_request(url)
 329
 330
 331 def del_node_request(node: str):
 332     url = URL_CONFIG_CLLI_NET + ("node/" + node)
 333     return delete_request(url)
 334
 335
 336 def portmapping_request(suffix: str):
 337     url = URL_PORTMAPPING + suffix
 338     return get_request(url)
 339
 340
 341 def get_notifications_service_request(attr):
 342     return post_request(URL_GET_NBINOTIFICATIONS_SERV, attr)
 343
 344
 345 def get_service_list_request(suffix: str):
 346     url = URL_OPER_SERV_LIST + suffix
 347     return get_request(url)
 348
 349
 350 def service_create_request(attr):
 351     return post_request(URL_SERV_CREATE, attr)
 352
 353
 354 def service_delete_request(servicename: str,
 355                            requestid="e3028bae-a90f-4ddd-a83f-cf224eba0e58",
 356                            notificationurl="http://localhost:8585/NotificationServer/notify"):
 357     attr = {"input": {
 358         "sdnc-request-header": {
 359             "request-id": requestid,
 360             "rpc-action": "service-delete",
 361             "request-system-id": "appname",
 362             "notification-url": notificationurl},
 363         "service-delete-req-info": {
 364             "service-name": servicename,
 365             "tail-retention": "no"}}}
 366     return post_request(URL_SERV_DELETE, attr)
 367
 368
 369 def service_path_request(operation: str, servicename: str, wavenumber: str, nodes, centerfreq: str,
 370                          slotwidth: int, minfreq: float, maxfreq: float, lowerslotnumber: int,
 371                          higherslotnumber: int):
 372     attr = {"renderer:input": {
 373         "renderer:service-name": servicename,
 374         "renderer:wave-number": wavenumber,
 375         "renderer:modulation-format": "dp-qpsk",
 376         "renderer:operation": operation,
 377         "renderer:nodes": nodes,
 378         "renderer:center-freq": centerfreq,
 379         "renderer:width": slotwidth,
 380         "renderer:min-freq": minfreq,
 381         "renderer:max-freq": maxfreq,
 382         "renderer:lower-spectral-slot-number": lowerslotnumber,
 383         "renderer:higher-spectral-slot-number": higherslotnumber}}
 384     return post_request(URL_SERVICE_PATH, attr)
 385
 386
 387 def otn_service_path_request(operation: str, servicename: str, servicerate: str, servicetype: str, nodes,
 388                              eth_attr=None):
 389     attr = {"service-name": servicename,
 390             "operation": operation,
 391             "service-rate": servicerate,
 392             "service-type": servicetype,
 393             "nodes": nodes}
 394     if eth_attr:
 395         attr.update(eth_attr)
 396     return post_request(URL_OTN_SERVICE_PATH, {"renderer:input": attr})
 397
 398
 399 def create_ots_oms_request(nodeid: str, lcp: str):
 400     attr = {"input": {
 401         "node-id": nodeid,
 402         "logical-connection-point": lcp}}
 403     return post_request(URL_CREATE_OTS_OMS, attr)
 404
 405
 406 def path_computation_request(requestid: str, servicename: str, serviceaend, servicezend,
 407                              hardconstraints=None, softconstraints=None, metric="hop-count", other_attr=None):
 408     attr = {"service-name": servicename,
 409             "resource-reserve": "true",
 410             "service-handler-header": {"request-id": requestid},
 411             "service-a-end": serviceaend,
 412             "service-z-end": servicezend,
 413             "pce-metric": metric}
 414     if hardconstraints:
 415         attr.update({"hard-constraints": hardconstraints})
 416     if softconstraints:
 417         attr.update({"soft-constraints": softconstraints})
 418     if other_attr:
 419         attr.update(other_attr)
 420     return post_request(URL_PATH_COMPUTATION_REQUEST, {"input": attr})
 421
 422
 423 def shutdown_process(process):
 424     if process is not None:
 425         for child in psutil.Process(process.pid).children():
 426             child.send_signal(signal.SIGINT)
 427             child.wait()
 428         process.send_signal(signal.SIGINT)
 429
 430
 431 def start_honeynode(log_file: str, node_port: str, node_config_file_name: str):
 432     if os.path.isfile(HONEYNODE_EXECUTABLE):
 433         with open(log_file, 'w') as outfile:
 434             return subprocess.Popen(
 435                 [HONEYNODE_EXECUTABLE, node_port, os.path.join(SAMPLES_DIRECTORY, node_config_file_name)],
 436                 stdout=outfile, stderr=outfile)
 437     return None
 438
 439
 440 def wait_until_log_contains(log_file, regexp, time_to_wait=20):
 441     # pylint: disable=lost-exception
 442     stringfound = False
 443     filefound = False
 444     line = None
 445     try:
 446         with TimeOut(seconds=time_to_wait):
 447             while not os.path.exists(log_file):
 448                 time.sleep(0.2)
 449             filelogs = open(log_file, 'r')
 450             filelogs.seek(0, 2)
 451             filefound = True
 452             print("Searching for pattern '" + regexp + "' in " + os.path.basename(log_file), end='... ', flush=True)
 453             compiled_regexp = re.compile(regexp)
 454             while True:
 455                 line = filelogs.readline()
 456                 if compiled_regexp.search(line):
 457                     print("Pattern found!", end=' ')
 458                     stringfound = True
 459                     break
 460                 if not line:
 461                     time.sleep(0.1)
 462     except TimeoutError:
 463         print("Pattern not found after " + str(time_to_wait), end=" seconds! ", flush=True)
 464     except PermissionError:
 465         print("Permission Error when trying to access the log file", end=" ... ", flush=True)
 466     finally:
 467         if filefound:
 468             filelogs.close()
 469         else:
 470             print("log file does not exist or is not accessible... ", flush=True)
 471         return stringfound
 472
 473
 474 class TimeOut:
 475     def __init__(self, seconds=1, error_message='Timeout'):
 476         self.seconds = seconds
 477         self.error_message = error_message
 478
 479     def handle_timeout(self, signum, frame):
 480         raise TimeoutError(self.error_message)
 481
 482     def __enter__(self):
 483         signal.signal(signal.SIGALRM, self.handle_timeout)
 484         signal.alarm(self.seconds)
 485
 486     def __exit__(self, type, value, traceback):
 487         # pylint: disable=W0622
 488         signal.alarm(0)
 489