tests/transportpce_tests/common/test_utils.py

   1 #!/usr/bin/env python
   2
   3 ##############################################################################
   4 # Copyright (c) 2020 Orange, Inc. and others.  All rights reserved.
   5 #
   6 # All rights reserved. This program and the accompanying materials
   7 # are made available under the terms of the Apache License, Version 2.0
   8 # which accompanies this distribution, and is available at
   9 # http://www.apache.org/licenses/LICENSE-2.0
  10 ##############################################################################
  11
  12 # pylint: disable=no-member
  13
  14 import json
  15 import os
  16 import sys
  17 import re
  18 import signal
  19 import subprocess
  20 import time
  21
  22 import psutil
  23 import requests
  24
  25 import simulators
  26
  27 SIMS = simulators.SIMS
  28
  29 HONEYNODE_OK_START_MSG = "Netconf SSH endpoint started successfully at 0.0.0.0"
  30 KARAF_OK_START_MSG = re.escape(
  31     "Blueprint container for bundle org.opendaylight.netconf.restconf")+".* was successfully created"
  32 LIGHTY_OK_START_MSG = re.escape("lighty.io and RESTCONF-NETCONF started")
  33
  34 RESTCONF_BASE_URL = "http://localhost:8181/restconf"
  35 ODL_LOGIN = "admin"
  36 ODL_PWD = "admin"
  37 NODES_LOGIN = "admin"
  38 NODES_PWD = "admin"
  39 URL_CONFIG_NETCONF_TOPO = "{}/config/network-topology:network-topology/topology/topology-netconf/"
  40 URL_CONFIG_ORDM_TOPO = "{}/config/ietf-network:networks/network/openroadm-topology/"
  41 URL_CONFIG_OTN_TOPO = "{}/config/ietf-network:networks/network/otn-topology/"
  42 URL_CONFIG_CLLI_NET = "{}/config/ietf-network:networks/network/clli-network/"
  43 URL_CONFIG_ORDM_NET = "{}/config/ietf-network:networks/network/openroadm-network/"
  44 URL_PORTMAPPING = "{}/config/transportpce-portmapping:network/nodes/"
  45 URL_OPER_SERV_LIST = "{}/operational/org-openroadm-service:service-list/"
  46 URL_GET_NBINOTIFICATIONS_SERV = "{}/operations/nbi-notifications:get-notifications-service/"
  47 URL_GET_NBINOTIFICATIONS_ALARM_SERV = "{}/operations/nbi-notifications:get-notifications-alarm-service/"
  48 URL_SERV_CREATE = "{}/operations/org-openroadm-service:service-create"
  49 URL_SERV_DELETE = "{}/operations/org-openroadm-service:service-delete"
  50 URL_SERVICE_PATH = "{}/operations/transportpce-device-renderer:service-path"
  51 URL_OTN_SERVICE_PATH = "{}/operations/transportpce-device-renderer:otn-service-path"
  52 URL_CREATE_OTS_OMS = "{}/operations/transportpce-device-renderer:create-ots-oms"
  53 URL_PATH_COMPUTATION_REQUEST = "{}/operations/transportpce-pce:path-computation-request"
  54 URL_FULL_PORTMAPPING = "{}/config/transportpce-portmapping:network"
  55
  56 TYPE_APPLICATION_JSON = {'Content-Type': 'application/json', 'Accept': 'application/json'}
  57 TYPE_APPLICATION_XML = {'Content-Type': 'application/xml', 'Accept': 'application/xml'}
  58
  59 CODE_SHOULD_BE_200 = 'Http status code should be 200'
  60 CODE_SHOULD_BE_201 = 'Http status code should be 201'
  61
  62 SIM_LOG_DIRECTORY = os.path.join(os.path.dirname(os.path.realpath(__file__)), "log")
  63 KARAF_LOG = os.path.join(
  64     os.path.dirname(os.path.realpath(__file__)),
  65     "..", "..", "..", "karaf", "target", "assembly", "data", "log", "karaf.log")
  66
  67 process_list = []
  68
  69 if "USE_LIGHTY" in os.environ and os.environ['USE_LIGHTY'] == 'True':
  70     TPCE_LOG = 'odl.log'
  71 else:
  72     TPCE_LOG = KARAF_LOG
  73
  74
  75 def start_sims(sims_list):
  76     for sim in sims_list:
  77         print("starting simulator " + sim[0] + " in OpenROADM device version " + sim[1] + "...")
  78         log_file = os.path.join(SIM_LOG_DIRECTORY, SIMS[sim]['logfile'])
  79         process = start_honeynode(log_file, sim)
  80         if wait_until_log_contains(log_file, HONEYNODE_OK_START_MSG, 100):
  81             print("simulator for " + sim[0] + " started")
  82         else:
  83             print("simulator for " + sim[0] + " failed to start")
  84             shutdown_process(process)
  85             for pid in process_list:
  86                 shutdown_process(pid)
  87             sys.exit(3)
  88         process_list.append(process)
  89     return process_list
  90
  91
  92 def start_tpce():
  93     print("starting OpenDaylight...")
  94     if "USE_LIGHTY" in os.environ and os.environ['USE_LIGHTY'] == 'True':
  95         process = start_lighty()
  96         start_msg = LIGHTY_OK_START_MSG
  97     else:
  98         process = start_karaf()
  99         start_msg = KARAF_OK_START_MSG
 100     if wait_until_log_contains(TPCE_LOG, start_msg, time_to_wait=120):
 101         print("OpenDaylight started !")
 102     else:
 103         print("OpenDaylight failed to start !")
 104         shutdown_process(process)
 105         for pid in process_list:
 106             shutdown_process(pid)
 107         sys.exit(1)
 108     process_list.append(process)
 109     return process_list
 110
 111
 112 def start_karaf():
 113     print("starting KARAF TransportPCE build...")
 114     executable = os.path.join(
 115         os.path.dirname(os.path.realpath(__file__)),
 116         "..", "..", "..", "karaf", "target", "assembly", "bin", "karaf")
 117     with open('odl.log', 'w') as outfile:
 118         return subprocess.Popen(
 119             ["sh", executable, "server"], stdout=outfile, stderr=outfile, stdin=None)
 120
 121
 122 def start_lighty():
 123     print("starting LIGHTY.IO TransportPCE build...")
 124     executable = os.path.join(
 125         os.path.dirname(os.path.realpath(__file__)),
 126         "..", "..", "..", "lighty", "target", "tpce",
 127         "clean-start-controller.sh")
 128     with open('odl.log', 'w') as outfile:
 129         return subprocess.Popen(
 130             ["sh", executable], stdout=outfile, stderr=outfile, stdin=None)
 131
 132
 133 def install_karaf_feature(feature_name: str):
 134     print("installing feature " + feature_name)
 135     executable = os.path.join(
 136         os.path.dirname(os.path.realpath(__file__)),
 137         "..", "..", "..", "karaf", "target", "assembly", "bin", "client")
 138     return subprocess.run([executable],
 139                           input='feature:install ' + feature_name + '\n feature:list | grep '
 140                           + feature_name + ' \n logout \n',
 141                           universal_newlines=True, check=False)
 142
 143
 144 def get_request(url):
 145     return requests.request(
 146         "GET", url.format(RESTCONF_BASE_URL),
 147         headers=TYPE_APPLICATION_JSON,
 148         auth=(ODL_LOGIN, ODL_PWD))
 149
 150
 151 def post_request(url, data):
 152     if data:
 153         print(json.dumps(data))
 154         return requests.request(
 155             "POST", url.format(RESTCONF_BASE_URL),
 156             data=json.dumps(data),
 157             headers=TYPE_APPLICATION_JSON,
 158             auth=(ODL_LOGIN, ODL_PWD))
 159
 160     return requests.request(
 161         "POST", url.format(RESTCONF_BASE_URL),
 162         headers=TYPE_APPLICATION_JSON,
 163         auth=(ODL_LOGIN, ODL_PWD))
 164
 165
 166 def post_xmlrequest(url, data):
 167     if data:
 168         return requests.request(
 169             "POST", url.format(RESTCONF_BASE_URL),
 170             data=data,
 171             headers=TYPE_APPLICATION_XML,
 172             auth=(ODL_LOGIN, ODL_PWD))
 173     return None
 174
 175
 176 def put_request(url, data):
 177     return requests.request(
 178         "PUT", url.format(RESTCONF_BASE_URL),
 179         data=json.dumps(data),
 180         headers=TYPE_APPLICATION_JSON,
 181         auth=(ODL_LOGIN, ODL_PWD))
 182
 183
 184 def put_xmlrequest(url, data):
 185     return requests.request(
 186         "PUT", url.format(RESTCONF_BASE_URL),
 187         data=data,
 188         headers=TYPE_APPLICATION_XML,
 189         auth=(ODL_LOGIN, ODL_PWD))
 190
 191 def put_jsonrequest(url, data):
 192     return requests.request(
 193         "PUT", url.format(RESTCONF_BASE_URL),
 194         data=data,
 195         headers=TYPE_APPLICATION_JSON,
 196         auth=(ODL_LOGIN, ODL_PWD))
 197
 198 def rawput_request(url, data):
 199     return requests.request(
 200         "PUT", url.format(RESTCONF_BASE_URL),
 201         data=data,
 202         headers=TYPE_APPLICATION_JSON,
 203         auth=(ODL_LOGIN, ODL_PWD))
 204
 205 def rawpost_request(url, data):
 206     return requests.request(
 207         "POST", url.format(RESTCONF_BASE_URL),
 208         data=data,
 209         headers=TYPE_APPLICATION_JSON,
 210         auth=(ODL_LOGIN, ODL_PWD))
 211
 212
 213 def delete_request(url):
 214     return requests.request(
 215         "DELETE", url.format(RESTCONF_BASE_URL),
 216         headers=TYPE_APPLICATION_JSON,
 217         auth=(ODL_LOGIN, ODL_PWD))
 218
 219
 220 def mount_device(node_id, sim):
 221     url = URL_CONFIG_NETCONF_TOPO + "node/" + node_id
 222     body = {"node": [{
 223         "node-id": node_id,
 224         "netconf-node-topology:username": NODES_LOGIN,
 225         "netconf-node-topology:password": NODES_PWD,
 226         "netconf-node-topology:host": "127.0.0.1",
 227         "netconf-node-topology:port": SIMS[sim]['port'],
 228         "netconf-node-topology:tcp-only": "false",
 229         "netconf-node-topology:pass-through": {}}]}
 230     response = put_request(url, body)
 231     if wait_until_log_contains(TPCE_LOG, re.escape("Triggering notification stream NETCONF for node " + node_id), 60):
 232         print("Node " + node_id + " correctly added to tpce topology", end='... ', flush=True)
 233     else:
 234         print("Node " + node_id + " still not added to tpce topology", end='... ', flush=True)
 235         if response.status_code == requests.codes.ok:
 236             print("It was probably loaded at start-up", end='... ', flush=True)
 237         # TODO an else-clause to abort test would probably be nice here
 238     return response
 239
 240
 241 def unmount_device(node_id):
 242     url = URL_CONFIG_NETCONF_TOPO + "node/" + node_id
 243     response = delete_request(url)
 244     if wait_until_log_contains(TPCE_LOG, re.escape("onDeviceDisConnected: " + node_id), 60):
 245         print("Node " + node_id + " correctly deleted from tpce topology", end='... ', flush=True)
 246     else:
 247         print("Node " + node_id + " still not deleted from tpce topology", end='... ', flush=True)
 248     return response
 249
 250
 251 def connect_xpdr_to_rdm_request(xpdr_node: str, xpdr_num: str, network_num: str,
 252                                 rdm_node: str, srg_num: str, termination_num: str):
 253     url = "{}/operations/transportpce-networkutils:init-xpdr-rdm-links"
 254     data = {
 255         "networkutils:input": {
 256             "networkutils:links-input": {
 257                 "networkutils:xpdr-node": xpdr_node,
 258                 "networkutils:xpdr-num": xpdr_num,
 259                 "networkutils:network-num": network_num,
 260                 "networkutils:rdm-node": rdm_node,
 261                 "networkutils:srg-num": srg_num,
 262                 "networkutils:termination-point-num": termination_num
 263             }
 264         }
 265     }
 266     return post_request(url, data)
 267
 268
 269 def connect_rdm_to_xpdr_request(xpdr_node: str, xpdr_num: str, network_num: str,
 270                                 rdm_node: str, srg_num: str, termination_num: str):
 271     url = "{}/operations/transportpce-networkutils:init-rdm-xpdr-links"
 272     data = {
 273         "networkutils:input": {
 274             "networkutils:links-input": {
 275                 "networkutils:xpdr-node": xpdr_node,
 276                 "networkutils:xpdr-num": xpdr_num,
 277                 "networkutils:network-num": network_num,
 278                 "networkutils:rdm-node": rdm_node,
 279                 "networkutils:srg-num": srg_num,
 280                 "networkutils:termination-point-num": termination_num
 281             }
 282         }
 283     }
 284     return post_request(url, data)
 285
 286
 287 def check_netconf_node_request(node: str, suffix: str):
 288     url = URL_CONFIG_NETCONF_TOPO + (
 289         "node/" + node + "/yang-ext:mount/org-openroadm-device:org-openroadm-device/" + suffix
 290     )
 291     return get_request(url)
 292
 293
 294 def get_netconf_oper_request(node: str):
 295     url = "{}/operational/network-topology:network-topology/topology/topology-netconf/node/" + node
 296     return get_request(url)
 297
 298
 299 def get_ordm_topo_request(suffix: str):
 300     url = URL_CONFIG_ORDM_TOPO + suffix
 301     return get_request(url)
 302
 303
 304 def add_oms_attr_request(link: str, attr):
 305     url = URL_CONFIG_ORDM_TOPO + (
 306         "ietf-network-topology:link/" + link + "/org-openroadm-network-topology:OMS-attributes/span"
 307     )
 308     return put_request(url, attr)
 309
 310
 311 def del_oms_attr_request(link: str):
 312     url = URL_CONFIG_ORDM_TOPO + (
 313         "ietf-network-topology:link/" + link + "/org-openroadm-network-topology:OMS-attributes/span"
 314     )
 315     return delete_request(url)
 316
 317
 318 def get_clli_net_request():
 319     return get_request(URL_CONFIG_CLLI_NET)
 320
 321
 322 def get_ordm_net_request():
 323     return get_request(URL_CONFIG_ORDM_NET)
 324
 325
 326 def get_otn_topo_request():
 327     return get_request(URL_CONFIG_OTN_TOPO)
 328
 329
 330 def del_link_request(link: str):
 331     url = URL_CONFIG_ORDM_TOPO + ("ietf-network-topology:link/" + link)
 332     return delete_request(url)
 333
 334
 335 def del_node_request(node: str):
 336     url = URL_CONFIG_CLLI_NET + ("node/" + node)
 337     return delete_request(url)
 338
 339
 340 def portmapping_request(suffix: str):
 341     url = URL_PORTMAPPING + suffix
 342     return get_request(url)
 343
 344
 345 def get_notifications_service_request(attr):
 346     return post_request(URL_GET_NBINOTIFICATIONS_SERV, attr)
 347
 348
 349 def get_notifications_alarm_service_request(attr):
 350     return post_request(URL_GET_NBINOTIFICATIONS_ALARM_SERV, attr)
 351
 352
 353 def get_service_list_request(suffix: str):
 354     url = URL_OPER_SERV_LIST + suffix
 355     return get_request(url)
 356
 357
 358 def service_create_request(attr):
 359     return post_request(URL_SERV_CREATE, attr)
 360
 361
 362 def service_delete_request(servicename: str,
 363                            requestid="e3028bae-a90f-4ddd-a83f-cf224eba0e58",
 364                            notificationurl="http://localhost:8585/NotificationServer/notify"):
 365     attr = {"input": {
 366         "sdnc-request-header": {
 367             "request-id": requestid,
 368             "rpc-action": "service-delete",
 369             "request-system-id": "appname",
 370             "notification-url": notificationurl},
 371         "service-delete-req-info": {
 372             "service-name": servicename,
 373             "tail-retention": "no"}}}
 374     return post_request(URL_SERV_DELETE, attr)
 375
 376
 377 def service_path_request(operation: str, servicename: str, wavenumber: str, nodes, centerfreq: str,
 378                          slotwidth: int, minfreq: float, maxfreq: float, lowerslotnumber: int,
 379                          higherslotnumber: int):
 380     attr = {"renderer:input": {
 381         "renderer:service-name": servicename,
 382         "renderer:wave-number": wavenumber,
 383         "renderer:modulation-format": "dp-qpsk",
 384         "renderer:operation": operation,
 385         "renderer:nodes": nodes,
 386         "renderer:center-freq": centerfreq,
 387         "renderer:width": slotwidth,
 388         "renderer:min-freq": minfreq,
 389         "renderer:max-freq": maxfreq,
 390         "renderer:lower-spectral-slot-number": lowerslotnumber,
 391         "renderer:higher-spectral-slot-number": higherslotnumber}}
 392     return post_request(URL_SERVICE_PATH, attr)
 393
 394
 395 def otn_service_path_request(operation: str, servicename: str, servicerate: str, serviceformat: str, nodes,
 396                              eth_attr=None):
 397     attr = {"service-name": servicename,
 398             "operation": operation,
 399             "service-rate": servicerate,
 400             "service-format": serviceformat,
 401             "nodes": nodes}
 402     if eth_attr:
 403         attr.update(eth_attr)
 404     return post_request(URL_OTN_SERVICE_PATH, {"renderer:input": attr})
 405
 406
 407 def create_ots_oms_request(nodeid: str, lcp: str):
 408     attr = {"input": {
 409         "node-id": nodeid,
 410         "logical-connection-point": lcp}}
 411     return post_request(URL_CREATE_OTS_OMS, attr)
 412
 413
 414 def path_computation_request(requestid: str, servicename: str, serviceaend, servicezend,
 415                              hardconstraints=None, softconstraints=None, metric="hop-count", other_attr=None):
 416     attr = {"service-name": servicename,
 417             "resource-reserve": "true",
 418             "service-handler-header": {"request-id": requestid},
 419             "service-a-end": serviceaend,
 420             "service-z-end": servicezend,
 421             "pce-metric": metric}
 422     if hardconstraints:
 423         attr.update({"hard-constraints": hardconstraints})
 424     if softconstraints:
 425         attr.update({"soft-constraints": softconstraints})
 426     if other_attr:
 427         attr.update(other_attr)
 428     return post_request(URL_PATH_COMPUTATION_REQUEST, {"input": attr})
 429
 430
 431 def shutdown_process(process):
 432     if process is not None:
 433         for child in psutil.Process(process.pid).children():
 434             child.send_signal(signal.SIGINT)
 435             child.wait()
 436         process.send_signal(signal.SIGINT)
 437
 438
 439 def start_honeynode(log_file: str, sim):
 440     executable = os.path.join(os.path.dirname(os.path.realpath(__file__)),
 441                               "..", "..", "honeynode", sim[1], "honeynode-simulator", "honeycomb-tpce")
 442     sample_directory = os.path.join(os.path.dirname(os.path.realpath(__file__)),
 443                                     "..", "..", "sample_configs", "openroadm", sim[1])
 444     if os.path.isfile(executable):
 445         with open(log_file, 'w') as outfile:
 446             return subprocess.Popen(
 447                 [executable, SIMS[sim]['port'], os.path.join(sample_directory, SIMS[sim]['configfile'])],
 448                 stdout=outfile, stderr=outfile)
 449     return None
 450
 451
 452 def wait_until_log_contains(log_file, regexp, time_to_wait=20):
 453     # pylint: disable=lost-exception
 454     stringfound = False
 455     filefound = False
 456     line = None
 457     try:
 458         with TimeOut(seconds=time_to_wait):
 459             while not os.path.exists(log_file):
 460                 time.sleep(0.2)
 461             filelogs = open(log_file, 'r')
 462             filelogs.seek(0, 2)
 463             filefound = True
 464             print("Searching for pattern '" + regexp + "' in " + os.path.basename(log_file), end='... ', flush=True)
 465             compiled_regexp = re.compile(regexp)
 466             while True:
 467                 line = filelogs.readline()
 468                 if compiled_regexp.search(line):
 469                     print("Pattern found!", end=' ')
 470                     stringfound = True
 471                     break
 472                 if not line:
 473                     time.sleep(0.1)
 474     except TimeoutError:
 475         print("Pattern not found after " + str(time_to_wait), end=" seconds! ", flush=True)
 476     except PermissionError:
 477         print("Permission Error when trying to access the log file", end=" ... ", flush=True)
 478     finally:
 479         if filefound:
 480             filelogs.close()
 481         else:
 482             print("log file does not exist or is not accessible... ", flush=True)
 483         return stringfound
 484
 485
 486 class TimeOut:
 487     def __init__(self, seconds=1, error_message='Timeout'):
 488         self.seconds = seconds
 489         self.error_message = error_message
 490
 491     def handle_timeout(self, signum, frame):
 492         raise TimeoutError(self.error_message)
 493
 494     def __enter__(self):
 495         signal.signal(signal.SIGALRM, self.handle_timeout)
 496         signal.alarm(self.seconds)
 497
 498     def __exit__(self, type, value, traceback):
 499         # pylint: disable=W0622
 500         signal.alarm(0)