tests/transportpce_tests/common/test_utils.py

   1 #!/usr/bin/env python
   2 ##############################################################################
   3 # Copyright (c) 2020 Orange, Inc. and others.  All rights reserved.
   4 #
   5 # All rights reserved. This program and the accompanying materials
   6 # are made available under the terms of the Apache License, Version 2.0
   7 # which accompanies this distribution, and is available at
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 ##############################################################################
  10 import json
  11 import os
  12 import sys
  13 import re
  14 import signal
  15 import subprocess
  16 import time
  17
  18 import psutil
  19 import requests
  20
  21 import simulators
  22
  23 SIMS = simulators.SIMS
  24 HONEYNODE_EXECUTABLE = simulators.HONEYNODE_EXECUTABLE
  25 SAMPLES_DIRECTORY = simulators.SAMPLES_DIRECTORY
  26
  27 HONEYNODE_OK_START_MSG = "Netconf SSH endpoint started successfully at 0.0.0.0"
  28 KARAF_OK_START_MSG = re.escape(
  29     "Blueprint container for bundle org.opendaylight.netconf.restconf")+".* was successfully created"
  30
  31
  32 RESTCONF_BASE_URL = "http://localhost:8181/restconf"
  33 ODL_LOGIN = "admin"
  34 ODL_PWD = "admin"
  35 NODES_LOGIN = "admin"
  36 NODES_PWD = "admin"
  37 URL_CONFIG_NETCONF_TOPO = "{}/config/network-topology:network-topology/topology/topology-netconf/"
  38 URL_CONFIG_ORDM_TOPO = "{}/config/ietf-network:networks/network/openroadm-topology/"
  39 URL_CONFIG_OTN_TOPO = "{}/config/ietf-network:networks/network/otn-topology/"
  40 URL_CONFIG_CLLI_NET = "{}/config/ietf-network:networks/network/clli-network/"
  41 URL_CONFIG_ORDM_NET = "{}/config/ietf-network:networks/network/openroadm-network/"
  42 URL_PORTMAPPING = "{}/config/transportpce-portmapping:network/nodes/"
  43 URL_OPER_SERV_LIST = "{}/operational/org-openroadm-service:service-list/"
  44 URL_SERV_CREATE = "{}/operations/org-openroadm-service:service-create"
  45 URL_SERV_DELETE = "{}/operations/org-openroadm-service:service-delete"
  46 URL_SERVICE_PATH = "{}/operations/transportpce-device-renderer:service-path"
  47 URL_OTN_SERVICE_PATH = "{}/operations/transportpce-device-renderer:otn-service-path"
  48 URL_CREATE_OTS_OMS = "{}/operations/transportpce-device-renderer:create-ots-oms"
  49
  50 TYPE_APPLICATION_JSON = {'Content-Type': 'application/json', 'Accept': 'application/json'}
  51 TYPE_APPLICATION_XML = {'Content-Type': 'application/xml', 'Accept': 'application/xml'}
  52
  53 CODE_SHOULD_BE_200 = 'Http status code should be 200'
  54 CODE_SHOULD_BE_201 = 'Http status code should be 201'
  55
  56 LOG_DIRECTORY = os.path.dirname(os.path.realpath(__file__))
  57
  58 KARAF_LOG = os.path.join(
  59     os.path.dirname(os.path.realpath(__file__)),
  60     "..", "..", "..", "karaf", "target", "assembly", "data", "log", "karaf.log")
  61
  62 process_list = []
  63
  64 if "USE_LIGHTY" in os.environ and os.environ['USE_LIGHTY'] == 'True':
  65     TPCE_LOG = 'odl.log'
  66 else:
  67     TPCE_LOG = KARAF_LOG
  68
  69
  70 def start_sims(sims_list):
  71     for sim in sims_list:
  72         print("starting simulator for " + sim + "...")
  73         log_file = os.path.join(LOG_DIRECTORY, SIMS[sim]['logfile'])
  74         process = start_honeynode(log_file, SIMS[sim]['port'], SIMS[sim]['configfile'])
  75         if wait_until_log_contains(log_file, HONEYNODE_OK_START_MSG, 100):
  76             print("simulator for " + sim + " started")
  77         else:
  78             print("simulator for " + sim + " failed to start")
  79             shutdown_process(process)
  80             for pid in process_list:
  81                 shutdown_process(pid)
  82             sys.exit(3)
  83         process_list.append(process)
  84     return process_list
  85
  86
  87 def start_tpce():
  88     print("starting OpenDaylight...")
  89     if "USE_LIGHTY" in os.environ and os.environ['USE_LIGHTY'] == 'True':
  90         process = start_lighty()
  91         # TODO: add some sort of health check similar to Karaf below
  92     else:
  93         process = start_karaf()
  94         if wait_until_log_contains(KARAF_LOG, KARAF_OK_START_MSG, time_to_wait=60):
  95             print("OpenDaylight started !")
  96         else:
  97             print("OpenDaylight failed to start !")
  98             shutdown_process(process)
  99             for pid in process_list:
 100                 shutdown_process(pid)
 101             sys.exit(1)
 102     process_list.append(process)
 103     return process_list
 104
 105
 106 def start_karaf():
 107     print("starting KARAF TransportPCE build...")
 108     executable = os.path.join(
 109         os.path.dirname(os.path.realpath(__file__)),
 110         "..", "..", "..", "karaf", "target", "assembly", "bin", "karaf")
 111     with open('odl.log', 'w') as outfile:
 112         return subprocess.Popen(
 113             ["sh", executable, "server"], stdout=outfile, stderr=outfile, stdin=None)
 114
 115
 116 def start_lighty():
 117     print("starting LIGHTY.IO TransportPCE build...")
 118     executable = os.path.join(
 119         os.path.dirname(os.path.realpath(__file__)),
 120         "..", "..", "..", "lighty", "target", "tpce",
 121         "clean-start-controller.sh")
 122     with open('odl.log', 'w') as outfile:
 123         return subprocess.Popen(
 124             ["sh", executable], stdout=outfile, stderr=outfile, stdin=None)
 125
 126
 127 def install_karaf_feature(feature_name: str):
 128     print("installing feature " + feature_name)
 129     executable = os.path.join(
 130         os.path.dirname(os.path.realpath(__file__)),
 131         "..", "..", "..", "karaf", "target", "assembly", "bin", "client")
 132     return subprocess.run([executable],
 133                           input='feature:install ' + feature_name + '\n feature:list | grep tapi \n logout \n',
 134                           universal_newlines=True)
 135
 136
 137 def get_request(url):
 138     return requests.request(
 139         "GET", url.format(RESTCONF_BASE_URL),
 140         headers=TYPE_APPLICATION_JSON,
 141         auth=(ODL_LOGIN, ODL_PWD))
 142
 143
 144 def post_request(url, data):
 145     if data:
 146         return requests.request(
 147             "POST", url.format(RESTCONF_BASE_URL),
 148             data=json.dumps(data),
 149             headers=TYPE_APPLICATION_JSON,
 150             auth=(ODL_LOGIN, ODL_PWD))
 151     else:
 152         return requests.request(
 153             "POST", url.format(RESTCONF_BASE_URL),
 154             headers=TYPE_APPLICATION_JSON,
 155             auth=(ODL_LOGIN, ODL_PWD))
 156
 157
 158 def post_xmlrequest(url, data):
 159     if data:
 160         return requests.request(
 161             "POST", url.format(RESTCONF_BASE_URL),
 162             data=data,
 163             headers=TYPE_APPLICATION_XML,
 164             auth=(ODL_LOGIN, ODL_PWD))
 165
 166
 167 def put_request(url, data):
 168     return requests.request(
 169         "PUT", url.format(RESTCONF_BASE_URL),
 170         data=json.dumps(data),
 171         headers=TYPE_APPLICATION_JSON,
 172         auth=(ODL_LOGIN, ODL_PWD))
 173
 174
 175 def put_xmlrequest(url, data):
 176     return requests.request(
 177         "PUT", url.format(RESTCONF_BASE_URL),
 178         data=data,
 179         headers=TYPE_APPLICATION_XML,
 180         auth=(ODL_LOGIN, ODL_PWD))
 181
 182
 183 def rawput_request(url, data):
 184     return requests.request(
 185         "PUT", url.format(RESTCONF_BASE_URL),
 186         data=data,
 187         headers=TYPE_APPLICATION_JSON,
 188         auth=(ODL_LOGIN, ODL_PWD))
 189
 190
 191 def delete_request(url):
 192     return requests.request(
 193         "DELETE", url.format(RESTCONF_BASE_URL),
 194         headers=TYPE_APPLICATION_JSON,
 195         auth=(ODL_LOGIN, ODL_PWD))
 196
 197
 198 def mount_device(node_id, sim):
 199     url = URL_CONFIG_NETCONF_TOPO+"node/"+node_id
 200     body = {"node": [{
 201         "node-id": node_id,
 202         "netconf-node-topology:username": NODES_LOGIN,
 203         "netconf-node-topology:password": NODES_PWD,
 204         "netconf-node-topology:host": "127.0.0.1",
 205         "netconf-node-topology:port": SIMS[sim]['port'],
 206         "netconf-node-topology:tcp-only": "false",
 207         "netconf-node-topology:pass-through": {}}]}
 208     response = put_request(url, body)
 209     if wait_until_log_contains(TPCE_LOG, re.escape("Triggering notification stream NETCONF for node "+node_id), 60):
 210         print("Node "+node_id+" correctly added to tpce topology", end='... ', flush=True)
 211     else:
 212         print("Node "+node_id+" still not added to tpce topology", end='... ', flush=True)
 213         if response.status_code == requests.codes.ok:
 214             print("It was probably loaded at start-up", end='... ', flush=True)
 215         # TODO an else-clause to abort test would probably be nice here
 216     return response
 217
 218
 219 def unmount_device(node_id):
 220     url = URL_CONFIG_NETCONF_TOPO+"node/"+node_id
 221     response = delete_request(url)
 222     if wait_until_log_contains(TPCE_LOG, re.escape("onDeviceDisConnected: "+node_id), 60):
 223         print("Node "+node_id+" correctly deleted from tpce topology", end='... ', flush=True)
 224     else:
 225         print("Node "+node_id+" still not deleted from tpce topology", end='... ', flush=True)
 226     return response
 227
 228
 229 def connect_xpdr_to_rdm_request(xpdr_node: str, xpdr_num: str, network_num: str,
 230                                 rdm_node: str, srg_num: str, termination_num: str):
 231     url = "{}/operations/transportpce-networkutils:init-xpdr-rdm-links"
 232     data = {
 233         "networkutils:input": {
 234             "networkutils:links-input": {
 235                 "networkutils:xpdr-node": xpdr_node,
 236                 "networkutils:xpdr-num": xpdr_num,
 237                 "networkutils:network-num": network_num,
 238                 "networkutils:rdm-node": rdm_node,
 239                 "networkutils:srg-num": srg_num,
 240                 "networkutils:termination-point-num": termination_num
 241             }
 242         }
 243     }
 244     return post_request(url, data)
 245
 246
 247 def connect_rdm_to_xpdr_request(xpdr_node: str, xpdr_num: str, network_num: str,
 248                                 rdm_node: str, srg_num: str, termination_num: str):
 249     url = "{}/operations/transportpce-networkutils:init-rdm-xpdr-links"
 250     data = {
 251         "networkutils:input": {
 252             "networkutils:links-input": {
 253                 "networkutils:xpdr-node": xpdr_node,
 254                 "networkutils:xpdr-num": xpdr_num,
 255                 "networkutils:network-num": network_num,
 256                 "networkutils:rdm-node": rdm_node,
 257                 "networkutils:srg-num": srg_num,
 258                 "networkutils:termination-point-num": termination_num
 259             }
 260         }
 261     }
 262     return post_request(url, data)
 263
 264
 265 def check_netconf_node_request(node: str, suffix: str):
 266     url = URL_CONFIG_NETCONF_TOPO + (
 267         "node/" + node + "/yang-ext:mount/org-openroadm-device:org-openroadm-device/" + suffix
 268     )
 269     return get_request(url)
 270
 271
 272 def get_netconf_oper_request(node: str):
 273     url = "{}/operational/network-topology:network-topology/topology/topology-netconf/node/" + node
 274     return get_request(url)
 275
 276
 277 def get_ordm_topo_request(suffix: str):
 278     url = URL_CONFIG_ORDM_TOPO + suffix
 279     return get_request(url)
 280
 281
 282 def add_oms_attr_request(link: str, attr):
 283     url = URL_CONFIG_ORDM_TOPO + (
 284         "ietf-network-topology:link/" + link + "/org-openroadm-network-topology:OMS-attributes/span"
 285     )
 286     return put_request(url, attr)
 287
 288
 289 def del_oms_attr_request(link: str):
 290     url = URL_CONFIG_ORDM_TOPO + (
 291         "ietf-network-topology:link/" + link + "/org-openroadm-network-topology:OMS-attributes/span"
 292     )
 293     return delete_request(url)
 294
 295
 296 def get_clli_net_request():
 297     return get_request(URL_CONFIG_CLLI_NET)
 298
 299
 300 def get_ordm_net_request():
 301     return get_request(URL_CONFIG_ORDM_NET)
 302
 303
 304 def get_otn_topo_request():
 305     return get_request(URL_CONFIG_OTN_TOPO)
 306
 307
 308 def del_link_request(link: str):
 309     url = URL_CONFIG_ORDM_TOPO + ("ietf-network-topology:link/" + link)
 310     return delete_request(url)
 311
 312
 313 def del_node_request(node: str):
 314     url = URL_CONFIG_CLLI_NET + ("node/" + node)
 315     return delete_request(url)
 316
 317
 318 def portmapping_request(suffix: str):
 319     url = URL_PORTMAPPING + suffix
 320     return get_request(url)
 321
 322
 323 def get_service_list_request(suffix: str):
 324     url = URL_OPER_SERV_LIST + suffix
 325     return get_request(url)
 326
 327
 328 def service_create_request(attr):
 329     return post_request(URL_SERV_CREATE, attr)
 330
 331 def service_delete_request(servicename : str,
 332                            requestid = "e3028bae-a90f-4ddd-a83f-cf224eba0e58",
 333                            notificationurl="http://localhost:8585/NotificationServer/notify"):
 334     attr = {"input": {
 335                 "sdnc-request-header": {
 336                      "request-id": requestid,
 337                      "rpc-action": "service-delete",
 338                      "request-system-id": "appname",
 339                      "notification-url": notificationurl},
 340                 "service-delete-req-info": {
 341                      "service-name": servicename,
 342                      "tail-retention": "no"}}}
 343     return post_request(URL_SERV_DELETE, attr)
 344
 345
 346 def service_path_request(operation: str, servicename: str, wavenumber: str, nodes):
 347     attr = {"renderer:input": {
 348             "renderer:service-name": servicename,
 349             "renderer:wave-number": wavenumber,
 350             "renderer:modulation-format": "qpsk",
 351             "renderer:operation": operation,
 352             "renderer:nodes": nodes}}
 353     return post_request(URL_SERVICE_PATH, attr)
 354
 355
 356 def otn_service_path_request(operation: str, servicename: str, servicerate: str, servicetype: str, nodes, eth_attr=None):
 357     attr = {"service-name": servicename,
 358             "operation": operation,
 359             "service-rate": servicerate,
 360             "service-type": servicetype,
 361             "nodes": nodes}
 362     if eth_attr:
 363         attr.update(eth_attr)
 364     return post_request(URL_OTN_SERVICE_PATH, {"renderer:input": attr})
 365
 366
 367 def create_ots_oms_request(nodeid: str, lcp: str):
 368     attr = {"input": {
 369             "node-id": nodeid,
 370             "logical-connection-point": lcp}}
 371     return post_request(URL_CREATE_OTS_OMS, attr)
 372
 373
 374 def shutdown_process(process):
 375     if process is not None:
 376         for child in psutil.Process(process.pid).children():
 377             child.send_signal(signal.SIGINT)
 378             child.wait()
 379         process.send_signal(signal.SIGINT)
 380
 381
 382 def start_honeynode(log_file: str, node_port: str, node_config_file_name: str):
 383     if os.path.isfile(HONEYNODE_EXECUTABLE):
 384         with open(log_file, 'w') as outfile:
 385             return subprocess.Popen(
 386                 [HONEYNODE_EXECUTABLE, node_port, os.path.join(SAMPLES_DIRECTORY, node_config_file_name)],
 387                 stdout=outfile, stderr=outfile)
 388
 389
 390 def wait_until_log_contains(log_file, regexp, time_to_wait=20):
 391     stringfound = False
 392     filefound = False
 393     line = None
 394     try:
 395         with TimeOut(seconds=time_to_wait):
 396             while not os.path.exists(log_file):
 397                 time.sleep(0.2)
 398             filelogs = open(log_file, 'r')
 399             filelogs.seek(0, 2)
 400             filefound = True
 401             print("Searching for pattern '"+regexp+"' in "+os.path.basename(log_file), end='... ', flush=True)
 402             compiled_regexp = re.compile(regexp)
 403             while True:
 404                 line = filelogs.readline()
 405                 if compiled_regexp.search(line):
 406                     print("Pattern found!", end=' ')
 407                     stringfound = True
 408                     break
 409                 if not line:
 410                     time.sleep(0.1)
 411     except TimeoutError:
 412         print("Pattern not found after "+str(time_to_wait), end=" seconds! ", flush=True)
 413     except PermissionError:
 414         print("Permission Error when trying to access the log file", end=" ... ", flush=True)
 415     finally:
 416         if filefound:
 417             filelogs.close()
 418         else:
 419             print("log file does not exist or is not accessible... ", flush=True)
 420         return stringfound
 421
 422
 423 class TimeOut:
 424     def __init__(self, seconds=1, error_message='Timeout'):
 425         self.seconds = seconds
 426         self.error_message = error_message
 427
 428     def handle_timeout(self, signum, frame):
 429         raise TimeoutError(self.error_message)
 430
 431     def __enter__(self):
 432         signal.signal(signal.SIGALRM, self.handle_timeout)
 433         signal.alarm(self.seconds)
 434
 435     def __exit__(self, type, value, traceback):
 436         signal.alarm(0)