tests/transportpce_tests/common/test_utils.py

   1 #!/usr/bin/env python
   2 ##############################################################################
   3 # Copyright (c) 2020 Orange, Inc. and others.  All rights reserved.
   4 #
   5 # All rights reserved. This program and the accompanying materials
   6 # are made available under the terms of the Apache License, Version 2.0
   7 # which accompanies this distribution, and is available at
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 ##############################################################################
  10 import json
  11 import os
  12 import sys
  13 import re
  14 import signal
  15 import subprocess
  16 import time
  17
  18 import psutil
  19 import requests
  20
  21 import simulators
  22
  23 SIMS = simulators.SIMS
  24 HONEYNODE_EXECUTABLE = simulators.HONEYNODE_EXECUTABLE
  25 SAMPLES_DIRECTORY = simulators.SAMPLES_DIRECTORY
  26
  27 HONEYNODE_OK_START_MSG = "Netconf SSH endpoint started successfully at 0.0.0.0"
  28 KARAF_OK_START_MSG = re.escape(
  29     "Blueprint container for bundle org.opendaylight.netconf.restconf")+".* was successfully created"
  30
  31
  32 RESTCONF_BASE_URL = "http://localhost:8181/restconf"
  33 ODL_LOGIN = "admin"
  34 ODL_PWD = "admin"
  35 NODES_LOGIN = "admin"
  36 NODES_PWD = "admin"
  37 URL_CONFIG_NETCONF_TOPO = "{}/config/network-topology:network-topology/topology/topology-netconf/"
  38 URL_CONFIG_ORDM_TOPO = "{}/config/ietf-network:networks/network/openroadm-topology/"
  39 URL_CONFIG_OTN_TOPO = "{}/config/ietf-network:networks/network/otn-topology/"
  40 URL_CONFIG_CLLI_NET = "{}/config/ietf-network:networks/network/clli-network/"
  41 URL_CONFIG_ORDM_NET = "{}/config/ietf-network:networks/network/openroadm-network/"
  42 URL_PORTMAPPING = "{}/config/transportpce-portmapping:network/nodes/"
  43 URL_OPER_SERV_LIST = "{}/operational/org-openroadm-service:service-list/"
  44 URL_SERV_CREATE = "{}/operations/org-openroadm-service:service-create"
  45 URL_SERVICE_PATH = "{}/operations/transportpce-device-renderer:service-path"
  46 URL_OTN_SERVICE_PATH = "{}/operations/transportpce-device-renderer:otn-service-path"
  47 URL_CREATE_OTS_OMS = "{}/operations/transportpce-device-renderer:create-ots-oms"
  48
  49 TYPE_APPLICATION_JSON = {'Content-Type': 'application/json', 'Accept': 'application/json'}
  50 TYPE_APPLICATION_XML = {'Content-Type': 'application/xml', 'Accept': 'application/xml'}
  51
  52 CODE_SHOULD_BE_200 = 'Http status code should be 200'
  53 CODE_SHOULD_BE_201 = 'Http status code should be 201'
  54
  55 LOG_DIRECTORY = os.path.dirname(os.path.realpath(__file__))
  56
  57 KARAF_LOG = os.path.join(
  58     os.path.dirname(os.path.realpath(__file__)),
  59     "..", "..", "..", "karaf", "target", "assembly", "data", "log", "karaf.log")
  60
  61 process_list = []
  62
  63 if "USE_LIGHTY" in os.environ and os.environ['USE_LIGHTY'] == 'True':
  64     TPCE_LOG = 'odl.log'
  65 else:
  66     TPCE_LOG = KARAF_LOG
  67
  68
  69 def start_sims(sims_list):
  70     for sim in sims_list:
  71         print("starting simulator for " + sim + "...")
  72         log_file = os.path.join(LOG_DIRECTORY, SIMS[sim]['logfile'])
  73         process = start_honeynode(log_file, SIMS[sim]['port'], SIMS[sim]['configfile'])
  74         if wait_until_log_contains(log_file, HONEYNODE_OK_START_MSG, 100):
  75             print("simulator for " + sim + " started")
  76         else:
  77             print("simulator for " + sim + " failed to start")
  78             shutdown_process(process)
  79             for pid in process_list:
  80                 shutdown_process(pid)
  81             sys.exit(3)
  82         process_list.append(process)
  83     return process_list
  84
  85
  86 def start_tpce():
  87     print("starting OpenDaylight...")
  88     if "USE_LIGHTY" in os.environ and os.environ['USE_LIGHTY'] == 'True':
  89         process = start_lighty()
  90         # TODO: add some sort of health check similar to Karaf below
  91     else:
  92         process = start_karaf()
  93         if wait_until_log_contains(KARAF_LOG, KARAF_OK_START_MSG, time_to_wait=60):
  94             print("OpenDaylight started !")
  95         else:
  96             print("OpenDaylight failed to start !")
  97             shutdown_process(process)
  98             for pid in process_list:
  99                 shutdown_process(pid)
 100             sys.exit(1)
 101     process_list.append(process)
 102     return process_list
 103
 104
 105 def start_karaf():
 106     print("starting KARAF TransportPCE build...")
 107     executable = os.path.join(
 108         os.path.dirname(os.path.realpath(__file__)),
 109         "..", "..", "..", "karaf", "target", "assembly", "bin", "karaf")
 110     with open('odl.log', 'w') as outfile:
 111         return subprocess.Popen(
 112             ["sh", executable, "server"], stdout=outfile, stderr=outfile, stdin=None)
 113
 114
 115 def start_lighty():
 116     print("starting LIGHTY.IO TransportPCE build...")
 117     executable = os.path.join(
 118         os.path.dirname(os.path.realpath(__file__)),
 119         "..", "..", "..", "lighty", "target", "tpce",
 120         "clean-start-controller.sh")
 121     with open('odl.log', 'w') as outfile:
 122         return subprocess.Popen(
 123             ["sh", executable], stdout=outfile, stderr=outfile, stdin=None)
 124
 125
 126 def install_karaf_feature(feature_name: str):
 127     print("installing feature " + feature_name)
 128     executable = os.path.join(
 129         os.path.dirname(os.path.realpath(__file__)),
 130         "..", "..", "..", "karaf", "target", "assembly", "bin", "client")
 131     return subprocess.run([executable],
 132                           input='feature:install ' + feature_name + '\n feature:list | grep tapi \n logout \n',
 133                           universal_newlines=True)
 134
 135
 136 def get_request(url):
 137     return requests.request(
 138         "GET", url.format(RESTCONF_BASE_URL),
 139         headers=TYPE_APPLICATION_JSON,
 140         auth=(ODL_LOGIN, ODL_PWD))
 141
 142
 143 def post_request(url, data):
 144     if data:
 145         return requests.request(
 146             "POST", url.format(RESTCONF_BASE_URL),
 147             data=json.dumps(data),
 148             headers=TYPE_APPLICATION_JSON,
 149             auth=(ODL_LOGIN, ODL_PWD))
 150     else:
 151         return requests.request(
 152             "POST", url.format(RESTCONF_BASE_URL),
 153             headers=TYPE_APPLICATION_JSON,
 154             auth=(ODL_LOGIN, ODL_PWD))
 155
 156
 157 def post_xmlrequest(url, data):
 158     if data:
 159         return requests.request(
 160             "POST", url.format(RESTCONF_BASE_URL),
 161             data=data,
 162             headers=TYPE_APPLICATION_XML,
 163             auth=(ODL_LOGIN, ODL_PWD))
 164
 165
 166 def put_request(url, data):
 167     return requests.request(
 168         "PUT", url.format(RESTCONF_BASE_URL),
 169         data=json.dumps(data),
 170         headers=TYPE_APPLICATION_JSON,
 171         auth=(ODL_LOGIN, ODL_PWD))
 172
 173
 174 def put_xmlrequest(url, data):
 175     return requests.request(
 176         "PUT", url.format(RESTCONF_BASE_URL),
 177         data=data,
 178         headers=TYPE_APPLICATION_XML,
 179         auth=(ODL_LOGIN, ODL_PWD))
 180
 181
 182 def rawput_request(url, data):
 183     return requests.request(
 184         "PUT", url.format(RESTCONF_BASE_URL),
 185         data=data,
 186         headers=TYPE_APPLICATION_JSON,
 187         auth=(ODL_LOGIN, ODL_PWD))
 188
 189
 190 def delete_request(url):
 191     return requests.request(
 192         "DELETE", url.format(RESTCONF_BASE_URL),
 193         headers=TYPE_APPLICATION_JSON,
 194         auth=(ODL_LOGIN, ODL_PWD))
 195
 196
 197 def mount_device(node_id, sim):
 198     url = URL_CONFIG_NETCONF_TOPO+"node/"+node_id
 199     body = {"node": [{
 200         "node-id": node_id,
 201         "netconf-node-topology:username": NODES_LOGIN,
 202         "netconf-node-topology:password": NODES_PWD,
 203         "netconf-node-topology:host": "127.0.0.1",
 204         "netconf-node-topology:port": SIMS[sim]['port'],
 205         "netconf-node-topology:tcp-only": "false",
 206         "netconf-node-topology:pass-through": {}}]}
 207     response = put_request(url, body)
 208     if wait_until_log_contains(TPCE_LOG, re.escape("Triggering notification stream NETCONF for node "+node_id), 60):
 209         print("Node "+node_id+" correctly added to tpce topology", end='... ', flush=True)
 210     else:
 211         print("Node "+node_id+" still not added to tpce topology", end='... ', flush=True)
 212         if response.status_code == requests.codes.ok:
 213             print("It was probably loaded at start-up", end='... ', flush=True)
 214         # TODO an else-clause to abort test would probably be nice here
 215     return response
 216
 217
 218 def unmount_device(node_id):
 219     url = URL_CONFIG_NETCONF_TOPO+"node/"+node_id
 220     response = delete_request(url)
 221     if wait_until_log_contains(TPCE_LOG, re.escape("onDeviceDisConnected: "+node_id), 60):
 222         print("Node "+node_id+" correctly deleted from tpce topology", end='... ', flush=True)
 223     else:
 224         print("Node "+node_id+" still not deleted from tpce topology", end='... ', flush=True)
 225     return response
 226
 227
 228 def connect_xpdr_to_rdm_request(xpdr_node: str, xpdr_num: str, network_num: str,
 229                                 rdm_node: str, srg_num: str, termination_num: str):
 230     url = "{}/operations/transportpce-networkutils:init-xpdr-rdm-links"
 231     data = {
 232         "networkutils:input": {
 233             "networkutils:links-input": {
 234                 "networkutils:xpdr-node": xpdr_node,
 235                 "networkutils:xpdr-num": xpdr_num,
 236                 "networkutils:network-num": network_num,
 237                 "networkutils:rdm-node": rdm_node,
 238                 "networkutils:srg-num": srg_num,
 239                 "networkutils:termination-point-num": termination_num
 240             }
 241         }
 242     }
 243     return post_request(url, data)
 244
 245
 246 def connect_rdm_to_xpdr_request(xpdr_node: str, xpdr_num: str, network_num: str,
 247                                 rdm_node: str, srg_num: str, termination_num: str):
 248     url = "{}/operations/transportpce-networkutils:init-rdm-xpdr-links"
 249     data = {
 250         "networkutils:input": {
 251             "networkutils:links-input": {
 252                 "networkutils:xpdr-node": xpdr_node,
 253                 "networkutils:xpdr-num": xpdr_num,
 254                 "networkutils:network-num": network_num,
 255                 "networkutils:rdm-node": rdm_node,
 256                 "networkutils:srg-num": srg_num,
 257                 "networkutils:termination-point-num": termination_num
 258             }
 259         }
 260     }
 261     return post_request(url, data)
 262
 263
 264 def check_netconf_node_request(node: str, suffix: str):
 265     url = URL_CONFIG_NETCONF_TOPO + (
 266         "node/" + node + "/yang-ext:mount/org-openroadm-device:org-openroadm-device/" + suffix
 267     )
 268     return get_request(url)
 269
 270
 271 def get_netconf_oper_request(node: str):
 272     url = "{}/operational/network-topology:network-topology/topology/topology-netconf/node/" + node
 273     return get_request(url)
 274
 275
 276 def get_ordm_topo_request(suffix: str):
 277     url = URL_CONFIG_ORDM_TOPO + suffix
 278     return get_request(url)
 279
 280
 281 def add_oms_attr_request(link: str, attr):
 282     url = URL_CONFIG_ORDM_TOPO + (
 283         "ietf-network-topology:link/" + link + "/org-openroadm-network-topology:OMS-attributes/span"
 284     )
 285     return put_request(url, attr)
 286
 287
 288 def del_oms_attr_request(link: str):
 289     url = URL_CONFIG_ORDM_TOPO + (
 290         "ietf-network-topology:link/" + link + "/org-openroadm-network-topology:OMS-attributes/span"
 291     )
 292     return delete_request(url)
 293
 294
 295 def get_clli_net_request():
 296     return get_request(URL_CONFIG_CLLI_NET)
 297
 298
 299 def get_ordm_net_request():
 300     return get_request(URL_CONFIG_ORDM_NET)
 301
 302
 303 def get_otn_topo_request():
 304     return get_request(URL_CONFIG_OTN_TOPO)
 305
 306
 307 def del_link_request(link: str):
 308     url = URL_CONFIG_ORDM_TOPO + ("ietf-network-topology:link/" + link)
 309     return delete_request(url)
 310
 311
 312 def del_node_request(node: str):
 313     url = URL_CONFIG_CLLI_NET + ("node/" + node)
 314     return delete_request(url)
 315
 316
 317 def portmapping_request(suffix: str):
 318     url = URL_PORTMAPPING + suffix
 319     return get_request(url)
 320
 321
 322 def get_service_list_request(suffix: str):
 323     url = URL_OPER_SERV_LIST + suffix
 324     return get_request(url)
 325
 326
 327 def service_create_request(attr):
 328     return post_request(URL_SERV_CREATE, attr)
 329
 330
 331 def service_path_request(operation: str, servicename: str, wavenumber: str, nodes):
 332     attr = {"renderer:input": {
 333             "renderer:service-name": servicename,
 334             "renderer:wave-number": wavenumber,
 335             "renderer:modulation-format": "qpsk",
 336             "renderer:operation": operation,
 337             "renderer:nodes": nodes}}
 338     return post_request(URL_SERVICE_PATH, attr)
 339
 340
 341 def otn_service_path_request(operation: str, servicename: str, servicerate: str, servicetype: str, nodes, eth_attr=None):
 342     attr = {"service-name": servicename,
 343             "operation": operation,
 344             "service-rate": servicerate,
 345             "service-type": servicetype,
 346             "nodes": nodes}
 347     if eth_attr:
 348         attr.update(eth_attr)
 349     return post_request(URL_OTN_SERVICE_PATH, {"renderer:input": attr})
 350
 351
 352 def create_ots_oms_request(nodeid: str, lcp: str):
 353     attr = {"input": {
 354             "node-id": nodeid,
 355             "logical-connection-point": lcp}}
 356     return post_request(URL_CREATE_OTS_OMS, attr)
 357
 358
 359 def shutdown_process(process):
 360     if process is not None:
 361         for child in psutil.Process(process.pid).children():
 362             child.send_signal(signal.SIGINT)
 363             child.wait()
 364         process.send_signal(signal.SIGINT)
 365
 366
 367 def start_honeynode(log_file: str, node_port: str, node_config_file_name: str):
 368     if os.path.isfile(HONEYNODE_EXECUTABLE):
 369         with open(log_file, 'w') as outfile:
 370             return subprocess.Popen(
 371                 [HONEYNODE_EXECUTABLE, node_port, os.path.join(SAMPLES_DIRECTORY, node_config_file_name)],
 372                 stdout=outfile, stderr=outfile)
 373
 374
 375 def wait_until_log_contains(log_file, regexp, time_to_wait=20):
 376     stringfound = False
 377     filefound = False
 378     line = None
 379     try:
 380         with TimeOut(seconds=time_to_wait):
 381             while not os.path.exists(log_file):
 382                 time.sleep(0.2)
 383             filelogs = open(log_file, 'r')
 384             filelogs.seek(0, 2)
 385             filefound = True
 386             print("Searching for pattern '"+regexp+"' in "+os.path.basename(log_file), end='... ', flush=True)
 387             compiled_regexp = re.compile(regexp)
 388             while True:
 389                 line = filelogs.readline()
 390                 if compiled_regexp.search(line):
 391                     print("Pattern found!", end=' ')
 392                     stringfound = True
 393                     break
 394                 if not line:
 395                     time.sleep(0.1)
 396     except TimeoutError:
 397         print("Pattern not found after "+str(time_to_wait), end=" seconds! ", flush=True)
 398     except PermissionError:
 399         print("Permission Error when trying to access the log file", end=" ... ", flush=True)
 400     finally:
 401         if filefound:
 402             filelogs.close()
 403         else:
 404             print("log file does not exist or is not accessible... ", flush=True)
 405         return stringfound
 406
 407
 408 class TimeOut:
 409     def __init__(self, seconds=1, error_message='Timeout'):
 410         self.seconds = seconds
 411         self.error_message = error_message
 412
 413     def handle_timeout(self, signum, frame):
 414         raise TimeoutError(self.error_message)
 415
 416     def __enter__(self):
 417         signal.signal(signal.SIGALRM, self.handle_timeout)
 418         signal.alarm(self.seconds)
 419
 420     def __exit__(self, type, value, traceback):
 421         signal.alarm(0)