tests/transportpce_tests/common/test_utils.py

   1 #!/usr/bin/env python
   2 ##############################################################################
   3 # Copyright (c) 2020 Orange, Inc. and others.  All rights reserved.
   4 #
   5 # All rights reserved. This program and the accompanying materials
   6 # are made available under the terms of the Apache License, Version 2.0
   7 # which accompanies this distribution, and is available at
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 ##############################################################################
  10 import json
  11 import os
  12 import sys
  13 import re
  14 import signal
  15 import subprocess
  16 import time
  17
  18 import psutil
  19 import requests
  20
  21 import simulators
  22
  23 SIMS = simulators.SIMS
  24 HONEYNODE_EXECUTABLE = simulators.HONEYNODE_EXECUTABLE
  25 SAMPLES_DIRECTORY = simulators.SAMPLES_DIRECTORY
  26
  27 HONEYNODE_OK_START_MSG = "Netconf SSH endpoint started successfully at 0.0.0.0"
  28 KARAF_OK_START_MSG = re.escape(
  29     "Blueprint container for bundle org.opendaylight.netconf.restconf")+".* was successfully created"
  30
  31
  32 RESTCONF_BASE_URL = "http://localhost:8181/restconf"
  33 ODL_LOGIN = "admin"
  34 ODL_PWD = "admin"
  35 NODES_LOGIN = "admin"
  36 NODES_PWD = "admin"
  37 URL_CONFIG_NETCONF_TOPO = "{}/config/network-topology:network-topology/topology/topology-netconf/"
  38 URL_CONFIG_ORDM_TOPO = "{}/config/ietf-network:networks/network/openroadm-topology/"
  39 URL_CONFIG_OTN_TOPO = "{}/config/ietf-network:networks/network/otn-topology/"
  40 URL_CONFIG_CLLI_NET = "{}/config/ietf-network:networks/network/clli-network/"
  41 URL_CONFIG_ORDM_NET = "{}/config/ietf-network:networks/network/openroadm-network/"
  42
  43 TYPE_APPLICATION_JSON = {'Content-Type': 'application/json', 'Accept': 'application/json'}
  44 TYPE_APPLICATION_XML = {'Content-Type': 'application/xml', 'Accept': 'application/xml'}
  45
  46 CODE_SHOULD_BE_200 = 'Http status code should be 200'
  47 CODE_SHOULD_BE_201 = 'Http status code should be 201'
  48
  49 LOG_DIRECTORY = os.path.dirname(os.path.realpath(__file__))
  50
  51 KARAF_LOG = os.path.join(
  52     os.path.dirname(os.path.realpath(__file__)),
  53     "..", "..", "..", "karaf", "target", "assembly", "data", "log", "karaf.log")
  54
  55 process_list = []
  56
  57 if "USE_LIGHTY" in os.environ and os.environ['USE_LIGHTY'] == 'True':
  58     TPCE_LOG = 'odl.log'
  59 else:
  60     TPCE_LOG = KARAF_LOG
  61
  62
  63 def start_sims(sims_list):
  64     for sim in sims_list:
  65         print("starting simulator for " + sim + "...")
  66         log_file = os.path.join(LOG_DIRECTORY, SIMS[sim]['logfile'])
  67         process = start_honeynode(log_file, SIMS[sim]['port'], SIMS[sim]['configfile'])
  68         if wait_until_log_contains(log_file, HONEYNODE_OK_START_MSG, 100):
  69             print("simulator for " + sim + " started")
  70         else:
  71             print("simulator for " + sim + " failed to start")
  72             shutdown_process(process)
  73             for pid in process_list:
  74                 shutdown_process(pid)
  75             sys.exit(3)
  76         process_list.append(process)
  77     return process_list
  78
  79
  80 def start_tpce():
  81     print("starting OpenDaylight...")
  82     if "USE_LIGHTY" in os.environ and os.environ['USE_LIGHTY'] == 'True':
  83         process = start_lighty()
  84         # TODO: add some sort of health check similar to Karaf below
  85     else:
  86         process = start_karaf()
  87         if wait_until_log_contains(KARAF_LOG, KARAF_OK_START_MSG, time_to_wait=60):
  88             print("OpenDaylight started !")
  89         else:
  90             print("OpenDaylight failed to start !")
  91             shutdown_process(process)
  92             for pid in process_list:
  93                 shutdown_process(pid)
  94             sys.exit(1)
  95     process_list.append(process)
  96     return process_list
  97
  98
  99 def start_karaf():
 100     print("starting KARAF TransportPCE build...")
 101     executable = os.path.join(
 102         os.path.dirname(os.path.realpath(__file__)),
 103         "..", "..", "..", "karaf", "target", "assembly", "bin", "karaf")
 104     with open('odl.log', 'w') as outfile:
 105         return subprocess.Popen(
 106             ["sh", executable, "server"], stdout=outfile, stderr=outfile, stdin=None)
 107
 108
 109 def start_lighty():
 110     print("starting LIGHTY.IO TransportPCE build...")
 111     executable = os.path.join(
 112         os.path.dirname(os.path.realpath(__file__)),
 113         "..", "..", "..", "lighty", "target", "tpce",
 114         "clean-start-controller.sh")
 115     with open('odl.log', 'w') as outfile:
 116         return subprocess.Popen(
 117             ["sh", executable], stdout=outfile, stderr=outfile, stdin=None)
 118
 119
 120 def install_karaf_feature(feature_name: str):
 121     print("installing feature " + feature_name)
 122     executable = os.path.join(
 123         os.path.dirname(os.path.realpath(__file__)),
 124         "..", "..", "..", "karaf", "target", "assembly", "bin", "client")
 125     return subprocess.run([executable],
 126                           input='feature:install ' + feature_name + '\n feature:list | grep tapi \n logout \n',
 127                           universal_newlines=True)
 128
 129
 130 def get_request(url):
 131     return requests.request(
 132         "GET", url.format(RESTCONF_BASE_URL),
 133         headers=TYPE_APPLICATION_JSON,
 134         auth=(ODL_LOGIN, ODL_PWD))
 135
 136
 137 def post_request(url, data):
 138     if data:
 139         return requests.request(
 140             "POST", url.format(RESTCONF_BASE_URL),
 141             data=json.dumps(data),
 142             headers=TYPE_APPLICATION_JSON,
 143             auth=(ODL_LOGIN, ODL_PWD))
 144     else:
 145         return requests.request(
 146             "POST", url.format(RESTCONF_BASE_URL),
 147             headers=TYPE_APPLICATION_JSON,
 148             auth=(ODL_LOGIN, ODL_PWD))
 149
 150
 151 def post_xmlrequest(url, data):
 152     if data:
 153         return requests.request(
 154             "POST", url.format(RESTCONF_BASE_URL),
 155             data=data,
 156             headers=TYPE_APPLICATION_XML,
 157             auth=(ODL_LOGIN, ODL_PWD))
 158
 159
 160 def put_request(url, data):
 161     return requests.request(
 162         "PUT", url.format(RESTCONF_BASE_URL),
 163         data=json.dumps(data),
 164         headers=TYPE_APPLICATION_JSON,
 165         auth=(ODL_LOGIN, ODL_PWD))
 166
 167
 168 def put_xmlrequest(url, data):
 169     return requests.request(
 170         "PUT", url.format(RESTCONF_BASE_URL),
 171         data=data,
 172         headers=TYPE_APPLICATION_XML,
 173         auth=(ODL_LOGIN, ODL_PWD))
 174
 175
 176 def rawput_request(url, data):
 177     return requests.request(
 178         "PUT", url.format(RESTCONF_BASE_URL),
 179         data=data,
 180         headers=TYPE_APPLICATION_JSON,
 181         auth=(ODL_LOGIN, ODL_PWD))
 182
 183
 184 def delete_request(url):
 185     return requests.request(
 186         "DELETE", url.format(RESTCONF_BASE_URL),
 187         headers=TYPE_APPLICATION_JSON,
 188         auth=(ODL_LOGIN, ODL_PWD))
 189
 190
 191 def mount_device(node_id, sim):
 192     url = URL_CONFIG_NETCONF_TOPO+"node/"+node_id
 193     body = {"node": [{
 194         "node-id": node_id,
 195         "netconf-node-topology:username": NODES_LOGIN,
 196         "netconf-node-topology:password": NODES_PWD,
 197         "netconf-node-topology:host": "127.0.0.1",
 198         "netconf-node-topology:port": SIMS[sim]['port'],
 199         "netconf-node-topology:tcp-only": "false",
 200         "netconf-node-topology:pass-through": {}}]}
 201     response = put_request(url, body)
 202     if wait_until_log_contains(TPCE_LOG, re.escape("Triggering notification stream NETCONF for node "+node_id), 60):
 203         print("Node "+node_id+" correctly added to tpce topology", end='... ', flush=True)
 204     else:
 205         print("Node "+node_id+" still not added to tpce topology", end='... ', flush=True)
 206         if response.status_code == requests.codes.ok:
 207             print("It was probably loaded at start-up", end='... ', flush=True)
 208         # TODO an else-clause to abort test would probably be nice here
 209     return response
 210
 211
 212 def unmount_device(node_id):
 213     url = URL_CONFIG_NETCONF_TOPO+"node/"+node_id
 214     response = delete_request(url)
 215     if wait_until_log_contains(TPCE_LOG, re.escape("onDeviceDisConnected: "+node_id), 60):
 216         print("Node "+node_id+" correctly deleted from tpce topology", end='... ', flush=True)
 217     else:
 218         print("Node "+node_id+" still not deleted from tpce topology", end='... ', flush=True)
 219     return response
 220
 221
 222 def connect_xpdr_to_rdm_request(xpdr_node: str, xpdr_num: str, network_num: str,
 223                                 rdm_node: str, srg_num: str, termination_num: str):
 224     url = "{}/operations/transportpce-networkutils:init-xpdr-rdm-links"
 225     data = {
 226         "networkutils:input": {
 227             "networkutils:links-input": {
 228                 "networkutils:xpdr-node": xpdr_node,
 229                 "networkutils:xpdr-num": xpdr_num,
 230                 "networkutils:network-num": network_num,
 231                 "networkutils:rdm-node": rdm_node,
 232                 "networkutils:srg-num": srg_num,
 233                 "networkutils:termination-point-num": termination_num
 234             }
 235         }
 236     }
 237     return post_request(url, data)
 238
 239
 240 def connect_rdm_to_xpdr_request(xpdr_node: str, xpdr_num: str, network_num: str,
 241                                 rdm_node: str, srg_num: str, termination_num: str):
 242     url = "{}/operations/transportpce-networkutils:init-rdm-xpdr-links"
 243     data = {
 244         "networkutils:input": {
 245             "networkutils:links-input": {
 246                 "networkutils:xpdr-node": xpdr_node,
 247                 "networkutils:xpdr-num": xpdr_num,
 248                 "networkutils:network-num": network_num,
 249                 "networkutils:rdm-node": rdm_node,
 250                 "networkutils:srg-num": srg_num,
 251                 "networkutils:termination-point-num": termination_num
 252             }
 253         }
 254     }
 255     return post_request(url, data)
 256
 257
 258 def check_netconf_node_request(node: str, suffix: str):
 259     url = URL_CONFIG_NETCONF_TOPO + (
 260         "node/" + node + "/yang-ext:mount/org-openroadm-device:org-openroadm-device/" + suffix
 261     )
 262     return get_request(url)
 263
 264
 265 def get_netconf_oper_request(node: str):
 266     url = "{}/operational/network-topology:network-topology/topology/topology-netconf/node/" + node
 267     return get_request(url)
 268
 269
 270 def get_ordm_topo_request(suffix: str):
 271     url = URL_CONFIG_ORDM_TOPO + suffix
 272     return get_request(url)
 273
 274
 275 def add_oms_attr_request(link: str, attr):
 276     url = URL_CONFIG_ORDM_TOPO + (
 277         "ietf-network-topology:link/" + link + "/org-openroadm-network-topology:OMS-attributes/span"
 278     )
 279     return put_request(url, attr)
 280
 281
 282 def del_oms_attr_request(link: str):
 283     url = URL_CONFIG_ORDM_TOPO + (
 284         "ietf-network-topology:link/" + link + "/org-openroadm-network-topology:OMS-attributes/span"
 285     )
 286     return delete_request(url)
 287
 288
 289 def get_clli_net_request():
 290     return get_request(URL_CONFIG_CLLI_NET)
 291
 292
 293 def get_ordm_net_request():
 294     return get_request(URL_CONFIG_ORDM_NET)
 295
 296
 297 def get_otn_topo_request():
 298     return get_request(URL_CONFIG_OTN_TOPO)
 299
 300
 301 def del_link_request(link: str):
 302     url = URL_CONFIG_ORDM_TOPO + ("ietf-network-topology:link/" + link)
 303     return delete_request(url)
 304
 305
 306 def del_node_request(node: str):
 307     url = URL_CONFIG_CLLI_NET + ("node/" + node)
 308     return delete_request(url)
 309
 310
 311 def shutdown_process(process):
 312     if process is not None:
 313         for child in psutil.Process(process.pid).children():
 314             child.send_signal(signal.SIGINT)
 315             child.wait()
 316         process.send_signal(signal.SIGINT)
 317
 318
 319 def start_honeynode(log_file: str, node_port: str, node_config_file_name: str):
 320     if os.path.isfile(HONEYNODE_EXECUTABLE):
 321         with open(log_file, 'w') as outfile:
 322             return subprocess.Popen(
 323                 [HONEYNODE_EXECUTABLE, node_port, os.path.join(SAMPLES_DIRECTORY, node_config_file_name)],
 324                 stdout=outfile, stderr=outfile)
 325
 326
 327 def wait_until_log_contains(log_file, regexp, time_to_wait=20):
 328     stringfound = False
 329     filefound = False
 330     line = None
 331     try:
 332         with TimeOut(seconds=time_to_wait):
 333             while not os.path.exists(log_file):
 334                 time.sleep(0.2)
 335             filelogs = open(log_file, 'r')
 336             filelogs.seek(0, 2)
 337             filefound = True
 338             print("Searching for pattern '"+regexp+"' in "+os.path.basename(log_file), end='... ', flush=True)
 339             compiled_regexp = re.compile(regexp)
 340             while True:
 341                 line = filelogs.readline()
 342                 if compiled_regexp.search(line):
 343                     print("Pattern found!", end=' ')
 344                     stringfound = True
 345                     break
 346                 if not line:
 347                     time.sleep(0.1)
 348     except TimeoutError:
 349         print("Pattern not found after "+str(time_to_wait), end=" seconds! ", flush=True)
 350     except PermissionError:
 351         print("Permission Error when trying to access the log file", end=" ... ", flush=True)
 352     finally:
 353         if filefound:
 354             filelogs.close()
 355         else:
 356             print("log file does not exist or is not accessible... ", flush=True)
 357         return stringfound
 358
 359
 360 class TimeOut:
 361     def __init__(self, seconds=1, error_message='Timeout'):
 362         self.seconds = seconds
 363         self.error_message = error_message
 364
 365     def handle_timeout(self, signum, frame):
 366         raise TimeoutError(self.error_message)
 367
 368     def __enter__(self):
 369         signal.signal(signal.SIGALRM, self.handle_timeout)
 370         signal.alarm(self.seconds)
 371
 372     def __exit__(self, type, value, traceback):
 373         signal.alarm(0)