tests/transportpce_tests/common/test_utils.py

   1 #!/usr/bin/env python
   2
   3 ##############################################################################
   4 # Copyright (c) 2020 Orange, Inc. and others.  All rights reserved.
   5 #
   6 # All rights reserved. This program and the accompanying materials
   7 # are made available under the terms of the Apache License, Version 2.0
   8 # which accompanies this distribution, and is available at
   9 # http://www.apache.org/licenses/LICENSE-2.0
  10 ##############################################################################
  11
  12 # pylint: disable=no-member
  13
  14 import base64
  15 import json
  16 import os
  17 import sys
  18 import re
  19 import signal
  20 import subprocess
  21 import time
  22
  23 import psutil
  24 import requests
  25
  26 import simulators
  27
  28 SIMS = simulators.SIMS
  29 HONEYNODE_EXECUTABLE = simulators.HONEYNODE_EXECUTABLE
  30 SAMPLES_DIRECTORY = simulators.SAMPLES_DIRECTORY
  31
  32 HONEYNODE_OK_START_MSG = "Netconf SSH endpoint started successfully at 0.0.0.0"
  33 KARAF_OK_START_MSG = re.escape(
  34     "Blueprint container for bundle org.opendaylight.netconf.restconf")+".* was successfully created"
  35 LIGHTY_OK_START_MSG = re.escape("lighty.io and RESTCONF-NETCONF started")
  36
  37 RESTCONF_BASE_URL = "http://localhost:8181/restconf"
  38 ODL_LOGIN = "admin"
  39 ODL_PWD = "admin"
  40 NODES_LOGIN = "admin"
  41 NODES_PWD = "admin"
  42 URL_CONFIG_NETCONF_TOPO = "{}/config/network-topology:network-topology/topology/topology-netconf/"
  43 URL_CONFIG_ORDM_TOPO = "{}/config/ietf-network:networks/network/openroadm-topology/"
  44 URL_CONFIG_OTN_TOPO = "{}/config/ietf-network:networks/network/otn-topology/"
  45 URL_CONFIG_CLLI_NET = "{}/config/ietf-network:networks/network/clli-network/"
  46 URL_CONFIG_ORDM_NET = "{}/config/ietf-network:networks/network/openroadm-network/"
  47 URL_PORTMAPPING = "{}/config/transportpce-portmapping:network/nodes/"
  48 URL_OPER_SERV_LIST = "{}/operational/org-openroadm-service:service-list/"
  49 URL_SERV_CREATE = "{}/operations/org-openroadm-service:service-create"
  50 URL_SERV_DELETE = "{}/operations/org-openroadm-service:service-delete"
  51 URL_SERVICE_PATH = "{}/operations/transportpce-device-renderer:service-path"
  52 URL_OTN_SERVICE_PATH = "{}/operations/transportpce-device-renderer:otn-service-path"
  53 URL_CREATE_OTS_OMS = "{}/operations/transportpce-device-renderer:create-ots-oms"
  54 URL_PATH_COMPUTATION_REQUEST = "{}/operations/transportpce-pce:path-computation-request"
  55
  56 TYPE_APPLICATION_JSON = {'Content-Type': 'application/json', 'Accept': 'application/json'}
  57 TYPE_APPLICATION_XML = {'Content-Type': 'application/xml', 'Accept': 'application/xml'}
  58
  59 CODE_SHOULD_BE_200 = 'Http status code should be 200'
  60 CODE_SHOULD_BE_201 = 'Http status code should be 201'
  61
  62 LOG_DIRECTORY = os.path.dirname(os.path.realpath(__file__))
  63
  64 KARAF_LOG = os.path.join(
  65     os.path.dirname(os.path.realpath(__file__)),
  66     "..", "..", "..", "karaf", "target", "assembly", "data", "log", "karaf.log")
  67
  68 process_list = []
  69
  70 if "USE_LIGHTY" in os.environ and os.environ['USE_LIGHTY'] == 'True':
  71     TPCE_LOG = 'odl.log'
  72 else:
  73     TPCE_LOG = KARAF_LOG
  74
  75
  76 def start_sims(sims_list):
  77     for sim in sims_list:
  78         print("starting simulator for " + sim + "...")
  79         log_file = os.path.join(LOG_DIRECTORY, SIMS[sim]['logfile'])
  80         process = start_honeynode(log_file, SIMS[sim]['port'], SIMS[sim]['configfile'])
  81         if wait_until_log_contains(log_file, HONEYNODE_OK_START_MSG, 100):
  82             print("simulator for " + sim + " started")
  83         else:
  84             print("simulator for " + sim + " failed to start")
  85             shutdown_process(process)
  86             for pid in process_list:
  87                 shutdown_process(pid)
  88             sys.exit(3)
  89         process_list.append(process)
  90     return process_list
  91
  92
  93 def start_tpce():
  94     print("starting OpenDaylight...")
  95     if "USE_LIGHTY" in os.environ and os.environ['USE_LIGHTY'] == 'True':
  96         process = start_lighty()
  97         start_msg = LIGHTY_OK_START_MSG
  98     else:
  99         process = start_karaf()
 100         start_msg = KARAF_OK_START_MSG
 101     if wait_until_log_contains(TPCE_LOG, start_msg, time_to_wait=60):
 102         print("OpenDaylight started !")
 103     else:
 104         print("OpenDaylight failed to start !")
 105         shutdown_process(process)
 106         for pid in process_list:
 107             shutdown_process(pid)
 108         sys.exit(1)
 109     process_list.append(process)
 110     return process_list
 111
 112
 113 def start_karaf():
 114     print("starting KARAF TransportPCE build...")
 115     executable = os.path.join(
 116         os.path.dirname(os.path.realpath(__file__)),
 117         "..", "..", "..", "karaf", "target", "assembly", "bin", "karaf")
 118     with open('odl.log', 'w') as outfile:
 119         return subprocess.Popen(
 120             ["sh", executable, "server"], stdout=outfile, stderr=outfile, stdin=None)
 121
 122
 123 def start_lighty():
 124     print("starting LIGHTY.IO TransportPCE build...")
 125     executable = os.path.join(
 126         os.path.dirname(os.path.realpath(__file__)),
 127         "..", "..", "..", "lighty", "target", "tpce",
 128         "clean-start-controller.sh")
 129     with open('odl.log', 'w') as outfile:
 130         return subprocess.Popen(
 131             ["sh", executable], stdout=outfile, stderr=outfile, stdin=None)
 132
 133
 134 def install_karaf_feature(feature_name: str):
 135     print("installing feature " + feature_name)
 136     executable = os.path.join(
 137         os.path.dirname(os.path.realpath(__file__)),
 138         "..", "..", "..", "karaf", "target", "assembly", "bin", "client")
 139     return subprocess.run([executable],
 140                           input='feature:install ' + feature_name + '\n feature:list | grep tapi \n logout \n',
 141                           universal_newlines=True, check=False)
 142
 143
 144 def get_request(url):
 145     return requests.request(
 146         "GET", url.format(RESTCONF_BASE_URL),
 147         headers=TYPE_APPLICATION_JSON,
 148         auth=(ODL_LOGIN, ODL_PWD))
 149
 150
 151 def post_request(url, data):
 152     if data:
 153         return requests.request(
 154             "POST", url.format(RESTCONF_BASE_URL),
 155             data=json.dumps(data),
 156             headers=TYPE_APPLICATION_JSON,
 157             auth=(ODL_LOGIN, ODL_PWD))
 158
 159     return requests.request(
 160         "POST", url.format(RESTCONF_BASE_URL),
 161         headers=TYPE_APPLICATION_JSON,
 162         auth=(ODL_LOGIN, ODL_PWD))
 163
 164
 165 def post_xmlrequest(url, data):
 166     if data:
 167         return requests.request(
 168             "POST", url.format(RESTCONF_BASE_URL),
 169             data=data,
 170             headers=TYPE_APPLICATION_XML,
 171             auth=(ODL_LOGIN, ODL_PWD))
 172     return None
 173
 174
 175 def put_request(url, data):
 176     return requests.request(
 177         "PUT", url.format(RESTCONF_BASE_URL),
 178         data=json.dumps(data),
 179         headers=TYPE_APPLICATION_JSON,
 180         auth=(ODL_LOGIN, ODL_PWD))
 181
 182
 183 def put_xmlrequest(url, data):
 184     return requests.request(
 185         "PUT", url.format(RESTCONF_BASE_URL),
 186         data=data,
 187         headers=TYPE_APPLICATION_XML,
 188         auth=(ODL_LOGIN, ODL_PWD))
 189
 190
 191 def rawput_request(url, data):
 192     return requests.request(
 193         "PUT", url.format(RESTCONF_BASE_URL),
 194         data=data,
 195         headers=TYPE_APPLICATION_JSON,
 196         auth=(ODL_LOGIN, ODL_PWD))
 197
 198
 199 def delete_request(url):
 200     return requests.request(
 201         "DELETE", url.format(RESTCONF_BASE_URL),
 202         headers=TYPE_APPLICATION_JSON,
 203         auth=(ODL_LOGIN, ODL_PWD))
 204
 205
 206 def mount_device(node_id, sim):
 207     url = URL_CONFIG_NETCONF_TOPO + "node/" + node_id
 208     body = {"node": [{
 209         "node-id": node_id,
 210         "netconf-node-topology:username": NODES_LOGIN,
 211         "netconf-node-topology:password": NODES_PWD,
 212         "netconf-node-topology:host": "127.0.0.1",
 213         "netconf-node-topology:port": SIMS[sim]['port'],
 214         "netconf-node-topology:tcp-only": "false",
 215         "netconf-node-topology:pass-through": {}}]}
 216     response = put_request(url, body)
 217     if wait_until_log_contains(TPCE_LOG, re.escape("Triggering notification stream NETCONF for node " + node_id), 60):
 218         print("Node " + node_id + " correctly added to tpce topology", end='... ', flush=True)
 219     else:
 220         print("Node " + node_id + " still not added to tpce topology", end='... ', flush=True)
 221         if response.status_code == requests.codes.ok:
 222             print("It was probably loaded at start-up", end='... ', flush=True)
 223         # TODO an else-clause to abort test would probably be nice here
 224     return response
 225
 226
 227 def unmount_device(node_id):
 228     url = URL_CONFIG_NETCONF_TOPO + "node/" + node_id
 229     response = delete_request(url)
 230     if wait_until_log_contains(TPCE_LOG, re.escape("onDeviceDisConnected: " + node_id), 60):
 231         print("Node " + node_id + " correctly deleted from tpce topology", end='... ', flush=True)
 232     else:
 233         print("Node " + node_id + " still not deleted from tpce topology", end='... ', flush=True)
 234     return response
 235
 236
 237 def connect_xpdr_to_rdm_request(xpdr_node: str, xpdr_num: str, network_num: str,
 238                                 rdm_node: str, srg_num: str, termination_num: str):
 239     url = "{}/operations/transportpce-networkutils:init-xpdr-rdm-links"
 240     data = {
 241         "networkutils:input": {
 242             "networkutils:links-input": {
 243                 "networkutils:xpdr-node": xpdr_node,
 244                 "networkutils:xpdr-num": xpdr_num,
 245                 "networkutils:network-num": network_num,
 246                 "networkutils:rdm-node": rdm_node,
 247                 "networkutils:srg-num": srg_num,
 248                 "networkutils:termination-point-num": termination_num
 249             }
 250         }
 251     }
 252     return post_request(url, data)
 253
 254
 255 def connect_rdm_to_xpdr_request(xpdr_node: str, xpdr_num: str, network_num: str,
 256                                 rdm_node: str, srg_num: str, termination_num: str):
 257     url = "{}/operations/transportpce-networkutils:init-rdm-xpdr-links"
 258     data = {
 259         "networkutils:input": {
 260             "networkutils:links-input": {
 261                 "networkutils:xpdr-node": xpdr_node,
 262                 "networkutils:xpdr-num": xpdr_num,
 263                 "networkutils:network-num": network_num,
 264                 "networkutils:rdm-node": rdm_node,
 265                 "networkutils:srg-num": srg_num,
 266                 "networkutils:termination-point-num": termination_num
 267             }
 268         }
 269     }
 270     return post_request(url, data)
 271
 272
 273 def check_netconf_node_request(node: str, suffix: str):
 274     url = URL_CONFIG_NETCONF_TOPO + (
 275         "node/" + node + "/yang-ext:mount/org-openroadm-device:org-openroadm-device/" + suffix
 276     )
 277     return get_request(url)
 278
 279
 280 def get_netconf_oper_request(node: str):
 281     url = "{}/operational/network-topology:network-topology/topology/topology-netconf/node/" + node
 282     return get_request(url)
 283
 284
 285 def get_ordm_topo_request(suffix: str):
 286     url = URL_CONFIG_ORDM_TOPO + suffix
 287     return get_request(url)
 288
 289
 290 def add_oms_attr_request(link: str, attr):
 291     url = URL_CONFIG_ORDM_TOPO + (
 292         "ietf-network-topology:link/" + link + "/org-openroadm-network-topology:OMS-attributes/span"
 293     )
 294     return put_request(url, attr)
 295
 296
 297 def del_oms_attr_request(link: str):
 298     url = URL_CONFIG_ORDM_TOPO + (
 299         "ietf-network-topology:link/" + link + "/org-openroadm-network-topology:OMS-attributes/span"
 300     )
 301     return delete_request(url)
 302
 303
 304 def get_clli_net_request():
 305     return get_request(URL_CONFIG_CLLI_NET)
 306
 307
 308 def get_ordm_net_request():
 309     return get_request(URL_CONFIG_ORDM_NET)
 310
 311
 312 def get_otn_topo_request():
 313     return get_request(URL_CONFIG_OTN_TOPO)
 314
 315
 316 def del_link_request(link: str):
 317     url = URL_CONFIG_ORDM_TOPO + ("ietf-network-topology:link/" + link)
 318     return delete_request(url)
 319
 320
 321 def del_node_request(node: str):
 322     url = URL_CONFIG_CLLI_NET + ("node/" + node)
 323     return delete_request(url)
 324
 325
 326 def portmapping_request(suffix: str):
 327     url = URL_PORTMAPPING + suffix
 328     return get_request(url)
 329
 330
 331 def get_service_list_request(suffix: str):
 332     url = URL_OPER_SERV_LIST + suffix
 333     return get_request(url)
 334
 335
 336 def service_create_request(attr):
 337     return post_request(URL_SERV_CREATE, attr)
 338
 339
 340 def service_delete_request(servicename: str,
 341                            requestid="e3028bae-a90f-4ddd-a83f-cf224eba0e58",
 342                            notificationurl="http://localhost:8585/NotificationServer/notify"):
 343     attr = {"input": {
 344         "sdnc-request-header": {
 345             "request-id": requestid,
 346             "rpc-action": "service-delete",
 347             "request-system-id": "appname",
 348             "notification-url": notificationurl},
 349         "service-delete-req-info": {
 350             "service-name": servicename,
 351             "tail-retention": "no"}}}
 352     return post_request(URL_SERV_DELETE, attr)
 353
 354
 355 def service_path_request(operation: str, servicename: str, wavenumber: str, nodes):
 356     attr = {"renderer:input": {
 357         "renderer:service-name": servicename,
 358         "renderer:wave-number": wavenumber,
 359         "renderer:modulation-format": "qpsk",
 360         "renderer:operation": operation,
 361         "renderer:nodes": nodes}}
 362     return post_request(URL_SERVICE_PATH, attr)
 363
 364
 365 def otn_service_path_request(operation: str, servicename: str, servicerate: str, servicetype: str, nodes,
 366                              eth_attr=None):
 367     attr = {"service-name": servicename,
 368             "operation": operation,
 369             "service-rate": servicerate,
 370             "service-type": servicetype,
 371             "nodes": nodes}
 372     if eth_attr:
 373         attr.update(eth_attr)
 374     return post_request(URL_OTN_SERVICE_PATH, {"renderer:input": attr})
 375
 376
 377 def create_ots_oms_request(nodeid: str, lcp: str):
 378     attr = {"input": {
 379         "node-id": nodeid,
 380         "logical-connection-point": lcp}}
 381     return post_request(URL_CREATE_OTS_OMS, attr)
 382
 383
 384 def path_computation_request(requestid: str, servicename: str, serviceaend, servicezend,
 385                              hardconstraints=None, softconstraints=None, metric="hop-count", other_attr=None):
 386     attr = {"service-name": servicename,
 387             "resource-reserve": "true",
 388             "service-handler-header": {"request-id": requestid},
 389             "service-a-end": serviceaend,
 390             "service-z-end": servicezend,
 391             "pce-metric": metric}
 392     if hardconstraints:
 393         attr.update({"hard-constraints": hardconstraints})
 394     if softconstraints:
 395         attr.update({"soft-constraints": softconstraints})
 396     if other_attr:
 397         attr.update(other_attr)
 398     return post_request(URL_PATH_COMPUTATION_REQUEST, {"input": attr})
 399
 400
 401 def shutdown_process(process):
 402     if process is not None:
 403         for child in psutil.Process(process.pid).children():
 404             child.send_signal(signal.SIGINT)
 405             child.wait()
 406         process.send_signal(signal.SIGINT)
 407
 408
 409 def start_honeynode(log_file: str, node_port: str, node_config_file_name: str):
 410     if os.path.isfile(HONEYNODE_EXECUTABLE):
 411         with open(log_file, 'w') as outfile:
 412             return subprocess.Popen(
 413                 [HONEYNODE_EXECUTABLE, node_port, os.path.join(SAMPLES_DIRECTORY, node_config_file_name)],
 414                 stdout=outfile, stderr=outfile)
 415     return None
 416
 417
 418 def wait_until_log_contains(log_file, regexp, time_to_wait=20):
 419     # pylint: disable=lost-exception
 420     stringfound = False
 421     filefound = False
 422     line = None
 423     try:
 424         with TimeOut(seconds=time_to_wait):
 425             while not os.path.exists(log_file):
 426                 time.sleep(0.2)
 427             filelogs = open(log_file, 'r')
 428             filelogs.seek(0, 2)
 429             filefound = True
 430             print("Searching for pattern '" + regexp + "' in " + os.path.basename(log_file), end='... ', flush=True)
 431             compiled_regexp = re.compile(regexp)
 432             while True:
 433                 line = filelogs.readline()
 434                 if compiled_regexp.search(line):
 435                     print("Pattern found!", end=' ')
 436                     stringfound = True
 437                     break
 438                 if not line:
 439                     time.sleep(0.1)
 440     except TimeoutError:
 441         print("Pattern not found after " + str(time_to_wait), end=" seconds! ", flush=True)
 442     except PermissionError:
 443         print("Permission Error when trying to access the log file", end=" ... ", flush=True)
 444     finally:
 445         if filefound:
 446             filelogs.close()
 447         else:
 448             print("log file does not exist or is not accessible... ", flush=True)
 449         return stringfound
 450
 451
 452 class TimeOut:
 453     def __init__(self, seconds=1, error_message='Timeout'):
 454         self.seconds = seconds
 455         self.error_message = error_message
 456
 457     def handle_timeout(self, signum, frame):
 458         raise TimeoutError(self.error_message)
 459
 460     def __enter__(self):
 461         signal.signal(signal.SIGALRM, self.handle_timeout)
 462         signal.alarm(self.seconds)
 463
 464     def __exit__(self, type, value, traceback):
 465         # pylint: disable=W0622
 466         signal.alarm(0)
 467
 468
 469 def check_freq_map(freq_map):
 470     freq_map_array = [int(x) for x in freq_map]
 471     return freq_map_array[0] == 255 and freq_map_array[1] == 255
 472
 473
 474 def set_used_index_for_freq_map(freq_map, index):
 475     freq_map[index] = 0
 476     return freq_map
 477
 478
 479 INDEX_1_USED_FREQ_MAP = base64.b64encode(set_used_index_for_freq_map(bytearray(b'\xFF' * 96), 0)).decode('UTF-8')
 480
 481 INDEX_1_2_USED_FREQ_MAP = base64.b64encode(set_used_index_for_freq_map(
 482     set_used_index_for_freq_map(bytearray(b'\xFF' * 96), 0), 1)).decode('utf-8')
 483
 484 AVAILABLE_FREQ_MAP = base64.b64encode(bytearray(b'\xFF' * 96)).decode('UTF-8')