Add tests for autonomous service rerouting
[transportpce.git] / tests / transportpce_tests / common / test_utils.py
1 #!/usr/bin/env python
2
3 ##############################################################################
4 # Copyright (c) 2020 Orange, Inc. and others.  All rights reserved.
5 #
6 # All rights reserved. This program and the accompanying materials
7 # are made available under the terms of the Apache License, Version 2.0
8 # which accompanies this distribution, and is available at
9 # http://www.apache.org/licenses/LICENSE-2.0
10 ##############################################################################
11
12 # pylint: disable=no-member
13 # pylint: disable=too-many-arguments
14
15 import json
16 import os
17 # pylint: disable=wrong-import-order
18 import sys
19 import re
20 import signal
21 import subprocess
22 import time
23
24 import psutil
25 import requests
26
27 # pylint: disable=import-error
28 import simulators
29
30 SIMS = simulators.SIMS
31
32 HONEYNODE_OK_START_MSG = "Netconf SSH endpoint started successfully at 0.0.0.0"
33 KARAF_OK_START_MSG = re.escape(
34     "Blueprint container for bundle org.opendaylight.netconf.restconf")+".* was successfully created"
35 LIGHTY_OK_START_MSG = re.escape("lighty.io and RESTCONF-NETCONF started")
36
37 ODL_LOGIN = "admin"
38 ODL_PWD = "admin"
39 NODES_LOGIN = "admin"
40 NODES_PWD = "admin"
41 URL_CONFIG_NETCONF_TOPO = "{}/config/network-topology:network-topology/topology/topology-netconf/"
42 URL_CONFIG_ORDM_TOPO = "{}/config/ietf-network:networks/network/openroadm-topology/"
43 URL_CONFIG_OTN_TOPO = "{}/config/ietf-network:networks/network/otn-topology/"
44 URL_CONFIG_CLLI_NET = "{}/config/ietf-network:networks/network/clli-network/"
45 URL_CONFIG_ORDM_NET = "{}/config/ietf-network:networks/network/openroadm-network/"
46 URL_PORTMAPPING = "{}/config/transportpce-portmapping:network/nodes/"
47 URL_OPER_SERV_LIST = "{}/operational/org-openroadm-service:service-list/"
48 URL_OPER_SERV_PATH_LIST = "{}/operational/transportpce-service-path:service-path-list/"
49 URL_GET_NBINOTIFICATIONS_PROCESS_SERV = "{}/operations/nbi-notifications:get-notifications-process-service/"
50 URL_GET_NBINOTIFICATIONS_ALARM_SERV = "{}/operations/nbi-notifications:get-notifications-alarm-service/"
51 URL_SERV_CREATE = "{}/operations/org-openroadm-service:service-create"
52 URL_SERV_DELETE = "{}/operations/org-openroadm-service:service-delete"
53 URL_SERVICE_PATH = "{}/operations/transportpce-device-renderer:service-path"
54 URL_OTN_SERVICE_PATH = "{}/operations/transportpce-device-renderer:otn-service-path"
55 URL_TAPI_CREATE_CONNECTIVITY = "{}/operations/tapi-connectivity:create-connectivity-service"
56 URL_TAPI_DELETE_CONNECTIVITY = "{}/operations/tapi-connectivity:delete-connectivity-service"
57 URL_CREATE_OTS_OMS = "{}/operations/transportpce-device-renderer:create-ots-oms"
58 URL_PATH_COMPUTATION_REQUEST = "{}/operations/transportpce-pce:path-computation-request"
59 URL_FULL_PORTMAPPING = "{}/config/transportpce-portmapping:network"
60 URL_TAPI_TOPOLOGY_DETAILS = "{}/operations/tapi-topology:get-topology-details"
61 URL_TAPI_NODE_DETAILS = "{}/operations/tapi-topology:get-node-details"
62 URL_TAPI_SIP_LIST = "{}/operations/tapi-common:get-service-interface-point-list"
63 URL_TAPI_SERVICE_LIST = "{}/operations/tapi-connectivity:get-connectivity-service-list"
64
65 TYPE_APPLICATION_JSON = {'Content-Type': 'application/json', 'Accept': 'application/json'}
66 TYPE_APPLICATION_XML = {'Content-Type': 'application/xml', 'Accept': 'application/xml'}
67
68 CODE_SHOULD_BE_200 = 'Http status code should be 200'
69 CODE_SHOULD_BE_201 = 'Http status code should be 201'
70
71 SIM_LOG_DIRECTORY = os.path.join(os.path.dirname(os.path.realpath(__file__)), "log")
72
73 process_list = []
74
75
76 if "USE_ODL_ALT_RESTCONF_PORT" in os.environ:
77     RESTCONF_BASE_URL = "http://localhost:" + os.environ['USE_ODL_ALT_RESTCONF_PORT'] + "/restconf"
78 else:
79     RESTCONF_BASE_URL = "http://localhost:8181/restconf"
80
81 if "USE_ODL_ALT_KARAF_INSTALL_DIR" in os.environ:
82     KARAF_INSTALLDIR = os.environ['USE_ODL_ALT_KARAF_INSTALL_DIR']
83 else:
84     KARAF_INSTALLDIR = "karaf"
85
86 KARAF_LOG = os.path.join(
87     os.path.dirname(os.path.realpath(__file__)),
88     "..", "..", "..", KARAF_INSTALLDIR, "target", "assembly", "data", "log", "karaf.log")
89
90 if "USE_LIGHTY" in os.environ and os.environ['USE_LIGHTY'] == 'True':
91     TPCE_LOG = 'odl-' + str(os.getpid()) + '.log'
92 else:
93     TPCE_LOG = KARAF_LOG
94
95
96 def start_sims(sims_list):
97     for sim in sims_list:
98         print("starting simulator " + sim[0] + " in OpenROADM device version " + sim[1] + "...")
99         log_file = os.path.join(SIM_LOG_DIRECTORY, SIMS[sim]['logfile'])
100         process = start_honeynode(log_file, sim)
101         if wait_until_log_contains(log_file, HONEYNODE_OK_START_MSG, 100):
102             print("simulator for " + sim[0] + " started")
103         else:
104             print("simulator for " + sim[0] + " failed to start")
105             shutdown_process(process)
106             for pid in process_list:
107                 shutdown_process(pid)
108             sys.exit(3)
109         process_list.append(process)
110     return process_list
111
112
113 def start_tpce():
114     print("starting OpenDaylight...")
115     if "USE_LIGHTY" in os.environ and os.environ['USE_LIGHTY'] == 'True':
116         process = start_lighty()
117         start_msg = LIGHTY_OK_START_MSG
118     else:
119         process = start_karaf()
120         start_msg = KARAF_OK_START_MSG
121     if wait_until_log_contains(TPCE_LOG, start_msg, time_to_wait=300):
122         print("OpenDaylight started !")
123     else:
124         print("OpenDaylight failed to start !")
125         shutdown_process(process)
126         for pid in process_list:
127             shutdown_process(pid)
128         sys.exit(1)
129     process_list.append(process)
130     return process_list
131
132
133 def start_karaf():
134     print("starting KARAF TransportPCE build...")
135     executable = os.path.join(
136         os.path.dirname(os.path.realpath(__file__)),
137         "..", "..", "..", KARAF_INSTALLDIR, "target", "assembly", "bin", "karaf")
138     with open('odl.log', 'w', encoding='utf-8') as outfile:
139         return subprocess.Popen(
140             ["sh", executable, "server"], stdout=outfile, stderr=outfile, stdin=None)
141
142
143 def start_lighty():
144     print("starting LIGHTY.IO TransportPCE build...")
145     executable = os.path.join(
146         os.path.dirname(os.path.realpath(__file__)),
147         "..", "..", "..", "lighty", "target", "tpce",
148         "clean-start-controller.sh")
149     with open(TPCE_LOG, 'w', encoding='utf-8') as outfile:
150         return subprocess.Popen(
151             ["sh", executable], stdout=outfile, stderr=outfile, stdin=None)
152
153
154 def install_karaf_feature(feature_name: str):
155     print("installing feature " + feature_name)
156     executable = os.path.join(
157         os.path.dirname(os.path.realpath(__file__)),
158         "..", "..", "..", KARAF_INSTALLDIR, "target", "assembly", "bin", "client")
159     return subprocess.run([executable],
160                           input='feature:install ' + feature_name + '\n feature:list | grep '
161                           + feature_name + ' \n logout \n',
162                           universal_newlines=True, check=False)
163
164
165 def get_request(url):
166     return requests.request(
167         "GET", url.format(RESTCONF_BASE_URL),
168         headers=TYPE_APPLICATION_JSON,
169         auth=(ODL_LOGIN, ODL_PWD))
170
171
172 def post_request(url, data):
173     if data:
174         print(json.dumps(data))
175         return requests.request(
176             "POST", url.format(RESTCONF_BASE_URL),
177             data=json.dumps(data),
178             headers=TYPE_APPLICATION_JSON,
179             auth=(ODL_LOGIN, ODL_PWD))
180
181     return requests.request(
182         "POST", url.format(RESTCONF_BASE_URL),
183         headers=TYPE_APPLICATION_JSON,
184         auth=(ODL_LOGIN, ODL_PWD))
185
186
187 def post_xmlrequest(url, data):
188     if data:
189         return requests.request(
190             "POST", url.format(RESTCONF_BASE_URL),
191             data=data,
192             headers=TYPE_APPLICATION_XML,
193             auth=(ODL_LOGIN, ODL_PWD))
194     return None
195
196
197 def put_request(url, data):
198     return requests.request(
199         "PUT", url.format(RESTCONF_BASE_URL),
200         data=json.dumps(data),
201         headers=TYPE_APPLICATION_JSON,
202         auth=(ODL_LOGIN, ODL_PWD))
203
204
205 def put_xmlrequest(url, data):
206     return requests.request(
207         "PUT", url.format(RESTCONF_BASE_URL),
208         data=data,
209         headers=TYPE_APPLICATION_XML,
210         auth=(ODL_LOGIN, ODL_PWD))
211
212
213 def put_jsonrequest(url, data):
214     return requests.request(
215         "PUT", url.format(RESTCONF_BASE_URL),
216         data=data,
217         headers=TYPE_APPLICATION_JSON,
218         auth=(ODL_LOGIN, ODL_PWD))
219
220
221 def rawput_request(url, data):
222     return requests.request(
223         "PUT", url.format(RESTCONF_BASE_URL),
224         data=data,
225         headers=TYPE_APPLICATION_JSON,
226         auth=(ODL_LOGIN, ODL_PWD))
227
228
229 def rawpost_request(url, data):
230     return requests.request(
231         "POST", url.format(RESTCONF_BASE_URL),
232         data=data,
233         headers=TYPE_APPLICATION_JSON,
234         auth=(ODL_LOGIN, ODL_PWD))
235
236
237 def delete_request(url):
238     return requests.request(
239         "DELETE", url.format(RESTCONF_BASE_URL),
240         headers=TYPE_APPLICATION_JSON,
241         auth=(ODL_LOGIN, ODL_PWD))
242
243
244 def mount_device(node_id, sim):
245     url = URL_CONFIG_NETCONF_TOPO + "node/" + node_id
246     body = {"node": [{
247         "node-id": node_id,
248         "netconf-node-topology:username": NODES_LOGIN,
249         "netconf-node-topology:password": NODES_PWD,
250         "netconf-node-topology:host": "127.0.0.1",
251         "netconf-node-topology:port": SIMS[sim]['port'],
252         "netconf-node-topology:tcp-only": "false",
253         "netconf-node-topology:pass-through": {}}]}
254     response = put_request(url, body)
255     if wait_until_log_contains(TPCE_LOG, re.escape("Triggering notification stream NETCONF for node " + node_id), 180):
256         print("Node " + node_id + " correctly added to tpce topology", end='... ', flush=True)
257     else:
258         print("Node " + node_id + " still not added to tpce topology", end='... ', flush=True)
259         if response.status_code == requests.codes.ok:
260             print("It was probably loaded at start-up", end='... ', flush=True)
261         # TODO an else-clause to abort test would probably be nice here
262     return response
263
264
265 def mount_tapi_device(node_id, sim):
266     url = URL_CONFIG_NETCONF_TOPO + "node/" + node_id
267     body = {"node": [{
268         "node-id": node_id,
269         "netconf-node-topology:username": NODES_LOGIN,
270         "netconf-node-topology:password": NODES_PWD,
271         "netconf-node-topology:host": "127.0.0.1",
272         "netconf-node-topology:port": SIMS[sim]['port'],
273         "netconf-node-topology:tcp-only": "false",
274         "netconf-node-topology:pass-through": {}}]}
275     response = put_request(url, body)
276     if wait_until_log_contains(TPCE_LOG, re.escape(f"TAPI node for or node {node_id} successfully merged"), 200):
277         print("Node " + node_id + " correctly added to tpce topology", end='... ', flush=True)
278     else:
279         print("Node " + node_id + " still not added to tpce topology", end='... ', flush=True)
280         if response.status_code == requests.codes.ok:
281             print("It was probably loaded at start-up", end='... ', flush=True)
282         # TODO an else-clause to abort test would probably be nice here
283     return response
284
285
286 def unmount_device(node_id):
287     url = URL_CONFIG_NETCONF_TOPO + "node/" + node_id
288     response = delete_request(url)
289     if wait_until_log_contains(TPCE_LOG, re.escape("onDeviceDisConnected: " + node_id), 180):
290         print("Node " + node_id + " correctly deleted from tpce topology", end='... ', flush=True)
291     else:
292         print("Node " + node_id + " still not deleted from tpce topology", end='... ', flush=True)
293     return response
294
295
296 def connect_xpdr_to_rdm_request(xpdr_node: str, xpdr_num: str, network_num: str,
297                                 rdm_node: str, srg_num: str, termination_num: str):
298     url = "{}/operations/transportpce-networkutils:init-xpdr-rdm-links"
299     data = {
300         "networkutils:input": {
301             "networkutils:links-input": {
302                 "networkutils:xpdr-node": xpdr_node,
303                 "networkutils:xpdr-num": xpdr_num,
304                 "networkutils:network-num": network_num,
305                 "networkutils:rdm-node": rdm_node,
306                 "networkutils:srg-num": srg_num,
307                 "networkutils:termination-point-num": termination_num
308             }
309         }
310     }
311     return post_request(url, data)
312
313
314 def connect_rdm_to_xpdr_request(xpdr_node: str, xpdr_num: str, network_num: str,
315                                 rdm_node: str, srg_num: str, termination_num: str):
316     url = "{}/operations/transportpce-networkutils:init-rdm-xpdr-links"
317     data = {
318         "networkutils:input": {
319             "networkutils:links-input": {
320                 "networkutils:xpdr-node": xpdr_node,
321                 "networkutils:xpdr-num": xpdr_num,
322                 "networkutils:network-num": network_num,
323                 "networkutils:rdm-node": rdm_node,
324                 "networkutils:srg-num": srg_num,
325                 "networkutils:termination-point-num": termination_num
326             }
327         }
328     }
329     return post_request(url, data)
330
331
332 def connect_xpdr_to_rdm_tapi_request(xpdr_node: str, xpdr_num: str, rdm_node: str, srg_num: str):
333     url = "{}/operations/transportpce-tapinetworkutils:init-xpdr-rdm-tapi-link"
334     data = {
335         "input": {
336             "xpdr-node": xpdr_node,
337             "network-tp": xpdr_num,
338             "rdm-node": rdm_node,
339             "add-drop-tp": srg_num
340         }
341     }
342     return post_request(url, data)
343
344
345 def check_netconf_node_request(node: str, suffix: str):
346     url = URL_CONFIG_NETCONF_TOPO + (
347         "node/" + node + "/yang-ext:mount/org-openroadm-device:org-openroadm-device/" + suffix
348     )
349     return get_request(url)
350
351
352 def get_netconf_oper_request(node: str):
353     url = "{}/operational/network-topology:network-topology/topology/topology-netconf/node/" + node
354     return get_request(url)
355
356
357 def get_ordm_topo_request(suffix: str):
358     url = URL_CONFIG_ORDM_TOPO + suffix
359     return get_request(url)
360
361
362 def add_oms_attr_request(link: str, attr):
363     url = URL_CONFIG_ORDM_TOPO + (
364         "ietf-network-topology:link/" + link + "/org-openroadm-network-topology:OMS-attributes/span"
365     )
366     return put_request(url, attr)
367
368
369 def del_oms_attr_request(link: str):
370     url = URL_CONFIG_ORDM_TOPO + (
371         "ietf-network-topology:link/" + link + "/org-openroadm-network-topology:OMS-attributes/span"
372     )
373     return delete_request(url)
374
375
376 def get_clli_net_request():
377     return get_request(URL_CONFIG_CLLI_NET)
378
379
380 def get_ordm_net_request():
381     return get_request(URL_CONFIG_ORDM_NET)
382
383
384 def get_otn_topo_request():
385     return get_request(URL_CONFIG_OTN_TOPO)
386
387
388 def del_link_request(link: str):
389     url = URL_CONFIG_ORDM_TOPO + ("ietf-network-topology:link/" + link)
390     return delete_request(url)
391
392
393 def del_node_request(node: str):
394     url = URL_CONFIG_CLLI_NET + ("node/" + node)
395     return delete_request(url)
396
397
398 def portmapping_request(suffix: str):
399     url = URL_PORTMAPPING + suffix
400     return get_request(url)
401
402
403 def get_notifications_process_service_request(attr):
404     return post_request(URL_GET_NBINOTIFICATIONS_PROCESS_SERV, attr)
405
406
407 def get_notifications_alarm_service_request(attr):
408     return post_request(URL_GET_NBINOTIFICATIONS_ALARM_SERV, attr)
409
410
411 def get_service_list_request(suffix: str):
412     return get_request(URL_OPER_SERV_LIST + suffix)
413
414
415 def get_service_path_list_request(suffix: str):
416     return get_request(URL_OPER_SERV_PATH_LIST + suffix)
417
418
419 def service_create_request(attr):
420     return post_request(URL_SERV_CREATE, attr)
421
422
423 def service_delete_request(servicename: str,
424                            requestid="e3028bae-a90f-4ddd-a83f-cf224eba0e58",
425                            notificationurl="http://localhost:8585/NotificationServer/notify"):
426     attr = {"input": {
427         "sdnc-request-header": {
428             "request-id": requestid,
429             "rpc-action": "service-delete",
430             "request-system-id": "appname",
431             "notification-url": notificationurl},
432         "service-delete-req-info": {
433             "service-name": servicename,
434             "tail-retention": "no"}}}
435     return post_request(URL_SERV_DELETE, attr)
436
437
438 def service_path_request(operation: str, servicename: str, wavenumber: str, nodes, centerfreq: str,
439                          slotwidth: int, minfreq: float, maxfreq: float, lowerslotnumber: int,
440                          higherslotnumber: int, modulation_format="dp-qpsk"):
441     attr = {"renderer:input": {
442         "renderer:service-name": servicename,
443         "renderer:wave-number": wavenumber,
444         "renderer:modulation-format": modulation_format,
445         "renderer:operation": operation,
446         "renderer:nodes": nodes,
447         "renderer:center-freq": centerfreq,
448         "renderer:nmc-width": slotwidth,
449         "renderer:min-freq": minfreq,
450         "renderer:max-freq": maxfreq,
451         "renderer:lower-spectral-slot-number": lowerslotnumber,
452         "renderer:higher-spectral-slot-number": higherslotnumber}}
453     return post_request(URL_SERVICE_PATH, attr)
454
455
456 def otn_service_path_request(operation: str, servicename: str, servicerate: str, serviceformat: str, nodes,
457                              eth_attr=None):
458     attr = {"service-name": servicename,
459             "operation": operation,
460             "service-rate": servicerate,
461             "service-format": serviceformat,
462             "nodes": nodes}
463     if eth_attr:
464         attr.update(eth_attr)
465     return post_request(URL_OTN_SERVICE_PATH, {"renderer:input": attr})
466
467
468 def create_ots_oms_request(nodeid: str, lcp: str):
469     attr = {"input": {
470         "node-id": nodeid,
471         "logical-connection-point": lcp}}
472     return post_request(URL_CREATE_OTS_OMS, attr)
473
474
475 def path_computation_request(requestid: str, servicename: str, serviceaend, servicezend,
476                              hardconstraints=None, softconstraints=None, metric="hop-count", other_attr=None):
477     attr = {"service-name": servicename,
478             "resource-reserve": "true",
479             "service-handler-header": {"request-id": requestid},
480             "service-a-end": serviceaend,
481             "service-z-end": servicezend,
482             "pce-routing-metric": metric}
483     if hardconstraints:
484         attr.update({"hard-constraints": hardconstraints})
485     if softconstraints:
486         attr.update({"soft-constraints": softconstraints})
487     if other_attr:
488         attr.update(other_attr)
489     return post_request(URL_PATH_COMPUTATION_REQUEST, {"input": attr})
490
491
492 def tapi_create_connectivity_request(topologyidorname):
493     return post_request(URL_TAPI_CREATE_CONNECTIVITY, topologyidorname)
494
495
496 def tapi_delete_connectivity_request(serviceidorname):
497     attr = {
498         "input": {
499             "service-id-or-name": serviceidorname}}
500     return post_request(URL_TAPI_DELETE_CONNECTIVITY, attr)
501
502
503 def tapi_get_topology_details_request(topologyidorname):
504     attr = {
505         "input": {
506             "topology-id-or-name": topologyidorname}}
507     return post_request(URL_TAPI_TOPOLOGY_DETAILS, attr)
508
509
510 def tapi_get_node_details_request(topologyidorname, nodeidorname):
511     attr = {
512         "input": {
513             "topology-id-or-name": topologyidorname,
514             "node-id-or-name": nodeidorname}}
515     return post_request(URL_TAPI_NODE_DETAILS, attr)
516
517
518 def tapi_get_sip_details_request():
519     return post_request(URL_TAPI_SIP_LIST, "")
520
521
522 def tapi_get_service_list_request():
523     return post_request(URL_TAPI_SERVICE_LIST, "")
524
525
526 def shutdown_process(process):
527     if process is not None:
528         for child in psutil.Process(process.pid).children():
529             child.send_signal(signal.SIGINT)
530             child.wait()
531         process.send_signal(signal.SIGINT)
532
533
534 def start_honeynode(log_file: str, sim):
535     executable = os.path.join(os.path.dirname(os.path.realpath(__file__)),
536                               "..", "..", "honeynode", sim[1], "honeynode-simulator", "honeycomb-tpce")
537     sample_directory = os.path.join(os.path.dirname(os.path.realpath(__file__)),
538                                     "..", "..", "sample_configs", "openroadm", sim[1])
539     if os.path.isfile(executable):
540         with open(log_file, 'w', encoding='utf-8') as outfile:
541             return subprocess.Popen(
542                 [executable, SIMS[sim]['port'], os.path.join(sample_directory, SIMS[sim]['configfile'])],
543                 stdout=outfile, stderr=outfile)
544     return None
545
546
547 def wait_until_log_contains(log_file, regexp, time_to_wait=60):
548     # pylint: disable=lost-exception
549     # pylint: disable=consider-using-with
550     stringfound = False
551     filefound = False
552     line = None
553     try:
554         with TimeOut(seconds=time_to_wait):
555             while not os.path.exists(log_file):
556                 time.sleep(0.2)
557             filelogs = open(log_file, 'r', encoding='utf-8')
558             filelogs.seek(0, 2)
559             filefound = True
560             print("Searching for pattern '" + regexp + "' in " + os.path.basename(log_file), end='... ', flush=True)
561             compiled_regexp = re.compile(regexp)
562             while True:
563                 line = filelogs.readline()
564                 if compiled_regexp.search(line):
565                     print("Pattern found!", end=' ')
566                     stringfound = True
567                     break
568                 if not line:
569                     time.sleep(0.1)
570     except TimeoutError:
571         print("Pattern not found after " + str(time_to_wait), end=" seconds! ", flush=True)
572     except PermissionError:
573         print("Permission Error when trying to access the log file", end=" ... ", flush=True)
574     finally:
575         if filefound:
576             filelogs.close()
577         else:
578             print("log file does not exist or is not accessible... ", flush=True)
579         return stringfound
580
581
582 class TimeOut:
583     def __init__(self, seconds=1, error_message='Timeout'):
584         self.seconds = seconds
585         self.error_message = error_message
586
587     def handle_timeout(self, signum, frame):
588         raise TimeoutError(self.error_message)
589
590     def __enter__(self):
591         signal.signal(signal.SIGALRM, self.handle_timeout)
592         signal.alarm(self.seconds)
593
594     def __exit__(self, type, value, traceback):
595         # pylint: disable=W0622
596         signal.alarm(0)