b6eb71cae984a48e667c214325739246cd8ee19e
[transportpce.git] / tests / transportpce_tests / common / test_utils.py
1 #!/usr/bin/env python
2
3 ##############################################################################
4 # Copyright (c) 2020 Orange, Inc. and others.  All rights reserved.
5 #
6 # All rights reserved. This program and the accompanying materials
7 # are made available under the terms of the Apache License, Version 2.0
8 # which accompanies this distribution, and is available at
9 # http://www.apache.org/licenses/LICENSE-2.0
10 ##############################################################################
11
12 # pylint: disable=no-member
13 # pylint: disable=too-many-arguments
14
15 import json
16 import os
17 # pylint: disable=wrong-import-order
18 import sys
19 import re
20 import signal
21 import subprocess
22 import time
23
24 import psutil
25 import requests
26
27 # pylint: disable=import-error
28 import simulators
29
30 SIMS = simulators.SIMS
31
32 HONEYNODE_OK_START_MSG = "Netconf SSH endpoint started successfully at 0.0.0.0"
33 KARAF_OK_START_MSG = re.escape(
34     "Blueprint container for bundle org.opendaylight.netconf.restconf")+".* was successfully created"
35 LIGHTY_OK_START_MSG = re.escape("lighty.io and RESTCONF-NETCONF started")
36
37 ODL_LOGIN = "admin"
38 ODL_PWD = "admin"
39 NODES_LOGIN = "admin"
40 NODES_PWD = "admin"
41 URL_CONFIG_NETCONF_TOPO = "{}/config/network-topology:network-topology/topology/topology-netconf/"
42 URL_CONFIG_ORDM_TOPO = "{}/config/ietf-network:networks/network/openroadm-topology/"
43 URL_CONFIG_OTN_TOPO = "{}/config/ietf-network:networks/network/otn-topology/"
44 URL_CONFIG_CLLI_NET = "{}/config/ietf-network:networks/network/clli-network/"
45 URL_CONFIG_ORDM_NET = "{}/config/ietf-network:networks/network/openroadm-network/"
46 URL_PORTMAPPING = "{}/config/transportpce-portmapping:network/nodes/"
47 URL_OPER_SERV_LIST = "{}/operational/org-openroadm-service:service-list/"
48 URL_GET_NBINOTIFICATIONS_PROCESS_SERV = "{}/operations/nbi-notifications:get-notifications-process-service/"
49 URL_GET_NBINOTIFICATIONS_ALARM_SERV = "{}/operations/nbi-notifications:get-notifications-alarm-service/"
50 URL_SERV_CREATE = "{}/operations/org-openroadm-service:service-create"
51 URL_SERV_DELETE = "{}/operations/org-openroadm-service:service-delete"
52 URL_SERVICE_PATH = "{}/operations/transportpce-device-renderer:service-path"
53 URL_OTN_SERVICE_PATH = "{}/operations/transportpce-device-renderer:otn-service-path"
54 URL_TAPI_CREATE_CONNECTIVITY = "{}/operations/tapi-connectivity:create-connectivity-service"
55 URL_TAPI_DELETE_CONNECTIVITY = "{}/operations/tapi-connectivity:delete-connectivity-service"
56 URL_CREATE_OTS_OMS = "{}/operations/transportpce-device-renderer:create-ots-oms"
57 URL_PATH_COMPUTATION_REQUEST = "{}/operations/transportpce-pce:path-computation-request"
58 URL_FULL_PORTMAPPING = "{}/config/transportpce-portmapping:network"
59 URL_TAPI_TOPOLOGY_DETAILS = "{}/operations/tapi-topology:get-topology-details"
60 URL_TAPI_NODE_DETAILS = "{}/operations/tapi-topology:get-node-details"
61 URL_TAPI_SIP_LIST = "{}/operations/tapi-common:get-service-interface-point-list"
62 URL_TAPI_SERVICE_LIST = "{}/operations/tapi-connectivity:get-connectivity-service-list"
63
64 TYPE_APPLICATION_JSON = {'Content-Type': 'application/json', 'Accept': 'application/json'}
65 TYPE_APPLICATION_XML = {'Content-Type': 'application/xml', 'Accept': 'application/xml'}
66
67 CODE_SHOULD_BE_200 = 'Http status code should be 200'
68 CODE_SHOULD_BE_201 = 'Http status code should be 201'
69
70 SIM_LOG_DIRECTORY = os.path.join(os.path.dirname(os.path.realpath(__file__)), "log")
71
72 process_list = []
73
74
75 if "USE_ODL_ALT_RESTCONF_PORT" in os.environ:
76     RESTCONF_BASE_URL = "http://localhost:" + os.environ['USE_ODL_ALT_RESTCONF_PORT'] + "/restconf"
77 else:
78     RESTCONF_BASE_URL = "http://localhost:8181/restconf"
79
80 if "USE_ODL_ALT_KARAF_INSTALL_DIR" in os.environ:
81     KARAF_INSTALLDIR = os.environ['USE_ODL_ALT_KARAF_INSTALL_DIR']
82 else:
83     KARAF_INSTALLDIR = "karaf"
84
85 KARAF_LOG = os.path.join(
86     os.path.dirname(os.path.realpath(__file__)),
87     "..", "..", "..", KARAF_INSTALLDIR, "target", "assembly", "data", "log", "karaf.log")
88
89 if "USE_LIGHTY" in os.environ and os.environ['USE_LIGHTY'] == 'True':
90     TPCE_LOG = 'odl-' + str(os.getpid()) + '.log'
91 else:
92     TPCE_LOG = KARAF_LOG
93
94
95 def start_sims(sims_list):
96     for sim in sims_list:
97         print("starting simulator " + sim[0] + " in OpenROADM device version " + sim[1] + "...")
98         log_file = os.path.join(SIM_LOG_DIRECTORY, SIMS[sim]['logfile'])
99         process = start_honeynode(log_file, sim)
100         if wait_until_log_contains(log_file, HONEYNODE_OK_START_MSG, 100):
101             print("simulator for " + sim[0] + " started")
102         else:
103             print("simulator for " + sim[0] + " failed to start")
104             shutdown_process(process)
105             for pid in process_list:
106                 shutdown_process(pid)
107             sys.exit(3)
108         process_list.append(process)
109     return process_list
110
111
112 def start_tpce():
113     print("starting OpenDaylight...")
114     if "USE_LIGHTY" in os.environ and os.environ['USE_LIGHTY'] == 'True':
115         process = start_lighty()
116         start_msg = LIGHTY_OK_START_MSG
117     else:
118         process = start_karaf()
119         start_msg = KARAF_OK_START_MSG
120     if wait_until_log_contains(TPCE_LOG, start_msg, time_to_wait=300):
121         print("OpenDaylight started !")
122     else:
123         print("OpenDaylight failed to start !")
124         shutdown_process(process)
125         for pid in process_list:
126             shutdown_process(pid)
127         sys.exit(1)
128     process_list.append(process)
129     return process_list
130
131
132 def start_karaf():
133     print("starting KARAF TransportPCE build...")
134     executable = os.path.join(
135         os.path.dirname(os.path.realpath(__file__)),
136         "..", "..", "..", KARAF_INSTALLDIR, "target", "assembly", "bin", "karaf")
137     with open('odl.log', 'w', encoding='utf-8') as outfile:
138         return subprocess.Popen(
139             ["sh", executable, "server"], stdout=outfile, stderr=outfile, stdin=None)
140
141
142 def start_lighty():
143     print("starting LIGHTY.IO TransportPCE build...")
144     executable = os.path.join(
145         os.path.dirname(os.path.realpath(__file__)),
146         "..", "..", "..", "lighty", "target", "tpce",
147         "clean-start-controller.sh")
148     with open(TPCE_LOG, 'w', encoding='utf-8') as outfile:
149         return subprocess.Popen(
150             ["sh", executable], stdout=outfile, stderr=outfile, stdin=None)
151
152
153 def install_karaf_feature(feature_name: str):
154     print("installing feature " + feature_name)
155     executable = os.path.join(
156         os.path.dirname(os.path.realpath(__file__)),
157         "..", "..", "..", KARAF_INSTALLDIR, "target", "assembly", "bin", "client")
158     return subprocess.run([executable],
159                           input='feature:install ' + feature_name + '\n feature:list | grep '
160                           + feature_name + ' \n logout \n',
161                           universal_newlines=True, check=False)
162
163
164 def get_request(url):
165     return requests.request(
166         "GET", url.format(RESTCONF_BASE_URL),
167         headers=TYPE_APPLICATION_JSON,
168         auth=(ODL_LOGIN, ODL_PWD))
169
170
171 def post_request(url, data):
172     if data:
173         print(json.dumps(data))
174         return requests.request(
175             "POST", url.format(RESTCONF_BASE_URL),
176             data=json.dumps(data),
177             headers=TYPE_APPLICATION_JSON,
178             auth=(ODL_LOGIN, ODL_PWD))
179
180     return requests.request(
181         "POST", url.format(RESTCONF_BASE_URL),
182         headers=TYPE_APPLICATION_JSON,
183         auth=(ODL_LOGIN, ODL_PWD))
184
185
186 def post_xmlrequest(url, data):
187     if data:
188         return requests.request(
189             "POST", url.format(RESTCONF_BASE_URL),
190             data=data,
191             headers=TYPE_APPLICATION_XML,
192             auth=(ODL_LOGIN, ODL_PWD))
193     return None
194
195
196 def put_request(url, data):
197     return requests.request(
198         "PUT", url.format(RESTCONF_BASE_URL),
199         data=json.dumps(data),
200         headers=TYPE_APPLICATION_JSON,
201         auth=(ODL_LOGIN, ODL_PWD))
202
203
204 def put_xmlrequest(url, data):
205     return requests.request(
206         "PUT", url.format(RESTCONF_BASE_URL),
207         data=data,
208         headers=TYPE_APPLICATION_XML,
209         auth=(ODL_LOGIN, ODL_PWD))
210
211
212 def put_jsonrequest(url, data):
213     return requests.request(
214         "PUT", url.format(RESTCONF_BASE_URL),
215         data=data,
216         headers=TYPE_APPLICATION_JSON,
217         auth=(ODL_LOGIN, ODL_PWD))
218
219
220 def rawput_request(url, data):
221     return requests.request(
222         "PUT", url.format(RESTCONF_BASE_URL),
223         data=data,
224         headers=TYPE_APPLICATION_JSON,
225         auth=(ODL_LOGIN, ODL_PWD))
226
227
228 def rawpost_request(url, data):
229     return requests.request(
230         "POST", url.format(RESTCONF_BASE_URL),
231         data=data,
232         headers=TYPE_APPLICATION_JSON,
233         auth=(ODL_LOGIN, ODL_PWD))
234
235
236 def delete_request(url):
237     return requests.request(
238         "DELETE", url.format(RESTCONF_BASE_URL),
239         headers=TYPE_APPLICATION_JSON,
240         auth=(ODL_LOGIN, ODL_PWD))
241
242
243 def mount_device(node_id, sim):
244     url = URL_CONFIG_NETCONF_TOPO + "node/" + node_id
245     body = {"node": [{
246         "node-id": node_id,
247         "netconf-node-topology:username": NODES_LOGIN,
248         "netconf-node-topology:password": NODES_PWD,
249         "netconf-node-topology:host": "127.0.0.1",
250         "netconf-node-topology:port": SIMS[sim]['port'],
251         "netconf-node-topology:tcp-only": "false",
252         "netconf-node-topology:pass-through": {}}]}
253     response = put_request(url, body)
254     if wait_until_log_contains(TPCE_LOG, re.escape("Triggering notification stream NETCONF for node " + node_id), 180):
255         print("Node " + node_id + " correctly added to tpce topology", end='... ', flush=True)
256     else:
257         print("Node " + node_id + " still not added to tpce topology", end='... ', flush=True)
258         if response.status_code == requests.codes.ok:
259             print("It was probably loaded at start-up", end='... ', flush=True)
260         # TODO an else-clause to abort test would probably be nice here
261     return response
262
263
264 def mount_tapi_device(node_id, sim):
265     url = URL_CONFIG_NETCONF_TOPO + "node/" + node_id
266     body = {"node": [{
267         "node-id": node_id,
268         "netconf-node-topology:username": NODES_LOGIN,
269         "netconf-node-topology:password": NODES_PWD,
270         "netconf-node-topology:host": "127.0.0.1",
271         "netconf-node-topology:port": SIMS[sim]['port'],
272         "netconf-node-topology:tcp-only": "false",
273         "netconf-node-topology:pass-through": {}}]}
274     response = put_request(url, body)
275     if wait_until_log_contains(TPCE_LOG, re.escape(f"TAPI node for or node {node_id} successfully merged"), 200):
276         print("Node " + node_id + " correctly added to tpce topology", end='... ', flush=True)
277     else:
278         print("Node " + node_id + " still not added to tpce topology", end='... ', flush=True)
279         if response.status_code == requests.codes.ok:
280             print("It was probably loaded at start-up", end='... ', flush=True)
281         # TODO an else-clause to abort test would probably be nice here
282     return response
283
284
285 def unmount_device(node_id):
286     url = URL_CONFIG_NETCONF_TOPO + "node/" + node_id
287     response = delete_request(url)
288     if wait_until_log_contains(TPCE_LOG, re.escape("onDeviceDisConnected: " + node_id), 180):
289         print("Node " + node_id + " correctly deleted from tpce topology", end='... ', flush=True)
290     else:
291         print("Node " + node_id + " still not deleted from tpce topology", end='... ', flush=True)
292     return response
293
294
295 def connect_xpdr_to_rdm_request(xpdr_node: str, xpdr_num: str, network_num: str,
296                                 rdm_node: str, srg_num: str, termination_num: str):
297     url = "{}/operations/transportpce-networkutils:init-xpdr-rdm-links"
298     data = {
299         "networkutils:input": {
300             "networkutils:links-input": {
301                 "networkutils:xpdr-node": xpdr_node,
302                 "networkutils:xpdr-num": xpdr_num,
303                 "networkutils:network-num": network_num,
304                 "networkutils:rdm-node": rdm_node,
305                 "networkutils:srg-num": srg_num,
306                 "networkutils:termination-point-num": termination_num
307             }
308         }
309     }
310     return post_request(url, data)
311
312
313 def connect_rdm_to_xpdr_request(xpdr_node: str, xpdr_num: str, network_num: str,
314                                 rdm_node: str, srg_num: str, termination_num: str):
315     url = "{}/operations/transportpce-networkutils:init-rdm-xpdr-links"
316     data = {
317         "networkutils:input": {
318             "networkutils:links-input": {
319                 "networkutils:xpdr-node": xpdr_node,
320                 "networkutils:xpdr-num": xpdr_num,
321                 "networkutils:network-num": network_num,
322                 "networkutils:rdm-node": rdm_node,
323                 "networkutils:srg-num": srg_num,
324                 "networkutils:termination-point-num": termination_num
325             }
326         }
327     }
328     return post_request(url, data)
329
330
331 def connect_xpdr_to_rdm_tapi_request(xpdr_node: str, xpdr_num: str, rdm_node: str, srg_num: str):
332     url = "{}/operations/transportpce-tapinetworkutils:init-xpdr-rdm-tapi-link"
333     data = {
334         "input": {
335             "xpdr-node": xpdr_node,
336             "network-tp": xpdr_num,
337             "rdm-node": rdm_node,
338             "add-drop-tp": srg_num
339         }
340     }
341     return post_request(url, data)
342
343
344 def check_netconf_node_request(node: str, suffix: str):
345     url = URL_CONFIG_NETCONF_TOPO + (
346         "node/" + node + "/yang-ext:mount/org-openroadm-device:org-openroadm-device/" + suffix
347     )
348     return get_request(url)
349
350
351 def get_netconf_oper_request(node: str):
352     url = "{}/operational/network-topology:network-topology/topology/topology-netconf/node/" + node
353     return get_request(url)
354
355
356 def get_ordm_topo_request(suffix: str):
357     url = URL_CONFIG_ORDM_TOPO + suffix
358     return get_request(url)
359
360
361 def add_oms_attr_request(link: str, attr):
362     url = URL_CONFIG_ORDM_TOPO + (
363         "ietf-network-topology:link/" + link + "/org-openroadm-network-topology:OMS-attributes/span"
364     )
365     return put_request(url, attr)
366
367
368 def del_oms_attr_request(link: str):
369     url = URL_CONFIG_ORDM_TOPO + (
370         "ietf-network-topology:link/" + link + "/org-openroadm-network-topology:OMS-attributes/span"
371     )
372     return delete_request(url)
373
374
375 def get_clli_net_request():
376     return get_request(URL_CONFIG_CLLI_NET)
377
378
379 def get_ordm_net_request():
380     return get_request(URL_CONFIG_ORDM_NET)
381
382
383 def get_otn_topo_request():
384     return get_request(URL_CONFIG_OTN_TOPO)
385
386
387 def del_link_request(link: str):
388     url = URL_CONFIG_ORDM_TOPO + ("ietf-network-topology:link/" + link)
389     return delete_request(url)
390
391
392 def del_node_request(node: str):
393     url = URL_CONFIG_CLLI_NET + ("node/" + node)
394     return delete_request(url)
395
396
397 def portmapping_request(suffix: str):
398     url = URL_PORTMAPPING + suffix
399     return get_request(url)
400
401
402 def get_notifications_process_service_request(attr):
403     return post_request(URL_GET_NBINOTIFICATIONS_PROCESS_SERV, attr)
404
405
406 def get_notifications_alarm_service_request(attr):
407     return post_request(URL_GET_NBINOTIFICATIONS_ALARM_SERV, attr)
408
409
410 def get_service_list_request(suffix: str):
411     url = URL_OPER_SERV_LIST + suffix
412     return get_request(url)
413
414
415 def service_create_request(attr):
416     return post_request(URL_SERV_CREATE, attr)
417
418
419 def service_delete_request(servicename: str,
420                            requestid="e3028bae-a90f-4ddd-a83f-cf224eba0e58",
421                            notificationurl="http://localhost:8585/NotificationServer/notify"):
422     attr = {"input": {
423         "sdnc-request-header": {
424             "request-id": requestid,
425             "rpc-action": "service-delete",
426             "request-system-id": "appname",
427             "notification-url": notificationurl},
428         "service-delete-req-info": {
429             "service-name": servicename,
430             "tail-retention": "no"}}}
431     return post_request(URL_SERV_DELETE, attr)
432
433
434 def service_path_request(operation: str, servicename: str, wavenumber: str, nodes, centerfreq: str,
435                          slotwidth: int, minfreq: float, maxfreq: float, lowerslotnumber: int,
436                          higherslotnumber: int):
437     attr = {"renderer:input": {
438         "renderer:service-name": servicename,
439         "renderer:wave-number": wavenumber,
440         "renderer:modulation-format": "dp-qpsk",
441         "renderer:operation": operation,
442         "renderer:nodes": nodes,
443         "renderer:center-freq": centerfreq,
444         "renderer:nmc-width": slotwidth,
445         "renderer:min-freq": minfreq,
446         "renderer:max-freq": maxfreq,
447         "renderer:lower-spectral-slot-number": lowerslotnumber,
448         "renderer:higher-spectral-slot-number": higherslotnumber}}
449     return post_request(URL_SERVICE_PATH, attr)
450
451
452 def otn_service_path_request(operation: str, servicename: str, servicerate: str, serviceformat: str, nodes,
453                              eth_attr=None):
454     attr = {"service-name": servicename,
455             "operation": operation,
456             "service-rate": servicerate,
457             "service-format": serviceformat,
458             "nodes": nodes}
459     if eth_attr:
460         attr.update(eth_attr)
461     return post_request(URL_OTN_SERVICE_PATH, {"renderer:input": attr})
462
463
464 def create_ots_oms_request(nodeid: str, lcp: str):
465     attr = {"input": {
466         "node-id": nodeid,
467         "logical-connection-point": lcp}}
468     return post_request(URL_CREATE_OTS_OMS, attr)
469
470
471 def path_computation_request(requestid: str, servicename: str, serviceaend, servicezend,
472                              hardconstraints=None, softconstraints=None, metric="hop-count", other_attr=None):
473     attr = {"service-name": servicename,
474             "resource-reserve": "true",
475             "service-handler-header": {"request-id": requestid},
476             "service-a-end": serviceaend,
477             "service-z-end": servicezend,
478             "pce-metric": metric}
479     if hardconstraints:
480         attr.update({"hard-constraints": hardconstraints})
481     if softconstraints:
482         attr.update({"soft-constraints": softconstraints})
483     if other_attr:
484         attr.update(other_attr)
485     return post_request(URL_PATH_COMPUTATION_REQUEST, {"input": attr})
486
487
488 def tapi_create_connectivity_request(topologyidorname):
489     return post_request(URL_TAPI_CREATE_CONNECTIVITY, topologyidorname)
490
491
492 def tapi_delete_connectivity_request(serviceidorname):
493     attr = {
494         "input": {
495             "service-id-or-name": serviceidorname}}
496     return post_request(URL_TAPI_DELETE_CONNECTIVITY, attr)
497
498
499 def tapi_get_topology_details_request(topologyidorname):
500     attr = {
501         "input": {
502             "topology-id-or-name": topologyidorname}}
503     return post_request(URL_TAPI_TOPOLOGY_DETAILS, attr)
504
505
506 def tapi_get_node_details_request(topologyidorname, nodeidorname):
507     attr = {
508         "input": {
509             "topology-id-or-name": topologyidorname,
510             "node-id-or-name": nodeidorname}}
511     return post_request(URL_TAPI_NODE_DETAILS, attr)
512
513
514 def tapi_get_sip_details_request():
515     return post_request(URL_TAPI_SIP_LIST, "")
516
517
518 def tapi_get_service_list_request():
519     return post_request(URL_TAPI_SERVICE_LIST, "")
520
521
522 def shutdown_process(process):
523     if process is not None:
524         for child in psutil.Process(process.pid).children():
525             child.send_signal(signal.SIGINT)
526             child.wait()
527         process.send_signal(signal.SIGINT)
528
529
530 def start_honeynode(log_file: str, sim):
531     executable = os.path.join(os.path.dirname(os.path.realpath(__file__)),
532                               "..", "..", "honeynode", sim[1], "honeynode-simulator", "honeycomb-tpce")
533     sample_directory = os.path.join(os.path.dirname(os.path.realpath(__file__)),
534                                     "..", "..", "sample_configs", "openroadm", sim[1])
535     if os.path.isfile(executable):
536         with open(log_file, 'w', encoding='utf-8') as outfile:
537             return subprocess.Popen(
538                 [executable, SIMS[sim]['port'], os.path.join(sample_directory, SIMS[sim]['configfile'])],
539                 stdout=outfile, stderr=outfile)
540     return None
541
542
543 def wait_until_log_contains(log_file, regexp, time_to_wait=60):
544     # pylint: disable=lost-exception
545     # pylint: disable=consider-using-with
546     stringfound = False
547     filefound = False
548     line = None
549     try:
550         with TimeOut(seconds=time_to_wait):
551             while not os.path.exists(log_file):
552                 time.sleep(0.2)
553             filelogs = open(log_file, 'r', encoding='utf-8')
554             filelogs.seek(0, 2)
555             filefound = True
556             print("Searching for pattern '" + regexp + "' in " + os.path.basename(log_file), end='... ', flush=True)
557             compiled_regexp = re.compile(regexp)
558             while True:
559                 line = filelogs.readline()
560                 if compiled_regexp.search(line):
561                     print("Pattern found!", end=' ')
562                     stringfound = True
563                     break
564                 if not line:
565                     time.sleep(0.1)
566     except TimeoutError:
567         print("Pattern not found after " + str(time_to_wait), end=" seconds! ", flush=True)
568     except PermissionError:
569         print("Permission Error when trying to access the log file", end=" ... ", flush=True)
570     finally:
571         if filefound:
572             filelogs.close()
573         else:
574             print("log file does not exist or is not accessible... ", flush=True)
575         return stringfound
576
577
578 class TimeOut:
579     def __init__(self, seconds=1, error_message='Timeout'):
580         self.seconds = seconds
581         self.error_message = error_message
582
583     def handle_timeout(self, signum, frame):
584         raise TimeoutError(self.error_message)
585
586     def __enter__(self):
587         signal.signal(signal.SIGALRM, self.handle_timeout)
588         signal.alarm(self.seconds)
589
590     def __exit__(self, type, value, traceback):
591         # pylint: disable=W0622
592         signal.alarm(0)