Merge "Fix func. tests utils base URL generation issue"
[transportpce.git] / tests / transportpce_tests / common / test_utils_rfc8040.py
1 #!/usr/bin/env python
2
3 ##############################################################################
4 # Copyright (c) 2021 Orange, Inc. and others.  All rights reserved.
5 #
6 # All rights reserved. This program and the accompanying materials
7 # are made available under the terms of the Apache License, Version 2.0
8 # which accompanies this distribution, and is available at
9 # http://www.apache.org/licenses/LICENSE-2.0
10 ##############################################################################
11
12 # pylint: disable=no-member
13
14 import json
15 import os
16 # pylint: disable=wrong-import-order
17 import sys
18 import re
19 import signal
20 import subprocess
21 import time
22
23 import psutil
24 import requests
25
26 # pylint: disable=import-error
27 import simulators
28
29 SIMS = simulators.SIMS
30
31 HONEYNODE_OK_START_MSG = 'Netconf SSH endpoint started successfully at 0.0.0.0'
32 KARAF_OK_START_MSG = re.escape(
33     "Blueprint container for bundle org.opendaylight.netconf.restconf")+".* was successfully created"
34 LIGHTY_OK_START_MSG = re.escape("lighty.io and RESTCONF-NETCONF started")
35
36 ODL_LOGIN = 'admin'
37 ODL_PWD = 'admin'
38 NODES_LOGIN = 'admin'
39 NODES_PWD = 'admin'
40
41 TYPE_APPLICATION_JSON = {'Content-Type': 'application/json', 'Accept': 'application/json'}
42 TYPE_APPLICATION_XML = {'Content-Type': 'application/xml', 'Accept': 'application/xml'}
43
44 CODE_SHOULD_BE_200 = 'Http status code should be 200'
45 CODE_SHOULD_BE_201 = 'Http status code should be 201'
46
47 SIM_LOG_DIRECTORY = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'log')
48
49 process_list = []
50
51 if 'USE_ODL_ALT_RESTCONF_PORT' in os.environ:
52     RESTCONF_PORT = os.environ['USE_ODL_ALT_RESTCONF_PORT']
53 else:
54     RESTCONF_PORT = 8181
55
56 RESTCONF_PATH_PREFIX = {'rfc8040': '/rests',
57                         'draft-bierman02': '/restconf'}
58 if 'USE_ODL_RESTCONF_VERSION' in os.environ:
59     RESTCONF_VERSION = os.environ['USE_ODL_RESTCONF_VERSION']
60     if RESTCONF_VERSION not in RESTCONF_PATH_PREFIX:
61         print('unsupported RESTCONF version ' + RESTCONF_VERSION)
62         sys.exit(3)
63 else:
64     RESTCONF_VERSION = 'rfc8040'
65
66 RESTCONF_BASE_URL = 'http://localhost:' + str(RESTCONF_PORT) + RESTCONF_PATH_PREFIX[RESTCONF_VERSION]
67
68 if 'USE_ODL_ALT_KARAF_INSTALL_DIR' in os.environ:
69     KARAF_INSTALLDIR = os.environ['USE_ODL_ALT_KARAF_INSTALL_DIR']
70 else:
71     KARAF_INSTALLDIR = 'karaf'
72
73 KARAF_LOG = os.path.join(
74     os.path.dirname(os.path.realpath(__file__)),
75     '..', '..', '..', KARAF_INSTALLDIR, 'target', 'assembly', 'data', 'log', 'karaf.log')
76
77 if 'USE_LIGHTY' in os.environ and os.environ['USE_LIGHTY'] == 'True':
78     TPCE_LOG = 'odl-' + str(os.getpid()) + '.log'
79 else:
80     TPCE_LOG = KARAF_LOG
81
82 #
83 # Basic HTTP operations
84 #
85
86
87 def get_request(url):
88     return requests.request(
89         'GET', url.format(RESTCONF_BASE_URL),
90         headers=TYPE_APPLICATION_JSON,
91         auth=(ODL_LOGIN, ODL_PWD))
92
93
94 def put_request(url, data):
95     return requests.request(
96         'PUT', url.format(RESTCONF_BASE_URL),
97         data=json.dumps(data),
98         headers=TYPE_APPLICATION_JSON,
99         auth=(ODL_LOGIN, ODL_PWD))
100
101
102 def delete_request(url):
103     return requests.request(
104         'DELETE', url.format(RESTCONF_BASE_URL),
105         headers=TYPE_APPLICATION_JSON,
106         auth=(ODL_LOGIN, ODL_PWD))
107
108
109 def post_request(url, data):
110     if data:
111         print(json.dumps(data))
112         return requests.request(
113             "POST", url.format(RESTCONF_BASE_URL),
114             data=json.dumps(data),
115             headers=TYPE_APPLICATION_JSON,
116             auth=(ODL_LOGIN, ODL_PWD))
117     return requests.request(
118         "POST", url.format(RESTCONF_BASE_URL),
119         headers=TYPE_APPLICATION_JSON,
120         auth=(ODL_LOGIN, ODL_PWD))
121
122 #
123 # Process management
124 #
125
126
127 def start_sims(sims_list):
128     for sim in sims_list:
129         print('starting simulator ' + sim[0] + ' in OpenROADM device version ' + sim[1] + '...')
130         log_file = os.path.join(SIM_LOG_DIRECTORY, SIMS[sim]['logfile'])
131         process = start_honeynode(log_file, sim)
132         if wait_until_log_contains(log_file, HONEYNODE_OK_START_MSG, 100):
133             print('simulator for ' + sim[0] + ' started')
134         else:
135             print('simulator for ' + sim[0] + ' failed to start')
136             shutdown_process(process)
137             for pid in process_list:
138                 shutdown_process(pid)
139             sys.exit(3)
140         process_list.append(process)
141     return process_list
142
143
144 def start_tpce():
145     print('starting OpenDaylight...')
146     if 'USE_LIGHTY' in os.environ and os.environ['USE_LIGHTY'] == 'True':
147         process = start_lighty()
148         start_msg = LIGHTY_OK_START_MSG
149     else:
150         process = start_karaf()
151         start_msg = KARAF_OK_START_MSG
152     if wait_until_log_contains(TPCE_LOG, start_msg, time_to_wait=300):
153         print('OpenDaylight started !')
154     else:
155         print('OpenDaylight failed to start !')
156         shutdown_process(process)
157         for pid in process_list:
158             shutdown_process(pid)
159         sys.exit(1)
160     process_list.append(process)
161     return process_list
162
163
164 def start_karaf():
165     print('starting KARAF TransportPCE build...')
166     executable = os.path.join(
167         os.path.dirname(os.path.realpath(__file__)),
168         '..', '..', '..', KARAF_INSTALLDIR, 'target', 'assembly', 'bin', 'karaf')
169     with open('odl.log', 'w', encoding='utf-8') as outfile:
170         return subprocess.Popen(
171             ['sh', executable, 'server'], stdout=outfile, stderr=outfile, stdin=None)
172
173
174 def start_lighty():
175     print('starting LIGHTY.IO TransportPCE build...')
176     executable = os.path.join(
177         os.path.dirname(os.path.realpath(__file__)),
178         '..', '..', '..', 'lighty', 'target', 'tpce',
179         'clean-start-controller.sh')
180     with open(TPCE_LOG, 'w', encoding='utf-8') as outfile:
181         return subprocess.Popen(
182             ['sh', executable], stdout=outfile, stderr=outfile, stdin=None)
183
184
185 def install_karaf_feature(feature_name: str):
186     print('installing feature ' + feature_name)
187     executable = os.path.join(
188         os.path.dirname(os.path.realpath(__file__)),
189         '..', '..', '..', KARAF_INSTALLDIR, 'target', 'assembly', 'bin', 'client')
190     return subprocess.run([executable],
191                           input='feature:install ' + feature_name + '\n feature:list | grep '
192                           + feature_name + ' \n logout \n',
193                           universal_newlines=True, check=False)
194
195
196 def shutdown_process(process):
197     if process is not None:
198         for child in psutil.Process(process.pid).children():
199             child.send_signal(signal.SIGINT)
200             child.wait()
201         process.send_signal(signal.SIGINT)
202
203
204 def start_honeynode(log_file: str, sim):
205     executable = os.path.join(os.path.dirname(os.path.realpath(__file__)),
206                               '..', '..', 'honeynode', sim[1], 'honeynode-simulator', 'honeycomb-tpce')
207     sample_directory = os.path.join(os.path.dirname(os.path.realpath(__file__)),
208                                     '..', '..', 'sample_configs', 'openroadm', sim[1])
209     if os.path.isfile(executable):
210         with open(log_file, 'w', encoding='utf-8') as outfile:
211             return subprocess.Popen(
212                 [executable, SIMS[sim]['port'], os.path.join(sample_directory, SIMS[sim]['configfile'])],
213                 stdout=outfile, stderr=outfile)
214     return None
215
216
217 def wait_until_log_contains(log_file, regexp, time_to_wait=60):
218     # pylint: disable=lost-exception
219     # pylint: disable=consider-using-with
220     stringfound = False
221     filefound = False
222     line = None
223     try:
224         with TimeOut(seconds=time_to_wait):
225             while not os.path.exists(log_file):
226                 time.sleep(0.2)
227             filelogs = open(log_file, 'r', encoding='utf-8')
228             filelogs.seek(0, 2)
229             filefound = True
230             print("Searching for pattern '" + regexp + "' in " + os.path.basename(log_file), end='... ', flush=True)
231             compiled_regexp = re.compile(regexp)
232             while True:
233                 line = filelogs.readline()
234                 if compiled_regexp.search(line):
235                     print('Pattern found!', end=' ')
236                     stringfound = True
237                     break
238                 if not line:
239                     time.sleep(0.1)
240     except TimeoutError:
241         print('Pattern not found after ' + str(time_to_wait), end=' seconds! ', flush=True)
242     except PermissionError:
243         print('Permission Error when trying to access the log file', end=' ... ', flush=True)
244     finally:
245         if filefound:
246             filelogs.close()
247         else:
248             print('log file does not exist or is not accessible... ', flush=True)
249         return stringfound
250
251
252 class TimeOut:
253     def __init__(self, seconds=1, error_message='Timeout'):
254         self.seconds = seconds
255         self.error_message = error_message
256
257     def handle_timeout(self, signum, frame):
258         raise TimeoutError(self.error_message)
259
260     def __enter__(self):
261         signal.signal(signal.SIGALRM, self.handle_timeout)
262         signal.alarm(self.seconds)
263
264     def __exit__(self, type, value, traceback):
265         # pylint: disable=W0622
266         signal.alarm(0)
267
268 #
269 # Basic NetCONF device operations
270 #
271
272
273 def mount_device(node: str, sim: str):
274     url = {'rfc8040': '{}/data/network-topology:network-topology/topology=topology-netconf/node={}',
275            'draft-bierman02': '{}/config/network-topology:network-topology/topology/topology-netconf/node/{}'}
276     body = {'node': [{
277         'node-id': node,
278         'netconf-node-topology:username': NODES_LOGIN,
279         'netconf-node-topology:password': NODES_PWD,
280         'netconf-node-topology:host': '127.0.0.1',
281         'netconf-node-topology:port': SIMS[sim]['port'],
282         'netconf-node-topology:tcp-only': 'false',
283         'netconf-node-topology:pass-through': {}}]}
284     response = put_request(url[RESTCONF_VERSION].format('{}', node), body)
285     if wait_until_log_contains(TPCE_LOG, re.escape('Triggering notification stream NETCONF for node ' + node), 180):
286         print('Node ' + node + ' correctly added to tpce topology', end='... ', flush=True)
287     else:
288         print('Node ' + node + ' still not added to tpce topology', end='... ', flush=True)
289         if response.status_code == requests.codes.ok:
290             print('It was probably loaded at start-up', end='... ', flush=True)
291         # TODO an else-clause to abort test would probably be nice here
292     return response
293
294
295 def unmount_device(node: str):
296     url = {'rfc8040': '{}/data/network-topology:network-topology/topology=topology-netconf/node={}',
297            'draft-bierman02': '{}/config/network-topology:network-topology/topology/topology-netconf/node/{}'}
298     response = delete_request(url[RESTCONF_VERSION].format('{}', node))
299     if wait_until_log_contains(TPCE_LOG, re.escape("onDeviceDisConnected: " + node), 180):
300         print('Node ' + node + ' correctly deleted from tpce topology', end='... ', flush=True)
301     else:
302         print('Node ' + node + ' still not deleted from tpce topology', end='... ', flush=True)
303     return response
304
305
306 def check_device_connection(node: str):
307     url = {'rfc8040': '{}/data/network-topology:network-topology/topology=topology-netconf/node={}?content=nonconfig',
308            'draft-bierman02': '{}/operational/network-topology:network-topology/topology/topology-netconf/node/{}'}
309     response = get_request(url[RESTCONF_VERSION].format('{}', node))
310     res = response.json()
311     return_key = {'rfc8040': 'network-topology:node',
312                   'draft-bierman02': 'node'}
313     if return_key[RESTCONF_VERSION] in res.keys():
314         connection_status = res[return_key[RESTCONF_VERSION]][0]['netconf-node-topology:connection-status']
315     else:
316         connection_status = res['errors']['error'][0]
317     return {'status_code': response.status_code,
318             'connection-status': connection_status}
319
320
321 def check_node_attribute_request(node: str, attribute: str, attribute_value: str):
322     # pylint: disable=line-too-long
323     url = {'rfc8040': '{}/data/network-topology:network-topology/topology=topology-netconf/node={}/yang-ext:mount/org-openroadm-device:org-openroadm-device/{}={}?content=nonconfig',  # nopep8
324            'draft-bierman02': '{}/operational/network-topology:network-topology/topology/topology-netconf/node/{}/yang-ext:mount/org-openroadm-device:org-openroadm-device/{}/{}'}  # nopep8
325     response = get_request(url[RESTCONF_VERSION].format('{}', node, attribute, attribute_value))
326     res = response.json()
327     return_key = {'rfc8040': 'org-openroadm-device:' + attribute,
328                   'draft-bierman02': attribute}
329     if return_key[RESTCONF_VERSION] in res.keys():
330         response_attribute = res[return_key[RESTCONF_VERSION]]
331     else:
332         response_attribute = res['errors']['error'][0]
333     return {'status_code': response.status_code,
334             attribute: response_attribute}
335
336
337 def del_node_attribute_request(node: str, attribute: str, attribute_value: str):
338     # pylint: disable=line-too-long
339     url = {'rfc8040': '{}/data/network-topology:network-topology/topology=topology-netconf/node={}/yang-ext:mount/org-openroadm-device:org-openroadm-device/{}={}',  # nopep8
340            'draft-bierman02': '{}/config/network-topology:network-topology/topology/topology-netconf/node/{}/yang-ext:mount/org-openroadm-device:org-openroadm-device/{}/{}'}  # nopep8
341     response = delete_request(url[RESTCONF_VERSION].format('{}', node, attribute, attribute_value))
342     return response
343
344 #
345 # Portmapping operations
346 #
347
348
349 def get_portmapping(node: str):
350     url = {'rfc8040': '{}/data/transportpce-portmapping:network/nodes={}',
351            'draft-bierman02': '{}/config/transportpce-portmapping:network/nodes/{}'}
352     response = get_request(url[RESTCONF_VERSION].format('{}', node))
353     res = response.json()
354     return_key = {'rfc8040': 'transportpce-portmapping:nodes',
355                   'draft-bierman02': 'nodes'}
356     nodes = res[return_key[RESTCONF_VERSION]]
357     return {'status_code': response.status_code,
358             'nodes': nodes}
359
360
361 def get_portmapping_node_info(node: str):
362     url = {'rfc8040': '{}/data/transportpce-portmapping:network/nodes={}/node-info',
363            'draft-bierman02': '{}/config/transportpce-portmapping:network/nodes/{}/node-info'}
364     response = get_request(url[RESTCONF_VERSION].format('{}', node))
365     res = response.json()
366     return_key = {'rfc8040': 'transportpce-portmapping:node-info',
367                   'draft-bierman02': 'node-info'}
368     if return_key[RESTCONF_VERSION] in res.keys():
369         node_info = res[return_key[RESTCONF_VERSION]]
370     else:
371         node_info = res['errors']['error'][0]
372     return {'status_code': response.status_code,
373             'node-info': node_info}
374
375
376 def portmapping_request(node: str, mapping: str):
377     url = {'rfc8040': '{}/data/transportpce-portmapping:network/nodes={}/mapping={}',
378            'draft-bierman02': '{}/config/transportpce-portmapping:network/nodes/{}/mapping/{}'}
379     response = get_request(url[RESTCONF_VERSION].format('{}', node, mapping))
380     res = response.json()
381     return_key = {'rfc8040': 'transportpce-portmapping:mapping',
382                   'draft-bierman02': 'mapping'}
383     mapping = res[return_key[RESTCONF_VERSION]]
384     return {'status_code': response.status_code,
385             'mapping': mapping}
386
387
388 def portmapping_mc_capa_request(node: str, mc_capa: str):
389     url = {'rfc8040': '{}/data/transportpce-portmapping:network/nodes={}/mc-capabilities={}',
390            'draft-bierman02': '{}/config/transportpce-portmapping:network/nodes/{}/mc-capabilities/{}'}
391     response = get_request(url[RESTCONF_VERSION].format('{}', node, mc_capa))
392     res = response.json()
393     return_key = {'rfc8040': 'transportpce-portmapping:mc-capabilities',
394                   'draft-bierman02': 'mc-capabilities'}
395     capabilities = res[return_key[RESTCONF_VERSION]]
396     return {'status_code': response.status_code,
397             'mc-capabilities': capabilities}
398
399 #
400 # Topology operations
401 #
402
403
404 def get_ietf_network_request(network: str, content: str):
405     url = {'rfc8040': '{}/data/ietf-network:networks/network={}?content={}',
406            'draft-bierman02': '{}/{}/ietf-network:networks/network/{}'}
407     if RESTCONF_VERSION == 'rfc8040':
408         format_args = ('{}', network, content)
409     elif content == 'config':
410         format_args = ('{}', content, network)
411     else:
412         format_args = ('{}', 'operational', network)
413     response = get_request(url[RESTCONF_VERSION].format(*format_args))
414     res = response.json()
415     return_key = {'rfc8040': 'ietf-network:network',
416                   'draft-bierman02': 'network'}
417     networks = res[return_key[RESTCONF_VERSION]]
418     return {'status_code': response.status_code,
419             'network': networks}
420
421
422 def get_ietf_network_link_request(network: str, link: str, content: str):
423     url = {'rfc8040': '{}/data/ietf-network:networks/network={}/ietf-network-topology:link={}?content={}',
424            'draft-bierman02': '{}/{}/ietf-network:networks/network/{}/ietf-network-topology:link/{}'}
425     if RESTCONF_VERSION == 'rfc8040':
426         format_args = ('{}', network, link, content)
427     elif content == 'config':
428         format_args = ('{}', content, network, link)
429     else:
430         format_args = ('{}', 'operational', network, link)
431     response = get_request(url[RESTCONF_VERSION].format(*format_args))
432     res = response.json()
433     return_key = {'rfc8040': 'ietf-network-topology:link',
434                   'draft-bierman02': 'ietf-network-topology:link'}
435     link = res[return_key[RESTCONF_VERSION]][0]
436     return {'status_code': response.status_code,
437             'link': link}
438
439
440 def del_ietf_network_link_request(network: str, link: str, content: str):
441     url = {'rfc8040': '{}/data/ietf-network:networks/network={}/ietf-network-topology:link={}?content={}',
442            'draft-bierman02': '{}/{}/ietf-network:networks/network/{}/ietf-network-topology:link/{}'}
443     if RESTCONF_VERSION == 'rfc8040':
444         format_args = ('{}', network, link, content)
445     elif content == 'config':
446         format_args = ('{}', content, network, link)
447     else:
448         format_args = ('{}', 'operational', network, link)
449     response = delete_request(url[RESTCONF_VERSION].format(*format_args))
450     return response
451
452
453 def add_oms_attr_request(link: str, oms_attr: str):
454     url = {'rfc8040': '{}/data/ietf-network:networks/network={}/ietf-network-topology:link={}',
455            'draft-bierman02': '{}/config/ietf-network:networks/network/{}/ietf-network-topology:link/{}'}
456     url2 = url[RESTCONF_VERSION] + '/org-openroadm-network-topology:OMS-attributes/span'
457     network = 'openroadm-topology'
458     response = put_request(url2.format('{}', network, link), oms_attr)
459     return response
460
461
462 def del_oms_attr_request(link: str,):
463     url = {'rfc8040': '{}/data/ietf-network:networks/network={}/ietf-network-topology:link={}',
464            'draft-bierman02': '{}/config/ietf-network:networks/network/{}/ietf-network-topology:link/{}'}
465     url2 = url[RESTCONF_VERSION] + '/org-openroadm-network-topology:OMS-attributes/span'
466     network = 'openroadm-topology'
467     response = delete_request(url2.format('{}', network, link))
468     return response
469
470
471 def del_ietf_network_node_request(network: str, node: str, content: str):
472     url = {'rfc8040': '{}/data/ietf-network:networks/network={}/node={}?content={}',
473            'draft-bierman02': '{}/{}/ietf-network:networks/network/{}/node/{}'}
474     if RESTCONF_VERSION == 'rfc8040':
475         format_args = ('{}', network, node, content)
476     elif content == 'config':
477         format_args = ('{}', content, network, node)
478     else:
479         format_args = ('{}', 'operational', network, node)
480     response = delete_request(url[RESTCONF_VERSION].format(*format_args))
481     return response
482
483 #
484 # TransportPCE network-utils and service-path operations
485 #
486
487
488 def prepend_dict_keys(input_dict: dict, prefix: str):
489     return_dict = {}
490     for key, value in input_dict.items():
491         newkey = prefix + key
492         if isinstance(value, dict):
493             return_dict[newkey] = prepend_dict_keys(value, prefix)
494             # TODO: perhaps some recursion depth limit or another solution has to be considered here
495             # even if recursion depth is given by the input_dict argument
496             # direct (self-)recursive functions may carry unwanted side-effects such as ressource consumptions
497         else:
498             return_dict[newkey] = value
499     return return_dict
500
501
502 def connect_xpdr_to_rdm_request(payload: dict):
503     url = "{}/operations/transportpce-networkutils:init-xpdr-rdm-links"
504     if RESTCONF_VERSION == 'draft-bierman02':
505         data = prepend_dict_keys({'input': {'links-input': payload}}, 'networkutils:')
506     else:
507         data = {'input': {'links-input': payload}}
508     return post_request(url, data)
509
510
511 def connect_rdm_to_xpdr_request(payload: dict):
512     url = "{}/operations/transportpce-networkutils:init-rdm-xpdr-links"
513     if RESTCONF_VERSION == 'draft-bierman02':
514         data = prepend_dict_keys({'input': {'links-input': payload}}, 'networkutils:')
515     else:
516         data = {'input': {'links-input': payload}}
517     return post_request(url, data)
518
519
520 def device_renderer_service_path_request(payload: dict):
521     url = "{}/operations/transportpce-device-renderer:service-path"
522     if RESTCONF_VERSION == 'draft-bierman02':
523         data = prepend_dict_keys(payload, 'transportpce-device-renderer:')
524     else:
525         data = payload
526     response = post_request(url, data)
527     res = response.json()
528     return_key = {'rfc8040': 'transportpce-device-renderer:output',
529                   'draft-bierman02': 'output'}
530     return_output = res[return_key[RESTCONF_VERSION]]
531     return {'status_code': response.status_code,
532             'output': return_output}
533
534
535 def device_renderer_otn_service_path_request(payload: dict):
536     url = "{}/operations/transportpce-device-renderer:otn-service-path"
537     if RESTCONF_VERSION == 'draft-bierman02':
538         data = prepend_dict_keys(payload, 'transportpce-device-renderer:')
539     else:
540         data = payload
541     response = post_request(url, data)
542     res = response.json()
543     return_key = {'rfc8040': 'transportpce-device-renderer:output',
544                   'draft-bierman02': 'output'}
545     return_output = res[return_key[RESTCONF_VERSION]]
546     return {'status_code': response.status_code,
547             'output': return_output}