Upgrade to OpenROADM 7.1.0 network models
[transportpce.git] / tests / transportpce_tests / common / test_utils.py
1 #!/usr/bin/env python
2
3 ##############################################################################
4 # Copyright (c) 2020 Orange, Inc. and others.  All rights reserved.
5 #
6 # All rights reserved. This program and the accompanying materials
7 # are made available under the terms of the Apache License, Version 2.0
8 # which accompanies this distribution, and is available at
9 # http://www.apache.org/licenses/LICENSE-2.0
10 ##############################################################################
11
12 # pylint: disable=no-member
13
14 import base64
15 import json
16 import os
17 import sys
18 import re
19 import signal
20 import subprocess
21 import time
22
23 import psutil
24 import requests
25
26 import simulators
27
28 SIMS = simulators.SIMS
29 HONEYNODE_EXECUTABLE = simulators.HONEYNODE_EXECUTABLE
30 SAMPLES_DIRECTORY = simulators.SAMPLES_DIRECTORY
31
32 HONEYNODE_OK_START_MSG = "Netconf SSH endpoint started successfully at 0.0.0.0"
33 KARAF_OK_START_MSG = re.escape(
34     "Blueprint container for bundle org.opendaylight.netconf.restconf")+".* was successfully created"
35 LIGHTY_OK_START_MSG = re.escape("lighty.io and RESTCONF-NETCONF started")
36
37 RESTCONF_BASE_URL = "http://localhost:8181/restconf"
38 ODL_LOGIN = "admin"
39 ODL_PWD = "admin"
40 NODES_LOGIN = "admin"
41 NODES_PWD = "admin"
42 URL_CONFIG_NETCONF_TOPO = "{}/config/network-topology:network-topology/topology/topology-netconf/"
43 URL_CONFIG_ORDM_TOPO = "{}/config/ietf-network:networks/network/openroadm-topology/"
44 URL_CONFIG_OTN_TOPO = "{}/config/ietf-network:networks/network/otn-topology/"
45 URL_CONFIG_CLLI_NET = "{}/config/ietf-network:networks/network/clli-network/"
46 URL_CONFIG_ORDM_NET = "{}/config/ietf-network:networks/network/openroadm-network/"
47 URL_PORTMAPPING = "{}/config/transportpce-portmapping:network/nodes/"
48 URL_OPER_SERV_LIST = "{}/operational/org-openroadm-service:service-list/"
49 URL_SERV_CREATE = "{}/operations/org-openroadm-service:service-create"
50 URL_SERV_DELETE = "{}/operations/org-openroadm-service:service-delete"
51 URL_SERVICE_PATH = "{}/operations/transportpce-device-renderer:service-path"
52 URL_OTN_SERVICE_PATH = "{}/operations/transportpce-device-renderer:otn-service-path"
53 URL_CREATE_OTS_OMS = "{}/operations/transportpce-device-renderer:create-ots-oms"
54 URL_PATH_COMPUTATION_REQUEST = "{}/operations/transportpce-pce:path-computation-request"
55
56 TYPE_APPLICATION_JSON = {'Content-Type': 'application/json', 'Accept': 'application/json'}
57 TYPE_APPLICATION_XML = {'Content-Type': 'application/xml', 'Accept': 'application/xml'}
58
59 CODE_SHOULD_BE_200 = 'Http status code should be 200'
60 CODE_SHOULD_BE_201 = 'Http status code should be 201'
61
62 LOG_DIRECTORY = os.path.dirname(os.path.realpath(__file__))
63
64 KARAF_LOG = os.path.join(
65     os.path.dirname(os.path.realpath(__file__)),
66     "..", "..", "..", "karaf", "target", "assembly", "data", "log", "karaf.log")
67
68 process_list = []
69
70 if "USE_LIGHTY" in os.environ and os.environ['USE_LIGHTY'] == 'True':
71     TPCE_LOG = 'odl.log'
72 else:
73     TPCE_LOG = KARAF_LOG
74
75
76 def start_sims(sims_list):
77     for sim in sims_list:
78         print("starting simulator for " + sim + "...")
79         log_file = os.path.join(LOG_DIRECTORY, SIMS[sim]['logfile'])
80         process = start_honeynode(log_file, SIMS[sim]['port'], SIMS[sim]['configfile'])
81         if wait_until_log_contains(log_file, HONEYNODE_OK_START_MSG, 100):
82             print("simulator for " + sim + " started")
83         else:
84             print("simulator for " + sim + " failed to start")
85             shutdown_process(process)
86             for pid in process_list:
87                 shutdown_process(pid)
88             sys.exit(3)
89         process_list.append(process)
90     return process_list
91
92
93 def start_tpce():
94     print("starting OpenDaylight...")
95     if "USE_LIGHTY" in os.environ and os.environ['USE_LIGHTY'] == 'True':
96         process = start_lighty()
97         start_msg = LIGHTY_OK_START_MSG
98     else:
99         process = start_karaf()
100         start_msg = KARAF_OK_START_MSG
101     if wait_until_log_contains(TPCE_LOG, start_msg, time_to_wait=60):
102         print("OpenDaylight started !")
103     else:
104         print("OpenDaylight failed to start !")
105         shutdown_process(process)
106         for pid in process_list:
107             shutdown_process(pid)
108         sys.exit(1)
109     process_list.append(process)
110     return process_list
111
112
113 def start_karaf():
114     print("starting KARAF TransportPCE build...")
115     executable = os.path.join(
116         os.path.dirname(os.path.realpath(__file__)),
117         "..", "..", "..", "karaf", "target", "assembly", "bin", "karaf")
118     with open('odl.log', 'w') as outfile:
119         return subprocess.Popen(
120             ["sh", executable, "server"], stdout=outfile, stderr=outfile, stdin=None)
121
122
123 def start_lighty():
124     print("starting LIGHTY.IO TransportPCE build...")
125     executable = os.path.join(
126         os.path.dirname(os.path.realpath(__file__)),
127         "..", "..", "..", "lighty", "target", "tpce",
128         "clean-start-controller.sh")
129     with open('odl.log', 'w') as outfile:
130         return subprocess.Popen(
131             ["sh", executable], stdout=outfile, stderr=outfile, stdin=None)
132
133
134 def install_karaf_feature(feature_name: str):
135     print("installing feature " + feature_name)
136     executable = os.path.join(
137         os.path.dirname(os.path.realpath(__file__)),
138         "..", "..", "..", "karaf", "target", "assembly", "bin", "client")
139     return subprocess.run([executable],
140                           input='feature:install ' + feature_name + '\n feature:list | grep tapi \n logout \n',
141                           universal_newlines=True, check=False)
142
143
144 def get_request(url):
145     return requests.request(
146         "GET", url.format(RESTCONF_BASE_URL),
147         headers=TYPE_APPLICATION_JSON,
148         auth=(ODL_LOGIN, ODL_PWD))
149
150
151 def post_request(url, data):
152     if data:
153         return requests.request(
154             "POST", url.format(RESTCONF_BASE_URL),
155             data=json.dumps(data),
156             headers=TYPE_APPLICATION_JSON,
157             auth=(ODL_LOGIN, ODL_PWD))
158
159     return requests.request(
160         "POST", url.format(RESTCONF_BASE_URL),
161         headers=TYPE_APPLICATION_JSON,
162         auth=(ODL_LOGIN, ODL_PWD))
163
164
165 def post_xmlrequest(url, data):
166     if data:
167         return requests.request(
168             "POST", url.format(RESTCONF_BASE_URL),
169             data=data,
170             headers=TYPE_APPLICATION_XML,
171             auth=(ODL_LOGIN, ODL_PWD))
172     return None
173
174
175 def put_request(url, data):
176     return requests.request(
177         "PUT", url.format(RESTCONF_BASE_URL),
178         data=json.dumps(data),
179         headers=TYPE_APPLICATION_JSON,
180         auth=(ODL_LOGIN, ODL_PWD))
181
182
183 def put_xmlrequest(url, data):
184     return requests.request(
185         "PUT", url.format(RESTCONF_BASE_URL),
186         data=data,
187         headers=TYPE_APPLICATION_XML,
188         auth=(ODL_LOGIN, ODL_PWD))
189
190
191 def rawput_request(url, data):
192     return requests.request(
193         "PUT", url.format(RESTCONF_BASE_URL),
194         data=data,
195         headers=TYPE_APPLICATION_JSON,
196         auth=(ODL_LOGIN, ODL_PWD))
197
198
199 def delete_request(url):
200     return requests.request(
201         "DELETE", url.format(RESTCONF_BASE_URL),
202         headers=TYPE_APPLICATION_JSON,
203         auth=(ODL_LOGIN, ODL_PWD))
204
205
206 def mount_device(node_id, sim):
207     url = URL_CONFIG_NETCONF_TOPO + "node/" + node_id
208     body = {"node": [{
209         "node-id": node_id,
210         "netconf-node-topology:username": NODES_LOGIN,
211         "netconf-node-topology:password": NODES_PWD,
212         "netconf-node-topology:host": "127.0.0.1",
213         "netconf-node-topology:port": SIMS[sim]['port'],
214         "netconf-node-topology:tcp-only": "false",
215         "netconf-node-topology:pass-through": {}}]}
216     response = put_request(url, body)
217     if wait_until_log_contains(TPCE_LOG, re.escape("Triggering notification stream NETCONF for node " + node_id), 60):
218         print("Node " + node_id + " correctly added to tpce topology", end='... ', flush=True)
219     else:
220         print("Node " + node_id + " still not added to tpce topology", end='... ', flush=True)
221         if response.status_code == requests.codes.ok:
222             print("It was probably loaded at start-up", end='... ', flush=True)
223         # TODO an else-clause to abort test would probably be nice here
224     return response
225
226
227 def unmount_device(node_id):
228     url = URL_CONFIG_NETCONF_TOPO + "node/" + node_id
229     response = delete_request(url)
230     if wait_until_log_contains(TPCE_LOG, re.escape("onDeviceDisConnected: " + node_id), 60):
231         print("Node " + node_id + " correctly deleted from tpce topology", end='... ', flush=True)
232     else:
233         print("Node " + node_id + " still not deleted from tpce topology", end='... ', flush=True)
234     return response
235
236
237 def connect_xpdr_to_rdm_request(xpdr_node: str, xpdr_num: str, network_num: str,
238                                 rdm_node: str, srg_num: str, termination_num: str):
239     url = "{}/operations/transportpce-networkutils:init-xpdr-rdm-links"
240     data = {
241         "networkutils:input": {
242             "networkutils:links-input": {
243                 "networkutils:xpdr-node": xpdr_node,
244                 "networkutils:xpdr-num": xpdr_num,
245                 "networkutils:network-num": network_num,
246                 "networkutils:rdm-node": rdm_node,
247                 "networkutils:srg-num": srg_num,
248                 "networkutils:termination-point-num": termination_num
249             }
250         }
251     }
252     return post_request(url, data)
253
254
255 def connect_rdm_to_xpdr_request(xpdr_node: str, xpdr_num: str, network_num: str,
256                                 rdm_node: str, srg_num: str, termination_num: str):
257     url = "{}/operations/transportpce-networkutils:init-rdm-xpdr-links"
258     data = {
259         "networkutils:input": {
260             "networkutils:links-input": {
261                 "networkutils:xpdr-node": xpdr_node,
262                 "networkutils:xpdr-num": xpdr_num,
263                 "networkutils:network-num": network_num,
264                 "networkutils:rdm-node": rdm_node,
265                 "networkutils:srg-num": srg_num,
266                 "networkutils:termination-point-num": termination_num
267             }
268         }
269     }
270     return post_request(url, data)
271
272
273 def check_netconf_node_request(node: str, suffix: str):
274     url = URL_CONFIG_NETCONF_TOPO + (
275         "node/" + node + "/yang-ext:mount/org-openroadm-device:org-openroadm-device/" + suffix
276     )
277     return get_request(url)
278
279
280 def get_netconf_oper_request(node: str):
281     url = "{}/operational/network-topology:network-topology/topology/topology-netconf/node/" + node
282     return get_request(url)
283
284
285 def get_ordm_topo_request(suffix: str):
286     url = URL_CONFIG_ORDM_TOPO + suffix
287     return get_request(url)
288
289
290 def add_oms_attr_request(link: str, attr):
291     url = URL_CONFIG_ORDM_TOPO + (
292         "ietf-network-topology:link/" + link + "/org-openroadm-network-topology:OMS-attributes/span"
293     )
294     return put_request(url, attr)
295
296
297 def del_oms_attr_request(link: str):
298     url = URL_CONFIG_ORDM_TOPO + (
299         "ietf-network-topology:link/" + link + "/org-openroadm-network-topology:OMS-attributes/span"
300     )
301     return delete_request(url)
302
303
304 def get_clli_net_request():
305     return get_request(URL_CONFIG_CLLI_NET)
306
307
308 def get_ordm_net_request():
309     return get_request(URL_CONFIG_ORDM_NET)
310
311
312 def get_otn_topo_request():
313     return get_request(URL_CONFIG_OTN_TOPO)
314
315
316 def del_link_request(link: str):
317     url = URL_CONFIG_ORDM_TOPO + ("ietf-network-topology:link/" + link)
318     return delete_request(url)
319
320
321 def del_node_request(node: str):
322     url = URL_CONFIG_CLLI_NET + ("node/" + node)
323     return delete_request(url)
324
325
326 def portmapping_request(suffix: str):
327     url = URL_PORTMAPPING + suffix
328     return get_request(url)
329
330
331 def get_service_list_request(suffix: str):
332     url = URL_OPER_SERV_LIST + suffix
333     return get_request(url)
334
335
336 def service_create_request(attr):
337     return post_request(URL_SERV_CREATE, attr)
338
339
340 def service_delete_request(servicename: str,
341                            requestid="e3028bae-a90f-4ddd-a83f-cf224eba0e58",
342                            notificationurl="http://localhost:8585/NotificationServer/notify"):
343     attr = {"input": {
344         "sdnc-request-header": {
345             "request-id": requestid,
346             "rpc-action": "service-delete",
347             "request-system-id": "appname",
348             "notification-url": notificationurl},
349         "service-delete-req-info": {
350             "service-name": servicename,
351             "tail-retention": "no"}}}
352     return post_request(URL_SERV_DELETE, attr)
353
354
355 def service_path_request(operation: str, servicename: str, wavenumber: str, nodes):
356     attr = {"renderer:input": {
357         "renderer:service-name": servicename,
358         "renderer:wave-number": wavenumber,
359         "renderer:modulation-format": "qpsk",
360         "renderer:operation": operation,
361         "renderer:nodes": nodes}}
362     return post_request(URL_SERVICE_PATH, attr)
363
364
365 def otn_service_path_request(operation: str, servicename: str, servicerate: str, servicetype: str, nodes,
366                              eth_attr=None):
367     attr = {"service-name": servicename,
368             "operation": operation,
369             "service-rate": servicerate,
370             "service-type": servicetype,
371             "nodes": nodes}
372     if eth_attr:
373         attr.update(eth_attr)
374     return post_request(URL_OTN_SERVICE_PATH, {"renderer:input": attr})
375
376
377 def create_ots_oms_request(nodeid: str, lcp: str):
378     attr = {"input": {
379         "node-id": nodeid,
380         "logical-connection-point": lcp}}
381     return post_request(URL_CREATE_OTS_OMS, attr)
382
383
384 def path_computation_request(requestid: str, servicename: str, serviceaend, servicezend,
385                              hardconstraints=None, softconstraints=None, metric="hop-count", other_attr=None):
386     attr = {"service-name": servicename,
387             "resource-reserve": "true",
388             "service-handler-header": {"request-id": requestid},
389             "service-a-end": serviceaend,
390             "service-z-end": servicezend,
391             "pce-metric": metric}
392     if hardconstraints:
393         attr.update({"hard-constraints": hardconstraints})
394     if softconstraints:
395         attr.update({"soft-constraints": softconstraints})
396     if other_attr:
397         attr.update(other_attr)
398     return post_request(URL_PATH_COMPUTATION_REQUEST, {"input": attr})
399
400
401 def shutdown_process(process):
402     if process is not None:
403         for child in psutil.Process(process.pid).children():
404             child.send_signal(signal.SIGINT)
405             child.wait()
406         process.send_signal(signal.SIGINT)
407
408
409 def start_honeynode(log_file: str, node_port: str, node_config_file_name: str):
410     if os.path.isfile(HONEYNODE_EXECUTABLE):
411         with open(log_file, 'w') as outfile:
412             return subprocess.Popen(
413                 [HONEYNODE_EXECUTABLE, node_port, os.path.join(SAMPLES_DIRECTORY, node_config_file_name)],
414                 stdout=outfile, stderr=outfile)
415     return None
416
417
418 def wait_until_log_contains(log_file, regexp, time_to_wait=20):
419     # pylint: disable=lost-exception
420     stringfound = False
421     filefound = False
422     line = None
423     try:
424         with TimeOut(seconds=time_to_wait):
425             while not os.path.exists(log_file):
426                 time.sleep(0.2)
427             filelogs = open(log_file, 'r')
428             filelogs.seek(0, 2)
429             filefound = True
430             print("Searching for pattern '" + regexp + "' in " + os.path.basename(log_file), end='... ', flush=True)
431             compiled_regexp = re.compile(regexp)
432             while True:
433                 line = filelogs.readline()
434                 if compiled_regexp.search(line):
435                     print("Pattern found!", end=' ')
436                     stringfound = True
437                     break
438                 if not line:
439                     time.sleep(0.1)
440     except TimeoutError:
441         print("Pattern not found after " + str(time_to_wait), end=" seconds! ", flush=True)
442     except PermissionError:
443         print("Permission Error when trying to access the log file", end=" ... ", flush=True)
444     finally:
445         if filefound:
446             filelogs.close()
447         else:
448             print("log file does not exist or is not accessible... ", flush=True)
449         return stringfound
450
451
452 class TimeOut:
453     def __init__(self, seconds=1, error_message='Timeout'):
454         self.seconds = seconds
455         self.error_message = error_message
456
457     def handle_timeout(self, signum, frame):
458         raise TimeoutError(self.error_message)
459
460     def __enter__(self):
461         signal.signal(signal.SIGALRM, self.handle_timeout)
462         signal.alarm(self.seconds)
463
464     def __exit__(self, type, value, traceback):
465         # pylint: disable=W0622
466         signal.alarm(0)
467
468
469 def check_freq_map(freq_map):
470     freq_map_array = [int(x) for x in freq_map]
471     return freq_map_array[0] == 255 and freq_map_array[1] == 255
472
473
474 def set_used_index_for_freq_map(freq_map, index):
475     freq_map[index] = 0
476     return freq_map
477
478
479 INDEX_1_USED_FREQ_MAP = base64.b64encode(set_used_index_for_freq_map(bytearray(b'\xFF' * 96), 0)).decode('UTF-8')
480
481 INDEX_1_2_USED_FREQ_MAP = base64.b64encode(set_used_index_for_freq_map(
482     set_used_index_for_freq_map(bytearray(b'\xFF' * 96), 0), 1)).decode('utf-8')
483
484 AVAILABLE_FREQ_MAP = base64.b64encode(bytearray(b'\xFF' * 96)).decode('UTF-8')