f34448cb1c646a89c2b3bcbff2fafea264c71c85
[demo.git] / tutorials / vFWDT / workflow / workflow.py
1 '''
2 /*-
3 * ============LICENSE_START=======================================================
4 * Copyright (C) 2019 Orange
5 * ================================================================================
6 * Licensed under the Apache License, Version 2.0 (the "License");
7 * you may not use this file except in compliance with the License.
8 * You may obtain a copy of the License at
9 *
10 *      http://www.apache.org/licenses/LICENSE-2.0
11 *
12 * Unless required by applicable law or agreed to in writing, software
13 * distributed under the License is distributed on an "AS IS" BASIS,
14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 * See the License for the specific language governing permissions and
16 * limitations under the License.
17 *
18 * ============LICENSE_END=========================================================
19 */
20 '''
21
22 import os
23 import json
24 import sys
25 import uuid
26 import time
27 import copy
28 import netifaces as ni
29 import warnings
30 import contextlib
31 import requests
32 from datetime import datetime
33 from datetime import timedelta
34 from simple_rest_client.api import API
35 from simple_rest_client.resource import Resource
36 from basicauth import encode
37 from pprint import pprint
38 from random import randint
39 from urllib3.exceptions import InsecureRequestWarning
40
41
42 old_merge_environment_settings = requests.Session.merge_environment_settings
43
44 hostname_cache = []
45 ansible_inventory = {}
46
47
48 @contextlib.contextmanager
49 def _no_ssl_verification():
50     opened_adapters = set()
51
52     def merge_environment_settings(self, url, proxies, stream, verify, cert):
53         # Verification happens only once per connection so we need to close
54         # all the opened adapters once we're done. Otherwise, the effects of
55         # verify=False persist beyond the end of this context manager.
56         opened_adapters.add(self.get_adapter(url))
57
58         settings = old_merge_environment_settings(self, url, proxies, stream, verify, cert)
59         settings['verify'] = False
60
61         return settings
62
63     requests.Session.merge_environment_settings = merge_environment_settings
64
65     try:
66         with warnings.catch_warnings():
67             warnings.simplefilter('ignore', InsecureRequestWarning)
68             yield
69     finally:
70         requests.Session.merge_environment_settings = old_merge_environment_settings
71
72         for adapter in opened_adapters:
73             try:
74                 adapter.close()
75             except:
76                 pass
77
78
79 def _get_aai_rel_link_data(data, related_to, search_key=None, match_dict=None):
80     # some strings that we will encounter frequently
81     rel_lst = "relationship-list"
82     rkey = "relationship-key"
83     rval = "relationship-value"
84     rdata = "relationship-data"
85     response = list()
86     if match_dict:
87         m_key = match_dict.get('key')
88         m_value = match_dict.get('value')
89     else:
90         m_key = None
91         m_value = None
92     rel_dict = data.get(rel_lst)
93     if rel_dict:  # check if data has relationship lists
94         for key, rel_list in rel_dict.items():
95             for rel in rel_list:
96                 if rel.get("related-to") == related_to:
97                     dval = None
98                     matched = False
99                     link = rel.get("related-link")
100                     r_data = rel.get(rdata, [])
101                     if search_key:
102                         for rd in r_data:
103                             if rd.get(rkey) == search_key:
104                                 dval = rd.get(rval)
105                                 if not match_dict:  # return first match
106                                     response.append(
107                                         {"link": link, "d_value": dval}
108                                     )
109                                     break  # go to next relation
110                             if rd.get(rkey) == m_key \
111                                     and rd.get(rval) == m_value:
112                                 matched = True
113                         if match_dict and matched:  # if matching required
114                             response.append(
115                                 {"link": link, "d_value": dval}
116                             )
117                             # matched, return search value corresponding
118                             # to the matched r_data group
119                     else:  # no search key; just return the link
120                         response.append(
121                             {"link": link, "d_value": dval}
122                         )
123     if len(response) == 0:
124         response.append(
125             {"link": None, "d_value": None}
126         )
127     return response
128
129
130 class AAIApiResource(Resource):
131     actions = {
132         'generic_vnf': {'method': 'GET', 'url': 'network/generic-vnfs/generic-vnf/{}'},
133         'link': {'method': 'GET', 'url': '{}'},
134         'service_instance': {'method': 'GET',
135                              'url': 'business/customers/customer/{}/service-subscriptions/service-subscription/{}/service-instances/service-instance/{}'}
136     }
137
138
139 class HASApiResource(Resource):
140     actions = {
141         'plans': {'method': 'POST', 'url': 'plans/'},
142         'plan': {'method': 'GET', 'url': 'plans/{}'}
143     }
144
145
146 class APPCLcmApiResource(Resource):
147     actions = {
148         'distribute_traffic': {'method': 'POST', 'url': 'appc-provider-lcm:distribute-traffic/'},
149         'distribute_traffic_check': {'method': 'POST', 'url': 'appc-provider-lcm:distribute-traffic-check/'},
150         'action_status': {'method': 'POST', 'url': 'appc-provider-lcm:action-status/'},
151     }
152
153
154 def _init_python_aai_api(onap_ip):
155     api = API(
156         api_root_url="https://{}:30233/aai/v14/".format(onap_ip),
157         params={},
158         headers={
159             'Authorization': encode("AAI", "AAI"),
160             'X-FromAppId': 'SCRIPT',
161             'Accept': 'application/json',
162             'Content-Type': 'application/json',
163             'X-TransactionId': str(uuid.uuid4()),
164         },
165         timeout=30,
166         append_slash=False,
167         json_encode_body=True # encode body as json
168     )
169     api.add_resource(resource_name='aai', resource_class=AAIApiResource)
170     return api
171
172
173 def _init_python_has_api(onap_ip):
174     api = API(
175         api_root_url="https://{}:30275/v1/".format(onap_ip),
176         params={},
177         headers={
178             'Authorization': encode("admin1", "plan.15"),
179             'X-FromAppId': 'SCRIPT',
180             'Accept': 'application/json',
181             'Content-Type': 'application/json',
182             'X-TransactionId': str(uuid.uuid4()),
183         },
184         timeout=30,
185         append_slash=False,
186         json_encode_body=True # encode body as json
187     )
188     api.add_resource(resource_name='has', resource_class=HASApiResource)
189     return api
190
191
192 def _init_python_appc_lcm_api(onap_ip):
193     api = API(
194         api_root_url="http://{}:30230/restconf/operations/".format(onap_ip),
195         params={},
196         headers={
197             'Authorization': encode("admin", "Kp8bJ4SXszM0WXlhak3eHlcse2gAw84vaoGGmJvUy2U"),
198             'X-FromAppId': 'SCRIPT',
199             'Accept': 'application/json',
200             'Content-Type': 'application/json',
201         },
202         timeout=300,
203         append_slash=False,
204         json_encode_body=True # encode body as json
205     )
206     api.add_resource(resource_name='lcm', resource_class=APPCLcmApiResource)
207     return api
208
209
210 def load_aai_data(vfw_vnf_id, onap_ip):
211     api = _init_python_aai_api(onap_ip)
212     aai_data = {}
213     aai_data['service-info'] = {'global-customer-id': '', 'service-instance-id': '', 'service-type': ''}
214     aai_data['vfw-model-info'] = {'model-invariant-id': '', 'model-version-id': '', 'vnf-name': '', 'vnf-type': ''}
215     aai_data['vpgn-model-info'] = {'model-invariant-id': '', 'model-version-id': '', 'vnf-name': '', 'vnf-type': ''}
216     with _no_ssl_verification():
217         response = api.aai.generic_vnf(vfw_vnf_id, body=None, params={'depth': 2}, headers={})
218         aai_data['vfw-model-info']['model-invariant-id'] = response.body.get('model-invariant-id')
219         aai_data['vfw-model-info']['model-version-id'] = response.body.get('model-version-id')
220         aai_data['vfw-model-info']['vnf-name'] = response.body.get('vnf-name')
221         aai_data['vfw-model-info']['vnf-type'] = response.body.get('vnf-type')
222         aai_data['vf-module-id'] = response.body['vf-modules']['vf-module'][0]['vf-module-id']
223
224         related_to = "service-instance"
225         search_key = "customer.global-customer-id"
226         rl_data_list = _get_aai_rel_link_data(data=response.body, related_to=related_to, search_key=search_key)
227         aai_data['service-info']['global-customer-id'] = rl_data_list[0]['d_value']
228
229         search_key = "service-subscription.service-type"
230         rl_data_list = _get_aai_rel_link_data(data=response.body, related_to=related_to, search_key=search_key)
231         aai_data['service-info']['service-type'] = rl_data_list[0]['d_value']
232
233         search_key = "service-instance.service-instance-id"
234         rl_data_list = _get_aai_rel_link_data(data=response.body, related_to=related_to, search_key=search_key)
235         aai_data['service-info']['service-instance-id'] = rl_data_list[0]['d_value']
236
237         service_link = rl_data_list[0]['link']
238         response = api.aai.link(service_link, body=None, params={}, headers={})
239
240         related_to = "generic-vnf"
241         search_key = "generic-vnf.vnf-id"
242         rl_data_list = _get_aai_rel_link_data(data=response.body, related_to=related_to, search_key=search_key)
243         for i in range(0, len(rl_data_list)):
244             vnf_id = rl_data_list[i]['d_value']
245
246             if vnf_id != vfw_vnf_id:
247                 vnf_link = rl_data_list[i]['link']
248                 response = api.aai.link(vnf_link, body=None, params={}, headers={})
249                 if aai_data['vfw-model-info']['model-invariant-id'] != response.body.get('model-invariant-id'):
250                     aai_data['vpgn-model-info']['model-invariant-id'] = response.body.get('model-invariant-id')
251                     aai_data['vpgn-model-info']['model-version-id'] = response.body.get('model-version-id')
252                     aai_data['vpgn-model-info']['vnf-name'] = response.body.get('vnf-name')
253                     aai_data['vpgn-model-info']['vnf-type'] = response.body.get('vnf-type')
254                     break
255     return aai_data
256
257
258 def _has_request(onap_ip, aai_data, exclude, use_oof_cache):
259     dirname = os.path.join('templates/oof-cache/', aai_data['vf-module-id'])
260     if exclude:
261         file = os.path.join(dirname, 'sample-has-excluded.json')
262     else:
263         file = os.path.join(dirname, 'sample-has-required.json')
264     if use_oof_cache and os.path.exists(file):
265         migrate_from = json.loads(open(file).read())
266         return migrate_from
267
268     print('Making HAS request for excluded {}'.format(str(exclude)))
269     api = _init_python_has_api(onap_ip)
270     request_id = str(uuid.uuid4())
271     template = json.loads(open('templates/hasRequest.json').read())
272     result = {}
273     template['name'] = request_id
274     node = template['template']['parameters']
275     node['chosen_customer_id'] = aai_data['service-info']['global-customer-id']
276     node['service_id'] = aai_data['service-info']['service-instance-id']
277     node = template['template']['demands']['vFW-SINK'][0]
278     node['attributes']['model-invariant-id'] = aai_data['vfw-model-info']['model-invariant-id']
279     node['attributes']['model-version-id'] = aai_data['vfw-model-info']['model-version-id']
280     if exclude:
281         node['excluded_candidates'][0]['candidate_id'] = aai_data['vf-module-id']
282         del node['required_candidates']
283     else:
284         node['required_candidates'][0]['candidate_id'] = aai_data['vf-module-id']
285         del node['excluded_candidates']
286     node = template['template']['demands']['vPGN'][0]
287     node['attributes']['model-invariant-id'] = aai_data['vpgn-model-info']['model-invariant-id']
288     node['attributes']['model-version-id'] = aai_data['vpgn-model-info']['model-version-id']
289
290     with _no_ssl_verification():
291         response = api.has.plans(body=template, params={}, headers={})
292         if response.body.get('error_message') is not None:
293             raise Exception(response.body['error_message']['explanation'])
294         else:
295             plan_id = response.body['id']
296             response = api.has.plan(plan_id, body=None, params={}, headers={})
297             status = response.body['plans'][0]['status']
298             while status != 'done' and status != 'error':
299                 print(status)
300                 response = api.has.plan(plan_id, body=None, params={}, headers={})
301                 status = response.body['plans'][0]['status']
302             if status == 'done':
303                 result = response.body['plans'][0]['recommendations'][0]
304             else:
305                 raise Exception(response.body['plans'][0]['message'])
306
307     if not os.path.exists(dirname):
308         os.makedirs(dirname)
309     f = open(file, 'w+')
310     f.write(json.dumps(result, indent=4))
311     f.close()
312     return result
313
314
315 def _extract_has_appc_identifiers(has_result, demand):
316     if demand == 'vPGN':
317         v_server = has_result[demand]['attributes']['vservers'][0]
318     else:
319         if len(has_result[demand]['attributes']['vservers'][0]['l-interfaces']) == 4:
320             v_server = has_result[demand]['attributes']['vservers'][0]
321         else:
322             v_server = has_result[demand]['attributes']['vservers'][1]
323     for itf in v_server['l-interfaces']:
324         if itf['ipv4-addresses'][0].startswith("10.0."):
325             ip = itf['ipv4-addresses'][0]
326             break
327
328     if v_server['vserver-name'] in hostname_cache and demand != 'vPGN':
329         v_server['vserver-name'] = v_server['vserver-name'].replace("01", "02")
330     hostname_cache.append(v_server['vserver-name'])
331
332     config = {
333         'vnf-id': has_result[demand]['attributes']['nf-id'],
334         'vf-module-id': has_result[demand]['attributes']['vf-module-id'],
335         'ip': ip,
336         'vserver-id': v_server['vserver-id'],
337         'vserver-name': v_server['vserver-name'],
338         'vnfc-type': demand.lower(),
339         'physical-location-id': has_result[demand]['attributes']['physical-location-id']
340     }
341     ansible_inventory_entry = "{} ansible_ssh_host={} ansible_ssh_user=ubuntu".format(config['vserver-name'], config['ip'])
342     if demand.lower() not in ansible_inventory:
343         ansible_inventory[demand.lower()] = {}
344     ansible_inventory[demand.lower()][config['vserver-name']] = ansible_inventory_entry
345     return config
346
347
348 def _extract_has_appc_dt_config(has_result, demand):
349     if demand == 'vPGN':
350         return {}
351     else:
352         config = {
353             "nf-type": has_result[demand]['attributes']['nf-type'],
354             "nf-name": has_result[demand]['attributes']['nf-name'],
355             "vf-module-name": has_result[demand]['attributes']['vf-module-name'],
356             "vnf-type": has_result[demand]['attributes']['vnf-type'],
357             "service_instance_id": "319e60ef-08b1-47aa-ae92-51b97f05e1bc",
358             "cloudClli": has_result[demand]['attributes']['physical-location-id'],
359             "nf-id": has_result[demand]['attributes']['nf-id'],
360             "vf-module-id": has_result[demand]['attributes']['vf-module-id'],
361             "aic_version": has_result[demand]['attributes']['aic_version'],
362             "ipv4-oam-address": has_result[demand]['attributes']['ipv4-oam-address'],
363             "vnfHostName": has_result[demand]['candidate']['host_id'],
364             "ipv6-oam-address": has_result[demand]['attributes']['ipv6-oam-address'],
365             "cloudOwner": has_result[demand]['candidate']['cloud_owner'],
366             "isRehome": has_result[demand]['candidate']['is_rehome'],
367             "locationId": has_result[demand]['candidate']['location_id'],
368             "locationType": has_result[demand]['candidate']['location_type'],
369             'vservers': has_result[demand]['attributes']['vservers']
370         }
371         return config
372
373
374 def _build_config_from_has(has_result):
375     v_pgn_result = _extract_has_appc_identifiers(has_result, 'vPGN')
376     v_fw_result = _extract_has_appc_identifiers(has_result, 'vFW-SINK')
377     dt_config = _extract_has_appc_dt_config(has_result, 'vFW-SINK')
378
379     config = {
380         'vPGN': v_pgn_result,
381         'vFW-SINK': v_fw_result
382     }
383     #print(json.dumps(config, indent=4))
384     config['dt-config'] = {
385         'destinations': [dt_config]
386     }
387     return config
388
389
390 def _build_appc_lcm_dt_payload(is_vpkg, oof_config, book_name, traffic_presence):
391     is_check = traffic_presence is not None
392     oof_config = copy.deepcopy(oof_config)
393     #if is_vpkg:
394     #    node_list = "[ {} ]".format(oof_config['vPGN']['vserver-id'])
395     #else:
396     #    node_list = "[ {} ]".format(oof_config['vFW-SINK']['vserver-id'])
397
398     if is_vpkg:
399         config = oof_config['vPGN']
400     else:
401         config = oof_config['vFW-SINK']
402     #node = {
403     #    'site': config['physical-location-id'],
404     #    'vnfc_type': config['vnfc-type'],
405     #    'vm_info': [{
406     #        'ne_id': config['vserver-name'],
407     #        'fixed_ip_address': config['ip']
408     #   }]
409     #}
410     #node_list = list()
411     #node_list.append(node)
412
413     if is_check:
414         oof_config['dt-config']['trafficpresence'] = traffic_presence
415
416     file_content = oof_config['dt-config']
417
418     config = {
419         "configuration-parameters": {
420             #"node_list": node_list,
421             "ne_id": config['vserver-name'],
422             "fixed_ip_address": config['ip'],
423             "file_parameter_content":  json.dumps(file_content)
424         }
425     }
426     if book_name != '':
427         config["configuration-parameters"]["book_name"] = book_name
428     payload = json.dumps(config)
429     return payload
430
431
432 def _build_appc_lcm_status_body(req):
433     payload = {
434         'request-id': req['input']['common-header']['request-id'],
435         'sub-request-id': req['input']['common-header']['sub-request-id'],
436         'originator-id': req['input']['common-header']['originator-id']
437     }
438     payload = json.dumps(payload)
439     template = json.loads(open('templates/appcRestconfLcm.json').read())
440     template['input']['action'] = 'ActionStatus'
441     template['input']['payload'] = payload
442     template['input']['common-header']['request-id'] = req['input']['common-header']['request-id']
443     template['input']['common-header']['sub-request-id'] = str(uuid.uuid4())
444     template['input']['action-identifiers']['vnf-id'] = req['input']['action-identifiers']['vnf-id']
445     return template
446
447
448 def _build_appc_lcm_request_body(is_vpkg, config, req_id, action, traffic_presence=None):
449     if is_vpkg:
450         demand = 'vPGN'
451     else:
452         demand = 'vFW-SINK'
453
454     book_name = "{}/latest/ansible/{}/site.yml".format(demand.lower(), action.lower())
455     payload = _build_appc_lcm_dt_payload(is_vpkg, config, book_name, traffic_presence)
456     template = json.loads(open('templates/appcRestconfLcm.json').read())
457     template['input']['action'] = action
458     template['input']['payload'] = payload
459     template['input']['common-header']['request-id'] = req_id
460     template['input']['common-header']['sub-request-id'] = str(uuid.uuid4())
461     template['input']['action-identifiers']['vnf-id'] = config[demand]['vnf-id']
462     return template
463
464
465 def _set_appc_lcm_timestamp(body, timestamp=None):
466     if timestamp is None:
467         t = datetime.utcnow() + timedelta(seconds=-10)
468         timestamp = t.strftime('%Y-%m-%dT%H:%M:%S.244Z')
469     body['input']['common-header']['timestamp'] = timestamp
470
471
472 def build_appc_lcms_requests_body(onap_ip, aai_data, use_oof_cache, if_close_loop_vfw):
473     migrate_from = _has_request(onap_ip, aai_data, False, use_oof_cache)
474
475     if if_close_loop_vfw:
476         migrate_to = migrate_from
477     else:
478         migrate_to = _has_request(onap_ip, aai_data, True, use_oof_cache)
479
480     migrate_from = _build_config_from_has(migrate_from)
481     migrate_to = _build_config_from_has(migrate_to)
482     req_id = str(uuid.uuid4())
483     payload_dt_check_vpkg = _build_appc_lcm_request_body(True, migrate_from, req_id, 'DistributeTrafficCheck', True)
484     payload_dt_vpkg_to = _build_appc_lcm_request_body(True, migrate_to, req_id, 'DistributeTraffic')
485     payload_dt_check_vfw_from = _build_appc_lcm_request_body(False, migrate_from, req_id, 'DistributeTrafficCheck',
486                                                              False)
487     payload_dt_check_vfw_to = _build_appc_lcm_request_body(False, migrate_to, req_id, 'DistributeTrafficCheck', True)
488
489     result = list()
490     result.append(payload_dt_check_vpkg)
491     result.append(payload_dt_vpkg_to)
492     result.append(payload_dt_check_vfw_from)
493     result.append(payload_dt_check_vfw_to)
494     return result
495
496
497 def appc_lcm_request(onap_ip, req):
498     api = _init_python_appc_lcm_api(onap_ip)
499     #print(json.dumps(req, indent=4))
500     if req['input']['action'] == "DistributeTraffic":
501         result = api.lcm.distribute_traffic(body=req, params={}, headers={})
502     elif req['input']['action'] == "DistributeTrafficCheck":
503         result = api.lcm.distribute_traffic_check(body=req, params={}, headers={})
504     else:
505         raise Exception("{} action not supported".format(req['input']['action']))
506
507     if result.body['output']['status']['code'] == 400:
508         print("Request Completed")
509     elif result.body['output']['status']['code'] == 100:
510         print("Request Accepted. Receiving result status...")
511 #    elif result.body['output']['status']['code'] == 311:
512 #        timestamp = result.body['output']['common-header']['timestamp']
513 #        _set_appc_lcm_timestamp(req, timestamp)
514 #        appc_lcm_request(onap_ip, req)
515 #        return
516     else:
517         raise Exception("{} - {}".format(result.body['output']['status']['code'],
518                                          result.body['output']['status']['message']))
519     #print(result)
520     return result.body['output']['status']['code']
521
522
523 def appc_lcm_status_request(onap_ip, req):
524     api = _init_python_appc_lcm_api(onap_ip)
525     status_body = _build_appc_lcm_status_body(req)
526     _set_appc_lcm_timestamp(status_body)
527
528     result = api.lcm.action_status(body=status_body, params={}, headers={})
529
530     if result.body['output']['status']['code'] == 400:
531         status = json.loads(result.body['output']['payload'])
532         return status
533     else:
534         raise Exception("{} - {}".format(result.body['output']['status']['code'],
535                                          result.body['output']['status']['message']))
536
537
538 def confirm_appc_lcm_action(onap_ip, req, check_appc_result):
539     print("Checking LCM {} Status".format(req['input']['action']))
540
541     while True:
542         time.sleep(2)
543         status = appc_lcm_status_request(onap_ip, req)
544         print(status['status'])
545         if status['status'] == 'SUCCESSFUL':
546             return
547         elif status['status'] == 'IN_PROGRESS':
548             continue
549         elif check_appc_result:
550             raise Exception("LCM {} {} - {}".format(req['input']['action'], status['status'], status['status-reason']))
551         else:
552             return
553
554
555 def execute_workflow(vfw_vnf_id, onap_ip, use_oof_cache, if_close_loop_vfw, info_only, check_result):
556     print("\nExecuting workflow for VNF ID '{}' on ONAP with IP {}".format(vfw_vnf_id, onap_ip))
557     print("\nOOF Cache {}, is CL vFW {}, only info {}, check LCM result {}".format(use_oof_cache, if_close_loop_vfw,
558                                                                                    info_only, check_result))
559     aai_data = load_aai_data(vfw_vnf_id, onap_ip)
560     print("\nvFWDT Service Information:")
561     print(json.dumps(aai_data, indent=4))
562     lcm_requests = build_appc_lcms_requests_body(onap_ip, aai_data, use_oof_cache, if_close_loop_vfw)
563     print("\nAnsible Inventory:")
564     for key in ansible_inventory:
565         print("[{}]".format(key))
566         for host in ansible_inventory[key]:
567             print(ansible_inventory[key][host])
568
569     if info_only:
570         return
571     print("\nDistribute Traffic Workflow Execution:")
572     for i in range(len(lcm_requests)):
573         req = lcm_requests[i]
574         print("APPC REQ {} - {}".format(i, req['input']['action']))
575         _set_appc_lcm_timestamp(req)
576         result = appc_lcm_request(onap_ip, req)
577         if result == 100:
578             confirm_appc_lcm_action(onap_ip, req, check_result)
579             #time.sleep(30)
580
581
582 #vnf_id, K8s node IP, use OOF cache, if close loop vfw, if info_only, if check APPC result
583 execute_workflow(sys.argv[1], sys.argv[2], sys.argv[3].lower() == 'true', sys.argv[4].lower() == 'true',
584                  sys.argv[5].lower() == 'true', sys.argv[6].lower() == 'true')