1 # Copyright 2017 ZTE Corporation.
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
7 # http://www.apache.org/licenses/LICENSE-2.0
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
24 from toscaparser.tosca_template import ToscaTemplate
25 from toscaparser.properties import Property
26 from toscaparser.functions import Function, Concat, GetInput, get_function, function_mappings
27 from catalog.pub.utils.toscaparser.graph import Graph
29 from catalog.pub.utils.toscaparser.dataentityext import DataEntityExt
31 logger = logging.getLogger(__name__)
34 PROPERTIES = "properties"
35 DESCRIPTION = "description"
36 REQUIREMENTS = "requirements"
37 INTERFACES = "interfaces"
38 TOPOLOGY_TEMPLATE = "topology_template"
40 CAPABILITIES = "capabilities"
41 ATTRIBUTES = "attributes"
42 ARTIFACTS = "artifacts"
43 DERIVED_FROM = "derived_from"
46 NODE_TYPE = "nodeType"
47 NODE_ROOT = "tosca.nodes.Root"
48 GROUP_TYPE = "groupType"
49 GROUPS_ROOT = "tosca.groups.Root"
52 class BaseInfoModel(object):
54 def __init__(self, path=None, params=None, tosca=None):
58 _tosca = self.buildToscaTemplate(path, params)
59 self.description = getattr(_tosca, "description", "")
60 self.parseModel(_tosca)
62 def parseModel(self, tosca):
65 def buildInputs(self, tosca):
66 topo = tosca.tpl.get(TOPOLOGY_TEMPLATE, None)
67 return topo.get(INPUTS, {}) if topo else {}
69 def buildToscaTemplate(self, path, params):
72 file_name = self._check_download_file(path)
73 valid_params = self._validate_input_params(file_name, params)
74 return self._create_tosca_template(file_name, valid_params)
76 if file_name is not None and file_name != path and os.path.exists(file_name):
79 except Exception as e:
80 logger.error("Failed to parse package, error: %s", e.args[0])
82 def _validate_input_params(self, path, params):
85 if isinstance(params, list):
87 key = param.get('key', 'undefined')
88 value = param.get('value', 'undefined')
93 tmp = self._create_tosca_template(path, None)
94 if isinstance(params, dict):
95 for key, value in list(params.items()):
96 if hasattr(tmp, 'inputs') and len(tmp.inputs) > 0:
97 for input_def in tmp.inputs:
98 if (input_def.name == key):
99 valid_params[key] = DataEntityExt.validate_datatype(input_def.type, value)
102 def _create_tosca_template(self, file_name, valid_params):
105 tosca_tpl = ToscaTemplate(path=file_name,
106 parsed_params=valid_params,
107 no_required_paras_check=True,
109 except Exception as e:
112 if tosca_tpl is not None and hasattr(tosca_tpl, "temp_dir") and os.path.exists(tosca_tpl.temp_dir):
114 shutil.rmtree(tosca_tpl.temp_dir)
115 except Exception as e:
116 logger.error("Failed to create tosca template, error: %s", e.args[0])
117 print("-----------------------------")
118 print('\n'.join(['%s:%s' % item for item in list(tosca_tpl.__dict__.items())]))
119 print("-----------------------------")
122 def _check_download_file(self, path):
123 if (path.startswith("ftp") or path.startswith("sftp")):
124 return self.downloadFileFromFtpServer(path)
125 elif (path.startswith("http")):
126 return self.download_file_from_httpserver(path)
129 def download_file_from_httpserver(self, path):
130 path = path.encode("utf-8")
131 tmps = str.split(path, '/')
132 localFileName = tmps[len(tmps) - 1]
133 urllib.request.urlretrieve(path, localFileName)
136 def downloadFileFromFtpServer(self, path):
137 path = path.encode("utf-8")
138 tmp = str.split(path, '://')
140 tmp = str.split(tmp[1], ':')
143 tmp = str.split(tmp[1], '@')
145 index = tmp[1].index('/')
146 hostIp = tmp[1][0:index]
147 remoteFileName = tmp[1][index:len(tmp[1])]
148 if protocol.lower() == 'ftp':
155 userPwd = str.split(tmp[1], '@')[0]
156 hostIp = str.split(tmp[1], '@')[1]
157 index = tmp[2].index('/')
158 hostPort = tmp[2][0:index]
159 remoteFileName = tmp[2][index:len(tmp[2])]
161 localFileName = str.split(remoteFileName, '/')
162 localFileName = localFileName[len(localFileName) - 1]
164 if protocol.lower() == 'sftp':
165 self.sftp_get(userName, userPwd, hostIp, hostPort, remoteFileName, localFileName)
167 self.ftp_get(userName, userPwd, hostIp, hostPort, remoteFileName, localFileName)
170 # def sftp_get(self, userName, userPwd, hostIp, hostPort, remoteFileName, localFileName):
174 # t = paramiko.Transport(hostIp, int(hostPort))
175 # t.connect(username=userName, password=userPwd)
176 # sftp = paramiko.SFTPClient.from_transport(t)
177 # sftp.get(remoteFileName, localFileName)
182 def ftp_get(self, userName, userPwd, hostIp, hostPort, remoteFileName, localFileName):
186 ftp.connect(hostIp, hostPort)
187 ftp.login(userName, userPwd)
188 f = open(localFileName, 'wb')
189 ftp.retrbinary('RETR ' + remoteFileName, f.write, 1024)
195 def buildMetadata(self, tosca):
196 return tosca.tpl.get(METADATA, {}) if tosca else {}
198 def buildNode(self, nodeTemplate, tosca):
199 inputs = tosca.inputs
200 parsed_params = tosca.parsed_params
202 ret[NODE_NAME] = nodeTemplate.name
203 ret[NODE_TYPE] = nodeTemplate.type
204 if DESCRIPTION in nodeTemplate.entity_tpl:
205 ret[DESCRIPTION] = nodeTemplate.entity_tpl[DESCRIPTION]
207 ret[DESCRIPTION] = ''
208 if METADATA in nodeTemplate.entity_tpl:
209 ret[METADATA] = nodeTemplate.entity_tpl[METADATA]
212 props = self.buildProperties_ex(nodeTemplate, tosca.topology_template)
213 ret[PROPERTIES] = self.verify_properties(props, inputs, parsed_params)
214 ret[REQUIREMENTS] = self.build_requirements(nodeTemplate)
215 self.buildCapabilities(nodeTemplate, inputs, ret)
216 self.buildArtifacts(nodeTemplate, inputs, ret)
217 interfaces = self.build_interfaces(nodeTemplate)
219 ret[INTERFACES] = interfaces
222 def buildProperties(self, nodeTemplate, parsed_params):
224 isMappingParams = parsed_params and len(parsed_params) > 0
225 for k, item in list(nodeTemplate.get_properties().items()):
226 properties[k] = item.value
227 if isinstance(item.value, GetInput):
228 if item.value.result() and isMappingParams:
229 properties[k] = DataEntityExt.validate_datatype(item.type, item.value.result())
232 tmp[item.value.name] = item.value.input_name
234 if ATTRIBUTES in nodeTemplate.entity_tpl:
235 for k, item in list(nodeTemplate.entity_tpl[ATTRIBUTES].items()):
236 properties[k] = str(item)
239 def buildProperties_ex(self, nodeTemplate, topology_template, properties=None):
240 if properties is None:
241 properties = nodeTemplate.get_properties()
243 if isinstance(properties, dict):
244 for name, prop in list(properties.items()):
245 if isinstance(prop, Property):
246 if isinstance(prop.value, Function):
247 if isinstance(prop.value, Concat): # support one layer inner function.
249 for arg in prop.value.args:
250 if isinstance(arg, str):
252 elif isinstance(arg, dict):
254 for k, v in list(arg.items()):
257 raw_func[k] = func_args
258 func = get_function(topology_template, nodeTemplate, raw_func)
259 value_str += str(func.result())
260 _properties[name] = value_str
262 _properties[name] = prop.value.result()
263 elif isinstance(prop.value, dict) or isinstance(prop.value, list):
264 _properties[name] = self.buildProperties_ex(nodeTemplate, topology_template, prop.value)
265 elif prop.type == 'string':
266 _properties[name] = prop.value
268 _properties[name] = json.dumps(prop.value)
269 elif isinstance(prop, dict):
270 _properties[name] = self.buildProperties_ex(nodeTemplate, topology_template, prop)
271 elif isinstance(prop, list):
272 _properties[name] = self.buildProperties_ex(nodeTemplate, topology_template, prop)
273 elif name in function_mappings:
276 func_args.append(prop)
277 raw_func[name] = func_args
281 if isinstance(arg, str):
283 elif isinstance(arg, dict):
285 for k, v in list(arg.items()):
288 raw_func[k] = func_args
290 get_function(topology_template, nodeTemplate, raw_func).result())
293 return get_function(topology_template, nodeTemplate, raw_func).result()
295 _properties[name] = prop
296 elif isinstance(properties, list):
298 for para in properties:
299 if isinstance(para, dict) or isinstance(para, list):
300 value.append(self.buildProperties_ex(nodeTemplate, topology_template, para))
306 def verify_properties(self, props, inputs, parsed_params):
308 if (props and len(props) > 0):
309 for key, value in list(props.items()):
310 ret_props[key] = self._verify_value(value, inputs, parsed_params)
311 # if isinstance(value, str):
312 # ret_props[key] = self._verify_string(inputs, parsed_params, value);
314 # if isinstance(value, list):
315 # ret_props[key] = map(lambda x: self._verify_dict(inputs, parsed_params, x), value)
317 # if isinstance(value, dict):
318 # ret_props[key] = self._verify_map(inputs, parsed_params, value)
320 # ret_props[key] = value
323 def build_requirements(self, node_template):
325 for req in node_template.requirements:
326 for req_name, req_value in list(req.items()):
327 if (isinstance(req_value, dict)):
328 if ('node' in req_value and req_value['node'] not in node_template.templates):
329 continue # No target requirement for aria parser, not add to result.
330 rets.append({req_name: req_value})
333 def buildCapabilities(self, nodeTemplate, inputs, ret):
334 capabilities = json.dumps(nodeTemplate.entity_tpl.get(CAPABILITIES, None))
335 match = re.findall(r'\{"get_input":\s*"([\w|\-]+)"\}', capabilities)
337 aa = [input_def for input_def in inputs if m == input_def.name][0]
338 capabilities = re.sub(r'\{"get_input":\s*"([\w|\-]+)"\}', json.dumps(aa.default), capabilities, 1)
339 if capabilities != 'null':
340 ret[CAPABILITIES] = json.loads(capabilities)
342 def buildArtifacts(self, nodeTemplate, inputs, ret):
343 artifacts = json.dumps(nodeTemplate.entity_tpl.get('artifacts', None))
344 match = re.findall(r'\{"get_input":\s*"([\w|\-]+)"\}', artifacts)
346 aa = [input_def for input_def in inputs if m == input_def.name][0]
347 artifacts = re.sub(r'\{"get_input":\s*"([\w|\-]+)"\}', json.dumps(aa.default), artifacts, 1)
348 if artifacts != 'null':
349 ret[ARTIFACTS] = json.loads(artifacts)
351 def build_interfaces(self, node_template):
352 if INTERFACES in node_template.entity_tpl:
353 return node_template.entity_tpl[INTERFACES]
356 def isNodeTypeX(self, node, nodeTypes, x):
357 node_type = node[NODE_TYPE]
358 while node_type != x:
359 node_type_derived = node_type
360 node_type = nodeTypes[node_type][DERIVED_FROM]
361 if node_type == NODE_ROOT or node_type == node_type_derived:
365 def get_requirement_node_name(self, req_value):
366 return self.get_prop_from_obj(req_value, 'node')
368 def getRequirementByNodeName(self, nodeTemplates, storage_name, prop):
369 for node in nodeTemplates:
370 if node[NODE_NAME] == storage_name:
374 def get_prop_from_obj(self, obj, prop):
375 if isinstance(obj, str):
377 if (isinstance(obj, dict) and prop in obj):
381 def getNodeDependencys(self, node):
382 return self.getRequirementByName(node, 'dependency')
384 def getRequirementByName(self, node, requirementName):
386 if REQUIREMENTS in node:
387 for item in node[REQUIREMENTS]:
388 for key, value in list(item.items()):
389 if key == requirementName:
390 requirements.append(value)
393 def _verify_value(self, value, inputs, parsed_params):
396 if isinstance(value, str):
397 return self._verify_string(inputs, parsed_params, value)
398 if isinstance(value, list) or isinstance(value, dict):
399 return self._verify_object(value, inputs, parsed_params)
402 def _verify_object(self, value, inputs, parsed_params):
403 s = self._verify_string(inputs, parsed_params, json.dumps(value))
406 def _get_input_name(self, getInput):
407 input_name = getInput.split(':')[1]
408 input_name = input_name.strip()
409 return input_name.replace('"', '').replace('}', '')
411 def _verify_string(self, inputs, parsed_params, value):
412 getInputs = re.findall(r'{"get_input": "[a-zA-Z_0-9]+"}', value)
413 for getInput in getInputs:
414 input_name = self._get_input_name(getInput)
415 if parsed_params and input_name in parsed_params:
416 value = value.replace(getInput, json.dumps(parsed_params[input_name]))
418 for input_def in inputs:
419 if input_def.default and input_name == input_def.name:
420 value = value.replace(getInput, json.dumps(input_def.default))
423 def get_node_by_name(self, node_templates, name):
424 for node in node_templates:
425 if node[NODE_NAME] == name:
429 def getCapabilityByName(self, node, capabilityName):
430 if CAPABILITIES in node and capabilityName in node[CAPABILITIES]:
431 return node[CAPABILITIES][capabilityName]
434 def get_base_path(self, tosca):
435 fpath, fname = os.path.split(tosca.path)
438 def build_artifacts(self, node):
440 if ARTIFACTS in node and len(node[ARTIFACTS]) > 0:
441 artifacts = node[ARTIFACTS]
442 for name, value in list(artifacts.items()):
444 ret['artifact_name'] = name
446 if isinstance(value, dict):
450 # TODO It is workaround for SDC-1900.
451 logger.error("VCPE specific code")
453 ret['artifact_name'] = "sw_image"
454 ret['file'] = "ubuntu_16.04"
455 ret['type'] = "tosca.artifacts.nfv.SwImage"
460 def get_node_by_req(self, node_templates, req):
461 req_node_name = self.get_requirement_node_name(req)
462 return self.get_node_by_name(node_templates, req_node_name)
464 def isGroupTypeX(self, group, groupTypes, x):
465 group_type = group[GROUP_TYPE]
466 while group_type != x:
467 group_type_derived = group_type
468 group_type = groupTypes[group_type][DERIVED_FROM]
469 if group_type == GROUPS_ROOT or group_type == group_type_derived:
473 def setTargetValues(self, dict_target, target_keys, dict_source, source_keys):
475 for item in source_keys:
476 dict_target[target_keys[i]] = dict_source.get(item, "")
480 def get_deploy_graph(self, tosca, relations):
481 nodes = tosca.graph.nodetemplates
484 self._build_deploy_path(node, [], graph, relations)
485 return graph.to_dict()
487 def _build_deploy_path(self, node, node_parent, graph, relations):
488 graph.add_node(node.name, node_parent)
489 type_require_set = {}
490 type_requires = node.type_definition.requirements
491 for type_require in type_requires:
492 type_require_set.update(type_require)
493 for requirement in node.requirements:
494 for k in list(requirement.keys()):
495 if type_require_set[k].get('relationship', None) in relations[0] or type_require_set[k].get('capability', None) in relations[0]:
496 if isinstance(requirement[k], dict):
497 next_node = requirement[k].get('node', None)
499 next_node = requirement[k]
500 graph.add_node(next_node, [node.name])
501 if type_require_set[k].get('relationship', None) in relations[1]:
502 if isinstance(requirement[k], dict):
503 next_node = requirement[k].get('node', None)
505 next_node = requirement[k]
506 graph.add_node(next_node, [node.name])
508 def get_substitution_mappings(self, tosca):
516 substitution_mappings = tosca.tpl['topology_template'].get('substitution_mappings', None)
517 if substitution_mappings:
518 nodeType = substitution_mappings['node_type']
519 logger.debug("nodeType %s", nodeType)
520 if "type" not in node or node['type'] == "":
521 node['type'] = nodeType
522 node['properties'] = substitution_mappings.get('properties', {})
523 node['requirements'] = substitution_mappings.get('requirements', {})
524 node['capabilities'] = substitution_mappings.get('capabilities', {})
525 metadata = substitution_mappings.get('metadata', {})
527 if "node_types" in tosca.tpl:
528 node_types = tosca.tpl['node_types'].get(nodeType, None)
529 derivedFrom = node_types.get('derived_from', "")
530 node['type'] = derivedFrom
531 node['properties'] = node_types.get('properties', {})
533 node['metadata'] = metadata if metadata and metadata != {} else self.buildMetadata(tosca)