genericparser/pub/utils/toscaparsers/basemodel.py

   1 # Copyright 2017 ZTE Corporation.
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #         http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14
  15 import ftplib
  16 import json
  17 import logging
  18 import os
  19 import re
  20 import shutil
  21 import urllib
  22
  23 # import paramiko
  24 from toscaparser.tosca_template import ToscaTemplate
  25 from toscaparser.properties import Property
  26 from toscaparser.functions import Function, Concat, GetInput, get_function, function_mappings
  27 from genericparser.pub.utils.toscaparsers.graph import Graph
  28
  29 from genericparser.pub.utils.toscaparsers.dataentityext import DataEntityExt
  30
  31 logger = logging.getLogger(__name__)
  32
  33 METADATA = "metadata"
  34 PROPERTIES = "properties"
  35 DESCRIPTION = "description"
  36 REQUIREMENTS = "requirements"
  37 INTERFACES = "interfaces"
  38 TOPOLOGY_TEMPLATE = "topology_template"
  39 INPUTS = "inputs"
  40 CAPABILITIES = "capabilities"
  41 ATTRIBUTES = "attributes"
  42 ARTIFACTS = "artifacts"
  43 DERIVED_FROM = "derived_from"
  44
  45 NODE_NAME = "name"
  46 NODE_TYPE = "nodeType"
  47 NODE_ROOT = "tosca.nodes.Root"
  48 GROUP_TYPE = "groupType"
  49 GROUPS_ROOT = "tosca.groups.Root"
  50
  51
  52 class BaseInfoModel(object):
  53
  54     def __init__(self, path=None, params=None, tosca=None):
  55         if tosca:
  56             _tosca = tosca
  57         else:
  58             _tosca = self.buildToscaTemplate(path, params)
  59         self.description = getattr(_tosca, "description", "")
  60         self.parseModel(_tosca)
  61
  62     def parseModel(self, tosca):
  63         pass
  64
  65     def buildInputs(self, tosca):
  66         topo = tosca.tpl.get(TOPOLOGY_TEMPLATE, None)
  67         return topo.get(INPUTS, {}) if topo else {}
  68
  69     def buildToscaTemplate(self, path, params):
  70         file_name = None
  71         try:
  72             file_name = self._check_download_file(path)
  73             valid_params = self._validate_input_params(file_name, params)
  74             return self._create_tosca_template(file_name, valid_params)
  75         finally:
  76             if file_name is not None and file_name != path and os.path.exists(file_name):
  77                 try:
  78                     os.remove(file_name)
  79                 except Exception as e:
  80                     logger.error("Failed to parse package, error: %s", e.message)
  81
  82     def _validate_input_params(self, path, params):
  83         valid_params = {}
  84         inputs = {}
  85         if isinstance(params, list):
  86             for param in params:
  87                 key = param.get('key', 'undefined')
  88                 value = param.get('value', 'undefined')
  89                 inputs[key] = value
  90             params = inputs
  91
  92         if params:
  93             tmp = self._create_tosca_template(path, None)
  94             if isinstance(params, dict):
  95                 for key, value in params.items():
  96                     if hasattr(tmp, 'inputs') and len(tmp.inputs) > 0:
  97                         for input_def in tmp.inputs:
  98                             if (input_def.name == key):
  99                                 valid_params[key] = DataEntityExt.validate_datatype(input_def.type, value)
 100         return valid_params
 101
 102     def _create_tosca_template(self, file_name, valid_params):
 103         tosca_tpl = None
 104         try:
 105             tosca_tpl = ToscaTemplate(path=file_name,
 106                                       parsed_params=valid_params,
 107                                       no_required_paras_check=True,
 108                                       debug_mode=True)
 109         except Exception as e:
 110             print e.message
 111         finally:
 112             if tosca_tpl is not None and hasattr(tosca_tpl, "temp_dir") and os.path.exists(tosca_tpl.temp_dir):
 113                 try:
 114                     shutil.rmtree(tosca_tpl.temp_dir)
 115                 except Exception as e:
 116                     logger.error("Failed to create tosca template, error: %s", e.message)
 117                 print "-----------------------------"
 118                 print '\n'.join(['%s:%s' % item for item in tosca_tpl.__dict__.items()])
 119                 print "-----------------------------"
 120             return tosca_tpl
 121
 122     def _check_download_file(self, path):
 123         if (path.startswith("ftp") or path.startswith("sftp")):
 124             return self.downloadFileFromFtpServer(path)
 125         elif (path.startswith("http")):
 126             return self.download_file_from_httpserver(path)
 127         return path
 128
 129     def download_file_from_httpserver(self, path):
 130         path = path.encode("utf-8")
 131         tmps = str.split(path, '/')
 132         localFileName = tmps[len(tmps) - 1]
 133         urllib.urlretrieve(path, localFileName)
 134         return localFileName
 135
 136     def downloadFileFromFtpServer(self, path):
 137         path = path.encode("utf-8")
 138         tmp = str.split(path, '://')
 139         protocol = tmp[0]
 140         tmp = str.split(tmp[1], ':')
 141         if len(tmp) == 2:
 142             userName = tmp[0]
 143             tmp = str.split(tmp[1], '@')
 144             userPwd = tmp[0]
 145             index = tmp[1].index('/')
 146             hostIp = tmp[1][0:index]
 147             remoteFileName = tmp[1][index:len(tmp[1])]
 148             if protocol.lower() == 'ftp':
 149                 hostPort = 21
 150             else:
 151                 hostPort = 22
 152
 153         if len(tmp) == 3:
 154             userName = tmp[0]
 155             userPwd = str.split(tmp[1], '@')[0]
 156             hostIp = str.split(tmp[1], '@')[1]
 157             index = tmp[2].index('/')
 158             hostPort = tmp[2][0:index]
 159             remoteFileName = tmp[2][index:len(tmp[2])]
 160
 161         localFileName = str.split(remoteFileName, '/')
 162         localFileName = localFileName[len(localFileName) - 1]
 163
 164         if protocol.lower() == 'sftp':
 165             self.sftp_get(userName, userPwd, hostIp, hostPort, remoteFileName, localFileName)
 166         else:
 167             self.ftp_get(userName, userPwd, hostIp, hostPort, remoteFileName, localFileName)
 168         return localFileName
 169
 170     # def sftp_get(self, userName, userPwd, hostIp, hostPort, remoteFileName, localFileName):
 171     #     # return
 172     #     t = None
 173     #     try:
 174     #         t = paramiko.Transport(hostIp, int(hostPort))
 175     #         t.connect(username=userName, password=userPwd)
 176     #         sftp = paramiko.SFTPClient.from_transport(t)
 177     #         sftp.get(remoteFileName, localFileName)
 178     #     finally:
 179     #         if t is not None:
 180     #             t.close()
 181
 182     def ftp_get(self, userName, userPwd, hostIp, hostPort, remoteFileName, localFileName):
 183         f = None
 184         try:
 185             ftp = ftplib.FTP()
 186             ftp.connect(hostIp, hostPort)
 187             ftp.login(userName, userPwd)
 188             f = open(localFileName, 'wb')
 189             ftp.retrbinary('RETR ' + remoteFileName, f.write, 1024)
 190             f.close()
 191         finally:
 192             if f is not None:
 193                 f.close()
 194
 195     def buildMetadata(self, tosca):
 196         return tosca.tpl.get(METADATA, {}) if tosca else {}
 197
 198     def buildNode(self, nodeTemplate, tosca):
 199         inputs = tosca.inputs
 200         parsed_params = tosca.parsed_params
 201         ret = {}
 202         ret[NODE_NAME] = nodeTemplate.name
 203         ret[NODE_TYPE] = nodeTemplate.type
 204         if DESCRIPTION in nodeTemplate.entity_tpl:
 205             ret[DESCRIPTION] = nodeTemplate.entity_tpl[DESCRIPTION]
 206         else:
 207             ret[DESCRIPTION] = ''
 208         if METADATA in nodeTemplate.entity_tpl:
 209             ret[METADATA] = nodeTemplate.entity_tpl[METADATA]
 210         else:
 211             ret[METADATA] = ''
 212         props = self.buildProperties_ex(nodeTemplate, tosca.topology_template)
 213         ret[PROPERTIES] = self.verify_properties(props, inputs, parsed_params)
 214         ret[REQUIREMENTS] = self.build_requirements(nodeTemplate)
 215         self.buildCapabilities(nodeTemplate, inputs, ret)
 216         self.buildArtifacts(nodeTemplate, inputs, ret)
 217         interfaces = self.build_interfaces(nodeTemplate)
 218         if interfaces:
 219             ret[INTERFACES] = interfaces
 220         return ret
 221
 222     def buildProperties(self, nodeTemplate, parsed_params):
 223         properties = {}
 224         isMappingParams = parsed_params and len(parsed_params) > 0
 225         for k, item in nodeTemplate.get_properties().items():
 226             properties[k] = item.value
 227             if isinstance(item.value, GetInput):
 228                 if item.value.result() and isMappingParams:
 229                     properties[k] = DataEntityExt.validate_datatype(item.type, item.value.result())
 230                 else:
 231                     tmp = {}
 232                     tmp[item.value.name] = item.value.input_name
 233                     properties[k] = tmp
 234         if ATTRIBUTES in nodeTemplate.entity_tpl:
 235             for k, item in nodeTemplate.entity_tpl[ATTRIBUTES].items():
 236                 properties[k] = str(item)
 237         return properties
 238
 239     def buildProperties_ex(self, nodeTemplate, topology_template, properties=None):
 240         if properties is None:
 241             properties = nodeTemplate.get_properties()
 242         _properties = {}
 243         if isinstance(properties, dict):
 244             for name, prop in properties.items():
 245                 if isinstance(prop, Property):
 246                     if isinstance(prop.value, Function):
 247                         if isinstance(prop.value, Concat):  # support one layer inner function.
 248                             value_str = ''
 249                             for arg in prop.value.args:
 250                                 if isinstance(arg, str):
 251                                     value_str += arg
 252                                 elif isinstance(arg, dict):
 253                                     raw_func = {}
 254                                     for k, v in arg.items():
 255                                         func_args = []
 256                                         func_args.append(v)
 257                                         raw_func[k] = func_args
 258                                     func = get_function(topology_template, nodeTemplate, raw_func)
 259                                     value_str += str(func.result())
 260                             _properties[name] = value_str
 261                         else:
 262                             _properties[name] = prop.value.result()
 263                     elif isinstance(prop.value, dict) or isinstance(prop.value, list):
 264                         _properties[name] = self.buildProperties_ex(nodeTemplate, topology_template, prop.value)
 265                     elif prop.type == 'string':
 266                         _properties[name] = prop.value
 267                     else:
 268                         _properties[name] = json.dumps(prop.value)
 269                 elif isinstance(prop, dict):
 270                     _properties[name] = self.buildProperties_ex(nodeTemplate, topology_template, prop)
 271                 elif isinstance(prop, list):
 272                     _properties[name] = self.buildProperties_ex(nodeTemplate, topology_template, prop)
 273                 elif name in function_mappings:
 274                     raw_func = {}
 275                     func_args = []
 276                     func_args.append(prop)
 277                     raw_func[name] = func_args
 278                     if name == 'CONCAT':
 279                         value_str = ''
 280                         for arg in prop:
 281                             if isinstance(arg, str):
 282                                 value_str += arg
 283                             elif isinstance(arg, dict):
 284                                 raw_func = {}
 285                                 for k, v in arg.items():
 286                                     func_args = []
 287                                     func_args.append(v)
 288                                     raw_func[k] = func_args
 289                                 value_str += str(
 290                                     get_function(topology_template, nodeTemplate, raw_func).result())
 291                                 value = value_str
 292                     else:
 293                         return get_function(topology_template, nodeTemplate, raw_func).result()
 294                 else:
 295                     _properties[name] = prop
 296         elif isinstance(properties, list):
 297             value = []
 298             for para in properties:
 299                 if isinstance(para, dict) or isinstance(para, list):
 300                     value.append(self.buildProperties_ex(nodeTemplate, topology_template, para))
 301                 else:
 302                     value.append(para)
 303             return value
 304         return _properties
 305
 306     def verify_properties(self, props, inputs, parsed_params):
 307         ret_props = {}
 308         if (props and len(props) > 0):
 309             for key, value in props.items():
 310                 ret_props[key] = self._verify_value(value, inputs, parsed_params)
 311                 #                 if isinstance(value, str):
 312                 #                     ret_props[key] = self._verify_string(inputs, parsed_params, value);
 313                 #                     continue
 314                 #                 if isinstance(value, list):
 315                 #                     ret_props[key] = map(lambda x: self._verify_dict(inputs, parsed_params, x), value)
 316                 #                     continue
 317                 #                 if isinstance(value, dict):
 318                 #                     ret_props[key] = self._verify_map(inputs, parsed_params, value)
 319                 #                     continue
 320                 #                 ret_props[key] = value
 321         return ret_props
 322
 323     def build_requirements(self, node_template):
 324         rets = []
 325         for req in node_template.requirements:
 326             for req_name, req_value in req.items():
 327                 if (isinstance(req_value, dict)):
 328                     if ('node' in req_value and req_value['node'] not in node_template.templates):
 329                         continue  # No target requirement for aria parser, not add to result.
 330                 rets.append({req_name: req_value})
 331         return rets
 332
 333     def buildCapabilities(self, nodeTemplate, inputs, ret):
 334         capabilities = json.dumps(nodeTemplate.entity_tpl.get(CAPABILITIES, None))
 335         match = re.findall(r'\{"get_input":\s*"([\w|\-]+)"\}', capabilities)
 336         for m in match:
 337             aa = [input_def for input_def in inputs if m == input_def.name][0]
 338             capabilities = re.sub(r'\{"get_input":\s*"([\w|\-]+)"\}', json.dumps(aa.default), capabilities, 1)
 339         if capabilities != 'null':
 340             ret[CAPABILITIES] = json.loads(capabilities)
 341
 342     def buildArtifacts(self, nodeTemplate, inputs, ret):
 343         artifacts = json.dumps(nodeTemplate.entity_tpl.get('artifacts', None))
 344         match = re.findall(r'\{"get_input":\s*"([\w|\-]+)"\}', artifacts)
 345         for m in match:
 346             aa = [input_def for input_def in inputs if m == input_def.name][0]
 347             artifacts = re.sub(r'\{"get_input":\s*"([\w|\-]+)"\}', json.dumps(aa.default), artifacts, 1)
 348         if artifacts != 'null':
 349             ret[ARTIFACTS] = json.loads(artifacts)
 350
 351     def build_interfaces(self, node_template):
 352         if INTERFACES in node_template.entity_tpl:
 353             return node_template.entity_tpl[INTERFACES]
 354         return None
 355
 356     def isNodeTypeX(self, node, nodeTypes, x):
 357         node_type = node[NODE_TYPE]
 358         while node_type != x:
 359             node_type_derived = node_type
 360             node_type = nodeTypes[node_type][DERIVED_FROM]
 361             if node_type == NODE_ROOT or node_type == node_type_derived:
 362                 return False
 363         return True
 364
 365     def get_requirement_node_name(self, req_value):
 366         return self.get_prop_from_obj(req_value, 'node')
 367
 368     def getRequirementByNodeName(self, nodeTemplates, storage_name, prop):
 369         for node in nodeTemplates:
 370             if node[NODE_NAME] == storage_name:
 371                 if prop in node:
 372                     return node[prop]
 373
 374     def get_prop_from_obj(self, obj, prop):
 375         if isinstance(obj, str):
 376             return obj
 377         if (isinstance(obj, dict) and prop in obj):
 378             return obj[prop]
 379         return None
 380
 381     def getNodeDependencys(self, node):
 382         return self.getRequirementByName(node, 'dependency')
 383
 384     def getRequirementByName(self, node, requirementName):
 385         requirements = []
 386         if REQUIREMENTS in node:
 387             for item in node[REQUIREMENTS]:
 388                 for key, value in item.items():
 389                     if key == requirementName:
 390                         requirements.append(value)
 391         return requirements
 392
 393     def _verify_value(self, value, inputs, parsed_params):
 394         if value == '{}':
 395             return ''
 396         if isinstance(value, str):
 397             return self._verify_string(inputs, parsed_params, value)
 398         if isinstance(value, list) or isinstance(value, dict):
 399             return self._verify_object(value, inputs, parsed_params)
 400         return value
 401
 402     def _verify_object(self, value, inputs, parsed_params):
 403         s = self._verify_string(inputs, parsed_params, json.dumps(value))
 404         return json.loads(s)
 405
 406     def _get_input_name(self, getInput):
 407         input_name = getInput.split(':')[1]
 408         input_name = input_name.strip()
 409         return input_name.replace('"', '').replace('}', '')
 410
 411     def _verify_string(self, inputs, parsed_params, value):
 412         getInputs = re.findall(r'{"get_input": "[a-zA-Z_0-9]+"}', value)
 413         for getInput in getInputs:
 414             input_name = self._get_input_name(getInput)
 415             if parsed_params and input_name in parsed_params:
 416                 value = value.replace(getInput, json.dumps(parsed_params[input_name]))
 417             else:
 418                 for input_def in inputs:
 419                     if input_def.default and input_name == input_def.name:
 420                         value = value.replace(getInput, json.dumps(input_def.default))
 421         return value
 422
 423     def get_node_by_name(self, node_templates, name):
 424         for node in node_templates:
 425             if node[NODE_NAME] == name:
 426                 return node
 427         return None
 428
 429     def getCapabilityByName(self, node, capabilityName):
 430         if CAPABILITIES in node and capabilityName in node[CAPABILITIES]:
 431             return node[CAPABILITIES][capabilityName]
 432         return None
 433
 434     def get_base_path(self, tosca):
 435         fpath, fname = os.path.split(tosca.path)
 436         return fpath
 437
 438     def build_artifacts(self, node):
 439         rets = []
 440         if ARTIFACTS in node and len(node[ARTIFACTS]) > 0:
 441             artifacts = node[ARTIFACTS]
 442             for name, value in artifacts.items():
 443                 ret = {}
 444                 ret['artifact_name'] = name
 445                 ret['file'] = value
 446                 if isinstance(value, dict):
 447                     ret.update(value)
 448                 rets.append(ret)
 449         else:
 450             # TODO It is workaround for SDC-1900.
 451             logger.error("VCPE specific code")
 452             ret = {}
 453             ret['artifact_name'] = "sw_image"
 454             ret['file'] = "ubuntu_16.04"
 455             ret['type'] = "tosca.artifacts.nfv.SwImage"
 456             rets.append(ret)
 457
 458         return rets
 459
 460     def get_node_by_req(self, node_templates, req):
 461         req_node_name = self.get_requirement_node_name(req)
 462         return self.get_node_by_name(node_templates, req_node_name)
 463
 464     def isGroupTypeX(self, group, groupTypes, x):
 465         group_type = group[GROUP_TYPE]
 466         while group_type != x:
 467             group_type_derived = group_type
 468             group_type = groupTypes[group_type][DERIVED_FROM]
 469             if group_type == GROUPS_ROOT or group_type == group_type_derived:
 470                 return False
 471         return True
 472
 473     def setTargetValues(self, dict_target, target_keys, dict_source, source_keys):
 474         i = 0
 475         for item in source_keys:
 476             dict_target[target_keys[i]] = dict_source.get(item, "")
 477             i += 1
 478         return dict_target
 479
 480     def get_deploy_graph(self, tosca, relations):
 481         nodes = tosca.graph.nodetemplates
 482         graph = Graph()
 483         for node in nodes:
 484             self._build_deploy_path(node, [], graph, relations)
 485         return graph.to_dict()
 486
 487     def _build_deploy_path(self, node, node_parent, graph, relations):
 488         graph.add_node(node.name, node_parent)
 489         type_require_set = {}
 490         type_requires = node.type_definition.requirements
 491         for type_require in type_requires:
 492             type_require_set.update(type_require)
 493         for requirement in node.requirements:
 494             for k in requirement.keys():
 495                 if type_require_set[k].get('relationship', None) in relations[0] or type_require_set[k].get('capability', None) in relations[0]:
 496                     if isinstance(requirement[k], dict):
 497                         next_node = requirement[k].get('node', None)
 498                     else:
 499                         next_node = requirement[k]
 500                     graph.add_node(next_node, [node.name])
 501                 if type_require_set[k].get('relationship', None) in relations[1]:
 502                     if isinstance(requirement[k], dict):
 503                         next_node = requirement[k].get('node', None)
 504                     else:
 505                         next_node = requirement[k]
 506                     graph.add_node(next_node, [node.name])
 507
 508     def get_substitution_mappings(self, tosca):
 509         node = {
 510             'properties': {},
 511             'requirements': {},
 512             'capabilities': {},
 513             'metadata': {}
 514         }
 515         metadata = None
 516         substitution_mappings = tosca.tpl['topology_template'].get('substitution_mappings', None)
 517         if substitution_mappings:
 518             nodeType = substitution_mappings['node_type']
 519             logger.debug("nodeType %s", nodeType)
 520             if "node_types" in tosca.tpl:
 521                 node_types = tosca.tpl['node_types'].get(nodeType, None)
 522                 derivedFrom = node_types.get('derived_from', "")
 523                 node['type'] = derivedFrom
 524                 node['properties'] = node_types.get('properties', {})
 525                 node['requirements'] = node_types.get('requirements', {})
 526                 node['capabilities'] = node_types.get('capabilities', {})
 527                 metadata = node_types.get('metadata', {})
 528
 529             if "type" not in node or node['type'] == "":
 530                 node['type'] = nodeType
 531                 node['properties'] = substitution_mappings.get('properties', {})
 532                 node['requirements'] = substitution_mappings.get('requirements', {})
 533                 node['capabilities'] = substitution_mappings.get('capabilities', {})
 534                 metadata = substitution_mappings.get('metadata', {})
 535
 536         node['metadata'] = metadata if metadata and metadata != {} else self.buildMetadata(tosca)
 537         return node