genericparser/pub/utils/toscaparsers/basemodel.py

   1 # Copyright 2017 ZTE Corporation.
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #         http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14
  15 import ftplib
  16 import json
  17 import logging
  18 import os
  19 import re
  20 import shutil
  21 import urllib
  22
  23 import paramiko
  24 from toscaparser.tosca_template import ToscaTemplate
  25 from toscaparser.properties import Property
  26 from toscaparser.functions import Function, Concat, GetInput, get_function, function_mappings
  27 from genericparser.pub.utils.toscaparsers.graph import Graph
  28
  29 from genericparser.pub.utils.toscaparsers.dataentityext import DataEntityExt
  30
  31 logger = logging.getLogger(__name__)
  32
  33 METADATA = "metadata"
  34 PROPERTIES = "properties"
  35 DESCRIPTION = "description"
  36 REQUIREMENTS = "requirements"
  37 INTERFACES = "interfaces"
  38 TOPOLOGY_TEMPLATE = "topology_template"
  39 INPUTS = "inputs"
  40 CAPABILITIES = "capabilities"
  41 ATTRIBUTES = "attributes"
  42 ARTIFACTS = "artifacts"
  43 DERIVED_FROM = "derived_from"
  44
  45 NODE_NAME = "name"
  46 NODE_TYPE = "nodeType"
  47 NODE_ROOT = "tosca.nodes.Root"
  48 GROUP_TYPE = "groupType"
  49 GROUPS_ROOT = "tosca.groups.Root"
  50
  51
  52 class BaseInfoModel(object):
  53
  54     def __init__(self, path=None, params=None, tosca=None):
  55         if tosca:
  56             _tosca = tosca
  57         else:
  58             _tosca = self.buildToscaTemplate(path, params)
  59         self.description = getattr(_tosca, "description", "")
  60         self.parseModel(_tosca)
  61
  62     def parseModel(self, tosca):
  63         pass
  64
  65     def buildInputs(self, tosca):
  66         topo = tosca.tpl.get(TOPOLOGY_TEMPLATE, None)
  67         return topo.get(INPUTS, {}) if topo else {}
  68
  69     def buildToscaTemplate(self, path, params):
  70         file_name = None
  71         try:
  72             file_name = self._check_download_file(path)
  73             valid_params = self._validate_input_params(file_name, params)
  74             return self._create_tosca_template(file_name, valid_params)
  75         finally:
  76             if file_name is not None and file_name != path and os.path.exists(file_name):
  77                 try:
  78                     os.remove(file_name)
  79                 except Exception as e:
  80                     logger.error("Failed to parse package, error: %s", e.message)
  81
  82     def _validate_input_params(self, path, params):
  83         valid_params = {}
  84         inputs = {}
  85         if isinstance(params, list):
  86             for param in params:
  87                 key = param.get('key', 'undefined')
  88                 value = param.get('value', 'undefined')
  89                 inputs[key] = value
  90             params = inputs
  91
  92         if params:
  93             tmp = self._create_tosca_template(path, None)
  94             if isinstance(params, dict):
  95                 for key, value in params.items():
  96                     if hasattr(tmp, 'inputs') and len(tmp.inputs) > 0:
  97                         for input_def in tmp.inputs:
  98                             if (input_def.name == key):
  99                                 valid_params[key] = DataEntityExt.validate_datatype(input_def.type, value)
 100         return valid_params
 101
 102     def _create_tosca_template(self, file_name, valid_params):
 103         tosca_tpl = None
 104         try:
 105             tosca_tpl = ToscaTemplate(path=file_name,
 106                                       parsed_params=valid_params,
 107                                       no_required_paras_check=True,
 108                                       debug_mode=True)
 109         except Exception as e:
 110             print e.message
 111         finally:
 112             if tosca_tpl is not None and hasattr(tosca_tpl, "temp_dir") and os.path.exists(tosca_tpl.temp_dir):
 113                 try:
 114                     shutil.rmtree(tosca_tpl.temp_dir)
 115                 except Exception as e:
 116                     logger.error("Failed to create tosca template, error: %s", e.message)
 117                 print "-----------------------------"
 118                 print '\n'.join(['%s:%s' % item for item in tosca_tpl.__dict__.items()])
 119                 print "-----------------------------"
 120             return tosca_tpl
 121
 122     def _check_download_file(self, path):
 123         if (path.startswith("ftp") or path.startswith("sftp")):
 124             return self.downloadFileFromFtpServer(path)
 125         elif (path.startswith("http")):
 126             return self.download_file_from_httpserver(path)
 127         return path
 128
 129     def download_file_from_httpserver(self, path):
 130         path = path.encode("utf-8")
 131         tmps = str.split(path, '/')
 132         localFileName = tmps[len(tmps) - 1]
 133         urllib.urlretrieve(path, localFileName)
 134         return localFileName
 135
 136     def downloadFileFromFtpServer(self, path):
 137         path = path.encode("utf-8")
 138         tmp = str.split(path, '://')
 139         protocol = tmp[0]
 140         tmp = str.split(tmp[1], ':')
 141         if len(tmp) == 2:
 142             userName = tmp[0]
 143             tmp = str.split(tmp[1], '@')
 144             userPwd = tmp[0]
 145             index = tmp[1].index('/')
 146             hostIp = tmp[1][0:index]
 147             remoteFileName = tmp[1][index:len(tmp[1])]
 148             if protocol.lower() == 'ftp':
 149                 hostPort = 21
 150             else:
 151                 hostPort = 22
 152
 153         if len(tmp) == 3:
 154             userName = tmp[0]
 155             userPwd = str.split(tmp[1], '@')[0]
 156             hostIp = str.split(tmp[1], '@')[1]
 157             index = tmp[2].index('/')
 158             hostPort = tmp[2][0:index]
 159             remoteFileName = tmp[2][index:len(tmp[2])]
 160
 161         localFileName = str.split(remoteFileName, '/')
 162         localFileName = localFileName[len(localFileName) - 1]
 163
 164         if protocol.lower() == 'sftp':
 165             self.sftp_get(userName, userPwd, hostIp, hostPort, remoteFileName, localFileName)
 166         else:
 167             self.ftp_get(userName, userPwd, hostIp, hostPort, remoteFileName, localFileName)
 168         return localFileName
 169
 170     def sftp_get(self, userName, userPwd, hostIp, hostPort, remoteFileName, localFileName):
 171         # return
 172         t = None
 173         try:
 174             t = paramiko.Transport(hostIp, int(hostPort))
 175             t.connect(username=userName, password=userPwd)
 176             sftp = paramiko.SFTPClient.from_transport(t)
 177             sftp.get(remoteFileName, localFileName)
 178         finally:
 179             if t is not None:
 180                 t.close()
 181
 182     def ftp_get(self, userName, userPwd, hostIp, hostPort, remoteFileName, localFileName):
 183         f = None
 184         try:
 185             ftp = ftplib.FTP()
 186             ftp.connect(hostIp, hostPort)
 187             ftp.login(userName, userPwd)
 188             f = open(localFileName, 'wb')
 189             ftp.retrbinary('RETR ' + remoteFileName, f.write, 1024)
 190             f.close()
 191         finally:
 192             if f is not None:
 193                 f.close()
 194
 195     def buildMetadata(self, tosca):
 196         return tosca.tpl.get(METADATA, {}) if tosca else {}
 197
 198     def buildNode(self, nodeTemplate, tosca):
 199         inputs = tosca.inputs
 200         parsed_params = tosca.parsed_params
 201         ret = {}
 202         ret[NODE_NAME] = nodeTemplate.name
 203         ret[NODE_TYPE] = nodeTemplate.type
 204         if DESCRIPTION in nodeTemplate.entity_tpl:
 205             ret[DESCRIPTION] = nodeTemplate.entity_tpl[DESCRIPTION]
 206         else:
 207             ret[DESCRIPTION] = ''
 208         if METADATA in nodeTemplate.entity_tpl:
 209             ret[METADATA] = nodeTemplate.entity_tpl[METADATA]
 210         else:
 211             ret[METADATA] = ''
 212         props = self.buildProperties_ex(nodeTemplate, tosca.topology_template)
 213         ret[PROPERTIES] = self.verify_properties(props, inputs, parsed_params)
 214         ret[REQUIREMENTS] = self.build_requirements(nodeTemplate)
 215         self.buildCapabilities(nodeTemplate, inputs, ret)
 216         self.buildArtifacts(nodeTemplate, inputs, ret)
 217         interfaces = self.build_interfaces(nodeTemplate)
 218         if interfaces:
 219             ret[INTERFACES] = interfaces
 220         return ret
 221
 222     def buildProperties(self, nodeTemplate, parsed_params):
 223         properties = {}
 224         isMappingParams = parsed_params and len(parsed_params) > 0
 225         for k, item in nodeTemplate.get_properties().items():
 226             properties[k] = item.value
 227             if isinstance(item.value, GetInput):
 228                 if item.value.result() and isMappingParams:
 229                     properties[k] = DataEntityExt.validate_datatype(item.type, item.value.result())
 230                 else:
 231                     tmp = {}
 232                     tmp[item.value.name] = item.value.input_name
 233                     properties[k] = tmp
 234         if ATTRIBUTES in nodeTemplate.entity_tpl:
 235             for k, item in nodeTemplate.entity_tpl[ATTRIBUTES].items():
 236                 properties[k] = str(item)
 237         return properties
 238
 239     def buildProperties_ex(self, nodeTemplate, topology_template, properties=None):
 240         if properties is None:
 241             properties = nodeTemplate.get_properties()
 242         _properties = {}
 243         if isinstance(properties, dict):
 244             for name, prop in properties.items():
 245                 if isinstance(prop, Property):
 246                     if isinstance(prop.value, Function):
 247                         if isinstance(prop.value, Concat):  # support one layer inner function.
 248                             value_str = ''
 249                             for arg in prop.value.args:
 250                                 if isinstance(arg, str):
 251                                     value_str += arg
 252                                 elif isinstance(arg, dict):
 253                                     raw_func = {}
 254                                     for k, v in arg.items():
 255                                         func_args = []
 256                                         func_args.append(v)
 257                                         raw_func[k] = func_args
 258                                     func = get_function(topology_template, nodeTemplate, raw_func)
 259                                     value_str += str(func.result())
 260                             _properties[name] = value_str
 261                         else:
 262                             _properties[name] = prop.value.result()
 263                     elif isinstance(prop.value, dict) or isinstance(prop.value, list):
 264                         _properties[name] = self.buildProperties_ex(nodeTemplate, topology_template, prop.value)
 265                     elif prop.type == 'string':
 266                         _properties[name] = prop.value
 267                     else:
 268                         _properties[name] = json.dumps(prop.value)
 269                 elif isinstance(prop, dict):
 270                     _properties[name] = self.buildProperties_ex(nodeTemplate, topology_template, prop)
 271                 elif isinstance(prop, list):
 272                     _properties[name] = self.buildProperties_ex(nodeTemplate, topology_template, prop)
 273                 elif name in function_mappings:
 274                     raw_func = {}
 275                     func_args = []
 276                     func_args.append(prop)
 277                     raw_func[name] = func_args
 278                     if name == 'CONCAT':
 279                         value_str = ''
 280                         for arg in prop:
 281                             if isinstance(arg, str):
 282                                 value_str += arg
 283                             elif isinstance(arg, dict):
 284                                 raw_func = {}
 285                                 for k, v in arg.items():
 286                                     func_args = []
 287                                     func_args.append(v)
 288                                     raw_func[k] = func_args
 289                                 value_str += str(
 290                                     get_function(topology_template, nodeTemplate, raw_func).result())
 291                                 value = value_str
 292                     else:
 293                         return get_function(topology_template, nodeTemplate, raw_func).result()
 294                 else:
 295                     _properties[name] = prop
 296         elif isinstance(properties, list):
 297             value = []
 298             for para in properties:
 299                 if isinstance(para, dict) or isinstance(para, list):
 300                     value.append(self.buildProperties_ex(nodeTemplate, topology_template, para))
 301                 else:
 302                     value.append(para)
 303             return value
 304         return _properties
 305
 306     def verify_properties(self, props, inputs, parsed_params):
 307         ret_props = {}
 308         if (props and len(props) > 0):
 309             for key, value in props.items():
 310                 ret_props[key] = self._verify_value(value, inputs, parsed_params)
 311                 #                 if isinstance(value, str):
 312                 #                     ret_props[key] = self._verify_string(inputs, parsed_params, value);
 313                 #                     continue
 314                 #                 if isinstance(value, list):
 315                 #                     ret_props[key] = map(lambda x: self._verify_dict(inputs, parsed_params, x), value)
 316                 #                     continue
 317                 #                 if isinstance(value, dict):
 318                 #                     ret_props[key] = self._verify_map(inputs, parsed_params, value)
 319                 #                     continue
 320                 #                 ret_props[key] = value
 321         return ret_props
 322
 323     def build_requirements(self, node_template):
 324         rets = []
 325         for req in node_template.requirements:
 326             for req_name, req_value in req.items():
 327                 if (isinstance(req_value, dict)):
 328                     if ('node' in req_value and req_value['node'] not in node_template.templates):
 329                         continue  # No target requirement for aria parser, not add to result.
 330                 rets.append({req_name: req_value})
 331         return rets
 332
 333     def buildCapabilities(self, nodeTemplate, inputs, ret):
 334         capabilities = json.dumps(nodeTemplate.entity_tpl.get(CAPABILITIES, None))
 335         match = re.findall(r'\{"get_input":\s*"([\w|\-]+)"\}', capabilities)
 336         for m in match:
 337             aa = [input_def for input_def in inputs if m == input_def.name][0]
 338             capabilities = re.sub(r'\{"get_input":\s*"([\w|\-]+)"\}', json.dumps(aa.default), capabilities, 1)
 339         if capabilities != 'null':
 340             ret[CAPABILITIES] = json.loads(capabilities)
 341
 342     def buildArtifacts(self, nodeTemplate, inputs, ret):
 343         artifacts = json.dumps(nodeTemplate.entity_tpl.get('artifacts', None))
 344         match = re.findall(r'\{"get_input":\s*"([\w|\-]+)"\}', artifacts)
 345         for m in match:
 346             aa = [input_def for input_def in inputs if m == input_def.name][0]
 347             artifacts = re.sub(r'\{"get_input":\s*"([\w|\-]+)"\}', json.dumps(aa.default), artifacts, 1)
 348         if artifacts != 'null':
 349             ret[ARTIFACTS] = json.loads(artifacts)
 350
 351     def build_interfaces(self, node_template):
 352         if INTERFACES in node_template.entity_tpl:
 353             return node_template.entity_tpl[INTERFACES]
 354         return None
 355
 356     def isNodeTypeX(self, node, nodeTypes, x):
 357         node_type = node[NODE_TYPE]
 358         while node_type != x:
 359             node_type_derived = node_type
 360             node_type = nodeTypes[node_type][DERIVED_FROM]
 361             if node_type == NODE_ROOT or node_type == node_type_derived:
 362                 return False
 363         return True
 364
 365     def get_requirement_node_name(self, req_value):
 366         return self.get_prop_from_obj(req_value, 'node')
 367
 368     def getRequirementByNodeName(self, nodeTemplates, storage_name, prop):
 369         for node in nodeTemplates:
 370             if node[NODE_NAME] == storage_name:
 371                 if prop in node:
 372                     return node[prop]
 373
 374     def get_prop_from_obj(self, obj, prop):
 375         if isinstance(obj, str):
 376             return obj
 377         if (isinstance(obj, dict) and prop in obj):
 378             return obj[prop]
 379         return None
 380
 381     def getNodeDependencys(self, node):
 382         return self.getRequirementByName(node, 'dependency')
 383
 384     def getRequirementByName(self, node, requirementName):
 385         requirements = []
 386         if REQUIREMENTS in node:
 387             for item in node[REQUIREMENTS]:
 388                 for key, value in item.items():
 389                     if key == requirementName:
 390                         requirements.append(value)
 391         return requirements
 392
 393     def _verify_value(self, value, inputs, parsed_params):
 394         if value == '{}':
 395             return ''
 396         if isinstance(value, str):
 397             return self._verify_string(inputs, parsed_params, value)
 398         if isinstance(value, list) or isinstance(value, dict):
 399             return self._verify_object(value, inputs, parsed_params)
 400         return value
 401
 402     def _verify_object(self, value, inputs, parsed_params):
 403         s = self._verify_string(inputs, parsed_params, json.dumps(value))
 404         return json.loads(s)
 405
 406     def _get_input_name(self, getInput):
 407         input_name = getInput.split(':')[1]
 408         input_name = input_name.strip()
 409         return input_name.replace('"', '').replace('}', '')
 410
 411     def _verify_string(self, inputs, parsed_params, value):
 412         getInputs = re.findall(r'{"get_input": "[a-zA-Z_0-9]+"}', value)
 413         for getInput in getInputs:
 414             input_name = self._get_input_name(getInput)
 415             if parsed_params and input_name in parsed_params:
 416                 value = value.replace(getInput, json.dumps(parsed_params[input_name]))
 417             else:
 418                 for input_def in inputs:
 419                     if input_def.default and input_name == input_def.name:
 420                         value = value.replace(getInput, json.dumps(input_def.default))
 421         return value
 422
 423     def get_node_by_name(self, node_templates, name):
 424         for node in node_templates:
 425             if node[NODE_NAME] == name:
 426                 return node
 427         return None
 428
 429     def getCapabilityByName(self, node, capabilityName):
 430         if CAPABILITIES in node and capabilityName in node[CAPABILITIES]:
 431             return node[CAPABILITIES][capabilityName]
 432         return None
 433
 434     def get_base_path(self, tosca):
 435         fpath, fname = os.path.split(tosca.path)
 436         return fpath
 437
 438     def build_artifacts(self, node):
 439         rets = []
 440         if ARTIFACTS in node and len(node[ARTIFACTS]) > 0:
 441             artifacts = node[ARTIFACTS]
 442             for name, value in artifacts.items():
 443                 ret = {}
 444                 ret['artifact_name'] = name
 445                 ret['file'] = value
 446                 if isinstance(value, dict):
 447                     ret.update(value)
 448                 rets.append(ret)
 449         else:
 450             # TODO It is workaround for SDC-1900.
 451             logger.error("VCPE specific code")
 452             ret = {}
 453             ret['artifact_name'] = "sw_image"
 454             ret['file'] = "ubuntu_16.04"
 455             ret['type'] = "tosca.artifacts.nfv.SwImage"
 456             rets.append(ret)
 457
 458         return rets
 459
 460     def get_node_by_req(self, node_templates, req):
 461         req_node_name = self.get_requirement_node_name(req)
 462         return self.get_node_by_name(node_templates, req_node_name)
 463
 464     def isGroupTypeX(self, group, groupTypes, x):
 465         group_type = group[GROUP_TYPE]
 466         while group_type != x:
 467             group_type_derived = group_type
 468             group_type = groupTypes[group_type][DERIVED_FROM]
 469             if group_type == GROUPS_ROOT or group_type == group_type_derived:
 470                 return False
 471         return True
 472
 473     def setTargetValues(self, dict_target, target_keys, dict_source, source_keys):
 474         i = 0
 475         for item in source_keys:
 476             dict_target[target_keys[i]] = dict_source.get(item, "")
 477             i += 1
 478         return dict_target
 479
 480     def get_deploy_graph(self, tosca, relations):
 481         nodes = tosca.graph.nodetemplates
 482         graph = Graph()
 483         for node in nodes:
 484             self._build_deploy_path(node, [], graph, relations)
 485         return graph.to_dict()
 486
 487     def _build_deploy_path(self, node, node_parent, graph, relations):
 488         graph.add_node(node.name, node_parent)
 489         type_require_set = {}
 490         type_requires = node.type_definition.requirements
 491         for type_require in type_requires:
 492             type_require_set.update(type_require)
 493         for requirement in node.requirements:
 494             for k in requirement.keys():
 495                 if type_require_set[k].get('relationship', None) in relations[0] or type_require_set[k].get('capability', None) in relations[0]:
 496                     if isinstance(requirement[k], dict):
 497                         next_node = requirement[k].get('node', None)
 498                     else:
 499                         next_node = requirement[k]
 500                     graph.add_node(next_node, [node.name])
 501                 if type_require_set[k].get('relationship', None) in relations[1]:
 502                     if isinstance(requirement[k], dict):
 503                         next_node = requirement[k].get('node', None)
 504                     else:
 505                         next_node = requirement[k]
 506                     graph.add_node(next_node, [node.name])
 507
 508     def get_substitution_mappings(self, tosca):
 509         node = {
 510             'properties': {},
 511             'requirements': {},
 512             'capabilities': {},
 513             'metadata': {}
 514         }
 515         metadata = None
 516         substitution_mappings = tosca.tpl['topology_template'].get('substitution_mappings', None)
 517         if substitution_mappings:
 518             node['type'] = substitution_mappings['node_type']
 519             node['properties'] = substitution_mappings.get('properties', {})
 520             node['requirements'] = substitution_mappings.get('requirements', {})
 521             node['capabilities'] = substitution_mappings.get('capabilities', {})
 522             metadata = substitution_mappings.get('metadata', {})
 523         node['metadata'] = metadata if metadata and metadata != {} else self.buildMetadata(tosca)
 524         return node