catalog/pub/utils/toscaparser/basemodel.py

   1 # Copyright 2017 ZTE Corporation.
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #         http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14
  15 import ftplib
  16 import json
  17 import logging
  18 import os
  19 import re
  20 import shutil
  21 import urllib
  22
  23 import paramiko
  24 from toscaparser.tosca_template import ToscaTemplate
  25 from toscaparser.properties import Property
  26 from toscaparser.functions import Function, Concat, GetInput, get_function, function_mappings
  27 from catalog.pub.utils.toscaparser.graph import Graph
  28
  29 from catalog.pub.utils.toscaparser.dataentityext import DataEntityExt
  30
  31 logger = logging.getLogger(__name__)
  32
  33 METADATA = "metadata"
  34 PROPERTIES = "properties"
  35 DESCRIPTION = "description"
  36 REQUIREMENTS = "requirements"
  37 INTERFACES = "interfaces"
  38 TOPOLOGY_TEMPLATE = "topology_template"
  39 INPUTS = "inputs"
  40 CAPABILITIES = "capabilities"
  41 ATTRIBUTES = "attributes"
  42 ARTIFACTS = "artifacts"
  43 DERIVED_FROM = "derived_from"
  44
  45 NODE_NAME = "name"
  46 NODE_TYPE = "nodeType"
  47 NODE_ROOT = "tosca.nodes.Root"
  48 GROUP_TYPE = "groupType"
  49 GROUPS_ROOT = "tosca.groups.Root"
  50
  51
  52 class BaseInfoModel(object):
  53
  54     def __init__(self, path, params):
  55         tosca = self.buildToscaTemplate(path, params)
  56         self.parseModel(tosca)
  57
  58     def parseModel(self, tosca):
  59         pass
  60
  61     def buildInputs(self, tosca):
  62         topo = tosca.tpl.get(TOPOLOGY_TEMPLATE, None)
  63         return topo.get(INPUTS, {}) if topo else {}
  64
  65     def buildToscaTemplate(self, path, params):
  66         file_name = None
  67         try:
  68             file_name = self._check_download_file(path)
  69             valid_params = self._validate_input_params(file_name, params)
  70             return self._create_tosca_template(file_name, valid_params)
  71         finally:
  72             if file_name is not None and file_name != path and os.path.exists(file_name):
  73                 try:
  74                     os.remove(file_name)
  75                 except Exception as e:
  76                     logger.error("Failed to parse package, error: %s", e.message)
  77
  78     def _validate_input_params(self, path, params):
  79         valid_params = {}
  80         inputs = {}
  81         if isinstance(params, list):
  82             for param in params:
  83                 key = param.get('key', 'undefined')
  84                 value = param.get('value', 'undefined')
  85                 inputs[key] = value
  86             params = inputs
  87
  88         if params and len(params) > 0:
  89             tmp = self._create_tosca_template(path, None)
  90             if isinstance(params, dict):
  91                 for key, value in params.items():
  92                     if hasattr(tmp, 'inputs') and len(tmp.inputs) > 0:
  93                         for input_def in tmp.inputs:
  94                             if (input_def.name == key):
  95                                 valid_params[key] = DataEntityExt.validate_datatype(input_def.type, value)
  96         return valid_params
  97
  98     def _create_tosca_template(self, file_name, valid_params):
  99         tosca_tpl = None
 100         try:
 101             tosca_tpl = ToscaTemplate(path=file_name,
 102                                       parsed_params=valid_params,
 103                                       no_required_paras_check=True,
 104                                       debug_mode=True)
 105         except Exception as e:
 106             print e.message
 107         finally:
 108             if tosca_tpl is not None and hasattr(tosca_tpl, "temp_dir") and os.path.exists(tosca_tpl.temp_dir):
 109                 try:
 110                     shutil.rmtree(tosca_tpl.temp_dir)
 111                 except Exception as e:
 112                     logger.error("Failed to create tosca template, error: %s", e.message)
 113                 print "-----------------------------"
 114                 print '\n'.join(['%s:%s' % item for item in tosca_tpl.__dict__.items()])
 115                 print "-----------------------------"
 116             return tosca_tpl
 117
 118     def _check_download_file(self, path):
 119         if (path.startswith("ftp") or path.startswith("sftp")):
 120             return self.downloadFileFromFtpServer(path)
 121         elif (path.startswith("http")):
 122             return self.download_file_from_httpserver(path)
 123         return path
 124
 125     def download_file_from_httpserver(self, path):
 126         path = path.encode("utf-8")
 127         tmps = str.split(path, '/')
 128         localFileName = tmps[len(tmps) - 1]
 129         urllib.urlretrieve(path, localFileName)
 130         return localFileName
 131
 132     def downloadFileFromFtpServer(self, path):
 133         path = path.encode("utf-8")
 134         tmp = str.split(path, '://')
 135         protocol = tmp[0]
 136         tmp = str.split(tmp[1], ':')
 137         if len(tmp) == 2:
 138             userName = tmp[0]
 139             tmp = str.split(tmp[1], '@')
 140             userPwd = tmp[0]
 141             index = tmp[1].index('/')
 142             hostIp = tmp[1][0:index]
 143             remoteFileName = tmp[1][index:len(tmp[1])]
 144             if protocol.lower() == 'ftp':
 145                 hostPort = 21
 146             else:
 147                 hostPort = 22
 148
 149         if len(tmp) == 3:
 150             userName = tmp[0]
 151             userPwd = str.split(tmp[1], '@')[0]
 152             hostIp = str.split(tmp[1], '@')[1]
 153             index = tmp[2].index('/')
 154             hostPort = tmp[2][0:index]
 155             remoteFileName = tmp[2][index:len(tmp[2])]
 156
 157         localFileName = str.split(remoteFileName, '/')
 158         localFileName = localFileName[len(localFileName) - 1]
 159
 160         if protocol.lower() == 'sftp':
 161             self.sftp_get(userName, userPwd, hostIp, hostPort, remoteFileName, localFileName)
 162         else:
 163             self.ftp_get(userName, userPwd, hostIp, hostPort, remoteFileName, localFileName)
 164         return localFileName
 165
 166     def sftp_get(self, userName, userPwd, hostIp, hostPort, remoteFileName, localFileName):
 167         # return
 168         t = None
 169         try:
 170             t = paramiko.Transport(hostIp, int(hostPort))
 171             t.connect(username=userName, password=userPwd)
 172             sftp = paramiko.SFTPClient.from_transport(t)
 173             sftp.get(remoteFileName, localFileName)
 174         finally:
 175             if t is not None:
 176                 t.close()
 177
 178     def ftp_get(self, userName, userPwd, hostIp, hostPort, remoteFileName, localFileName):
 179         f = None
 180         try:
 181             ftp = ftplib.FTP()
 182             ftp.connect(hostIp, hostPort)
 183             ftp.login(userName, userPwd)
 184             f = open(localFileName, 'wb')
 185             ftp.retrbinary('RETR ' + remoteFileName, f.write, 1024)
 186             f.close()
 187         finally:
 188             if f is not None:
 189                 f.close()
 190
 191     def buildMetadata(self, tosca):
 192         return tosca.tpl.get(METADATA, {}) if tosca else {}
 193
 194     def buildNode(self, nodeTemplate, tosca):
 195         inputs = tosca.inputs
 196         parsed_params = tosca.parsed_params
 197         ret = {}
 198         ret[NODE_NAME] = nodeTemplate.name
 199         ret[NODE_TYPE] = nodeTemplate.type
 200         if DESCRIPTION in nodeTemplate.entity_tpl:
 201             ret[DESCRIPTION] = nodeTemplate.entity_tpl[DESCRIPTION]
 202         else:
 203             ret[DESCRIPTION] = ''
 204         if METADATA in nodeTemplate.entity_tpl:
 205             ret[METADATA] = nodeTemplate.entity_tpl[METADATA]
 206         else:
 207             ret[METADATA] = ''
 208         props = self.buildProperties_ex(nodeTemplate, tosca.topology_template)
 209         ret[PROPERTIES] = self.verify_properties(props, inputs, parsed_params)
 210         ret[REQUIREMENTS] = self.build_requirements(nodeTemplate)
 211         self.buildCapabilities(nodeTemplate, inputs, ret)
 212         self.buildArtifacts(nodeTemplate, inputs, ret)
 213         interfaces = self.build_interfaces(nodeTemplate)
 214         if interfaces:
 215             ret[INTERFACES] = interfaces
 216         return ret
 217
 218     def buildProperties(self, nodeTemplate, parsed_params):
 219         properties = {}
 220         isMappingParams = parsed_params and len(parsed_params) > 0
 221         for k, item in nodeTemplate.get_properties().items():
 222             properties[k] = item.value
 223             if isinstance(item.value, GetInput):
 224                 if item.value.result() and isMappingParams:
 225                     properties[k] = DataEntityExt.validate_datatype(item.type, item.value.result())
 226                 else:
 227                     tmp = {}
 228                     tmp[item.value.name] = item.value.input_name
 229                     properties[k] = tmp
 230         if ATTRIBUTES in nodeTemplate.entity_tpl:
 231             for k, item in nodeTemplate.entity_tpl[ATTRIBUTES].items():
 232                 properties[k] = str(item)
 233         return properties
 234
 235     def buildProperties_ex(self, nodeTemplate, topology_template, properties=None):
 236         if properties is None:
 237             properties = nodeTemplate.get_properties()
 238         _properties = {}
 239         if isinstance(properties, dict):
 240             for name, prop in properties.items():
 241                 if isinstance(prop, Property):
 242                     if isinstance(prop.value, Function):
 243                         if isinstance(prop.value, Concat):  # support one layer inner function.
 244                             value_str = ''
 245                             for arg in prop.value.args:
 246                                 if isinstance(arg, str):
 247                                     value_str += arg
 248                                 elif isinstance(arg, dict):
 249                                     raw_func = {}
 250                                     for k, v in arg.items():
 251                                         func_args = []
 252                                         func_args.append(v)
 253                                         raw_func[k] = func_args
 254                                     func = get_function(topology_template, nodeTemplate, raw_func)
 255                                     value_str += str(func.result())
 256                             _properties[name] = value_str
 257                         else:
 258                             _properties[name] = prop.value.result()
 259                     elif isinstance(prop.value, dict) or isinstance(prop.value, list):
 260                         _properties[name] = self.buildProperties_ex(nodeTemplate, topology_template, prop.value)
 261                     elif prop.type == 'string':
 262                         _properties[name] = prop.value
 263                     else:
 264                         _properties[name] = json.dumps(prop.value)
 265                 elif isinstance(prop, dict):
 266                     _properties[name] = self.buildProperties_ex(nodeTemplate, topology_template, prop)
 267                 elif isinstance(prop, list):
 268                     _properties[name] = self.buildProperties_ex(nodeTemplate, topology_template, prop)
 269                 elif name in function_mappings:
 270                     raw_func = {}
 271                     func_args = []
 272                     func_args.append(prop)
 273                     raw_func[name] = func_args
 274                     if name == 'CONCAT':
 275                         value_str = ''
 276                         for arg in prop:
 277                             if isinstance(arg, str):
 278                                 value_str += arg
 279                             elif isinstance(arg, dict):
 280                                 raw_func = {}
 281                                 for k, v in arg.items():
 282                                     func_args = []
 283                                     func_args.append(v)
 284                                     raw_func[k] = func_args
 285                                 value_str += str(
 286                                     get_function(topology_template, nodeTemplate, raw_func).result())
 287                                 value = value_str
 288                     else:
 289                         return get_function(topology_template, nodeTemplate, raw_func).result()
 290                 else:
 291                     _properties[name] = prop
 292         elif isinstance(properties, list):
 293             value = []
 294             for para in properties:
 295                 if isinstance(para, dict) or isinstance(para, list):
 296                     value.append(self.buildProperties_ex(nodeTemplate, topology_template, para))
 297                 else:
 298                     value.append(para)
 299             return value
 300         return _properties
 301
 302     def verify_properties(self, props, inputs, parsed_params):
 303         ret_props = {}
 304         if (props and len(props) > 0):
 305             for key, value in props.items():
 306                 ret_props[key] = self._verify_value(value, inputs, parsed_params)
 307                 #                 if isinstance(value, str):
 308                 #                     ret_props[key] = self._verify_string(inputs, parsed_params, value);
 309                 #                     continue
 310                 #                 if isinstance(value, list):
 311                 #                     ret_props[key] = map(lambda x: self._verify_dict(inputs, parsed_params, x), value)
 312                 #                     continue
 313                 #                 if isinstance(value, dict):
 314                 #                     ret_props[key] = self._verify_map(inputs, parsed_params, value)
 315                 #                     continue
 316                 #                 ret_props[key] = value
 317         return ret_props
 318
 319     def build_requirements(self, node_template):
 320         rets = []
 321         for req in node_template.requirements:
 322             for req_name, req_value in req.items():
 323                 if (isinstance(req_value, dict)):
 324                     if ('node' in req_value and req_value['node'] not in node_template.templates):
 325                         continue  # No target requirement for aria parser, not add to result.
 326                 rets.append({req_name: req_value})
 327         return rets
 328
 329     def buildCapabilities(self, nodeTemplate, inputs, ret):
 330         capabilities = json.dumps(nodeTemplate.entity_tpl.get(CAPABILITIES, None))
 331         match = re.findall(r'\{"get_input":\s*"([\w|\-]+)"\}', capabilities)
 332         for m in match:
 333             aa = [input_def for input_def in inputs if m == input_def.name][0]
 334             capabilities = re.sub(r'\{"get_input":\s*"([\w|\-]+)"\}', json.dumps(aa.default), capabilities, 1)
 335         if capabilities != 'null':
 336             ret[CAPABILITIES] = json.loads(capabilities)
 337
 338     def buildArtifacts(self, nodeTemplate, inputs, ret):
 339         artifacts = json.dumps(nodeTemplate.entity_tpl.get('artifacts', None))
 340         match = re.findall(r'\{"get_input":\s*"([\w|\-]+)"\}', artifacts)
 341         for m in match:
 342             aa = [input_def for input_def in inputs if m == input_def.name][0]
 343             artifacts = re.sub(r'\{"get_input":\s*"([\w|\-]+)"\}', json.dumps(aa.default), artifacts, 1)
 344         if artifacts != 'null':
 345             ret[ARTIFACTS] = json.loads(artifacts)
 346
 347     def build_interfaces(self, node_template):
 348         if INTERFACES in node_template.entity_tpl:
 349             return node_template.entity_tpl[INTERFACES]
 350         return None
 351
 352     def isNodeTypeX(self, node, nodeTypes, x):
 353         node_type = node[NODE_TYPE]
 354         while node_type != x:
 355             node_type_derived = node_type
 356             node_type = nodeTypes[node_type][DERIVED_FROM]
 357             if node_type == NODE_ROOT or node_type == node_type_derived:
 358                 return False
 359         return True
 360
 361     def get_requirement_node_name(self, req_value):
 362         return self.get_prop_from_obj(req_value, 'node')
 363
 364     def getRequirementByNodeName(self, nodeTemplates, storage_name, prop):
 365         for node in nodeTemplates:
 366             if node[NODE_NAME] == storage_name:
 367                 if prop in node:
 368                     return node[prop]
 369
 370     def get_prop_from_obj(self, obj, prop):
 371         if isinstance(obj, str):
 372             return obj
 373         if (isinstance(obj, dict) and prop in obj):
 374             return obj[prop]
 375         return None
 376
 377     def getNodeDependencys(self, node):
 378         return self.getRequirementByName(node, 'dependency')
 379
 380     def getRequirementByName(self, node, requirementName):
 381         requirements = []
 382         if REQUIREMENTS in node:
 383             for item in node[REQUIREMENTS]:
 384                 for key, value in item.items():
 385                     if key == requirementName:
 386                         requirements.append(value)
 387         return requirements
 388
 389     def _verify_value(self, value, inputs, parsed_params):
 390         if value == '{}':
 391             return ''
 392         if isinstance(value, str):
 393             return self._verify_string(inputs, parsed_params, value)
 394         if isinstance(value, list) or isinstance(value, dict):
 395             return self._verify_object(value, inputs, parsed_params)
 396         return value
 397
 398     def _verify_object(self, value, inputs, parsed_params):
 399         s = self._verify_string(inputs, parsed_params, json.dumps(value))
 400         return json.loads(s)
 401
 402     def _get_input_name(self, getInput):
 403         input_name = getInput.split(':')[1]
 404         input_name = input_name.strip()
 405         return input_name.replace('"', '').replace('}', '')
 406
 407     def _verify_string(self, inputs, parsed_params, value):
 408         getInputs = re.findall(r'{"get_input": "[a-zA-Z_0-9]+"}', value)
 409         for getInput in getInputs:
 410             input_name = self._get_input_name(getInput)
 411             if parsed_params and input_name in parsed_params:
 412                 value = value.replace(getInput, json.dumps(parsed_params[input_name]))
 413             else:
 414                 for input_def in inputs:
 415                     if input_def.default and input_name == input_def.name:
 416                         value = value.replace(getInput, json.dumps(input_def.default))
 417         return value
 418
 419     def get_node_by_name(self, node_templates, name):
 420         for node in node_templates:
 421             if node[NODE_NAME] == name:
 422                 return node
 423         return None
 424
 425     def getCapabilityByName(self, node, capabilityName):
 426         if CAPABILITIES in node and capabilityName in node[CAPABILITIES]:
 427             return node[CAPABILITIES][capabilityName]
 428         return None
 429
 430     def get_base_path(self, tosca):
 431         fpath, fname = os.path.split(tosca.path)
 432         return fpath
 433
 434     def build_artifacts(self, node):
 435         rets = []
 436         if ARTIFACTS in node and len(node[ARTIFACTS]) > 0:
 437             artifacts = node[ARTIFACTS]
 438             for name, value in artifacts.items():
 439                 ret = {}
 440                 ret['artifact_name'] = name
 441                 ret['file'] = value
 442                 if isinstance(value, dict):
 443                     ret.update(value)
 444                 rets.append(ret)
 445         return rets
 446
 447     def get_node_by_req(self, node_templates, req):
 448         req_node_name = self.get_requirement_node_name(req)
 449         return self.get_node_by_name(node_templates, req_node_name)
 450
 451     def isGroupTypeX(self, group, groupTypes, x):
 452         group_type = group[GROUP_TYPE]
 453         while group_type != x:
 454             group_type_derived = group_type
 455             group_type = groupTypes[group_type][DERIVED_FROM]
 456             if group_type == GROUPS_ROOT or group_type == group_type_derived:
 457                 return False
 458         return True
 459
 460     def setTargetValues(self, dict_target, target_keys, dict_source, source_keys):
 461         i = 0
 462         for item in source_keys:
 463             dict_target[target_keys[i]] = dict_source.get(item, "")
 464             i += 1
 465         return dict_target
 466
 467     def get_deploy_graph(self, tosca, relations):
 468         nodes = tosca.graph.nodetemplates
 469         graph = Graph()
 470         for node in nodes:
 471             self._build_deploy_path(node, [], graph, relations)
 472         return graph.to_dict()
 473
 474     def _build_deploy_path(self, node, node_parent, graph, relations):
 475         graph.add_node(node.name, node_parent)
 476         type_require_set = {}
 477         type_requires = node.type_definition.requirements
 478         for type_require in type_requires:
 479             type_require_set.update(type_require)
 480         for requirement in node.requirements:
 481             for k in requirement.keys():
 482                 if type_require_set[k].get('relationship', None) in relations[0] or type_require_set[k].get('capability', None) in relations[0]:
 483                     if isinstance(requirement[k], dict):
 484                         next_node = requirement[k].get('node', None)
 485                     else:
 486                         next_node = requirement[k]
 487                     graph.add_node(next_node, [node.name])
 488                 if type_require_set[k].get('relationship', None) in relations[1]:
 489                     if isinstance(requirement[k], dict):
 490                         next_node = requirement[k].get('node', None)
 491                     else:
 492                         next_node = requirement[k]
 493                     graph.add_node(next_node, [node.name])