2 * ============LICENSE_START=======================================================
4 * ================================================================================
5 * Copyright (C) 2019 AT&T Intellectual Property. All rights reserved.
6 * ================================================================================
7 * Licensed under the Apache License, Version 2.0 (the "License");
8 * you may not use this file except in compliance with the License.
9 * You may obtain a copy of the License at
11 * http://www.apache.org/licenses/LICENSE-2.0
13 * Unless required by applicable law or agreed to in writing, software
14 * distributed under the License is distributed on an "AS IS" BASIS,
15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 * See the License for the specific language governing permissions and
17 * limitations under the License.
18 * ============LICENSE_END=========================================================
21 package org.openecomp.sdc.asdctool.impl;
23 import com.google.gson.Gson;
24 import com.google.gson.GsonBuilder;
25 import com.google.gson.JsonArray;
26 import fj.data.Either;
27 import org.openecomp.sdc.asdctool.impl.validator.utils.VfModuleArtifactPayloadEx;
28 import org.openecomp.sdc.be.components.distribution.engine.VfModuleArtifactPayload;
29 import org.openecomp.sdc.be.dao.cassandra.ArtifactCassandraDao;
30 import org.openecomp.sdc.be.dao.cassandra.CassandraOperationStatus;
31 import org.openecomp.sdc.be.dao.janusgraph.JanusGraphOperationStatus;
32 import org.openecomp.sdc.be.dao.jsongraph.GraphVertex;
33 import org.openecomp.sdc.be.dao.jsongraph.JanusGraphDao;
34 import org.openecomp.sdc.be.dao.jsongraph.types.EdgeLabelEnum;
35 import org.openecomp.sdc.be.dao.jsongraph.types.JsonParseFlagEnum;
36 import org.openecomp.sdc.be.dao.jsongraph.types.VertexTypeEnum;
37 import org.openecomp.sdc.be.datatypes.elements.ArtifactDataDefinition;
38 import org.openecomp.sdc.be.datatypes.elements.GroupDataDefinition;
39 import org.openecomp.sdc.be.datatypes.elements.MapArtifactDataDefinition;
40 import org.openecomp.sdc.be.datatypes.elements.MapGroupsDataDefinition;
41 import org.openecomp.sdc.be.datatypes.enums.ComponentTypeEnum;
42 import org.openecomp.sdc.be.datatypes.enums.GraphPropertyEnum;
43 import org.openecomp.sdc.be.datatypes.enums.OriginTypeEnum;
44 import org.openecomp.sdc.be.datatypes.enums.ResourceTypeEnum;
45 import org.openecomp.sdc.be.datatypes.tosca.ToscaDataDefinition;
46 import org.openecomp.sdc.be.impl.ComponentsUtils;
47 import org.openecomp.sdc.be.model.ArtifactDefinition;
48 import org.openecomp.sdc.be.model.Component;
49 import org.openecomp.sdc.be.model.ComponentInstance;
50 import org.openecomp.sdc.be.model.ComponentParametersView;
51 import org.openecomp.sdc.be.model.DistributionStatusEnum;
52 import org.openecomp.sdc.be.model.GroupDefinition;
53 import org.openecomp.sdc.be.model.GroupInstance;
54 import org.openecomp.sdc.be.model.LifecycleStateEnum;
55 import org.openecomp.sdc.be.model.Resource;
56 import org.openecomp.sdc.be.model.Service;
57 import org.openecomp.sdc.be.model.jsonjanusgraph.datamodel.TopologyTemplate;
58 import org.openecomp.sdc.be.model.jsonjanusgraph.datamodel.ToscaElement;
59 import org.openecomp.sdc.be.model.jsonjanusgraph.operations.ToscaOperationFacade;
60 import org.openecomp.sdc.be.model.jsonjanusgraph.utils.ModelConverter;
61 import org.openecomp.sdc.be.model.operations.api.StorageOperationStatus;
62 import org.openecomp.sdc.be.model.operations.impl.DaoStatusConverter;
63 import org.openecomp.sdc.be.model.operations.impl.UniqueIdBuilder;
64 import org.openecomp.sdc.be.resources.data.DAOArtifactData;
65 import org.openecomp.sdc.be.tosca.CsarUtils;
66 import org.openecomp.sdc.be.tosca.ToscaError;
67 import org.openecomp.sdc.be.tosca.ToscaExportHandler;
68 import org.openecomp.sdc.be.tosca.ToscaRepresentation;
69 import org.openecomp.sdc.common.api.ArtifactGroupTypeEnum;
70 import org.openecomp.sdc.common.api.ArtifactTypeEnum;
71 import org.openecomp.sdc.common.api.Constants;
72 import org.openecomp.sdc.common.log.wrappers.Logger;
73 import org.openecomp.sdc.common.util.GeneralUtility;
74 import org.openecomp.sdc.exception.ResponseFormat;
75 import org.springframework.beans.factory.annotation.Autowired;
77 import java.io.BufferedWriter;
78 import java.io.FileOutputStream;
79 import java.io.IOException;
80 import java.io.OutputStreamWriter;
81 import java.io.Writer;
82 import java.util.ArrayList;
83 import java.util.Collections;
84 import java.util.EnumMap;
85 import java.util.HashMap;
86 import java.util.HashSet;
87 import java.util.List;
89 import java.util.Optional;
91 import java.util.stream.Collectors;
93 @org.springframework.stereotype.Component("artifactUuidFix")
94 public class ArtifactUuidFix {
96 private static final String MIGRATION1707_ARTIFACT_UUID_FIX = "Migration1707ArtifactUuidFix fix group: group name {} correct artifactId {} artifactUUID {} ";
98 private static final String FAILED_TO_FETCH_VF_RESOURCES = "Failed to fetch vf resources ";
100 private static final String UTF8 = "utf-8";
102 private JanusGraphDao janusGraphDao;
103 private ToscaOperationFacade toscaOperationFacade;
104 private ToscaExportHandler toscaExportUtils;
105 private ArtifactCassandraDao artifactCassandraDao;
106 private CsarUtils csarUtils;
108 private static Logger log = Logger.getLogger(ArtifactUuidFix.class.getName());
111 public ArtifactUuidFix(JanusGraphDao janusGraphDao,
112 ToscaOperationFacade toscaOperationFacade, ToscaExportHandler toscaExportUtils,
113 ArtifactCassandraDao artifactCassandraDao, CsarUtils csarUtils) {
114 this.janusGraphDao = janusGraphDao;
115 this.toscaOperationFacade = toscaOperationFacade;
116 this.toscaExportUtils = toscaExportUtils;
117 this.artifactCassandraDao = artifactCassandraDao;
118 this.csarUtils = csarUtils;
121 public boolean doFix(String fixComponent, String runMode) {
122 List<Resource> vfLst = new ArrayList<>();
123 List<Service> serviceList = new ArrayList<>();
124 Map<String, List<Component>> nodeToFixTosca = new HashMap<>();
125 Map<String, List<Component>> vfToFixTosca = new HashMap<>();
126 Map<String, List<Component>> serviceToFixTosca = new HashMap<>();
128 long time = System.currentTimeMillis();
130 doFixTosca(nodeToFixTosca, vfToFixTosca, serviceToFixTosca);
132 if ("vf_only".equals(fixComponent)) {
133 if (!fetchFaultVf(fixComponent, vfLst, time)) {
137 if (!fetchServices(fixComponent, serviceList, time)) {
141 if ("service_vf".equals(runMode) || "fix".equals(runMode)) {
142 log.info("Mode {}. Find problem VFs", runMode);
143 if (!fetchVf(serviceList, vfLst, time)) {
144 log.info("Mode {}. Find problem VFs finished with failure", runMode);
147 log.info("Mode {}. Find problem VFs finished with success", runMode);
149 if ("fix".equals(runMode) || "fix_only_services".equals(runMode)) {
150 log.info("Mode {}. Start fix", runMode);
151 if (!fix(vfLst, serviceList, nodeToFixTosca, vfToFixTosca, serviceToFixTosca)) {
152 log.info("Mode {}. Fix finished with failure", runMode);
155 log.info("Mode {}. Fix finished with success", runMode);
161 private boolean fetchFaultVf(String fixComponent, List<Resource> vfLst, long time) {
162 log.info("Find fault VF ");
163 String fileName = "fault_" + time + ".csv";
164 try(Writer writer = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(fileName), UTF8))) {
165 writer.write("vf name, vf id, state, version\n");
167 Map<GraphPropertyEnum, Object> hasProps = new EnumMap<>(GraphPropertyEnum.class);
168 hasProps.put(GraphPropertyEnum.COMPONENT_TYPE, ComponentTypeEnum.RESOURCE.name());
169 hasProps.put(GraphPropertyEnum.RESOURCE_TYPE, ResourceTypeEnum.VF.name());
171 Map<GraphPropertyEnum, Object> hasNotProps = new EnumMap<>(GraphPropertyEnum.class);
172 hasNotProps.put(GraphPropertyEnum.IS_DELETED, true);
173 log.info("Try to fetch resources with properties {} and not {}", hasProps, hasNotProps);
175 Either<List<GraphVertex>, JanusGraphOperationStatus> servicesByCriteria = janusGraphDao
176 .getByCriteria(VertexTypeEnum.TOPOLOGY_TEMPLATE, hasProps, hasNotProps, JsonParseFlagEnum.ParseAll);
177 if (servicesByCriteria.isRight()) {
178 log.info("Failed to fetch resources {}", servicesByCriteria.right().value());
181 List<GraphVertex> resources = servicesByCriteria.left().value();
182 for (GraphVertex gv : resources) {
183 ComponentParametersView filter = new ComponentParametersView(true);
184 filter.setIgnoreComponentInstances(false);
185 filter.setIgnoreArtifacts(false);
186 filter.setIgnoreGroups(false);
188 Either<Resource, StorageOperationStatus> toscaElement = toscaOperationFacade
189 .getToscaElement(gv.getUniqueId());
190 if (toscaElement.isRight()) {
191 log.info("Failed to fetch resources {} {}", gv.getUniqueId(), toscaElement.right().value());
195 Resource resource = toscaElement.left().value();
196 String resourceName = resource.getName();
197 Map<String, ArtifactDefinition> deploymentArtifacts = resource.getDeploymentArtifacts();
198 List<GroupDefinition> groups = resource.getGroups();
199 if (groups == null || groups.isEmpty()) {
200 log.info("No groups for resource {} id {} ", resourceName, gv.getUniqueId());
203 boolean isProblematic = false;
204 for (GroupDefinition gr : groups) {
205 if (gr.getType().equals(Constants.DEFAULT_GROUP_VF_MODULE)) {
206 if (isProblematicGroup(gr, resourceName, deploymentArtifacts)) {
207 isProblematic = true;
214 writeModuleResultToFile(writer, resource, null);
217 janusGraphDao.commit();
220 } catch (Exception e) {
221 log.info(FAILED_TO_FETCH_VF_RESOURCES, e);
224 janusGraphDao.commit();
229 private boolean fetchVf(List<Service> serviceList, List<Resource> vfLst, long time) {
230 log.info("Find problem VF ");
231 if (serviceList.isEmpty()) {
232 log.info("No services as input");
235 String fileName = "problemVf_" + time + ".csv";
236 try(Writer writer = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(fileName), UTF8))) {
237 writer.write("vf name, vf id, state, version, example service name\n");
238 Set<String> vfIds = new HashSet<>();
239 for (Service service : serviceList) {
240 List<ComponentInstance> componentInstances = service.getComponentInstances().stream()
241 .filter(ci -> ci.getOriginType().equals(OriginTypeEnum.VF)).collect(Collectors.toList());
242 for (ComponentInstance ci : componentInstances) {
243 if (!vfIds.contains(ci.getComponentUid())) {
244 vfIds.add(ci.getComponentUid());
245 ComponentParametersView filter = new ComponentParametersView(true);
246 filter.setIgnoreComponentInstances(false);
247 filter.setIgnoreArtifacts(false);
248 filter.setIgnoreGroups(false);
249 Either<Resource, StorageOperationStatus> toscaElement = toscaOperationFacade
250 .getToscaElement(ci.getComponentUid(), filter);
251 if (toscaElement.isRight()) {
252 log.info("Failed to fetch resource {} {}", ci.getComponentUid(),
253 toscaElement.right().value());
256 Resource resource = toscaElement.left().value();
257 if (resource.getResourceType().equals(ResourceTypeEnum.VF)) {
259 writeModuleResultToFile(writer, resource, service);
263 janusGraphDao.commit();
267 log.info("output file with list of Vf : {}", fileName);
268 } catch (Exception e) {
269 log.info("Failed to fetch services ", e);
272 janusGraphDao.commit();
277 private boolean fetchServices(String fixServices, List<Service> serviceList, long time) {
278 log.info("Find problem Services {}", fixServices);
279 String fileName = "problemService_" + time + ".csv";
280 try(Writer writer = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(fileName), UTF8))) {
281 writer.write("service name, service id, state, version\n");
283 Map<GraphPropertyEnum, Object> hasProps = new EnumMap<>(GraphPropertyEnum.class);
284 hasProps.put(GraphPropertyEnum.COMPONENT_TYPE, ComponentTypeEnum.SERVICE.name());
285 if ("distributed_only".equals(fixServices)) {
286 hasProps.put(GraphPropertyEnum.STATE, LifecycleStateEnum.CERTIFIED.name());
287 hasProps.put(GraphPropertyEnum.DISTRIBUTION_STATUS, DistributionStatusEnum.DISTRIBUTED.name());
290 Map<GraphPropertyEnum, Object> hasNotProps = new EnumMap<>(GraphPropertyEnum.class);
291 hasNotProps.put(GraphPropertyEnum.IS_DELETED, true);
292 log.info("Try to fetch services with properties {} and not {}", hasProps, hasNotProps);
294 Either<List<GraphVertex>, JanusGraphOperationStatus> servicesByCriteria = janusGraphDao
295 .getByCriteria(VertexTypeEnum.TOPOLOGY_TEMPLATE, hasProps, hasNotProps, JsonParseFlagEnum.ParseAll);
296 if (servicesByCriteria.isRight()) {
297 log.info("Failed to fetch services {}", servicesByCriteria.right().value());
300 List<GraphVertex> services = servicesByCriteria.left().value();
301 for (GraphVertex gv : services) {
302 ComponentParametersView filter = new ComponentParametersView(true);
303 filter.setIgnoreComponentInstances(false);
304 filter.setIgnoreArtifacts(false);
305 filter.setIgnoreGroups(false);
307 Either<Service, StorageOperationStatus> toscaElement = toscaOperationFacade
308 .getToscaElement(gv.getUniqueId(), filter);
309 if (toscaElement.isRight()) {
310 log.info("Failed to fetch service {} {}", gv.getUniqueId(), toscaElement.right().value());
313 Service service = toscaElement.left().value();
315 String serviceName = (String) gv.getMetadataProperty(GraphPropertyEnum.NAME);
317 boolean isProblematic = isProblematicService(service, serviceName);
319 serviceList.add(service);
320 writeModuleResultToFile(writer, service, null);
325 janusGraphDao.commit();
327 log.info("output file with list of services : {}", fileName);
328 } catch (Exception e) {
329 log.info("Failed to fetch services ", e);
332 janusGraphDao.commit();
337 private boolean isProblematicService( Service service, String serviceName) {
339 List<ComponentInstance> componentInstances = service.getComponentInstances();
341 if (componentInstances == null) {
342 log.info("No instances for service {} ", service.getUniqueId());
345 boolean isCheckVFModules = true;
346 if(service.getLifecycleState() == LifecycleStateEnum.NOT_CERTIFIED_CHECKIN ||
347 service.getLifecycleState() == LifecycleStateEnum.NOT_CERTIFIED_CHECKOUT){
348 isCheckVFModules = false;
350 for (ComponentInstance ci : componentInstances) {
351 Map<String, ArtifactDefinition> deploymentArtifacts = ci.getDeploymentArtifacts();
352 List<GroupInstance> groupInstances = ci.getGroupInstances();
353 if (groupInstances == null || groupInstances.isEmpty()) {
354 log.info("No instance groups for instance {} in service {} id {} ", ci.getName(), serviceName,
355 service.getUniqueId());
358 List<VfModuleArtifactPayloadEx> vfModules = null;
359 if(isCheckVFModules){
360 Optional<ArtifactDefinition> optionalVfModuleArtifact = deploymentArtifacts.values().stream()
361 .filter(p -> p.getArtifactType().equals(ArtifactTypeEnum.VF_MODULES_METADATA.name())).findAny();
363 if(!optionalVfModuleArtifact.isPresent())
366 ArtifactDefinition vfModuleArtifact = optionalVfModuleArtifact.get();
367 Either<List<VfModuleArtifactPayloadEx>, StorageOperationStatus> vfModulesEither = parseVFModuleJson(vfModuleArtifact);
368 if(vfModulesEither.isRight()){
369 log.error("Failed to parse vfModule for service {} status is {}", service.getUniqueId(), vfModulesEither.right().value());
372 vfModules = vfModulesEither.left().value();
373 if(vfModules == null || vfModules.isEmpty() ){
374 log.info("vfModules empty for service {}", service.getUniqueId());
379 for (GroupInstance gi : groupInstances) {
380 if (gi.getType().equals(Constants.DEFAULT_GROUP_VF_MODULE)) {
381 VfModuleArtifactPayloadEx vfModule= null;
382 if(isCheckVFModules && vfModules != null){
383 Optional<VfModuleArtifactPayloadEx> op = vfModules.stream().filter(vf -> vf.getVfModuleModelName().equals(gi.getGroupName())).findAny();
385 log.error("Failed to find vfModule for group {}", gi.getGroupName());
390 if (isProblematicGroupInstance(gi, ci.getName(), serviceName, deploymentArtifacts, vfModule)) {
400 private boolean isProblematicGroup(GroupDefinition gr, String resourceName,
401 Map<String, ArtifactDefinition> deploymentArtifacts) {
402 List<String> artifacts = gr.getArtifacts();
403 List<String> artifactsUuid = gr.getArtifactsUuid();
404 Set<String> artifactsSet = new HashSet<>();
406 if ((artifactsUuid == null || artifactsUuid.isEmpty()) && (artifacts == null || artifacts.isEmpty())) {
407 log.info("No groups in resource {} ", resourceName);
410 artifactsSet.addAll(artifacts);
411 if (artifactsSet.size() < artifacts.size()) {
412 log.info(" artifactsSet.size() < artifacts.size() group {} in resource {} ", gr.getName(), resourceName);
416 if ((artifactsUuid != null) && (artifacts.size() < artifactsUuid.size())) {
417 log.info(" artifacts.size() < artifactsUuid.size() group {} in resource {} ", gr.getName(), resourceName);
420 if (!artifacts.isEmpty() && (artifactsUuid == null || artifactsUuid.isEmpty())) {
422 " artifacts.size() > 0 && (artifactsUuid == null || artifactsUuid.isEmpty() group {} in resource {} ",
423 gr.getName(), resourceName);
426 if (artifactsUuid != null && artifactsUuid.contains(null)) {
427 log.info(" artifactsUuid.contains(null) group {} in resource {} ", gr.getName(), resourceName);
431 for (String artifactId : artifacts) {
432 String artifactlabel = findArtifactLabelFromArtifactId(artifactId);
433 ArtifactDefinition artifactDefinition = deploymentArtifacts.get(artifactlabel);
434 if (artifactDefinition == null) {
435 log.info(" artifactDefinition == null label {} group {} in resource {} ", artifactlabel, gr.getName(),
439 ArtifactTypeEnum artifactType = ArtifactTypeEnum.findType(artifactDefinition.getArtifactType());
440 if (artifactType != ArtifactTypeEnum.HEAT_ENV) {
441 if (!artifactId.equals(artifactDefinition.getUniqueId())) {
443 " !artifactId.equals(artifactDefinition.getUniqueId() artifact {} artId {} group {} in resource {} ",
444 artifactlabel, artifactId, gr.getName(), resourceName);
447 if (artifactsUuid != null && !artifactsUuid.contains(artifactDefinition.getArtifactUUID())) {
449 " artifactsUuid.contains(artifactDefinition.getArtifactUUID() label {} group {} in resource {} ",
450 artifactlabel, gr.getName(), resourceName);
455 for (String artifactUUID : artifactsUuid) {
456 String label = findArtifactLabelFromArtifactId(artifactUUID);
457 if (label != null && !label.isEmpty() && !label.equals("")) {
465 private boolean isProblematicGroupInstance(GroupInstance gi, String instName, String servicename,
466 Map<String, ArtifactDefinition> deploymentArtifacts, VfModuleArtifactPayloadEx vfModule) {
467 List<String> artifacts = gi.getArtifacts();
468 List<String> artifactsUuid = gi.getArtifactsUuid();
469 List<String> instArtifactsUuid = gi.getGroupInstanceArtifactsUuid();
470 List<String> instArtifactsId = gi.getGroupInstanceArtifacts();
471 Set<String> instArtifatIdSet = new HashSet<>();
472 Set<String> artifactsSet = new HashSet<>();
474 log.info("check group {} for instance {} ", gi.getGroupName(), instName);
475 if ((artifactsUuid == null || artifactsUuid.isEmpty()) && (artifacts == null || artifacts.isEmpty())) {
476 log.info("No instance groups for instance {} in service {} ", instName, servicename);
479 artifactsSet.addAll(artifacts);
480 if (artifactsSet.size() < artifacts.size()) {
481 log.info(" artifactsSet.size() < artifacts.size() group {} in resource {} ", instName, servicename);
485 if (instArtifactsId != null && !instArtifactsId.isEmpty()) {
486 instArtifatIdSet.addAll(instArtifactsId);
489 if ((artifactsUuid != null) && (artifacts.size() < artifactsUuid.size())) {
490 log.info(" artifacts.size() < artifactsUuid.size() inst {} in service {} ", instName, servicename);
493 if (!artifacts.isEmpty() && (artifactsUuid == null || artifactsUuid.isEmpty())) {
495 " artifacts.size() > 0 && (artifactsUuid == null || artifactsUuid.isEmpty() inst {} in service {} ",
496 instName, servicename);
499 if (artifactsUuid != null && artifactsUuid.contains(null)) {
500 log.info(" artifactsUuid.contains(null) inst {} in service {} ", instName, servicename);
503 if (instArtifactsId != null && instArtifatIdSet.size() < instArtifactsId.size()) {
504 log.info(" instArtifatIdSet.size() < instArtifactsId.size() inst {} in service {} ", instName, servicename);
508 if ((instArtifactsId != null && instArtifactsUuid != null)
509 && instArtifactsId.size() != instArtifactsUuid.size()) {
510 log.info(" instArtifactsId.size() != instArtifactsUuid.size() inst {} in service {} ", instName,
515 for (String artifactId : artifacts) {
516 String artifactlabel = findArtifactLabelFromArtifactId(artifactId);
517 ArtifactDefinition artifactDefinition = deploymentArtifacts.get(artifactlabel);
518 if (artifactDefinition == null) {
519 log.info(" artifactDefinition == null label {} inst {} in service {} ", artifactlabel, instName,
523 ArtifactTypeEnum artifactType = ArtifactTypeEnum.findType(artifactDefinition.getArtifactType());
524 if (artifactType != ArtifactTypeEnum.HEAT_ENV) {
525 if (!artifactId.equals(artifactDefinition.getUniqueId())) {
527 " !artifactId.equals(artifactDefinition.getUniqueId() artifact {} artId {} inst {} in service {} ",
528 artifactlabel, artifactId, instName, servicename);
531 if (artifactsUuid != null && !artifactsUuid.contains(artifactDefinition.getArtifactUUID())) {
533 " artifactsUuid.contains(artifactDefinition.getArtifactUUID() label {} inst {} in service {} ",
534 artifactlabel, instName, servicename);
538 if (instArtifactsUuid == null || instArtifactsUuid.isEmpty()) {
539 log.info(" instArtifactsUuid empty. label {} inst {} in service {} ", artifactlabel, instName,
543 if (!instArtifactsUuid.contains(artifactDefinition.getArtifactUUID())) {
545 " instArtifactsUuid.contains(artifactDefinition.getArtifactUUID() label {} inst {} in service {} ",
546 artifactlabel, instName, servicename);
551 for (String artifactUUID : artifactsUuid) {
552 String label = findArtifactLabelFromArtifactId(artifactUUID);
553 if (label != null && !label.isEmpty() && !"".equals(label)) {
557 if(vfModule != null && artifactsUuid != null){
558 return isProblematicVFModule(vfModule, artifactsUuid, instArtifactsUuid);
564 private boolean isProblematicVFModule(VfModuleArtifactPayloadEx vfModule, List<String> artifactsUuid,
565 List<String> instArtifactsUuid) {
566 log.info(" isProblematicVFModule {} ", vfModule.getVfModuleModelName());
567 List<String> vfModuleArtifacts = vfModule.getArtifacts();
568 List<String> allArtifacts = new ArrayList<>();
569 allArtifacts.addAll(artifactsUuid);
570 if(instArtifactsUuid != null)
571 allArtifacts.addAll(instArtifactsUuid);
572 if((vfModuleArtifacts == null || vfModuleArtifacts.isEmpty()) && !artifactsUuid.isEmpty()){
573 log.error(" vfModuleArtifacts == null || vfModuleArtifacts.isEmpty()) && !artifactsUuid.isEmpty()");
576 if(vfModuleArtifacts!= null){
577 if( vfModuleArtifacts.size() != allArtifacts.size()){
578 log.error(" vfModuleArtifacts.size() != allArtifacts.size()");
581 for(String vfModuleArtifact: vfModuleArtifacts){
582 Optional<String> op = allArtifacts.stream().filter(a -> a.equals(vfModuleArtifact)).findAny();
584 log.error("failed to find artifact {} in group artifacts {}", vfModuleArtifact, allArtifacts);
594 private boolean fix(List<Resource> vfLst, List<Service> serviceList, Map<String, List<Component>> nodesToFixTosca,
595 Map<String, List<Component>> vfToFixTosca, Map<String, List<Component>> servicesToFixTosca) {
597 log.info(" Fix started ***** ");
598 if (vfLst != null && !vfLst.isEmpty()) {
603 if (res && serviceList != null && !serviceList.isEmpty()) {
604 res = fixServices(serviceList);
608 Set<String> fixedIds = new HashSet<>();
610 long time = System.currentTimeMillis();
611 String fileName = "FailedGenerateTosca" + "_" + time + ".csv";
613 try(Writer writer = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(fileName), UTF8))) {
614 writer.write("componentType, name, version, UID, UUID, invariantUUID, state\n");
615 List<Component> failedList = new ArrayList<>();
617 if (res && nodesToFixTosca != null && !nodesToFixTosca.isEmpty()) {
619 generateAndSaveToscaArtifacts(nodesToFixTosca, fixedIds, null, failedList);
622 if (vfToFixTosca != null && !vfToFixTosca.isEmpty()) {
624 generateAndSaveToscaArtifacts(vfToFixTosca, fixedIds, vfLst, failedList);
628 for (Component component : vfLst) {
629 res = generateToscaPerComponent(fixedIds, component);
631 TopologyTemplate topologyTemplate = ModelConverter.convertToToscaElement(component);
632 Map<String, GroupDataDefinition> groups = topologyTemplate.getGroups();
633 res = fixDataOnGraph(component.getUniqueId(), VertexTypeEnum.GROUPS, EdgeLabelEnum.GROUPS, groups);
635 Map<String, ArtifactDataDefinition> arifacts = topologyTemplate.getDeploymentArtifacts();
636 res = fixDataOnGraph(component.getUniqueId(), VertexTypeEnum.DEPLOYMENT_ARTIFACTS,
637 EdgeLabelEnum.DEPLOYMENT_ARTIFACTS, arifacts);
640 Map<String, ArtifactDataDefinition> arifacts = topologyTemplate.getToscaArtifacts();
641 res = fixDataOnGraph(component.getUniqueId(), VertexTypeEnum.TOSCA_ARTIFACTS,
642 EdgeLabelEnum.TOSCA_ARTIFACTS, arifacts);
644 janusGraphDao.commit();
646 failedList.add(component);
650 if (servicesToFixTosca != null && !servicesToFixTosca.isEmpty()) {
651 generateAndSaveToscaArtifacts(servicesToFixTosca, fixedIds, serviceList, failedList);
656 for (Component component : serviceList) {
657 res = generateToscaPerComponent(fixedIds, component);
659 TopologyTemplate topologyTemplate = ModelConverter.convertToToscaElement(component);
660 Map<String, MapGroupsDataDefinition> groups = topologyTemplate.getInstGroups();
661 res = fixDataOnGraph(component.getUniqueId(), VertexTypeEnum.INST_GROUPS, EdgeLabelEnum.INST_GROUPS,
665 Map<String, MapArtifactDataDefinition> artifacts = topologyTemplate
666 .getInstDeploymentArtifacts();
667 res = fixDataOnGraph(component.getUniqueId(), VertexTypeEnum.INST_DEPLOYMENT_ARTIFACTS,
668 EdgeLabelEnum.INST_DEPLOYMENT_ARTIFACTS, artifacts);
671 Map<String, ArtifactDataDefinition> arifacts = topologyTemplate.getToscaArtifacts();
672 res = fixDataOnGraph(component.getUniqueId(), VertexTypeEnum.TOSCA_ARTIFACTS,
673 EdgeLabelEnum.TOSCA_ARTIFACTS, arifacts);
675 janusGraphDao.commit();
677 failedList.add(component);
681 if (!failedList.isEmpty()) {
682 for (Component component : failedList) {
683 StringBuilder sb = new StringBuilder(component.getComponentType().getValue());
684 sb.append(",").append(component.getName()).append(",").append(component.getVersion()).append(",")
685 .append(component.getUniqueId()).append(",").append(component.getUUID()).append(",")
686 .append(component.getInvariantUUID()).append(",").append(component.getLifecycleState());
689 writer.write(sb.toString());
693 } catch (IOException e) {
694 log.error(e.getMessage());
696 janusGraphDao.commit();
698 log.info(" Fix finished with res {} ***** ", res);
702 private boolean generateAndSaveToscaArtifacts(Map<String, List<Component>> nodesToFixTosca, Set<String> fixedIds,
703 List<? extends Component> componentsWithFailedGroups, List<Component> failedList) {
705 log.debug("Migration1707ArtifactUuidFix generateAndSaveToscaArtifacts started ");
706 for (Map.Entry<String, List<Component>> entry : nodesToFixTosca.entrySet()) {
708 List<Component> component = entry.getValue();
709 for (Component c : component) {
710 log.debug("Migration1707ArtifactUuidFix fix tosca on component : id {}, name {} ", c.getUniqueId(),
712 if (componentsWithFailedGroups != null) {
713 Optional<Component> op = (Optional<Component>) componentsWithFailedGroups.stream()
714 .filter(cg -> cg.getUniqueId().equals(c.getUniqueId())).findAny();
716 res = generateToscaPerComponent(fixedIds, c);
718 res = generateToscaPerComponent(fixedIds, c);
720 ToscaElement topologyTemplate = ModelConverter.convertToToscaElement(c);
721 Map<String, ArtifactDataDefinition> arifacts = topologyTemplate.getToscaArtifacts();
722 res = fixDataOnGraph(c.getUniqueId(), VertexTypeEnum.TOSCA_ARTIFACTS, EdgeLabelEnum.TOSCA_ARTIFACTS,
724 janusGraphDao.commit();
731 log.debug("Migration1707ArtifactUuidFix generateAndSaveToscaArtifacts finished with res {} ", res);
735 private boolean generateToscaPerComponent(Set<String> fixedIds, Component c) {
737 log.debug("Migration1707ArtifactUuidFix generateToscaPerComponent started component name {} id {}",
738 c.getName(), c.getUniqueId());
740 Either<Component, StorageOperationStatus> toscaElement = toscaOperationFacade
741 .getToscaFullElement(c.getUniqueId());
742 if (toscaElement.isRight()) {
743 log.info("Failed to fetch resources {} {}", c.getUniqueId(), toscaElement.right().value());
746 Component toscaElementFull = toscaElement.left().value();
747 toscaElementFull.setGroups(c.getGroups());
748 List<ComponentInstance> ciListFull = toscaElementFull.getComponentInstances();
749 List<ComponentInstance> ciList = c.getComponentInstances();
750 if (ciListFull != null && !ciListFull.isEmpty()) {
751 ciListFull.forEach(ciFull -> {
752 ComponentInstance compInst = ciList.stream()
753 .filter(ci -> ci.getUniqueId().equals(ciFull.getUniqueId())).findAny().get();
754 ciFull.setGroupInstances(compInst.getGroupInstances());
758 Either<Component, ToscaError> either = generateToscaArtifact(toscaElementFull);
760 if (either.isRight()) {
761 log.error("Couldn't generate and save tosca template component unique id {}, name {} error: {}",
762 toscaElementFull.getUniqueId(), toscaElementFull.getName(), either.right().value());
768 c.setToscaArtifacts(either.left().value().getToscaArtifacts());
769 fixedIds.add(toscaElementFull.getUniqueId());
773 janusGraphDao.commit();
775 janusGraphDao.rollback();
777 log.debug("Migration1707ArtifactUuidFix generateToscaPerComponent finished component name {} id {} res {}",
778 c.getName(), c.getUniqueId(), res);
782 private <T extends ToscaDataDefinition> boolean fixDataOnGraph(String componentId, VertexTypeEnum vertexTypeEnum,
783 EdgeLabelEnum edgeLabelEnum, Map<String, T> groups) {
784 log.debug("amount groups to update: VertexTypeEnum {} EdgeLabelEnum {} data size {}", vertexTypeEnum.getName(),
785 edgeLabelEnum, groups.size());
787 Either<GraphVertex, JanusGraphOperationStatus> getResponse = janusGraphDao.getVertexById(componentId,
788 JsonParseFlagEnum.NoParse);
789 if (getResponse.isRight()) {
790 log.debug("Couldn't fetch component unique id {}, error: {}", componentId, getResponse.right().value());
795 GraphVertex componentVertex = getResponse.left().value();
797 GraphVertex toscaDataVertex = null;
798 Either<GraphVertex, JanusGraphOperationStatus> groupVertexEither = janusGraphDao.getChildVertex(componentVertex,
799 edgeLabelEnum, JsonParseFlagEnum.ParseJson);
800 if (groupVertexEither.isRight() && groupVertexEither.right().value() == JanusGraphOperationStatus.NOT_FOUND) {
801 log.debug("no child {} vertex for component unique id {}, error: {}", edgeLabelEnum, componentId,
802 groupVertexEither.right().value());
805 if (groupVertexEither.isRight()) {
807 log.debug("failed to get child {} vertex for component unique id {}, error: {}", edgeLabelEnum,
808 componentId, groupVertexEither.right().value());
811 toscaDataVertex = groupVertexEither.left().value();
812 toscaDataVertex.setJson(groups);
813 Either<GraphVertex, JanusGraphOperationStatus> updatevertexEither = janusGraphDao.updateVertex(toscaDataVertex);
814 if (updatevertexEither.isRight()) {
815 log.debug("failed to update vertex for component unique id {}, error: {}", componentId,
816 updatevertexEither.right().value());
817 janusGraphDao.rollback();
822 log.debug("Fix data on graph finished: VertexTypeEnum {} EdgeLabelEnum {} res {}", vertexTypeEnum.getName(),
827 private boolean fixServices(List<Service> serviceList) {
828 for (Service service : serviceList) {
829 log.debug("Migration1707ArtifactUuidFix fix service: id {}, name {} ", service.getUniqueId(),
831 List<ComponentInstance> instances = service.getComponentInstances();
832 for (ComponentInstance instance : instances) {
833 fixComponentInstances(service, instance);
841 private void fixComponentInstances(Service service, ComponentInstance instance) {
842 Map<String, ArtifactDefinition> artifactsMap = instance.getDeploymentArtifacts();
843 List<GroupInstance> groupsList = instance.getGroupInstances();
844 if (groupsList != null && artifactsMap != null) {
845 List<GroupInstance> groupsToDelete = new ArrayList<>();
846 for (GroupInstance group : groupsList) {
847 fixGroupInstances(service, artifactsMap, groupsToDelete, group);
851 if (!groupsToDelete.isEmpty()) {
852 log.debug("Migration1707ArtifactUuidFix delete group: resource id {}, group instance to delete {} ",
853 service.getUniqueId(), groupsToDelete);
854 groupsList.removeAll(groupsToDelete);
858 Optional<ArtifactDefinition> optionalVfModuleArtifact = artifactsMap.values().stream()
859 .filter(p -> p.getArtifactType().equals(ArtifactTypeEnum.VF_MODULES_METADATA.name())).findAny();
860 ArtifactDefinition vfModuleArtifact;
861 if(!optionalVfModuleArtifact.isPresent()){
862 vfModuleArtifact = createVfModuleArtifact(instance, service);
863 artifactsMap.put(vfModuleArtifact.getArtifactLabel(), vfModuleArtifact);
866 vfModuleArtifact = optionalVfModuleArtifact.get();
868 fillVfModuleInstHeatEnvPayload(service, instance, groupsList, vfModuleArtifact);
872 private void fixGroupInstances(Service service, Map<String, ArtifactDefinition> artifactsMap,
873 List<GroupInstance> groupsToDelete, GroupInstance group) {
874 if (group.getType().equals(Constants.DEFAULT_GROUP_VF_MODULE)) {
875 log.debug("Migration1707ArtifactUuidFix fix group: resource id {}, group name {} ", service.getUniqueId(),
877 if(group.getArtifacts() != null){
878 Set<String> groupArtifactsSet = new HashSet<>(group.getArtifacts());
879 if(group.getGroupInstanceArtifacts() != null){
880 List<String> groupInsArtifacts = new ArrayList<>(group.getGroupInstanceArtifacts());
881 groupArtifactsSet.addAll(groupInsArtifacts);
883 List<String> groupArtifacts = new ArrayList<>(groupArtifactsSet);
885 clearGroupInstanceArtifacts(group);
887 for (String artifactId : groupArtifacts) {
888 fixArtifactUndergroupInstances(artifactsMap, group, groupArtifacts, artifactId);
891 if (group.getArtifacts() == null || group.getArtifacts().isEmpty()) {
893 "Migration1707ArtifactUuidFix fix groupInstance add to delete list: resource id {} name {} , group name {} ",
894 service.getUniqueId(), service.getName(), group.getName());
895 groupsToDelete.add(group);
900 private void clearGroupInstanceArtifacts(GroupInstance group) {
901 if(group.getArtifacts() != null)
902 group.getArtifacts().clear();
904 group.setArtifacts(new ArrayList<>());
905 if(group.getArtifactsUuid() != null)
906 group.getArtifactsUuid().clear();
908 group.setArtifactsUuid(new ArrayList<>());
910 if(group.getGroupInstanceArtifacts() != null)
911 group.getGroupInstanceArtifacts().clear();
913 group.setGroupInstanceArtifacts(new ArrayList<>());
915 if(group.getGroupInstanceArtifactsUuid() != null )
916 group.getGroupInstanceArtifactsUuid().clear();
918 group.setGroupInstanceArtifactsUuid(new ArrayList<>());
921 private void fixArtifactUndergroupInstances(Map<String, ArtifactDefinition> artifactsMap, GroupInstance group,
922 List<String> groupArtifacts, String artifactId) {
923 String artifactlabel = findArtifactLabelFromArtifactId(artifactId);
924 log.debug("Migration1707ArtifactUuidFix fix group: group name {} artifactId for fix {} artifactlabel {} ",
925 group.getName(), artifactId, artifactlabel);
926 if (!artifactlabel.isEmpty() && artifactsMap.containsKey(artifactlabel)) {
927 ArtifactDefinition artifact = artifactsMap.get(artifactlabel);
928 ArtifactTypeEnum artifactType = ArtifactTypeEnum.findType(artifact.getArtifactType());
929 String correctArtifactId = artifact.getUniqueId();
930 String correctArtifactUUID = artifact.getArtifactUUID();
931 if (artifactType != ArtifactTypeEnum.HEAT_ENV) {
932 boolean isAddToGroup = true;
933 if (groupArtifacts.size() == 1) {
934 if (artifactType == ArtifactTypeEnum.HEAT_ARTIFACT) {
935 isAddToGroup = false;
936 artifact.setArtifactType(ArtifactTypeEnum.OTHER.getType());
941 MIGRATION1707_ARTIFACT_UUID_FIX,
942 group.getName(), correctArtifactId, correctArtifactUUID);
943 group.getArtifacts().add(correctArtifactId);
944 if (correctArtifactUUID != null && !correctArtifactUUID.isEmpty()) {
945 group.getArtifactsUuid().add(correctArtifactUUID);
950 MIGRATION1707_ARTIFACT_UUID_FIX,
951 group.getName(), correctArtifactId, correctArtifactUUID);
952 Set<String> tmpSet = new HashSet<>(group.getGroupInstanceArtifacts());
953 tmpSet.add(correctArtifactId);
954 group.setGroupInstanceArtifacts(new ArrayList<>(tmpSet));
955 if (correctArtifactUUID != null && !correctArtifactUUID.isEmpty()) {
956 Set<String> tmpSetUUID = new HashSet<>(group.getGroupInstanceArtifactsUuid());
957 tmpSetUUID.add(correctArtifactUUID);
958 group.setGroupInstanceArtifactsUuid(new ArrayList<>(tmpSetUUID));
964 private boolean fixVf(List<Resource> vfLst) {
965 for (Resource resource : vfLst) {
966 log.debug("Migration1707ArtifactUuidFix fix resource: id {}, name {} ", resource.getUniqueId(),
968 Map<String, ArtifactDefinition> artifactsMap = resource.getDeploymentArtifacts();
969 List<GroupDefinition> groupsList = resource.getGroups();
970 List<GroupDefinition> groupsToDelete = new ArrayList<>();
971 if (groupsList != null && artifactsMap != null) {
972 for (GroupDefinition group : groupsList) {
973 if (group.getType().equals(Constants.DEFAULT_GROUP_VF_MODULE) && group.getArtifacts() != null) {
974 fixVfGroup(resource, artifactsMap, group);
976 if (group.getType().equals(Constants.DEFAULT_GROUP_VF_MODULE)
977 && (group.getArtifacts() == null || group.getArtifacts().isEmpty())) {
979 "Migration1707ArtifactUuidFix add group to delete list fix resource: id {}, name {} ",
980 resource.getUniqueId(), resource.getName(), group.getName());
981 groupsToDelete.add(group);
985 if (!groupsToDelete.isEmpty()) {
986 groupsList.removeAll(groupsToDelete);
996 private void fixVfGroup(Resource resource, Map<String, ArtifactDefinition> artifactsMap, GroupDefinition group) {
997 log.debug("Migration1707ArtifactUuidFix fix group: resource id {}, group name {} ", resource.getUniqueId(),
999 Set<String> groupArtifactsSet = new HashSet<>(group.getArtifacts());
1000 List<String> groupArtifacts = new ArrayList<>(groupArtifactsSet);
1001 group.getArtifacts().clear();
1002 group.getArtifactsUuid().clear();
1004 for (String artifactId : groupArtifacts) {
1005 fixArtifactUnderGroup(artifactsMap, group, groupArtifacts, artifactId);
1009 private void fixArtifactUnderGroup(Map<String, ArtifactDefinition> artifactsMap, GroupDefinition group,
1010 List<String> groupArtifacts, String artifactId) {
1012 String artifactlabel = findArtifactLabelFromArtifactId(artifactId);
1013 log.debug("Migration1707ArtifactUuidFix fix group: group name {} artifactId for fix {} artifactlabel {} ",
1014 group.getName(), artifactId, artifactlabel);
1015 if (!artifactlabel.isEmpty() && artifactsMap.containsKey(artifactlabel)) {
1016 ArtifactDefinition artifact = artifactsMap.get(artifactlabel);
1017 String correctArtifactId = artifact.getUniqueId();
1018 String correctArtifactUUID = artifact.getArtifactUUID();
1019 boolean isAddToGroup = true;
1020 if (groupArtifacts.size() == 1) {
1021 ArtifactTypeEnum artifactType = ArtifactTypeEnum.findType(artifact.getArtifactType());
1022 if (artifactType == ArtifactTypeEnum.HEAT_ARTIFACT) {
1023 isAddToGroup = false;
1024 artifact.setArtifactType(ArtifactTypeEnum.OTHER.getType());
1029 MIGRATION1707_ARTIFACT_UUID_FIX,
1030 group.getName(), correctArtifactId, correctArtifactUUID);
1031 group.getArtifacts().add(correctArtifactId);
1032 if (correctArtifactUUID != null && !correctArtifactUUID.isEmpty()) {
1033 group.getArtifactsUuid().add(correctArtifactUUID);
1040 private String findArtifactLabelFromArtifactId(String artifactId) {
1041 String artifactLabel = "";
1043 int index = artifactId.lastIndexOf('.');
1044 if (index > 0 && index + 1 < artifactId.length())
1045 artifactLabel = artifactId.substring(index + 1);
1046 return artifactLabel;
1049 private void writeModuleResultToFile(Writer writer, org.openecomp.sdc.be.model.Component component,
1052 // "service name, service id, state, version
1053 StringBuilder sb = new StringBuilder(component.getName());
1054 sb.append(",").append(component.getUniqueId()).append(",").append(component.getLifecycleState()).append(",")
1055 .append(component.getVersion());
1056 if (service != null) {
1057 sb.append(",").append(service.getName());
1060 writer.write(sb.toString());
1061 } catch (IOException e) {
1062 log.error(e.getMessage());
1066 private void writeModuleResultToFile(Writer writer, List<Component> components) {
1068 // "service name, service id, state, version
1069 for (Component component : components) {
1070 StringBuilder sb = new StringBuilder(component.getName());
1071 sb.append(",").append(component.getUniqueId()).append(",").append(component.getInvariantUUID())
1072 .append(",").append(component.getLifecycleState()).append(",").append(component.getVersion());
1075 writer.write(sb.toString());
1077 } catch (IOException e) {
1079 log.error(e.getMessage());
1083 public boolean doFixTosca(Map<String, List<Component>> nodeToFix, Map<String, List<Component>> vfToFix,
1084 Map<String, List<Component>> serviceToFix) {
1086 Map<GraphPropertyEnum, Object> hasProps = new EnumMap<>(GraphPropertyEnum.class);
1087 hasProps.put(GraphPropertyEnum.COMPONENT_TYPE, ComponentTypeEnum.RESOURCE.name());
1088 hasProps.put(GraphPropertyEnum.STATE, LifecycleStateEnum.CERTIFIED.name());
1090 Map<String, List<Component>> vertices = getVerticesToValidate(VertexTypeEnum.NODE_TYPE, hasProps);
1091 boolean result = validateTosca(vertices, nodeToFix, "RESOURCE_TOSCA_ARTIFACTS");//
1094 hasProps.put(GraphPropertyEnum.COMPONENT_TYPE, ComponentTypeEnum.RESOURCE.name());
1095 hasProps.put(GraphPropertyEnum.RESOURCE_TYPE, ResourceTypeEnum.VF);
1096 hasProps.put(GraphPropertyEnum.STATE, LifecycleStateEnum.CERTIFIED.name());
1098 vertices = getVerticesToValidate(VertexTypeEnum.TOPOLOGY_TEMPLATE, hasProps);
1099 result = validateTosca(vertices, vfToFix, "VF_TOSCA_ARTIFACTS");
1102 hasProps.put(GraphPropertyEnum.COMPONENT_TYPE, ComponentTypeEnum.SERVICE.name());
1103 hasProps.put(GraphPropertyEnum.STATE, LifecycleStateEnum.CERTIFIED.name());
1105 vertices = getVerticesToValidate(VertexTypeEnum.TOPOLOGY_TEMPLATE, hasProps);
1106 result = validateTosca(vertices, serviceToFix, "SERVICE_TOSCA_ARTIFACTS");
1111 public Map<String, List<Component>> getVerticesToValidate(VertexTypeEnum type,
1112 Map<GraphPropertyEnum, Object> hasProps) {
1114 Map<String, List<Component>> result = new HashMap<>();
1117 Either<List<GraphVertex>, JanusGraphOperationStatus> resultsEither = janusGraphDao
1118 .getByCriteria(type, hasProps);
1119 if (resultsEither.isRight()) {
1120 log.error("getVerticesToValidate failed {} ",resultsEither.right().value());
1123 log.info("getVerticesToValidate: {} vertices to scan", resultsEither.left().value().size());
1124 List<GraphVertex> componentsList = resultsEither.left().value();
1125 componentsList.forEach(vertex -> {
1126 String ivariantUuid = (String) vertex.getMetadataProperty(GraphPropertyEnum.INVARIANT_UUID);
1127 if (!result.containsKey(ivariantUuid)) {
1128 List<Component> compList = new ArrayList<>();
1129 result.put(ivariantUuid, compList);
1131 List<Component> compList = result.get(ivariantUuid);
1133 ComponentParametersView filter = new ComponentParametersView(true);
1134 filter.setIgnoreArtifacts(false);
1136 Either<Component, StorageOperationStatus> toscaElement = toscaOperationFacade
1137 .getToscaElement(vertex.getUniqueId(), filter);
1138 if (toscaElement.isRight()) {
1139 log.error("getVerticesToValidate: failed to find element {} staus is {}", vertex.getUniqueId()
1140 ,toscaElement.right().value());
1142 compList.add(toscaElement.left().value());
1144 janusGraphDao.commit();
1148 } catch (Exception e) {
1149 log.info(FAILED_TO_FETCH_VF_RESOURCES, e);
1152 janusGraphDao.commit();
1158 public boolean validateTosca(Map<String, List<Component>> vertices, Map<String, List<Component>> compToFix,
1160 boolean result = true;
1161 long time = System.currentTimeMillis();
1162 String fileName = name + "_" + time + ".csv";
1163 try(Writer writer = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(fileName), UTF8))) {
1164 writer.write("name, UUID, invariantUUID, state, version\n");
1165 for (Map.Entry<String, List<Component>> entry : vertices.entrySet()) {
1166 List<Component> compList = entry.getValue();
1167 Set<String> artifactEsId = new HashSet<>();
1168 for (Component component : compList) {
1169 Map<String, ArtifactDefinition> toscaArtifacts = component.getToscaArtifacts();
1170 Optional<ArtifactDefinition> op = toscaArtifacts.values().stream()
1171 .filter(a -> artifactEsId.contains(a.getEsId()) && a.getEsId() != null).findAny();
1172 if (op.isPresent()) {
1174 writeModuleResultToFile(writer, compList);
1178 artifactEsId.addAll(toscaArtifacts.values().stream().map(ArtifactDefinition::getEsId)
1179 .collect(Collectors.toList()));
1183 List<Component> compListfull = new ArrayList<>();
1184 for (Component c : compList) {
1185 ComponentParametersView filter = new ComponentParametersView(true);
1186 filter.setIgnoreComponentInstances(false);
1187 filter.setIgnoreArtifacts(false);
1188 filter.setIgnoreGroups(false);
1190 Either<Component, StorageOperationStatus> toscaElement = toscaOperationFacade
1191 .getToscaElement(c.getUniqueId(), filter);
1192 if (toscaElement.isRight()) {
1193 log.debug("getVerticesToValidate: failed to find element" + c.getUniqueId()
1194 + " status is" + toscaElement.right().value());
1196 compListfull.add(toscaElement.left().value());
1198 this.janusGraphDao.commit();
1201 compToFix.put(entry.getKey(), compListfull);
1207 } catch (Exception e) {
1208 log.info(FAILED_TO_FETCH_VF_RESOURCES, e);
1211 janusGraphDao.commit();
1216 private Either<Component, ToscaError> generateToscaArtifact(Component parent) {
1217 log.debug("tosca artifact generation");
1219 Map<String, ArtifactDefinition> toscaArtifacts = parent.getToscaArtifacts();
1221 ArtifactDefinition toscaArtifact = null;
1222 Optional<ArtifactDefinition> op = toscaArtifacts.values().stream()
1223 .filter(p -> p.getArtifactType().equals(ArtifactTypeEnum.TOSCA_TEMPLATE.getType())).findAny();
1225 if (op.isPresent()) {
1226 toscaArtifact = op.get();
1228 if (toscaArtifact != null) {
1229 log.debug("Migration1707ArtifactUuidFix generateToscaPerComponent artifact name {} id {} esId {}",
1230 toscaArtifact.getArtifactName(), toscaArtifact.getUniqueId(), toscaArtifact.getEsId());
1232 Either<ToscaRepresentation, ToscaError> exportComponent = toscaExportUtils.exportComponent(parent);
1233 if (exportComponent.isRight()) {
1234 log.debug("Failed export tosca yaml for component {} error {}", parent.getUniqueId(),
1235 exportComponent.right().value());
1237 return Either.right(exportComponent.right().value());
1239 log.debug("Tosca yaml exported for component {} ", parent.getUniqueId());
1240 String payload = exportComponent.left().value().getMainYaml();
1242 toscaArtifact.setPayloadData(payload);
1243 byte[] decodedPayload = toscaArtifact.getPayloadData();
1245 String uniqueId = UniqueIdBuilder.buildPropertyUniqueId(parent.getUniqueId(),
1246 toscaArtifact.getArtifactLabel());
1247 toscaArtifact.setUniqueId(uniqueId);
1248 toscaArtifact.setEsId(toscaArtifact.getUniqueId());
1250 toscaArtifact.setArtifactChecksum(GeneralUtility.calculateMD5Base64EncodedByByteArray(decodedPayload));
1251 DAOArtifactData artifactData = new DAOArtifactData(toscaArtifact.getEsId(), decodedPayload);
1252 artifactCassandraDao.saveArtifact(artifactData);
1254 log.debug("Tosca yaml artifact esId {} ", toscaArtifact.getEsId());
1256 ArtifactDefinition csarArtifact = null;
1257 op = toscaArtifacts.values().stream()
1258 .filter(p -> p.getArtifactType().equals(ArtifactTypeEnum.TOSCA_CSAR.getType())).findAny();
1260 if (op.isPresent()) {
1261 csarArtifact = op.get();
1264 if (csarArtifact != null) {
1265 Either<byte[], ResponseFormat> generated = csarUtils.createCsar(parent, true, true);
1267 if (generated.isRight()) {
1268 log.debug("Failed to export tosca csar for component {} error {}", parent.getUniqueId(),
1269 generated.right().value());
1271 return Either.right(ToscaError.GENERAL_ERROR);
1273 byte[] value = generated.left().value();
1274 csarArtifact.setPayload(value);
1275 byte[] decodedPayload = csarArtifact.getPayloadData();
1277 String uniqueId = UniqueIdBuilder.buildPropertyUniqueId(parent.getUniqueId(),
1278 csarArtifact.getArtifactLabel());
1279 csarArtifact.setUniqueId(uniqueId);
1280 csarArtifact.setEsId(csarArtifact.getUniqueId());
1282 csarArtifact.setArtifactChecksum(GeneralUtility.calculateMD5Base64EncodedByByteArray(decodedPayload));
1283 DAOArtifactData artifactData = new DAOArtifactData(csarArtifact.getEsId(), decodedPayload);
1284 artifactCassandraDao.saveArtifact(artifactData);
1285 log.debug("Tosca csar artifact esId {} ", csarArtifact.getEsId());
1289 } catch (Exception ex) {
1290 log.error("Failed to generate tosca atifact component id {} component name {} error {}",
1291 parent.getUniqueId(), parent.getName(), ex.getMessage());
1293 return Either.right(ToscaError.GENERAL_ERROR);
1296 return Either.left(parent);
1299 private ArtifactDefinition createVfModuleArtifact(ComponentInstance currVF, Service service) {
1301 ArtifactDefinition vfModuleArtifactDefinition = new ArtifactDefinition();
1303 vfModuleArtifactDefinition.setDescription("Auto-generated VF Modules information artifact");
1304 vfModuleArtifactDefinition.setArtifactDisplayName("Vf Modules Metadata");
1305 vfModuleArtifactDefinition.setArtifactType(ArtifactTypeEnum.VF_MODULES_METADATA.getType());
1306 vfModuleArtifactDefinition.setArtifactGroupType(ArtifactGroupTypeEnum.DEPLOYMENT);
1307 vfModuleArtifactDefinition.setArtifactLabel("vfModulesMetadata");
1308 vfModuleArtifactDefinition.setTimeout(0);
1309 vfModuleArtifactDefinition.setArtifactName(currVF.getNormalizedName() + "_modules.json");
1311 return vfModuleArtifactDefinition;
1315 private void fillVfModuleInstHeatEnvPayload(Component parent, ComponentInstance instance, List<GroupInstance> groupsForCurrVF,
1316 ArtifactDefinition vfModuleArtifact) {
1317 log.debug("generate new vf module for component. name {}, id {}, Version {}", instance.getName(), instance.getUniqueId());
1319 String uniqueId = UniqueIdBuilder.buildInstanceArtifactUniqueId(parent.getUniqueId(), instance.getUniqueId(), vfModuleArtifact.getArtifactLabel());
1321 vfModuleArtifact.setUniqueId(uniqueId);
1322 vfModuleArtifact.setEsId(vfModuleArtifact.getUniqueId());
1324 List<VfModuleArtifactPayload> vfModulePayloadForCurrVF = new ArrayList<>();
1325 if (groupsForCurrVF != null) {
1326 for (GroupInstance groupInstance : groupsForCurrVF) {
1327 VfModuleArtifactPayload modulePayload = new VfModuleArtifactPayload(groupInstance);
1328 vfModulePayloadForCurrVF.add(modulePayload);
1330 Collections.sort(vfModulePayloadForCurrVF,
1331 (art1, art2) -> VfModuleArtifactPayload.compareByGroupName(art1, art2));
1333 final Gson gson = new GsonBuilder().setPrettyPrinting().create();
1335 String vfModulePayloadString = gson.toJson(vfModulePayloadForCurrVF);
1336 log.debug("vfModulePayloadString {}", vfModulePayloadString);
1337 if (vfModulePayloadString != null) {
1338 String newCheckSum = GeneralUtility
1339 .calculateMD5Base64EncodedByByteArray(vfModulePayloadString.getBytes());
1340 vfModuleArtifact.setArtifactChecksum(newCheckSum);
1342 DAOArtifactData artifactData = new DAOArtifactData(vfModuleArtifact.getEsId(),
1343 vfModulePayloadString.getBytes());
1344 artifactCassandraDao.saveArtifact(artifactData);
1352 private Either<List<VfModuleArtifactPayloadEx>, StorageOperationStatus> parseVFModuleJson(ArtifactDefinition vfModuleArtifact) {
1353 log.info("Try to get vfModule json from cassandra {}", vfModuleArtifact.getEsId());
1354 Either<DAOArtifactData, CassandraOperationStatus> vfModuleData = artifactCassandraDao.getArtifact(vfModuleArtifact.getEsId());
1356 if (vfModuleData.isRight()) {
1357 CassandraOperationStatus resourceUploadStatus = vfModuleData.right().value();
1358 StorageOperationStatus storageResponse = DaoStatusConverter.convertCassandraStatusToStorageStatus(resourceUploadStatus);
1359 log.error("failed to fetch vfModule json {} from cassandra. Status is {}", vfModuleArtifact.getEsId(), storageResponse);
1360 return Either.right(storageResponse);
1364 DAOArtifactData DAOArtifactData = vfModuleData.left().value();
1365 String gsonData = new String( DAOArtifactData.getDataAsArray());
1366 final Gson gson = new GsonBuilder().setPrettyPrinting().create();
1367 JsonArray jsonElement = new JsonArray();
1368 jsonElement = gson.fromJson(gsonData, jsonElement.getClass());
1369 List<VfModuleArtifactPayloadEx> vfModules = new ArrayList<>();
1370 jsonElement.forEach(je ->{
1371 VfModuleArtifactPayloadEx vfModule = ComponentsUtils.parseJsonToObject(je.toString(), VfModuleArtifactPayloadEx.class);
1372 vfModules.add(vfModule);
1375 log.debug ("parse vf module finish {}", gsonData);
1376 return Either.left(vfModules);