catalog-model/src/main/java/org/openecomp/sdc/be/model/operations/impl/ModelOperation.java

   1 /*
   2  * ============LICENSE_START=======================================================
   3  *  Copyright (C) 2021 Nordix Foundation
   4  *  ================================================================================
   5  *  Licensed under the Apache License, Version 2.0 (the "License");
   6  *  you may not use this file except in compliance with the License.
   7  *  You may obtain a copy of the License at
   8  *
   9  *        http://www.apache.org/licenses/LICENSE-2.0
  10  *  Unless required by applicable law or agreed to in writing, software
  11  *  distributed under the License is distributed on an "AS IS" BASIS,
  12  *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  *  See the License for the specific language governing permissions and
  14  *  limitations under the License.
  15  *
  16  *  SPDX-License-Identifier: Apache-2.0
  17  *  ============LICENSE_END=========================================================
  18  */
  19 package org.openecomp.sdc.be.model.operations.impl;
  20
  21 import static org.openecomp.sdc.common.api.Constants.ADDITIONAL_TYPE_DEFINITIONS;
  22
  23 import fj.data.Either;
  24 import java.nio.charset.StandardCharsets;
  25 import java.nio.file.Path;
  26 import java.util.ArrayList;
  27 import java.util.Collections;
  28 import java.util.EnumMap;
  29 import java.util.HashMap;
  30 import java.util.LinkedHashMap;
  31 import java.util.List;
  32 import java.util.Map;
  33 import java.util.Map.Entry;
  34 import java.util.Objects;
  35 import java.util.Optional;
  36 import java.util.Set;
  37 import java.util.stream.Collectors;
  38 import org.apache.commons.collections.CollectionUtils;
  39 import org.apache.commons.collections.MapUtils;
  40 import org.apache.commons.lang3.StringUtils;
  41 import org.apache.commons.lang3.tuple.ImmutablePair;
  42 import org.onap.sdc.tosca.services.YamlUtil;
  43 import org.openecomp.sdc.be.dao.api.ActionStatus;
  44 import org.openecomp.sdc.be.dao.cassandra.ToscaModelImportCassandraDao;
  45 import org.openecomp.sdc.be.dao.graph.datatype.GraphEdge;
  46 import org.openecomp.sdc.be.dao.graph.datatype.GraphRelation;
  47 import org.openecomp.sdc.be.dao.janusgraph.JanusGraphDao;
  48 import org.openecomp.sdc.be.dao.janusgraph.JanusGraphGenericDao;
  49 import org.openecomp.sdc.be.dao.janusgraph.JanusGraphOperationStatus;
  50 import org.openecomp.sdc.be.dao.jsongraph.GraphVertex;
  51 import org.openecomp.sdc.be.dao.jsongraph.types.VertexTypeEnum;
  52 import org.openecomp.sdc.be.dao.neo4j.GraphEdgeLabels;
  53 import org.openecomp.sdc.be.data.model.ToscaImportByModel;
  54 import org.openecomp.sdc.be.datatypes.enums.GraphPropertyEnum;
  55 import org.openecomp.sdc.be.datatypes.enums.ModelTypeEnum;
  56 import org.openecomp.sdc.be.datatypes.enums.NodeTypeEnum;
  57 import org.openecomp.sdc.be.model.DataTypeDefinition;
  58 import org.openecomp.sdc.be.model.Model;
  59 import org.openecomp.sdc.be.model.jsonjanusgraph.operations.exception.ModelOperationExceptionSupplier;
  60 import org.openecomp.sdc.be.model.jsonjanusgraph.operations.exception.OperationException;
  61 import org.openecomp.sdc.be.model.normatives.ElementTypeEnum;
  62 import org.openecomp.sdc.be.model.operations.api.DerivedFromOperation;
  63 import org.openecomp.sdc.be.model.operations.api.StorageOperationStatus;
  64 import org.openecomp.sdc.be.resources.data.ModelData;
  65 import org.openecomp.sdc.common.log.enums.EcompLoggerErrorCode;
  66 import org.openecomp.sdc.common.log.wrappers.Logger;
  67 import org.springframework.beans.factory.annotation.Autowired;
  68 import org.springframework.stereotype.Component;
  69 import org.yaml.snakeyaml.Yaml;
  70
  71 @Component("model-operation")
  72 public class ModelOperation {
  73
  74     private static final Logger log = Logger.getLogger(ModelOperation.class);
  75     static final Path ADDITIONAL_TYPE_DEFINITIONS_PATH = Path.of(ADDITIONAL_TYPE_DEFINITIONS);
  76
  77     private final JanusGraphGenericDao janusGraphGenericDao;
  78     private final JanusGraphDao janusGraphDao;
  79     private final ToscaModelImportCassandraDao toscaModelImportCassandraDao;
  80     private final DerivedFromOperation derivedFromOperation;
  81     private ModelElementOperation modelElementOperation;
  82
  83     @Autowired
  84     public ModelOperation(final JanusGraphGenericDao janusGraphGenericDao,
  85                           final JanusGraphDao janusGraphDao,
  86                           final ToscaModelImportCassandraDao toscaModelImportCassandraDao,
  87                           final DerivedFromOperation derivedFromOperation) {
  88         this.janusGraphGenericDao = janusGraphGenericDao;
  89         this.janusGraphDao = janusGraphDao;
  90         this.toscaModelImportCassandraDao = toscaModelImportCassandraDao;
  91         this.derivedFromOperation = derivedFromOperation;
  92     }
  93
  94     public Model createModel(final Model model, final boolean inTransaction) {
  95         Model result = null;
  96         final var modelData = new ModelData(model.getName(), UniqueIdBuilder.buildModelUid(model.getName()), model.getModelType());
  97         try {
  98             final Either<ModelData, JanusGraphOperationStatus> createNode = janusGraphGenericDao.createNode(modelData, ModelData.class);
  99             if (createNode.isRight()) {
 100                 final var janusGraphOperationStatus = createNode.right().value();
 101                 log.error(EcompLoggerErrorCode.DATA_ERROR, ModelOperation.class.getName(), "Problem while creating model, reason {}",
 102                     janusGraphOperationStatus);
 103                 if (janusGraphOperationStatus == JanusGraphOperationStatus.JANUSGRAPH_SCHEMA_VIOLATION) {
 104                     throw ModelOperationExceptionSupplier.modelAlreadyExists(model.getName()).get();
 105                 }
 106                 throw new OperationException(ActionStatus.GENERAL_ERROR,
 107                     String.format("Failed to create model %s on JanusGraph with %s error", model, janusGraphOperationStatus));
 108             }
 109             addDerivedFromRelation(model);
 110             result = new Model(createNode.left().value().getName(), model.getDerivedFrom(), model.getModelType());
 111             return result;
 112         } finally {
 113             if (!inTransaction) {
 114                 if (Objects.nonNull(result)) {
 115                     janusGraphGenericDao.commit();
 116                 } else {
 117                     janusGraphGenericDao.rollback();
 118                 }
 119             }
 120         }
 121     }
 122
 123     private void addDerivedFromRelation(final Model model) {
 124         final String derivedFrom = model.getDerivedFrom();
 125         if (derivedFrom == null) {
 126             return;
 127         }
 128         log.debug("Adding derived from relation between model {} to its parent {}",
 129             model.getName(), derivedFrom);
 130         final Optional<Model> derivedFromModelOptional = this.findModelByName(derivedFrom);
 131         if (derivedFromModelOptional.isPresent()) {
 132             final Either<GraphRelation, StorageOperationStatus> result = derivedFromOperation.addDerivedFromRelation(
 133                 UniqueIdBuilder.buildModelUid(model.getName()),
 134                 UniqueIdBuilder.buildModelUid(derivedFromModelOptional.get().getName()), NodeTypeEnum.Model);
 135             if (result.isRight()) {
 136                 throw new OperationException(ActionStatus.GENERAL_ERROR,
 137                     String.format("Failed to create relationship from model %s to derived from model %s on JanusGraph with %s error", model,
 138                         derivedFrom, result.right().value()));
 139             }
 140         }
 141     }
 142
 143     public Optional<GraphVertex> findModelVertexByName(final String name) {
 144         if (StringUtils.isEmpty(name)) {
 145             return Optional.empty();
 146         }
 147         final Map<GraphPropertyEnum, Object> props = new EnumMap<>(GraphPropertyEnum.class);
 148         props.put(GraphPropertyEnum.NAME, name);
 149         props.put(GraphPropertyEnum.UNIQUE_ID, UniqueIdBuilder.buildModelUid(name));
 150         final List<GraphVertex> modelVerticesList = findModelVerticesByCriteria(props);
 151         if (modelVerticesList.isEmpty()) {
 152             return Optional.empty();
 153         }
 154         return Optional.ofNullable(modelVerticesList.get(0));
 155     }
 156
 157     public Optional<Model> findModelByName(final String name) {
 158         if (StringUtils.isEmpty(name)) {
 159             return Optional.empty();
 160         }
 161         final Optional<GraphVertex> modelVertexOpt = findModelVertexByName(name);
 162         if (modelVertexOpt.isEmpty()) {
 163             return Optional.empty();
 164         }
 165
 166         final GraphVertex graphVertex = modelVertexOpt.get();
 167         return Optional.of(convertToModel(graphVertex));
 168     }
 169
 170     public void createModelImports(final String modelId, final Map<String, byte[]> zipContent) {
 171         if (MapUtils.isEmpty(zipContent)) {
 172             return;
 173         }
 174         final List<ToscaImportByModel> toscaImportByModelList = zipContent.entrySet().stream()
 175             .map(entry -> {
 176                 final String path = entry.getKey();
 177                 final byte[] bytes = entry.getValue();
 178                 final String content = new String(bytes, StandardCharsets.UTF_8);
 179                 final var toscaImportByModel = new ToscaImportByModel();
 180                 toscaImportByModel.setModelId(modelId);
 181                 toscaImportByModel.setFullPath(path);
 182                 toscaImportByModel.setContent(content);
 183                 return toscaImportByModel;
 184             }).collect(Collectors.toList());
 185         toscaModelImportCassandraDao.replaceImports(modelId, toscaImportByModelList);
 186     }
 187
 188     /**
 189      * Find all the model default imports, with the option to include the default imports from the parent model.
 190      *
 191      * @param modelId       the model id
 192      * @param includeParent a flag to include the parent model imports.
 193      * @return the list of model default imports, or an empty list if no imports were found.
 194      */
 195     public List<ToscaImportByModel> findAllModelImports(final String modelId, final boolean includeParent) {
 196         final List<ToscaImportByModel> toscaImportByModelList = toscaModelImportCassandraDao.findAllByModel(modelId);
 197         if (includeParent) {
 198             findModelByName(modelId).ifPresent(model -> {
 199                 if (model.getDerivedFrom() != null) {
 200                     toscaImportByModelList.addAll(toscaModelImportCassandraDao.findAllByModel(model.getDerivedFrom()));
 201                 }
 202             });
 203         }
 204         toscaImportByModelList.sort((o1, o2) -> {
 205             final int modelIdComparison = o1.getModelId().compareTo(o2.getModelId());
 206             if (modelIdComparison == 0) {
 207                 return o1.getFullPath().compareTo(o2.getFullPath());
 208             }
 209             return modelIdComparison;
 210         });
 211         return toscaImportByModelList;
 212     }
 213
 214     /**
 215      * Finds all the models.
 216      *
 217      * @return the list of models
 218      */
 219     public List<Model> findAllModels() {
 220         return findModelsByCriteria(Collections.emptyMap());
 221     }
 222
 223     public List<Model> findModels(final ModelTypeEnum modelType) {
 224         final Map<GraphPropertyEnum, Object> propertyCriteria = new EnumMap<>(GraphPropertyEnum.class);
 225         propertyCriteria.put(GraphPropertyEnum.MODEL_TYPE, modelType.getValue());
 226
 227         return findModelsByCriteria(propertyCriteria);
 228     }
 229
 230     private List<Model> findModelsByCriteria(final Map<GraphPropertyEnum, Object> propertyCriteria) {
 231         final List<GraphVertex> modelVerticesByCriteria = findModelVerticesByCriteria(propertyCriteria);
 232         if (modelVerticesByCriteria.isEmpty()) {
 233             return Collections.emptyList();
 234         }
 235
 236         return modelVerticesByCriteria.stream().map(this::convertToModel).collect(Collectors.toList());
 237     }
 238
 239     private List<GraphVertex> findModelVerticesByCriteria(final Map<GraphPropertyEnum, Object> propertyCriteria) {
 240         final Either<List<GraphVertex>, JanusGraphOperationStatus> result = janusGraphDao.getByCriteria(VertexTypeEnum.MODEL, propertyCriteria);
 241         if (result.isRight()) {
 242             final var janusGraphOperationStatus = result.right().value();
 243             if (janusGraphOperationStatus == JanusGraphOperationStatus.NOT_FOUND) {
 244                 return Collections.emptyList();
 245             }
 246             final var operationException = ModelOperationExceptionSupplier.failedToRetrieveModels(janusGraphOperationStatus).get();
 247             log.error(EcompLoggerErrorCode.DATA_ERROR, this.getClass().getName(), operationException.getMessage());
 248             throw operationException;
 249         }
 250         return result.left().value();
 251     }
 252
 253     private Model convertToModel(final GraphVertex modelGraphVertex) {
 254         final String modelName = (String) modelGraphVertex.getMetadataProperty(GraphPropertyEnum.NAME);
 255         final String modelTypeProperty = (String) modelGraphVertex.getMetadataProperty(GraphPropertyEnum.MODEL_TYPE);
 256         ModelTypeEnum modelType = ModelTypeEnum.NORMATIVE;
 257         final Optional<ModelTypeEnum> optionalModelTypeEnum = ModelTypeEnum.findByValue(modelTypeProperty);
 258         if (optionalModelTypeEnum.isPresent()) {
 259             modelType = optionalModelTypeEnum.get();
 260         }
 261
 262         final Either<ImmutablePair<ModelData, GraphEdge>, JanusGraphOperationStatus> parentNode =
 263             janusGraphGenericDao.getChild(UniqueIdBuilder.getKeyByNodeType(NodeTypeEnum.Model), UniqueIdBuilder.buildModelUid(modelName),
 264                 GraphEdgeLabels.DERIVED_FROM, NodeTypeEnum.Model, ModelData.class);
 265         log.debug("After retrieving DERIVED_FROM node of {}. status is {}", modelName, parentNode);
 266         if (parentNode.isRight()) {
 267             final JanusGraphOperationStatus janusGraphOperationStatus = parentNode.right().value();
 268             if (janusGraphOperationStatus != JanusGraphOperationStatus.NOT_FOUND) {
 269                 final var operationException = ModelOperationExceptionSupplier.failedToRetrieveModels(janusGraphOperationStatus).get();
 270                 log.error(EcompLoggerErrorCode.DATA_ERROR, this.getClass().getName(), operationException.getMessage());
 271                 throw operationException;
 272             }
 273             return new Model((String) modelGraphVertex.getMetadataProperty(GraphPropertyEnum.NAME), modelType);
 274         } else {
 275             final ModelData parentModel = parentNode.left().value().getKey();
 276             return new Model((String) modelGraphVertex.getMetadataProperty(GraphPropertyEnum.NAME), parentModel.getName(), modelType);
 277         }
 278     }
 279
 280     public void addTypesToDefaultImports(final ElementTypeEnum elementTypeEnum, final String typesYaml, final String modelName) {
 281         final List<ToscaImportByModel> modelImportList = toscaModelImportCassandraDao.findAllByModel(modelName);
 282         final Optional<ToscaImportByModel> additionalTypeDefinitionsImportOptional = modelImportList.stream()
 283             .filter(t -> ADDITIONAL_TYPE_DEFINITIONS_PATH.equals(Path.of(t.getFullPath()))).findAny();
 284         final ToscaImportByModel additionalTypeDefinitionsImport;
 285         final List<ToscaImportByModel> rebuiltModelImportList;
 286         if (additionalTypeDefinitionsImportOptional.isPresent()) {
 287             additionalTypeDefinitionsImport = additionalTypeDefinitionsImportOptional.get();
 288             rebuiltModelImportList = modelImportList.stream()
 289                 .filter(toscaImportByModel -> !ADDITIONAL_TYPE_DEFINITIONS_PATH.equals(Path.of(toscaImportByModel.getFullPath())))
 290                 .collect(Collectors.toList());
 291         } else {
 292             additionalTypeDefinitionsImport = new ToscaImportByModel();
 293             additionalTypeDefinitionsImport.setModelId(modelName);
 294             additionalTypeDefinitionsImport.setFullPath(ADDITIONAL_TYPE_DEFINITIONS_PATH.toString());
 295             additionalTypeDefinitionsImport.setContent(createAdditionalTypeDefinitionsHeader());
 296             rebuiltModelImportList = new ArrayList<>(modelImportList);
 297         }
 298
 299         final Map<String, Object> typesYamlMap = new Yaml().load(typesYaml);
 300         removeExistingTypesFromDefaultImports(elementTypeEnum, typesYamlMap, rebuiltModelImportList);
 301
 302         final Map<String, Object> originalContent = new Yaml().load(additionalTypeDefinitionsImport.getContent());
 303         additionalTypeDefinitionsImport.setContent(buildAdditionalTypeDefinitionsContent(elementTypeEnum, typesYamlMap, originalContent));
 304         rebuiltModelImportList.add(additionalTypeDefinitionsImport);
 305
 306         toscaModelImportCassandraDao.saveAll(modelName, rebuiltModelImportList);
 307     }
 308
 309     private void removeExistingTypesFromDefaultImports(final ElementTypeEnum elementTypeEnum, final Map<String, Object> typesYaml,
 310                                                        final List<ToscaImportByModel> defaultImportList) {
 311         defaultImportList.forEach(toscaImportByModel -> {
 312             final Map<String, Object> existingImportYamlMap = new Yaml().load(toscaImportByModel.getContent());
 313             final Map<String, Object> currentTypeYamlMap = (Map<String, Object>) existingImportYamlMap.get(elementTypeEnum.getToscaEntryName());
 314             if (MapUtils.isNotEmpty(currentTypeYamlMap)) {
 315                 typesYaml.keySet().forEach(currentTypeYamlMap::remove);
 316             }
 317             toscaImportByModel.setContent(new YamlUtil().objectToYaml(existingImportYamlMap));
 318         });
 319     }
 320
 321     private String buildAdditionalTypeDefinitionsContent(final ElementTypeEnum elementTypeEnum, final Map<String, Object> typesYamlMap,
 322                                                          final Map<String, Object> originalContent) {
 323         final Map<String, Object> originalTypeContent = (Map<String, Object>) originalContent.get(elementTypeEnum.getToscaEntryName());
 324         if (MapUtils.isEmpty(originalTypeContent)) {
 325             originalContent.put(elementTypeEnum.getToscaEntryName(), new LinkedHashMap<>(typesYamlMap));
 326         } else {
 327             originalTypeContent.putAll(typesYamlMap);
 328         }
 329         return new YamlUtil().objectToYaml(originalContent);
 330     }
 331
 332     private String createAdditionalTypeDefinitionsHeader() {
 333         return "tosca_definitions_version: tosca_simple_yaml_1_3" + "\n"
 334             + "description: Auto-generated file that contains package custom types or types added after system installation." + "\n";
 335     }
 336
 337     /**
 338      * Deletes the given model if it exists, along with its MODEL_ELEMENT edges and import files.
 339      *
 340      * @param model         the model
 341      * @param inTransaction if the operation is called in the middle of a janusgraph transaction
 342      */
 343     public void deleteModel(final Model model, final boolean inTransaction) {
 344         boolean rollback = false;
 345
 346         try {
 347             final GraphVertex modelVertexByName = findModelVertexByName(model.getName()).orElse(null);
 348             if (modelVertexByName == null) {
 349                 return;
 350             }
 351             toscaModelImportCassandraDao.deleteAllByModel(model.getName());
 352             modelElementOperation.deleteModelElements(model, inTransaction);
 353             deleteModel(model);
 354         } catch (final OperationException e) {
 355             rollback = true;
 356             throw e;
 357         } catch (final Exception e) {
 358             rollback = true;
 359             throw new OperationException(e, ActionStatus.COULD_NOT_DELETE_MODEL, model.getName());
 360         } finally {
 361             if (!inTransaction) {
 362                 if (rollback) {
 363                     janusGraphGenericDao.rollback();
 364                 } else {
 365                     janusGraphGenericDao.commit();
 366                 }
 367             }
 368         }
 369     }
 370
 371     private void deleteModel(final Model model) {
 372         final var modelData = new ModelData(model.getName(), UniqueIdBuilder.buildModelUid(model.getName()), model.getModelType());
 373         final Either<ModelData, JanusGraphOperationStatus> deleteParentNodeByModel = janusGraphGenericDao.deleteNode(modelData, ModelData.class);
 374         if (deleteParentNodeByModel.isRight()) {
 375             final var janusGraphOperationStatus = deleteParentNodeByModel.right().value();
 376             log.error(EcompLoggerErrorCode.DATA_ERROR, ModelOperation.class.getName(),
 377                 "Failed to delete model {} on JanusGraph with status {}", new Object[] {model.getName(), janusGraphOperationStatus});
 378             throw new OperationException(ActionStatus.COULD_NOT_DELETE_MODEL, model.getName());
 379         }
 380     }
 381
 382     @Autowired
 383     public void setModelElementOperation(final ModelElementOperation modelElementOperation) {
 384         this.modelElementOperation = modelElementOperation;
 385     }
 386
 387     @SuppressWarnings("unchecked")
 388     public void updateTypesInAdditionalTypesImport(final ElementTypeEnum elementTypeEnum, final String typesYaml, final String modelName) {
 389         final Optional<ToscaImportByModel> additionalTypeDefinitionsImportOptional = getAdditionalTypes(modelName);
 390
 391         if (additionalTypeDefinitionsImportOptional.isPresent()) {
 392
 393             final Map<String, Object> existingTypeContent = getExistingTypes(elementTypeEnum, additionalTypeDefinitionsImportOptional.get());
 394             final Set<String> existingTypeNames = existingTypeContent.keySet();
 395
 396             final Map<String, Object> typesToUpate = new HashMap<>();
 397
 398             final Map<String, Object> newTypesYaml = new Yaml().load(typesYaml);
 399             newTypesYaml.entrySet().stream().filter(entry -> existingTypeNames.contains(entry.getKey())).forEach(newTypeToUpdate -> {
 400
 401                 final Map<String, Object> propertiesInNewDef = (Map<String, Object>) ((Map<String, Object>) newTypeToUpdate.getValue()).get("properties");
 402                 final Map<String, Object> existingProperties =
 403                         (Map<String, Object>) ((Map<String, Object>) existingTypeContent.get(newTypeToUpdate.getKey())).get("properties");
 404
 405                 final List<Entry<String, Object>> propertiesMissingFromNewDef = MapUtils.isEmpty(existingProperties) ? Collections.emptyList()
 406                         : existingProperties.entrySet().stream()
 407                                 .filter(existingPropEntry -> !propertiesInNewDef.keySet().contains(existingPropEntry.getKey()))
 408                                 .collect(Collectors.toList());
 409
 410                 if (CollectionUtils.isNotEmpty(propertiesMissingFromNewDef)) {
 411                     typesToUpate.put(newTypeToUpdate.getKey(), newTypeToUpdate.getValue());
 412
 413                     propertiesMissingFromNewDef
 414                             .forEach(existingPropToAdd -> propertiesInNewDef.put(existingPropToAdd.getKey(), existingPropToAdd.getValue()));
 415                 }
 416             });
 417             if (MapUtils.isNotEmpty(typesToUpate)) {
 418                 addTypesToDefaultImports(elementTypeEnum, new Yaml().dumpAsMap(typesToUpate), modelName);
 419             }
 420         }
 421     }
 422
 423     private  Optional<ToscaImportByModel> getAdditionalTypes(final String modelName) {
 424         final List<ToscaImportByModel> modelImportList = toscaModelImportCassandraDao.findAllByModel(modelName);
 425         return modelImportList.stream().filter(t -> ADDITIONAL_TYPE_DEFINITIONS_PATH.equals(Path.of(t.getFullPath()))).findAny();
 426     }
 427
 428     private Map<String, Object> getExistingTypes(final ElementTypeEnum elementTypeEnum, final ToscaImportByModel additionalTypeDefinitionsImport) {
 429         final Map<String, Object> existingContent = new Yaml().load(additionalTypeDefinitionsImport.getContent());
 430         return  (Map<String, Object>) existingContent.get(elementTypeEnum.getToscaEntryName());
 431     }
 432
 433 }