cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/NetworkCmProxyDataServiceImpl.java

   1 /*
   2  *  ============LICENSE_START=======================================================
   3  *  Copyright (C) 2021 highstreet technologies GmbH
   4  *  Modifications Copyright (C) 2021-2023 Nordix Foundation
   5  *  Modifications Copyright (C) 2021 Pantheon.tech
   6  *  Modifications Copyright (C) 2021-2022 Bell Canada
   7  *  Modifications Copyright (C) 2023 TechMahindra Ltd.
   8  *  ================================================================================
   9  *  Licensed under the Apache License, Version 2.0 (the "License");
  10  *  you may not use this file except in compliance with the License.
  11  *  You may obtain a copy of the License at
  12  *
  13  *        http://www.apache.org/licenses/LICENSE-2.0
  14  *
  15  *  Unless required by applicable law or agreed to in writing, software
  16  *  distributed under the License is distributed on an "AS IS" BASIS,
  17  *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  18  *  See the License for the specific language governing permissions and
  19  *  limitations under the License.
  20  *
  21  *  SPDX-License-Identifier: Apache-2.0
  22  *  ============LICENSE_END=========================================================
  23  */
  24
  25 package org.onap.cps.ncmp.api.impl;
  26
  27 import static org.onap.cps.ncmp.api.NcmpResponseStatus.CM_HANDLES_NOT_FOUND;
  28 import static org.onap.cps.ncmp.api.NcmpResponseStatus.CM_HANDLE_ALREADY_EXIST;
  29 import static org.onap.cps.ncmp.api.NcmpResponseStatus.CM_HANDLE_INVALID_ID;
  30 import static org.onap.cps.ncmp.api.impl.ncmppersistence.NcmpPersistence.NCMP_DMI_REGISTRY_PARENT;
  31 import static org.onap.cps.ncmp.api.impl.ncmppersistence.NcmpPersistence.NFP_OPERATIONAL_DATASTORE_DATASPACE_NAME;
  32 import static org.onap.cps.ncmp.api.impl.utils.RestQueryParametersValidator.validateCmHandleQueryParameters;
  33
  34 import com.google.common.collect.Lists;
  35 import com.hazelcast.map.IMap;
  36 import java.time.OffsetDateTime;
  37 import java.util.ArrayList;
  38 import java.util.Collection;
  39 import java.util.HashMap;
  40 import java.util.HashSet;
  41 import java.util.List;
  42 import java.util.Map;
  43 import java.util.Set;
  44 import java.util.stream.Collectors;
  45 import lombok.RequiredArgsConstructor;
  46 import lombok.extern.slf4j.Slf4j;
  47 import org.onap.cps.api.CpsDataService;
  48 import org.onap.cps.ncmp.api.NetworkCmProxyCmHandleQueryService;
  49 import org.onap.cps.ncmp.api.NetworkCmProxyDataService;
  50 import org.onap.cps.ncmp.api.impl.events.lcm.LcmEventsCmHandleStateHandler;
  51 import org.onap.cps.ncmp.api.impl.inventory.CmHandleQueries;
  52 import org.onap.cps.ncmp.api.impl.inventory.CmHandleState;
  53 import org.onap.cps.ncmp.api.impl.inventory.CompositeState;
  54 import org.onap.cps.ncmp.api.impl.inventory.CompositeStateUtils;
  55 import org.onap.cps.ncmp.api.impl.inventory.DataStoreSyncState;
  56 import org.onap.cps.ncmp.api.impl.inventory.InventoryPersistence;
  57 import org.onap.cps.ncmp.api.impl.operations.DmiDataOperations;
  58 import org.onap.cps.ncmp.api.impl.operations.OperationType;
  59 import org.onap.cps.ncmp.api.impl.trustlevel.TrustLevel;
  60 import org.onap.cps.ncmp.api.impl.utils.CmHandleQueryConditions;
  61 import org.onap.cps.ncmp.api.impl.utils.InventoryQueryConditions;
  62 import org.onap.cps.ncmp.api.impl.utils.YangDataConverter;
  63 import org.onap.cps.ncmp.api.impl.yangmodels.YangModelCmHandle;
  64 import org.onap.cps.ncmp.api.models.CmHandleQueryApiParameters;
  65 import org.onap.cps.ncmp.api.models.CmHandleQueryServiceParameters;
  66 import org.onap.cps.ncmp.api.models.CmHandleRegistrationResponse;
  67 import org.onap.cps.ncmp.api.models.DataOperationRequest;
  68 import org.onap.cps.ncmp.api.models.DmiPluginRegistration;
  69 import org.onap.cps.ncmp.api.models.DmiPluginRegistrationResponse;
  70 import org.onap.cps.ncmp.api.models.NcmpServiceCmHandle;
  71 import org.onap.cps.spi.FetchDescendantsOption;
  72 import org.onap.cps.spi.exceptions.AlreadyDefinedException;
  73 import org.onap.cps.spi.exceptions.CpsException;
  74 import org.onap.cps.spi.exceptions.DataNodeNotFoundException;
  75 import org.onap.cps.spi.exceptions.DataValidationException;
  76 import org.onap.cps.spi.model.ModuleDefinition;
  77 import org.onap.cps.spi.model.ModuleReference;
  78 import org.onap.cps.utils.JsonObjectMapper;
  79 import org.springframework.http.ResponseEntity;
  80 import org.springframework.stereotype.Service;
  81
  82 @Slf4j
  83 @Service
  84 @RequiredArgsConstructor
  85 public class NetworkCmProxyDataServiceImpl implements NetworkCmProxyDataService {
  86
  87     private static final int DELETE_BATCH_SIZE = 100;
  88     private final JsonObjectMapper jsonObjectMapper;
  89     private final DmiDataOperations dmiDataOperations;
  90     private final NetworkCmProxyDataServicePropertyHandler networkCmProxyDataServicePropertyHandler;
  91     private final InventoryPersistence inventoryPersistence;
  92     private final CmHandleQueries cmHandleQueries;
  93     private final NetworkCmProxyCmHandleQueryService networkCmProxyCmHandleQueryService;
  94     private final LcmEventsCmHandleStateHandler lcmEventsCmHandleStateHandler;
  95     private final CpsDataService cpsDataService;
  96     private final IMap<String, Object> moduleSyncStartedOnCmHandles;
  97     private final IMap<String, TrustLevel> trustLevelPerDmiPlugin;
  98
  99     @Override
 100     public DmiPluginRegistrationResponse updateDmiRegistrationAndSyncModule(
 101             final DmiPluginRegistration dmiPluginRegistration) {
 102         dmiPluginRegistration.validateDmiPluginRegistration();
 103         final DmiPluginRegistrationResponse dmiPluginRegistrationResponse = new DmiPluginRegistrationResponse();
 104
 105         if (!dmiPluginRegistration.getRemovedCmHandles().isEmpty()) {
 106             dmiPluginRegistrationResponse.setRemovedCmHandles(
 107                     parseAndRemoveCmHandlesInDmiRegistration(dmiPluginRegistration.getRemovedCmHandles()));
 108         }
 109
 110         if (!dmiPluginRegistration.getCreatedCmHandles().isEmpty()) {
 111             dmiPluginRegistrationResponse.setCreatedCmHandles(
 112                     parseAndCreateCmHandlesInDmiRegistrationAndSyncModules(dmiPluginRegistration));
 113         }
 114         if (!dmiPluginRegistration.getUpdatedCmHandles().isEmpty()) {
 115             dmiPluginRegistrationResponse.setUpdatedCmHandles(
 116                     networkCmProxyDataServicePropertyHandler
 117                             .updateCmHandleProperties(dmiPluginRegistration.getUpdatedCmHandles()));
 118         }
 119
 120         setTrustLevelPerDmiPlugin(dmiPluginRegistration);
 121
 122         return dmiPluginRegistrationResponse;
 123     }
 124
 125     @Override
 126     public Object getResourceDataForCmHandle(final String datastoreName,
 127                                              final String cmHandleId,
 128                                              final String resourceIdentifier,
 129                                              final String optionsParamInQuery,
 130                                              final String topicParamInQuery,
 131                                              final String requestId) {
 132         final ResponseEntity<?> responseEntity = dmiDataOperations.getResourceDataFromDmi(datastoreName, cmHandleId,
 133                 resourceIdentifier,
 134                 optionsParamInQuery,
 135                 topicParamInQuery,
 136                 requestId);
 137         return responseEntity.getBody();
 138     }
 139
 140     @Override
 141     public Object getResourceDataForCmHandle(final String datastoreName,
 142                                              final String cmHandleId,
 143                                              final String resourceIdentifier,
 144                                              final FetchDescendantsOption fetchDescendantsOption) {
 145         return cpsDataService.getDataNodes(datastoreName, cmHandleId, resourceIdentifier,
 146                 fetchDescendantsOption).iterator().next();
 147     }
 148
 149     @Override
 150     public void executeDataOperationForCmHandles(final String topicParamInQuery,
 151                                                  final DataOperationRequest
 152                                                          dataOperationRequest,
 153                                                  final String requestId) {
 154         dmiDataOperations.requestResourceDataFromDmi(topicParamInQuery, dataOperationRequest, requestId);
 155     }
 156
 157     @Override
 158     public Object writeResourceDataPassThroughRunningForCmHandle(final String cmHandleId,
 159                                                                  final String resourceIdentifier,
 160                                                                  final OperationType operationType,
 161                                                                  final String requestData,
 162                                                                  final String dataType) {
 163         return dmiDataOperations.writeResourceDataPassThroughRunningFromDmi(cmHandleId, resourceIdentifier,
 164                 operationType, requestData, dataType);
 165     }
 166
 167     @Override
 168     public Collection<ModuleReference> getYangResourcesModuleReferences(final String cmHandleId) {
 169         return inventoryPersistence.getYangResourcesModuleReferences(cmHandleId);
 170     }
 171
 172     @Override
 173     public Collection<ModuleDefinition> getModuleDefinitionsByCmHandleId(final String cmHandleId) {
 174         return inventoryPersistence.getModuleDefinitionsByCmHandleId(cmHandleId);
 175     }
 176
 177     /**
 178      * Retrieve cm handles with details for the given query parameters.
 179      *
 180      * @param cmHandleQueryApiParameters cm handle query parameters
 181      * @return cm handles with details
 182      */
 183     @Override
 184     public Collection<NcmpServiceCmHandle> executeCmHandleSearch(
 185             final CmHandleQueryApiParameters cmHandleQueryApiParameters) {
 186         final CmHandleQueryServiceParameters cmHandleQueryServiceParameters = jsonObjectMapper.convertToValueType(
 187                 cmHandleQueryApiParameters, CmHandleQueryServiceParameters.class);
 188         validateCmHandleQueryParameters(cmHandleQueryServiceParameters, CmHandleQueryConditions.ALL_CONDITION_NAMES);
 189         return networkCmProxyCmHandleQueryService.queryCmHandles(cmHandleQueryServiceParameters);
 190     }
 191
 192     /**
 193      * Retrieve cm handle ids for the given query parameters.
 194      *
 195      * @param cmHandleQueryApiParameters cm handle query parameters
 196      * @return cm handle ids
 197      */
 198     @Override
 199     public Collection<String> executeCmHandleIdSearch(final CmHandleQueryApiParameters cmHandleQueryApiParameters) {
 200         final CmHandleQueryServiceParameters cmHandleQueryServiceParameters = jsonObjectMapper.convertToValueType(
 201                 cmHandleQueryApiParameters, CmHandleQueryServiceParameters.class);
 202         validateCmHandleQueryParameters(cmHandleQueryServiceParameters, CmHandleQueryConditions.ALL_CONDITION_NAMES);
 203         return networkCmProxyCmHandleQueryService.queryCmHandleIds(cmHandleQueryServiceParameters);
 204     }
 205
 206     /**
 207      * Set the data sync enabled flag, along with the data sync state
 208      * based on the data sync enabled boolean for the cm handle id provided.
 209      *
 210      * @param cmHandleId      cm handle id
 211      * @param dataSyncEnabled data sync enabled flag
 212      */
 213     @Override
 214     public void setDataSyncEnabled(final String cmHandleId, final boolean dataSyncEnabled) {
 215         final CompositeState compositeState = inventoryPersistence
 216                 .getCmHandleState(cmHandleId);
 217         if (compositeState.getDataSyncEnabled().equals(dataSyncEnabled)) {
 218             log.info("Data-Sync Enabled flag is already: {} ", dataSyncEnabled);
 219         } else if (compositeState.getCmHandleState() != CmHandleState.READY) {
 220             throw new CpsException("State mismatch exception.", "Cm-Handle not in READY state. Cm handle state is: "
 221                     + compositeState.getCmHandleState());
 222         } else {
 223             final DataStoreSyncState dataStoreSyncState = compositeState.getDataStores()
 224                     .getOperationalDataStore().getDataStoreSyncState();
 225             if (!dataSyncEnabled && dataStoreSyncState == DataStoreSyncState.SYNCHRONIZED) {
 226                 cpsDataService.deleteDataNode(NFP_OPERATIONAL_DATASTORE_DATASPACE_NAME, cmHandleId,
 227                         "/netconf-state", OffsetDateTime.now());
 228             }
 229             CompositeStateUtils.setDataSyncEnabledFlagWithDataSyncState(dataSyncEnabled, compositeState);
 230             inventoryPersistence.saveCmHandleState(cmHandleId,
 231                     compositeState);
 232         }
 233     }
 234
 235     /**
 236      * Get all cm handle IDs by DMI plugin identifier.
 237      *
 238      * @param dmiPluginIdentifier DMI plugin identifier
 239      * @return set of cm handle IDs
 240      */
 241     @Override
 242     public Collection<String> getAllCmHandleIdsByDmiPluginIdentifier(final String dmiPluginIdentifier) {
 243         return cmHandleQueries.getCmHandleIdsByDmiPluginIdentifier(dmiPluginIdentifier);
 244     }
 245
 246     /**
 247      * Get all cm handle IDs by various properties.
 248      *
 249      * @param cmHandleQueryServiceParameters cm handle query parameters
 250      * @return set of cm handle IDs
 251      */
 252     @Override
 253     public Collection<String> executeCmHandleIdSearchForInventory(
 254             final CmHandleQueryServiceParameters cmHandleQueryServiceParameters) {
 255         validateCmHandleQueryParameters(cmHandleQueryServiceParameters, InventoryQueryConditions.ALL_CONDITION_NAMES);
 256         return networkCmProxyCmHandleQueryService.queryCmHandleIdsForInventory(cmHandleQueryServiceParameters);
 257     }
 258
 259     /**
 260      * Retrieve cm handle details for a given cm handle.
 261      *
 262      * @param cmHandleId cm handle identifier
 263      * @return cm handle details
 264      */
 265     @Override
 266     public NcmpServiceCmHandle getNcmpServiceCmHandle(final String cmHandleId) {
 267         return YangDataConverter.convertYangModelCmHandleToNcmpServiceCmHandle(
 268                 inventoryPersistence.getYangModelCmHandle(cmHandleId));
 269     }
 270
 271     /**
 272      * Get cm handle public properties for a given cm handle id.
 273      *
 274      * @param cmHandleId cm handle identifier
 275      * @return cm handle public properties
 276      */
 277     @Override
 278     public Map<String, String> getCmHandlePublicProperties(final String cmHandleId) {
 279         final YangModelCmHandle yangModelCmHandle =
 280                 inventoryPersistence.getYangModelCmHandle(cmHandleId);
 281         final List<YangModelCmHandle.Property> yangModelPublicProperties = yangModelCmHandle.getPublicProperties();
 282         final Map<String, String> cmHandlePublicProperties = new HashMap<>();
 283         YangDataConverter.asPropertiesMap(yangModelPublicProperties, cmHandlePublicProperties);
 284         return cmHandlePublicProperties;
 285     }
 286
 287     /**
 288      * Get cm handle composite state for a given cm handle id.
 289      *
 290      * @param cmHandleId cm handle identifier
 291      * @return cm handle state
 292      */
 293     @Override
 294     public CompositeState getCmHandleCompositeState(final String cmHandleId) {
 295         return inventoryPersistence.getYangModelCmHandle(cmHandleId).getCompositeState();
 296     }
 297
 298     /**
 299      * THis method registers a cm handle and initiates modules sync.
 300      *
 301      * @param dmiPluginRegistration dmi plugin registration information.
 302      * @return cm-handle registration response for create cm-handle requests.
 303      */
 304     public List<CmHandleRegistrationResponse> parseAndCreateCmHandlesInDmiRegistrationAndSyncModules(
 305             final DmiPluginRegistration dmiPluginRegistration) {
 306         final Map<YangModelCmHandle, CmHandleState> cmHandleStatePerCmHandle = new HashMap<>();
 307         dmiPluginRegistration.getCreatedCmHandles()
 308                 .forEach(cmHandle -> {
 309                     final YangModelCmHandle yangModelCmHandle = YangModelCmHandle.toYangModelCmHandle(
 310                             dmiPluginRegistration.getDmiPlugin(),
 311                             dmiPluginRegistration.getDmiDataPlugin(),
 312                             dmiPluginRegistration.getDmiModelPlugin(),
 313                             cmHandle);
 314                     cmHandleStatePerCmHandle.put(yangModelCmHandle, CmHandleState.ADVISED);
 315                 });
 316         return registerNewCmHandles(cmHandleStatePerCmHandle);
 317     }
 318
 319     protected List<CmHandleRegistrationResponse> parseAndRemoveCmHandlesInDmiRegistration(
 320             final List<String> tobeRemovedCmHandles) {
 321         final List<CmHandleRegistrationResponse> cmHandleRegistrationResponses =
 322                 new ArrayList<>(tobeRemovedCmHandles.size());
 323         final Collection<YangModelCmHandle> yangModelCmHandles =
 324             inventoryPersistence.getYangModelCmHandles(tobeRemovedCmHandles);
 325
 326         updateCmHandleStateBatch(yangModelCmHandles, CmHandleState.DELETING);
 327
 328         final Set<String> notDeletedCmHandles = new HashSet<>();
 329         for (final List<String> tobeRemovedCmHandleBatch : Lists.partition(tobeRemovedCmHandles, DELETE_BATCH_SIZE)) {
 330             try {
 331                 batchDeleteCmHandlesFromDbAndModuleSyncMap(tobeRemovedCmHandleBatch);
 332                 tobeRemovedCmHandleBatch.forEach(cmHandleId ->
 333                     cmHandleRegistrationResponses.add(CmHandleRegistrationResponse.createSuccessResponse(cmHandleId)));
 334
 335             } catch (final RuntimeException batchException) {
 336                 log.error("Unable to de-register cm-handle batch, retrying on each cm handle");
 337                 for (final String cmHandleId : tobeRemovedCmHandleBatch) {
 338                     final CmHandleRegistrationResponse cmHandleRegistrationResponse =
 339                         deleteCmHandleAndGetCmHandleRegistrationResponse(cmHandleId);
 340                     cmHandleRegistrationResponses.add(cmHandleRegistrationResponse);
 341                     if (cmHandleRegistrationResponse.getStatus() != CmHandleRegistrationResponse.Status.SUCCESS) {
 342                         notDeletedCmHandles.add(cmHandleId);
 343                     }
 344                 }
 345             }
 346         }
 347
 348         yangModelCmHandles.removeIf(yangModelCmHandle -> notDeletedCmHandles.contains(yangModelCmHandle.getId()));
 349         updateCmHandleStateBatch(yangModelCmHandles, CmHandleState.DELETED);
 350
 351         return cmHandleRegistrationResponses;
 352     }
 353
 354     private CmHandleRegistrationResponse deleteCmHandleAndGetCmHandleRegistrationResponse(final String cmHandleId) {
 355         try {
 356             deleteCmHandleFromDbAndModuleSyncMap(cmHandleId);
 357             return CmHandleRegistrationResponse.createSuccessResponse(cmHandleId);
 358         } catch (final DataNodeNotFoundException dataNodeNotFoundException) {
 359             log.error("Unable to find dataNode for cmHandleId : {} , caused by : {}",
 360                 cmHandleId, dataNodeNotFoundException.getMessage());
 361             return CmHandleRegistrationResponse.createFailureResponse(cmHandleId, CM_HANDLES_NOT_FOUND);
 362         } catch (final DataValidationException dataValidationException) {
 363             log.error("Unable to de-register cm-handle id: {}, caused by: {}",
 364                 cmHandleId, dataValidationException.getMessage());
 365             return CmHandleRegistrationResponse.createFailureResponse(cmHandleId, CM_HANDLE_INVALID_ID);
 366         } catch (final Exception exception) {
 367             log.error("Unable to de-register cm-handle id : {} , caused by : {}", cmHandleId, exception.getMessage());
 368             return CmHandleRegistrationResponse.createFailureResponse(cmHandleId, exception);
 369         }
 370     }
 371
 372     private void updateCmHandleStateBatch(final Collection<YangModelCmHandle> yangModelCmHandles,
 373                                           final CmHandleState cmHandleState) {
 374         final Map<YangModelCmHandle, CmHandleState> cmHandleStatePerCmHandle = new HashMap<>(yangModelCmHandles.size());
 375         yangModelCmHandles.forEach(yangModelCmHandle -> cmHandleStatePerCmHandle.put(yangModelCmHandle, cmHandleState));
 376         lcmEventsCmHandleStateHandler.updateCmHandleStateBatch(cmHandleStatePerCmHandle);
 377     }
 378
 379     private void deleteCmHandleFromDbAndModuleSyncMap(final String cmHandleId) {
 380         inventoryPersistence.deleteSchemaSetWithCascade(cmHandleId);
 381         inventoryPersistence.deleteDataNode(NCMP_DMI_REGISTRY_PARENT + "/cm-handles[@id='" + cmHandleId
 382                 + "']");
 383         removeDeletedCmHandleFromModuleSyncMap(cmHandleId);
 384     }
 385
 386     private void batchDeleteCmHandlesFromDbAndModuleSyncMap(final Collection<String> tobeRemovedCmHandles) {
 387         inventoryPersistence.deleteSchemaSetsWithCascade(tobeRemovedCmHandles);
 388         inventoryPersistence.deleteDataNodes(mapCmHandleIdsToXpaths(tobeRemovedCmHandles));
 389         tobeRemovedCmHandles.forEach(this::removeDeletedCmHandleFromModuleSyncMap);
 390     }
 391
 392     private Collection<String> mapCmHandleIdsToXpaths(final Collection<String> cmHandles) {
 393         return cmHandles.stream()
 394                 .map(cmHandleId -> NCMP_DMI_REGISTRY_PARENT + "/cm-handles[@id='" + cmHandleId + "']")
 395                 .collect(Collectors.toSet());
 396     }
 397
 398     // CPS-1239 Robustness cleaning of in progress cache
 399     private void removeDeletedCmHandleFromModuleSyncMap(final String deletedCmHandleId) {
 400         if (moduleSyncStartedOnCmHandles.remove(deletedCmHandleId) != null) {
 401             log.debug("{} removed from in progress map", deletedCmHandleId);
 402         }
 403     }
 404
 405     private List<CmHandleRegistrationResponse> registerNewCmHandles(final Map<YangModelCmHandle, CmHandleState>
 406                                                                             cmHandleStatePerCmHandle) {
 407         final List<String> cmHandleIds = cmHandleStatePerCmHandle.keySet().stream().map(YangModelCmHandle::getId)
 408                 .toList();
 409         try {
 410             lcmEventsCmHandleStateHandler.updateCmHandleStateBatch(cmHandleStatePerCmHandle);
 411             return CmHandleRegistrationResponse.createSuccessResponses(cmHandleIds);
 412         } catch (final AlreadyDefinedException alreadyDefinedException) {
 413             return CmHandleRegistrationResponse.createFailureResponses(
 414                     alreadyDefinedException.getAlreadyDefinedObjectNames(),
 415                     CM_HANDLE_ALREADY_EXIST);
 416         } catch (final Exception exception) {
 417             return CmHandleRegistrationResponse.createFailureResponses(cmHandleIds, exception);
 418         }
 419     }
 420
 421     private void setTrustLevelPerDmiPlugin(final DmiPluginRegistration dmiPluginRegistration) {
 422         if (DmiPluginRegistration.isNullEmptyOrBlank(dmiPluginRegistration.getDmiDataPlugin())) {
 423             trustLevelPerDmiPlugin.put(dmiPluginRegistration.getDmiPlugin(), TrustLevel.COMPLETE);
 424         } else {
 425             trustLevelPerDmiPlugin.put(dmiPluginRegistration.getDmiDataPlugin(), TrustLevel.COMPLETE);
 426         }
 427     }
 428
 429 }