cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/NetworkCmProxyDataServiceImpl.java

   1 /*
   2  *  ============LICENSE_START=======================================================
   3  *  Copyright (C) 2021 highstreet technologies GmbH
   4  *  Modifications Copyright (C) 2021-2023 Nordix Foundation
   5  *  Modifications Copyright (C) 2021 Pantheon.tech
   6  *  Modifications Copyright (C) 2021-2022 Bell Canada
   7  *  Modifications Copyright (C) 2023 TechMahindra Ltd.
   8  *  ================================================================================
   9  *  Licensed under the Apache License, Version 2.0 (the "License");
  10  *  you may not use this file except in compliance with the License.
  11  *  You may obtain a copy of the License at
  12  *
  13  *        http://www.apache.org/licenses/LICENSE-2.0
  14  *
  15  *  Unless required by applicable law or agreed to in writing, software
  16  *  distributed under the License is distributed on an "AS IS" BASIS,
  17  *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  18  *  See the License for the specific language governing permissions and
  19  *  limitations under the License.
  20  *
  21  *  SPDX-License-Identifier: Apache-2.0
  22  *  ============LICENSE_END=========================================================
  23  */
  24
  25 package org.onap.cps.ncmp.api.impl;
  26
  27 import static org.onap.cps.ncmp.api.impl.constants.DmiRegistryConstants.NFP_OPERATIONAL_DATASTORE_DATASPACE_NAME;
  28 import static org.onap.cps.ncmp.api.impl.utils.RestQueryParametersValidator.validateCmHandleQueryParameters;
  29
  30 import com.google.common.collect.Lists;
  31 import com.hazelcast.map.IMap;
  32 import java.time.OffsetDateTime;
  33 import java.util.ArrayList;
  34 import java.util.Collection;
  35 import java.util.HashMap;
  36 import java.util.HashSet;
  37 import java.util.List;
  38 import java.util.Map;
  39 import java.util.Set;
  40 import java.util.stream.Collectors;
  41 import lombok.RequiredArgsConstructor;
  42 import lombok.extern.slf4j.Slf4j;
  43 import org.onap.cps.api.CpsDataService;
  44 import org.onap.cps.ncmp.api.NetworkCmProxyCmHandleQueryService;
  45 import org.onap.cps.ncmp.api.NetworkCmProxyDataService;
  46 import org.onap.cps.ncmp.api.impl.events.lcm.LcmEventsCmHandleStateHandler;
  47 import org.onap.cps.ncmp.api.impl.operations.DmiDataOperations;
  48 import org.onap.cps.ncmp.api.impl.operations.OperationType;
  49 import org.onap.cps.ncmp.api.impl.trustlevel.TrustLevel;
  50 import org.onap.cps.ncmp.api.impl.utils.CmHandleQueryConditions;
  51 import org.onap.cps.ncmp.api.impl.utils.InventoryQueryConditions;
  52 import org.onap.cps.ncmp.api.impl.utils.YangDataConverter;
  53 import org.onap.cps.ncmp.api.impl.yangmodels.YangModelCmHandle;
  54 import org.onap.cps.ncmp.api.inventory.CmHandleQueries;
  55 import org.onap.cps.ncmp.api.inventory.CmHandleState;
  56 import org.onap.cps.ncmp.api.inventory.CompositeState;
  57 import org.onap.cps.ncmp.api.inventory.CompositeStateUtils;
  58 import org.onap.cps.ncmp.api.inventory.DataStoreSyncState;
  59 import org.onap.cps.ncmp.api.inventory.InventoryPersistence;
  60 import org.onap.cps.ncmp.api.models.CmHandleQueryApiParameters;
  61 import org.onap.cps.ncmp.api.models.CmHandleQueryServiceParameters;
  62 import org.onap.cps.ncmp.api.models.CmHandleRegistrationResponse;
  63 import org.onap.cps.ncmp.api.models.CmHandleRegistrationResponse.RegistrationError;
  64 import org.onap.cps.ncmp.api.models.DataOperationRequest;
  65 import org.onap.cps.ncmp.api.models.DmiPluginRegistration;
  66 import org.onap.cps.ncmp.api.models.DmiPluginRegistrationResponse;
  67 import org.onap.cps.ncmp.api.models.NcmpServiceCmHandle;
  68 import org.onap.cps.spi.FetchDescendantsOption;
  69 import org.onap.cps.spi.exceptions.AlreadyDefinedException;
  70 import org.onap.cps.spi.exceptions.CpsException;
  71 import org.onap.cps.spi.exceptions.DataNodeNotFoundException;
  72 import org.onap.cps.spi.exceptions.DataValidationException;
  73 import org.onap.cps.spi.model.ModuleDefinition;
  74 import org.onap.cps.spi.model.ModuleReference;
  75 import org.onap.cps.utils.JsonObjectMapper;
  76 import org.springframework.http.ResponseEntity;
  77 import org.springframework.stereotype.Service;
  78
  79 @Slf4j
  80 @Service
  81 @RequiredArgsConstructor
  82 public class NetworkCmProxyDataServiceImpl implements NetworkCmProxyDataService {
  83
  84     private static final int DELETE_BATCH_SIZE = 100;
  85     private final JsonObjectMapper jsonObjectMapper;
  86     private final DmiDataOperations dmiDataOperations;
  87     private final NetworkCmProxyDataServicePropertyHandler networkCmProxyDataServicePropertyHandler;
  88     private final InventoryPersistence inventoryPersistence;
  89     private final CmHandleQueries cmHandleQueries;
  90     private final NetworkCmProxyCmHandleQueryService networkCmProxyCmHandleQueryService;
  91     private final LcmEventsCmHandleStateHandler lcmEventsCmHandleStateHandler;
  92     private final CpsDataService cpsDataService;
  93     private final IMap<String, Object> moduleSyncStartedOnCmHandles;
  94     private final IMap<String, TrustLevel> trustLevelPerDmiPlugin;
  95
  96     @Override
  97     public DmiPluginRegistrationResponse updateDmiRegistrationAndSyncModule(
  98             final DmiPluginRegistration dmiPluginRegistration) {
  99         dmiPluginRegistration.validateDmiPluginRegistration();
 100         final DmiPluginRegistrationResponse dmiPluginRegistrationResponse = new DmiPluginRegistrationResponse();
 101
 102         if (!dmiPluginRegistration.getRemovedCmHandles().isEmpty()) {
 103             dmiPluginRegistrationResponse.setRemovedCmHandles(
 104                     parseAndRemoveCmHandlesInDmiRegistration(dmiPluginRegistration.getRemovedCmHandles()));
 105         }
 106
 107         if (!dmiPluginRegistration.getCreatedCmHandles().isEmpty()) {
 108             dmiPluginRegistrationResponse.setCreatedCmHandles(
 109                     parseAndCreateCmHandlesInDmiRegistrationAndSyncModules(dmiPluginRegistration));
 110         }
 111         if (!dmiPluginRegistration.getUpdatedCmHandles().isEmpty()) {
 112             dmiPluginRegistrationResponse.setUpdatedCmHandles(
 113                     networkCmProxyDataServicePropertyHandler
 114                             .updateCmHandleProperties(dmiPluginRegistration.getUpdatedCmHandles()));
 115         }
 116
 117         setTrustLevelPerDmiPlugin(dmiPluginRegistration);
 118
 119         return dmiPluginRegistrationResponse;
 120     }
 121
 122     @Override
 123     public Object getResourceDataForCmHandle(final String datastoreName,
 124                                              final String cmHandleId,
 125                                              final String resourceIdentifier,
 126                                              final String optionsParamInQuery,
 127                                              final String topicParamInQuery,
 128                                              final String requestId) {
 129         final ResponseEntity<?> responseEntity = dmiDataOperations.getResourceDataFromDmi(datastoreName, cmHandleId,
 130                 resourceIdentifier,
 131                 optionsParamInQuery,
 132                 topicParamInQuery,
 133                 requestId);
 134         return responseEntity.getBody();
 135     }
 136
 137     @Override
 138     public Object getResourceDataForCmHandle(final String datastoreName,
 139                                              final String cmHandleId,
 140                                              final String resourceIdentifier,
 141                                              final FetchDescendantsOption fetchDescendantsOption) {
 142         return cpsDataService.getDataNodes(datastoreName, cmHandleId, resourceIdentifier,
 143                 fetchDescendantsOption).iterator().next();
 144     }
 145
 146     @Override
 147     public void executeDataOperationForCmHandles(final String topicParamInQuery,
 148                                                  final DataOperationRequest
 149                                                          dataOperationRequest,
 150                                                  final String requestId) {
 151         dmiDataOperations.requestResourceDataFromDmi(topicParamInQuery, dataOperationRequest, requestId);
 152     }
 153
 154     @Override
 155     public Object writeResourceDataPassThroughRunningForCmHandle(final String cmHandleId,
 156                                                                  final String resourceIdentifier,
 157                                                                  final OperationType operationType,
 158                                                                  final String requestData,
 159                                                                  final String dataType) {
 160         return dmiDataOperations.writeResourceDataPassThroughRunningFromDmi(cmHandleId, resourceIdentifier,
 161                 operationType, requestData, dataType);
 162     }
 163
 164     @Override
 165     public Collection<ModuleReference> getYangResourcesModuleReferences(final String cmHandleId) {
 166         return inventoryPersistence.getYangResourcesModuleReferences(cmHandleId);
 167     }
 168
 169     @Override
 170     public Collection<ModuleDefinition> getModuleDefinitionsByCmHandleId(final String cmHandleId) {
 171         return inventoryPersistence.getModuleDefinitionsByCmHandleId(cmHandleId);
 172     }
 173
 174     /**
 175      * Retrieve cm handles with details for the given query parameters.
 176      *
 177      * @param cmHandleQueryApiParameters cm handle query parameters
 178      * @return cm handles with details
 179      */
 180     @Override
 181     public Collection<NcmpServiceCmHandle> executeCmHandleSearch(
 182             final CmHandleQueryApiParameters cmHandleQueryApiParameters) {
 183         final CmHandleQueryServiceParameters cmHandleQueryServiceParameters = jsonObjectMapper.convertToValueType(
 184                 cmHandleQueryApiParameters, CmHandleQueryServiceParameters.class);
 185         validateCmHandleQueryParameters(cmHandleQueryServiceParameters, CmHandleQueryConditions.ALL_CONDITION_NAMES);
 186         return networkCmProxyCmHandleQueryService.queryCmHandles(cmHandleQueryServiceParameters);
 187     }
 188
 189     /**
 190      * Retrieve cm handle ids for the given query parameters.
 191      *
 192      * @param cmHandleQueryApiParameters cm handle query parameters
 193      * @return cm handle ids
 194      */
 195     @Override
 196     public Collection<String> executeCmHandleIdSearch(final CmHandleQueryApiParameters cmHandleQueryApiParameters) {
 197         final CmHandleQueryServiceParameters cmHandleQueryServiceParameters = jsonObjectMapper.convertToValueType(
 198                 cmHandleQueryApiParameters, CmHandleQueryServiceParameters.class);
 199         validateCmHandleQueryParameters(cmHandleQueryServiceParameters, CmHandleQueryConditions.ALL_CONDITION_NAMES);
 200         return networkCmProxyCmHandleQueryService.queryCmHandleIds(cmHandleQueryServiceParameters);
 201     }
 202
 203     /**
 204      * Set the data sync enabled flag, along with the data sync state
 205      * based on the data sync enabled boolean for the cm handle id provided.
 206      *
 207      * @param cmHandleId      cm handle id
 208      * @param dataSyncEnabled data sync enabled flag
 209      */
 210     @Override
 211     public void setDataSyncEnabled(final String cmHandleId, final boolean dataSyncEnabled) {
 212         final CompositeState compositeState = inventoryPersistence
 213                 .getCmHandleState(cmHandleId);
 214         if (compositeState.getDataSyncEnabled().equals(dataSyncEnabled)) {
 215             log.info("Data-Sync Enabled flag is already: {} ", dataSyncEnabled);
 216         } else if (compositeState.getCmHandleState() != CmHandleState.READY) {
 217             throw new CpsException("State mismatch exception.", "Cm-Handle not in READY state. Cm handle state is: "
 218                     + compositeState.getCmHandleState());
 219         } else {
 220             final DataStoreSyncState dataStoreSyncState = compositeState.getDataStores()
 221                     .getOperationalDataStore().getDataStoreSyncState();
 222             if (!dataSyncEnabled && dataStoreSyncState == DataStoreSyncState.SYNCHRONIZED) {
 223                 cpsDataService.deleteDataNode(NFP_OPERATIONAL_DATASTORE_DATASPACE_NAME, cmHandleId,
 224                         "/netconf-state", OffsetDateTime.now());
 225             }
 226             CompositeStateUtils.setDataSyncEnabledFlagWithDataSyncState(dataSyncEnabled, compositeState);
 227             inventoryPersistence.saveCmHandleState(cmHandleId,
 228                     compositeState);
 229         }
 230     }
 231
 232     /**
 233      * Get all cm handle IDs by DMI plugin identifier.
 234      *
 235      * @param dmiPluginIdentifier DMI plugin identifier
 236      * @return set of cm handle IDs
 237      */
 238     @Override
 239     public Collection<String> getAllCmHandleIdsByDmiPluginIdentifier(final String dmiPluginIdentifier) {
 240         return cmHandleQueries.getCmHandleIdsByDmiPluginIdentifier(dmiPluginIdentifier);
 241     }
 242
 243     /**
 244      * Get all cm handle IDs by various properties.
 245      *
 246      * @param cmHandleQueryServiceParameters cm handle query parameters
 247      * @return set of cm handle IDs
 248      */
 249     @Override
 250     public Collection<String> executeCmHandleIdSearchForInventory(
 251             final CmHandleQueryServiceParameters cmHandleQueryServiceParameters) {
 252         validateCmHandleQueryParameters(cmHandleQueryServiceParameters, InventoryQueryConditions.ALL_CONDITION_NAMES);
 253         return networkCmProxyCmHandleQueryService.queryCmHandleIdsForInventory(cmHandleQueryServiceParameters);
 254     }
 255
 256     /**
 257      * Retrieve cm handle details for a given cm handle.
 258      *
 259      * @param cmHandleId cm handle identifier
 260      * @return cm handle details
 261      */
 262     @Override
 263     public NcmpServiceCmHandle getNcmpServiceCmHandle(final String cmHandleId) {
 264         return YangDataConverter.convertYangModelCmHandleToNcmpServiceCmHandle(
 265                 inventoryPersistence.getYangModelCmHandle(cmHandleId));
 266     }
 267
 268     /**
 269      * Get cm handle public properties for a given cm handle id.
 270      *
 271      * @param cmHandleId cm handle identifier
 272      * @return cm handle public properties
 273      */
 274     @Override
 275     public Map<String, String> getCmHandlePublicProperties(final String cmHandleId) {
 276         final YangModelCmHandle yangModelCmHandle =
 277                 inventoryPersistence.getYangModelCmHandle(cmHandleId);
 278         final List<YangModelCmHandle.Property> yangModelPublicProperties = yangModelCmHandle.getPublicProperties();
 279         final Map<String, String> cmHandlePublicProperties = new HashMap<>();
 280         YangDataConverter.asPropertiesMap(yangModelPublicProperties, cmHandlePublicProperties);
 281         return cmHandlePublicProperties;
 282     }
 283
 284     /**
 285      * Get cm handle composite state for a given cm handle id.
 286      *
 287      * @param cmHandleId cm handle identifier
 288      * @return cm handle state
 289      */
 290     @Override
 291     public CompositeState getCmHandleCompositeState(final String cmHandleId) {
 292         return inventoryPersistence.getYangModelCmHandle(cmHandleId).getCompositeState();
 293     }
 294
 295     /**
 296      * THis method registers a cm handle and initiates modules sync.
 297      *
 298      * @param dmiPluginRegistration dmi plugin registration information.
 299      * @return cm-handle registration response for create cm-handle requests.
 300      */
 301     public List<CmHandleRegistrationResponse> parseAndCreateCmHandlesInDmiRegistrationAndSyncModules(
 302             final DmiPluginRegistration dmiPluginRegistration) {
 303         final Map<YangModelCmHandle, CmHandleState> cmHandleStatePerCmHandle = new HashMap<>();
 304         dmiPluginRegistration.getCreatedCmHandles()
 305                 .forEach(cmHandle -> {
 306                     final YangModelCmHandle yangModelCmHandle = YangModelCmHandle.toYangModelCmHandle(
 307                             dmiPluginRegistration.getDmiPlugin(),
 308                             dmiPluginRegistration.getDmiDataPlugin(),
 309                             dmiPluginRegistration.getDmiModelPlugin(),
 310                             cmHandle);
 311                     cmHandleStatePerCmHandle.put(yangModelCmHandle, CmHandleState.ADVISED);
 312                 });
 313         return registerNewCmHandles(cmHandleStatePerCmHandle);
 314     }
 315
 316     protected List<CmHandleRegistrationResponse> parseAndRemoveCmHandlesInDmiRegistration(
 317             final List<String> tobeRemovedCmHandles) {
 318         final List<CmHandleRegistrationResponse> cmHandleRegistrationResponses =
 319                 new ArrayList<>(tobeRemovedCmHandles.size());
 320         final Collection<YangModelCmHandle> yangModelCmHandles =
 321             inventoryPersistence.getYangModelCmHandles(tobeRemovedCmHandles);
 322
 323         updateCmHandleStateBatch(yangModelCmHandles, CmHandleState.DELETING);
 324
 325         final Set<String> notDeletedCmHandles = new HashSet<>();
 326         for (final List<String> tobeRemovedCmHandleBatch : Lists.partition(tobeRemovedCmHandles, DELETE_BATCH_SIZE)) {
 327             try {
 328                 batchDeleteCmHandlesFromDbAndModuleSyncMap(tobeRemovedCmHandleBatch);
 329                 tobeRemovedCmHandleBatch.forEach(cmHandleId ->
 330                     cmHandleRegistrationResponses.add(CmHandleRegistrationResponse.createSuccessResponse(cmHandleId)));
 331
 332             } catch (final RuntimeException batchException) {
 333                 log.error("Unable to de-register cm-handle batch, retrying on each cm handle");
 334                 for (final String cmHandleId : tobeRemovedCmHandleBatch) {
 335                     final CmHandleRegistrationResponse cmHandleRegistrationResponse =
 336                         deleteCmHandleAndGetCmHandleRegistrationResponse(cmHandleId);
 337                     cmHandleRegistrationResponses.add(cmHandleRegistrationResponse);
 338                     if (cmHandleRegistrationResponse.getStatus() != CmHandleRegistrationResponse.Status.SUCCESS) {
 339                         notDeletedCmHandles.add(cmHandleId);
 340                     }
 341                 }
 342             }
 343         }
 344
 345         yangModelCmHandles.removeIf(yangModelCmHandle -> notDeletedCmHandles.contains(yangModelCmHandle.getId()));
 346         updateCmHandleStateBatch(yangModelCmHandles, CmHandleState.DELETED);
 347
 348         return cmHandleRegistrationResponses;
 349     }
 350
 351     private CmHandleRegistrationResponse deleteCmHandleAndGetCmHandleRegistrationResponse(final String cmHandleId) {
 352         try {
 353             deleteCmHandleFromDbAndModuleSyncMap(cmHandleId);
 354             return CmHandleRegistrationResponse.createSuccessResponse(cmHandleId);
 355         } catch (final DataNodeNotFoundException dataNodeNotFoundException) {
 356             log.error("Unable to find dataNode for cmHandleId : {} , caused by : {}",
 357                 cmHandleId, dataNodeNotFoundException.getMessage());
 358             return CmHandleRegistrationResponse.createFailureResponse(cmHandleId,
 359                 RegistrationError.CM_HANDLE_DOES_NOT_EXIST);
 360         } catch (final DataValidationException dataValidationException) {
 361             log.error("Unable to de-register cm-handle id: {}, caused by: {}",
 362                 cmHandleId, dataValidationException.getMessage());
 363             return CmHandleRegistrationResponse.createFailureResponse(cmHandleId,
 364                 RegistrationError.CM_HANDLE_INVALID_ID);
 365         } catch (final Exception exception) {
 366             log.error("Unable to de-register cm-handle id : {} , caused by : {}", cmHandleId, exception.getMessage());
 367             return CmHandleRegistrationResponse.createFailureResponse(cmHandleId, exception);
 368         }
 369     }
 370
 371     private void updateCmHandleStateBatch(final Collection<YangModelCmHandle> yangModelCmHandles,
 372                                           final CmHandleState cmHandleState) {
 373         final Map<YangModelCmHandle, CmHandleState> cmHandleStatePerCmHandle = new HashMap<>(yangModelCmHandles.size());
 374         yangModelCmHandles.forEach(yangModelCmHandle -> cmHandleStatePerCmHandle.put(yangModelCmHandle, cmHandleState));
 375         lcmEventsCmHandleStateHandler.updateCmHandleStateBatch(cmHandleStatePerCmHandle);
 376     }
 377
 378     private void deleteCmHandleFromDbAndModuleSyncMap(final String cmHandleId) {
 379         inventoryPersistence.deleteSchemaSetWithCascade(cmHandleId);
 380         inventoryPersistence.deleteDataNode("/dmi-registry/cm-handles[@id='" + cmHandleId + "']");
 381         removeDeletedCmHandleFromModuleSyncMap(cmHandleId);
 382     }
 383
 384     private void batchDeleteCmHandlesFromDbAndModuleSyncMap(final Collection<String> tobeRemovedCmHandles) {
 385         inventoryPersistence.deleteSchemaSetsWithCascade(tobeRemovedCmHandles);
 386         inventoryPersistence.deleteDataNodes(mapCmHandleIdsToXpaths(tobeRemovedCmHandles));
 387         tobeRemovedCmHandles.forEach(this::removeDeletedCmHandleFromModuleSyncMap);
 388     }
 389
 390     private Collection<String> mapCmHandleIdsToXpaths(final Collection<String> cmHandles) {
 391         return cmHandles.stream()
 392             .map(cmHandleId -> "/dmi-registry/cm-handles[@id='" + cmHandleId + "']")
 393             .collect(Collectors.toSet());
 394     }
 395
 396     // CPS-1239 Robustness cleaning of in progress cache
 397     private void removeDeletedCmHandleFromModuleSyncMap(final String deletedCmHandleId) {
 398         if (moduleSyncStartedOnCmHandles.remove(deletedCmHandleId) != null) {
 399             log.debug("{} removed from in progress map", deletedCmHandleId);
 400         }
 401     }
 402
 403     private List<CmHandleRegistrationResponse> registerNewCmHandles(final Map<YangModelCmHandle, CmHandleState>
 404                                                                             cmHandleStatePerCmHandle) {
 405         final List<String> cmHandleIds = cmHandleStatePerCmHandle.keySet().stream().map(YangModelCmHandle::getId)
 406                 .collect(Collectors.toList());
 407         try {
 408             lcmEventsCmHandleStateHandler.updateCmHandleStateBatch(cmHandleStatePerCmHandle);
 409             return CmHandleRegistrationResponse.createSuccessResponses(cmHandleIds);
 410         } catch (final AlreadyDefinedException alreadyDefinedException) {
 411             return CmHandleRegistrationResponse.createFailureResponses(
 412                     alreadyDefinedException.getAlreadyDefinedObjectNames(),
 413                     RegistrationError.CM_HANDLE_ALREADY_EXIST);
 414         } catch (final Exception exception) {
 415             return CmHandleRegistrationResponse.createFailureResponses(cmHandleIds, exception);
 416         }
 417     }
 418
 419     private void setTrustLevelPerDmiPlugin(final DmiPluginRegistration dmiPluginRegistration) {
 420         if (DmiPluginRegistration.isNullEmptyOrBlank(dmiPluginRegistration.getDmiDataPlugin())) {
 421             trustLevelPerDmiPlugin.put(dmiPluginRegistration.getDmiPlugin(), TrustLevel.COMPLETE);
 422         } else {
 423             trustLevelPerDmiPlugin.put(dmiPluginRegistration.getDmiDataPlugin(), TrustLevel.COMPLETE);
 424         }
 425     }
 426
 427 }