cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/NetworkCmProxyDataServiceImpl.java

   1 /*
   2  *  ============LICENSE_START=======================================================
   3  *  Copyright (C) 2021 highstreet technologies GmbH
   4  *  Modifications Copyright (C) 2021-2023 Nordix Foundation
   5  *  Modifications Copyright (C) 2021 Pantheon.tech
   6  *  Modifications Copyright (C) 2021-2022 Bell Canada
   7  *  Modifications Copyright (C) 2023 TechMahindra Ltd.
   8  *  ================================================================================
   9  *  Licensed under the Apache License, Version 2.0 (the "License");
  10  *  you may not use this file except in compliance with the License.
  11  *  You may obtain a copy of the License at
  12  *
  13  *        http://www.apache.org/licenses/LICENSE-2.0
  14  *
  15  *  Unless required by applicable law or agreed to in writing, software
  16  *  distributed under the License is distributed on an "AS IS" BASIS,
  17  *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  18  *  See the License for the specific language governing permissions and
  19  *  limitations under the License.
  20  *
  21  *  SPDX-License-Identifier: Apache-2.0
  22  *  ============LICENSE_END=========================================================
  23  */
  24
  25 package org.onap.cps.ncmp.api.impl;
  26
  27 import static org.onap.cps.ncmp.api.impl.constants.DmiRegistryConstants.NFP_OPERATIONAL_DATASTORE_DATASPACE_NAME;
  28 import static org.onap.cps.ncmp.api.impl.operations.DmiRequestBody.OperationEnum;
  29 import static org.onap.cps.ncmp.api.impl.utils.RestQueryParametersValidator.validateCmHandleQueryParameters;
  30
  31 import com.google.common.collect.Lists;
  32 import com.hazelcast.map.IMap;
  33 import java.time.OffsetDateTime;
  34 import java.util.ArrayList;
  35 import java.util.Collection;
  36 import java.util.HashMap;
  37 import java.util.HashSet;
  38 import java.util.List;
  39 import java.util.Map;
  40 import java.util.Set;
  41 import java.util.stream.Collectors;
  42 import lombok.RequiredArgsConstructor;
  43 import lombok.extern.slf4j.Slf4j;
  44 import org.onap.cps.api.CpsDataService;
  45 import org.onap.cps.ncmp.api.NetworkCmProxyCmHandleQueryService;
  46 import org.onap.cps.ncmp.api.NetworkCmProxyDataService;
  47 import org.onap.cps.ncmp.api.impl.events.lcm.LcmEventsCmHandleStateHandler;
  48 import org.onap.cps.ncmp.api.impl.operations.DmiDataOperations;
  49 import org.onap.cps.ncmp.api.impl.operations.DmiOperations;
  50 import org.onap.cps.ncmp.api.impl.utils.CmHandleQueryConditions;
  51 import org.onap.cps.ncmp.api.impl.utils.InventoryQueryConditions;
  52 import org.onap.cps.ncmp.api.impl.utils.YangDataConverter;
  53 import org.onap.cps.ncmp.api.impl.yangmodels.YangModelCmHandle;
  54 import org.onap.cps.ncmp.api.inventory.CmHandleQueries;
  55 import org.onap.cps.ncmp.api.inventory.CmHandleState;
  56 import org.onap.cps.ncmp.api.inventory.CompositeState;
  57 import org.onap.cps.ncmp.api.inventory.CompositeStateUtils;
  58 import org.onap.cps.ncmp.api.inventory.DataStoreSyncState;
  59 import org.onap.cps.ncmp.api.inventory.InventoryPersistence;
  60 import org.onap.cps.ncmp.api.models.CmHandleQueryApiParameters;
  61 import org.onap.cps.ncmp.api.models.CmHandleQueryServiceParameters;
  62 import org.onap.cps.ncmp.api.models.CmHandleRegistrationResponse;
  63 import org.onap.cps.ncmp.api.models.CmHandleRegistrationResponse.RegistrationError;
  64 import org.onap.cps.ncmp.api.models.DmiPluginRegistration;
  65 import org.onap.cps.ncmp.api.models.DmiPluginRegistrationResponse;
  66 import org.onap.cps.ncmp.api.models.NcmpServiceCmHandle;
  67 import org.onap.cps.spi.FetchDescendantsOption;
  68 import org.onap.cps.spi.exceptions.AlreadyDefinedExceptionBatch;
  69 import org.onap.cps.spi.exceptions.CpsException;
  70 import org.onap.cps.spi.exceptions.DataNodeNotFoundException;
  71 import org.onap.cps.spi.exceptions.DataValidationException;
  72 import org.onap.cps.spi.model.ModuleDefinition;
  73 import org.onap.cps.spi.model.ModuleReference;
  74 import org.onap.cps.utils.JsonObjectMapper;
  75 import org.springframework.http.ResponseEntity;
  76 import org.springframework.stereotype.Service;
  77
  78 @Slf4j
  79 @Service
  80 @RequiredArgsConstructor
  81 public class NetworkCmProxyDataServiceImpl implements NetworkCmProxyDataService {
  82
  83     private static final int DELETE_BATCH_SIZE = 100;
  84     private final JsonObjectMapper jsonObjectMapper;
  85     private final DmiDataOperations dmiDataOperations;
  86     private final NetworkCmProxyDataServicePropertyHandler networkCmProxyDataServicePropertyHandler;
  87     private final InventoryPersistence inventoryPersistence;
  88     private final CmHandleQueries cmHandleQueries;
  89     private final NetworkCmProxyCmHandleQueryService networkCmProxyCmHandleQueryService;
  90     private final LcmEventsCmHandleStateHandler lcmEventsCmHandleStateHandler;
  91     private final CpsDataService cpsDataService;
  92     private final IMap<String, Object> moduleSyncStartedOnCmHandles;
  93
  94     @Override
  95     public DmiPluginRegistrationResponse updateDmiRegistrationAndSyncModule(
  96             final DmiPluginRegistration dmiPluginRegistration) {
  97         dmiPluginRegistration.validateDmiPluginRegistration();
  98         final DmiPluginRegistrationResponse dmiPluginRegistrationResponse = new DmiPluginRegistrationResponse();
  99
 100         if (!dmiPluginRegistration.getRemovedCmHandles().isEmpty()) {
 101             dmiPluginRegistrationResponse.setRemovedCmHandles(
 102                     parseAndRemoveCmHandlesInDmiRegistration(dmiPluginRegistration.getRemovedCmHandles()));
 103         }
 104
 105         if (!dmiPluginRegistration.getCreatedCmHandles().isEmpty()) {
 106             dmiPluginRegistrationResponse.setCreatedCmHandles(
 107                     parseAndCreateCmHandlesInDmiRegistrationAndSyncModules(dmiPluginRegistration));
 108         }
 109         if (!dmiPluginRegistration.getUpdatedCmHandles().isEmpty()) {
 110             dmiPluginRegistrationResponse.setUpdatedCmHandles(
 111                     networkCmProxyDataServicePropertyHandler
 112                             .updateCmHandleProperties(dmiPluginRegistration.getUpdatedCmHandles()));
 113         }
 114         return dmiPluginRegistrationResponse;
 115     }
 116
 117     @Override
 118     public Object getResourceDataOperationalForCmHandle(final String cmHandleId,
 119                                                         final String resourceIdentifier,
 120                                                         final String optionsParamInQuery,
 121                                                         final String topicParamInQuery,
 122                                                         final String requestId) {
 123         final ResponseEntity<?> responseEntity = dmiDataOperations.getResourceDataFromDmi(cmHandleId,
 124                 resourceIdentifier,
 125                 optionsParamInQuery,
 126                 DmiOperations.DataStoreEnum.PASSTHROUGH_OPERATIONAL,
 127                 requestId, topicParamInQuery);
 128         return responseEntity.getBody();
 129     }
 130
 131     @Override
 132     public Object getResourceDataOperational(final String cmHandleId,
 133                                              final String resourceIdentifier,
 134                                              final FetchDescendantsOption fetchDescendantsOption) {
 135         return cpsDataService.getDataNodes(NFP_OPERATIONAL_DATASTORE_DATASPACE_NAME, cmHandleId, resourceIdentifier,
 136                 fetchDescendantsOption).iterator().next();
 137     }
 138
 139     @Override
 140     public Object getResourceDataPassThroughRunningForCmHandle(final String cmHandleId,
 141                                                                final String resourceIdentifier,
 142                                                                final String optionsParamInQuery,
 143                                                                final String topicParamInQuery,
 144                                                                final String requestId) {
 145         final ResponseEntity<?> responseEntity = dmiDataOperations.getResourceDataFromDmi(cmHandleId,
 146                 resourceIdentifier,
 147                 optionsParamInQuery,
 148                 DmiOperations.DataStoreEnum.PASSTHROUGH_RUNNING,
 149                 requestId, topicParamInQuery);
 150         return responseEntity.getBody();
 151     }
 152
 153     @Override
 154     public Object writeResourceDataPassThroughRunningForCmHandle(final String cmHandleId,
 155                                                                  final String resourceIdentifier,
 156                                                                  final OperationEnum operation,
 157                                                                  final String requestData,
 158                                                                  final String dataType) {
 159         return dmiDataOperations.writeResourceDataPassThroughRunningFromDmi(cmHandleId, resourceIdentifier, operation,
 160                 requestData, dataType);
 161     }
 162
 163     @Override
 164     public Collection<ModuleReference> getYangResourcesModuleReferences(final String cmHandleId) {
 165         return inventoryPersistence.getYangResourcesModuleReferences(cmHandleId);
 166     }
 167
 168     @Override
 169     public Collection<ModuleDefinition> getModuleDefinitionsByCmHandleId(final String cmHandleId) {
 170         return inventoryPersistence.getModuleDefinitionsByCmHandleId(cmHandleId);
 171     }
 172
 173     /**
 174      * Retrieve cm handles with details for the given query parameters.
 175      *
 176      * @param cmHandleQueryApiParameters cm handle query parameters
 177      * @return cm handles with details
 178      */
 179     @Override
 180     public Collection<NcmpServiceCmHandle> executeCmHandleSearch(
 181             final CmHandleQueryApiParameters cmHandleQueryApiParameters) {
 182         final CmHandleQueryServiceParameters cmHandleQueryServiceParameters = jsonObjectMapper.convertToValueType(
 183                 cmHandleQueryApiParameters, CmHandleQueryServiceParameters.class);
 184         validateCmHandleQueryParameters(cmHandleQueryServiceParameters, CmHandleQueryConditions.ALL_CONDITION_NAMES);
 185         return networkCmProxyCmHandleQueryService.queryCmHandles(cmHandleQueryServiceParameters);
 186     }
 187
 188     /**
 189      * Retrieve cm handle ids for the given query parameters.
 190      *
 191      * @param cmHandleQueryApiParameters cm handle query parameters
 192      * @return cm handle ids
 193      */
 194     @Override
 195     public Collection<String> executeCmHandleIdSearch(final CmHandleQueryApiParameters cmHandleQueryApiParameters) {
 196         final CmHandleQueryServiceParameters cmHandleQueryServiceParameters = jsonObjectMapper.convertToValueType(
 197                 cmHandleQueryApiParameters, CmHandleQueryServiceParameters.class);
 198         validateCmHandleQueryParameters(cmHandleQueryServiceParameters, CmHandleQueryConditions.ALL_CONDITION_NAMES);
 199         return networkCmProxyCmHandleQueryService.queryCmHandleIds(cmHandleQueryServiceParameters);
 200     }
 201
 202     /**
 203      * Set the data sync enabled flag, along with the data sync state
 204      * based on the data sync enabled boolean for the cm handle id provided.
 205      *
 206      * @param cmHandleId      cm handle id
 207      * @param dataSyncEnabled data sync enabled flag
 208      */
 209     @Override
 210     public void setDataSyncEnabled(final String cmHandleId, final boolean dataSyncEnabled) {
 211         final CompositeState compositeState = inventoryPersistence
 212                 .getCmHandleState(cmHandleId);
 213         if (compositeState.getDataSyncEnabled().equals(dataSyncEnabled)) {
 214             log.info("Data-Sync Enabled flag is already: {} ", dataSyncEnabled);
 215         } else if (compositeState.getCmHandleState() != CmHandleState.READY) {
 216             throw new CpsException("State mismatch exception.", "Cm-Handle not in READY state. Cm handle state is: "
 217                     + compositeState.getCmHandleState());
 218         } else {
 219             final DataStoreSyncState dataStoreSyncState = compositeState.getDataStores()
 220                     .getOperationalDataStore().getDataStoreSyncState();
 221             if (!dataSyncEnabled && dataStoreSyncState == DataStoreSyncState.SYNCHRONIZED) {
 222                 cpsDataService.deleteDataNode(NFP_OPERATIONAL_DATASTORE_DATASPACE_NAME, cmHandleId,
 223                         "/netconf-state", OffsetDateTime.now());
 224             }
 225             CompositeStateUtils.setDataSyncEnabledFlagWithDataSyncState(dataSyncEnabled, compositeState);
 226             inventoryPersistence.saveCmHandleState(cmHandleId,
 227                     compositeState);
 228         }
 229     }
 230
 231     /**
 232      * Get all cm handle IDs by DMI plugin identifier.
 233      *
 234      * @param dmiPluginIdentifier DMI plugin identifier
 235      * @return set of cm handle IDs
 236      */
 237     @Override
 238     public Collection<String> getAllCmHandleIdsByDmiPluginIdentifier(final String dmiPluginIdentifier) {
 239         return cmHandleQueries.getCmHandleIdsByDmiPluginIdentifier(dmiPluginIdentifier);
 240     }
 241
 242     /**
 243      * Get all cm handle IDs by various properties.
 244      *
 245      * @param cmHandleQueryServiceParameters cm handle query parameters
 246      * @return set of cm handle IDs
 247      */
 248     @Override
 249     public Collection<String> executeCmHandleIdSearchForInventory(
 250             final CmHandleQueryServiceParameters cmHandleQueryServiceParameters) {
 251         validateCmHandleQueryParameters(cmHandleQueryServiceParameters, InventoryQueryConditions.ALL_CONDITION_NAMES);
 252         return networkCmProxyCmHandleQueryService.queryCmHandleIdsForInventory(cmHandleQueryServiceParameters);
 253     }
 254
 255     /**
 256      * Retrieve cm handle details for a given cm handle.
 257      *
 258      * @param cmHandleId cm handle identifier
 259      * @return cm handle details
 260      */
 261     @Override
 262     public NcmpServiceCmHandle getNcmpServiceCmHandle(final String cmHandleId) {
 263         return YangDataConverter.convertYangModelCmHandleToNcmpServiceCmHandle(
 264                 inventoryPersistence.getYangModelCmHandle(cmHandleId));
 265     }
 266
 267     /**
 268      * Get cm handle public properties for a given cm handle id.
 269      *
 270      * @param cmHandleId cm handle identifier
 271      * @return cm handle public properties
 272      */
 273     @Override
 274     public Map<String, String> getCmHandlePublicProperties(final String cmHandleId) {
 275         final YangModelCmHandle yangModelCmHandle =
 276                 inventoryPersistence.getYangModelCmHandle(cmHandleId);
 277         final List<YangModelCmHandle.Property> yangModelPublicProperties = yangModelCmHandle.getPublicProperties();
 278         final Map<String, String> cmHandlePublicProperties = new HashMap<>();
 279         YangDataConverter.asPropertiesMap(yangModelPublicProperties, cmHandlePublicProperties);
 280         return cmHandlePublicProperties;
 281     }
 282
 283     /**
 284      * Get cm handle composite state for a given cm handle id.
 285      *
 286      * @param cmHandleId cm handle identifier
 287      * @return cm handle state
 288      */
 289     @Override
 290     public CompositeState getCmHandleCompositeState(final String cmHandleId) {
 291         return inventoryPersistence.getYangModelCmHandle(cmHandleId).getCompositeState();
 292     }
 293
 294     /**
 295      * THis method registers a cm handle and initiates modules sync.
 296      *
 297      * @param dmiPluginRegistration dmi plugin registration information.
 298      * @return cm-handle registration response for create cm-handle requests.
 299      */
 300     public List<CmHandleRegistrationResponse> parseAndCreateCmHandlesInDmiRegistrationAndSyncModules(
 301             final DmiPluginRegistration dmiPluginRegistration) {
 302         List<CmHandleRegistrationResponse> cmHandleRegistrationResponses = new ArrayList<>();
 303         final Map<YangModelCmHandle, CmHandleState> cmHandleStatePerCmHandle = new HashMap<>();
 304         try {
 305             dmiPluginRegistration.getCreatedCmHandles()
 306                     .forEach(cmHandle -> {
 307                         final YangModelCmHandle yangModelCmHandle = YangModelCmHandle.toYangModelCmHandle(
 308                                 dmiPluginRegistration.getDmiPlugin(),
 309                                 dmiPluginRegistration.getDmiDataPlugin(),
 310                                 dmiPluginRegistration.getDmiModelPlugin(),
 311                                 cmHandle);
 312                         cmHandleStatePerCmHandle.put(yangModelCmHandle, CmHandleState.ADVISED);
 313                     });
 314             cmHandleRegistrationResponses = registerNewCmHandles(cmHandleStatePerCmHandle);
 315         } catch (final DataValidationException dataValidationException) {
 316             cmHandleRegistrationResponses.add(CmHandleRegistrationResponse.createFailureResponse(dmiPluginRegistration
 317                             .getCreatedCmHandles().stream()
 318                             .map(NcmpServiceCmHandle::getCmHandleId).findFirst().orElse(null),
 319                     RegistrationError.CM_HANDLE_INVALID_ID));
 320         }
 321         return cmHandleRegistrationResponses;
 322     }
 323
 324     protected List<CmHandleRegistrationResponse> parseAndRemoveCmHandlesInDmiRegistration(
 325             final List<String> tobeRemovedCmHandles) {
 326         final List<CmHandleRegistrationResponse> cmHandleRegistrationResponses =
 327                 new ArrayList<>(tobeRemovedCmHandles.size());
 328         final Collection<YangModelCmHandle> yangModelCmHandles =
 329             inventoryPersistence.getYangModelCmHandles(tobeRemovedCmHandles);
 330
 331         updateCmHandleStateBatch(yangModelCmHandles, CmHandleState.DELETING);
 332
 333         final Set<String> notDeletedCmHandles = new HashSet<>();
 334         for (final List<String> tobeRemovedCmHandleBatch : Lists.partition(tobeRemovedCmHandles, DELETE_BATCH_SIZE)) {
 335             try {
 336                 batchDeleteCmHandlesFromDbAndModuleSyncMap(tobeRemovedCmHandleBatch);
 337                 tobeRemovedCmHandleBatch.forEach(cmHandleId ->
 338                     cmHandleRegistrationResponses.add(CmHandleRegistrationResponse.createSuccessResponse(cmHandleId)));
 339
 340             } catch (final RuntimeException batchException) {
 341                 log.error("Unable to de-register cm-handle batch, retrying on each cm handle");
 342                 for (final String cmHandleId : tobeRemovedCmHandleBatch) {
 343                     final CmHandleRegistrationResponse cmHandleRegistrationResponse =
 344                         deleteCmHandleAndGetCmHandleRegistrationResponse(cmHandleId);
 345                     cmHandleRegistrationResponses.add(cmHandleRegistrationResponse);
 346                     if (cmHandleRegistrationResponse.getStatus() != CmHandleRegistrationResponse.Status.SUCCESS) {
 347                         notDeletedCmHandles.add(cmHandleId);
 348                     }
 349                 }
 350             }
 351         }
 352
 353         yangModelCmHandles.removeIf(yangModelCmHandle -> notDeletedCmHandles.contains(yangModelCmHandle.getId()));
 354         updateCmHandleStateBatch(yangModelCmHandles, CmHandleState.DELETED);
 355
 356         return cmHandleRegistrationResponses;
 357     }
 358
 359     private CmHandleRegistrationResponse deleteCmHandleAndGetCmHandleRegistrationResponse(final String cmHandleId) {
 360         try {
 361             deleteCmHandleFromDbAndModuleSyncMap(cmHandleId);
 362             return CmHandleRegistrationResponse.createSuccessResponse(cmHandleId);
 363         } catch (final DataNodeNotFoundException dataNodeNotFoundException) {
 364             log.error("Unable to find dataNode for cmHandleId : {} , caused by : {}",
 365                 cmHandleId, dataNodeNotFoundException.getMessage());
 366             return CmHandleRegistrationResponse.createFailureResponse(cmHandleId,
 367                 RegistrationError.CM_HANDLE_DOES_NOT_EXIST);
 368         } catch (final DataValidationException dataValidationException) {
 369             log.error("Unable to de-register cm-handle id: {}, caused by: {}",
 370                 cmHandleId, dataValidationException.getMessage());
 371             return CmHandleRegistrationResponse.createFailureResponse(cmHandleId,
 372                 RegistrationError.CM_HANDLE_INVALID_ID);
 373         } catch (final Exception exception) {
 374             log.error("Unable to de-register cm-handle id : {} , caused by : {}", cmHandleId, exception.getMessage());
 375             return CmHandleRegistrationResponse.createFailureResponse(cmHandleId, exception);
 376         }
 377     }
 378
 379     private void updateCmHandleStateBatch(final Collection<YangModelCmHandle> yangModelCmHandles,
 380                                           final CmHandleState cmHandleState) {
 381         final Map<YangModelCmHandle, CmHandleState> cmHandleStatePerCmHandle = new HashMap<>(yangModelCmHandles.size());
 382         yangModelCmHandles.forEach(yangModelCmHandle -> cmHandleStatePerCmHandle.put(yangModelCmHandle, cmHandleState));
 383         lcmEventsCmHandleStateHandler.updateCmHandleStateBatch(cmHandleStatePerCmHandle);
 384     }
 385
 386     private void deleteCmHandleFromDbAndModuleSyncMap(final String cmHandleId) {
 387         inventoryPersistence.deleteSchemaSetWithCascade(cmHandleId);
 388         inventoryPersistence.deleteDataNode("/dmi-registry/cm-handles[@id='" + cmHandleId + "']");
 389         removeDeletedCmHandleFromModuleSyncMap(cmHandleId);
 390     }
 391
 392     private void batchDeleteCmHandlesFromDbAndModuleSyncMap(final Collection<String> tobeRemovedCmHandles) {
 393         inventoryPersistence.deleteSchemaSetsWithCascade(tobeRemovedCmHandles);
 394         inventoryPersistence.deleteDataNodes(mapCmHandleIdsToXpaths(tobeRemovedCmHandles));
 395         tobeRemovedCmHandles.forEach(this::removeDeletedCmHandleFromModuleSyncMap);
 396     }
 397
 398     private Collection<String> mapCmHandleIdsToXpaths(final Collection<String> cmHandles) {
 399         return cmHandles.stream()
 400             .map(cmHandleId -> "/dmi-registry/cm-handles[@id='" + cmHandleId + "']")
 401             .collect(Collectors.toSet());
 402     }
 403
 404     // CPS-1239 Robustness cleaning of in progress cache
 405     private void removeDeletedCmHandleFromModuleSyncMap(final String deletedCmHandleId) {
 406         if (moduleSyncStartedOnCmHandles.remove(deletedCmHandleId) != null) {
 407             log.debug("{} removed from in progress map", deletedCmHandleId);
 408         }
 409     }
 410
 411     private List<CmHandleRegistrationResponse> registerNewCmHandles(final Map<YangModelCmHandle, CmHandleState>
 412                                                                             cmHandleStatePerCmHandle) {
 413         final List<String> cmHandleIds = cmHandleStatePerCmHandle.keySet().stream().map(YangModelCmHandle::getId)
 414                 .collect(Collectors.toList());
 415         try {
 416             lcmEventsCmHandleStateHandler.updateCmHandleStateBatch(cmHandleStatePerCmHandle);
 417             return CmHandleRegistrationResponse.createSuccessResponses(cmHandleIds);
 418         } catch (final AlreadyDefinedExceptionBatch alreadyDefinedExceptionBatch) {
 419             return CmHandleRegistrationResponse.createFailureResponses(
 420                     alreadyDefinedExceptionBatch.getAlreadyDefinedXpaths(),
 421                     RegistrationError.CM_HANDLE_ALREADY_EXIST);
 422         } catch (final Exception exception) {
 423             return CmHandleRegistrationResponse.createFailureResponses(cmHandleIds, exception);
 424         }
 425     }
 426
 427 }