cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/NetworkCmProxyDataServiceImpl.java

   1 /*
   2  *  ============LICENSE_START=======================================================
   3  *  Copyright (C) 2021 highstreet technologies GmbH
   4  *  Modifications Copyright (C) 2021-2023 Nordix Foundation
   5  *  Modifications Copyright (C) 2021 Pantheon.tech
   6  *  Modifications Copyright (C) 2021-2022 Bell Canada
   7  *  ================================================================================
   8  *  Licensed under the Apache License, Version 2.0 (the "License");
   9  *  you may not use this file except in compliance with the License.
  10  *  You may obtain a copy of the License at
  11  *
  12  *        http://www.apache.org/licenses/LICENSE-2.0
  13  *
  14  *  Unless required by applicable law or agreed to in writing, software
  15  *  distributed under the License is distributed on an "AS IS" BASIS,
  16  *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  17  *  See the License for the specific language governing permissions and
  18  *  limitations under the License.
  19  *
  20  *  SPDX-License-Identifier: Apache-2.0
  21  *  ============LICENSE_END=========================================================
  22  */
  23
  24 package org.onap.cps.ncmp.api.impl;
  25
  26 import static org.onap.cps.ncmp.api.impl.constants.DmiRegistryConstants.NFP_OPERATIONAL_DATASTORE_DATASPACE_NAME;
  27 import static org.onap.cps.ncmp.api.impl.operations.DmiRequestBody.OperationEnum;
  28 import static org.onap.cps.ncmp.api.impl.utils.RestQueryParametersValidator.validateCmHandleQueryParameters;
  29
  30 import com.google.common.collect.Lists;
  31 import com.hazelcast.map.IMap;
  32 import java.time.OffsetDateTime;
  33 import java.util.ArrayList;
  34 import java.util.Collection;
  35 import java.util.HashMap;
  36 import java.util.HashSet;
  37 import java.util.List;
  38 import java.util.Map;
  39 import java.util.Set;
  40 import java.util.stream.Collectors;
  41 import lombok.RequiredArgsConstructor;
  42 import lombok.extern.slf4j.Slf4j;
  43 import org.onap.cps.api.CpsDataService;
  44 import org.onap.cps.ncmp.api.NetworkCmProxyCmHandlerQueryService;
  45 import org.onap.cps.ncmp.api.NetworkCmProxyDataService;
  46 import org.onap.cps.ncmp.api.impl.event.lcm.LcmEventsCmHandleStateHandler;
  47 import org.onap.cps.ncmp.api.impl.operations.DmiDataOperations;
  48 import org.onap.cps.ncmp.api.impl.operations.DmiOperations;
  49 import org.onap.cps.ncmp.api.impl.utils.CmHandleQueryConditions;
  50 import org.onap.cps.ncmp.api.impl.utils.InventoryQueryConditions;
  51 import org.onap.cps.ncmp.api.impl.utils.YangDataConverter;
  52 import org.onap.cps.ncmp.api.impl.yangmodels.YangModelCmHandle;
  53 import org.onap.cps.ncmp.api.inventory.CmHandleQueries;
  54 import org.onap.cps.ncmp.api.inventory.CmHandleState;
  55 import org.onap.cps.ncmp.api.inventory.CompositeState;
  56 import org.onap.cps.ncmp.api.inventory.CompositeStateUtils;
  57 import org.onap.cps.ncmp.api.inventory.DataStoreSyncState;
  58 import org.onap.cps.ncmp.api.inventory.InventoryPersistence;
  59 import org.onap.cps.ncmp.api.models.CmHandleQueryApiParameters;
  60 import org.onap.cps.ncmp.api.models.CmHandleQueryServiceParameters;
  61 import org.onap.cps.ncmp.api.models.CmHandleRegistrationResponse;
  62 import org.onap.cps.ncmp.api.models.CmHandleRegistrationResponse.RegistrationError;
  63 import org.onap.cps.ncmp.api.models.DmiPluginRegistration;
  64 import org.onap.cps.ncmp.api.models.DmiPluginRegistrationResponse;
  65 import org.onap.cps.ncmp.api.models.NcmpServiceCmHandle;
  66 import org.onap.cps.spi.FetchDescendantsOption;
  67 import org.onap.cps.spi.exceptions.AlreadyDefinedExceptionBatch;
  68 import org.onap.cps.spi.exceptions.CpsException;
  69 import org.onap.cps.spi.exceptions.DataNodeNotFoundException;
  70 import org.onap.cps.spi.exceptions.DataValidationException;
  71 import org.onap.cps.spi.model.ModuleDefinition;
  72 import org.onap.cps.spi.model.ModuleReference;
  73 import org.onap.cps.utils.JsonObjectMapper;
  74 import org.springframework.http.ResponseEntity;
  75 import org.springframework.stereotype.Service;
  76
  77 @Slf4j
  78 @Service
  79 @RequiredArgsConstructor
  80 public class NetworkCmProxyDataServiceImpl implements NetworkCmProxyDataService {
  81
  82     private static final int DELETE_BATCH_SIZE = 100;
  83     private final JsonObjectMapper jsonObjectMapper;
  84     private final DmiDataOperations dmiDataOperations;
  85     private final NetworkCmProxyDataServicePropertyHandler networkCmProxyDataServicePropertyHandler;
  86     private final InventoryPersistence inventoryPersistence;
  87     private final CmHandleQueries cmHandleQueries;
  88     private final NetworkCmProxyCmHandlerQueryService networkCmProxyCmHandlerQueryService;
  89     private final LcmEventsCmHandleStateHandler lcmEventsCmHandleStateHandler;
  90     private final CpsDataService cpsDataService;
  91     private final IMap<String, Object> moduleSyncStartedOnCmHandles;
  92
  93     @Override
  94     public DmiPluginRegistrationResponse updateDmiRegistrationAndSyncModule(
  95             final DmiPluginRegistration dmiPluginRegistration) {
  96         dmiPluginRegistration.validateDmiPluginRegistration();
  97         final DmiPluginRegistrationResponse dmiPluginRegistrationResponse = new DmiPluginRegistrationResponse();
  98
  99         if (!dmiPluginRegistration.getRemovedCmHandles().isEmpty()) {
 100             dmiPluginRegistrationResponse.setRemovedCmHandles(
 101                     parseAndRemoveCmHandlesInDmiRegistration(dmiPluginRegistration.getRemovedCmHandles()));
 102         }
 103
 104         if (!dmiPluginRegistration.getCreatedCmHandles().isEmpty()) {
 105             dmiPluginRegistrationResponse.setCreatedCmHandles(
 106                     parseAndCreateCmHandlesInDmiRegistrationAndSyncModules(dmiPluginRegistration));
 107         }
 108         if (!dmiPluginRegistration.getUpdatedCmHandles().isEmpty()) {
 109             dmiPluginRegistrationResponse.setUpdatedCmHandles(
 110                     networkCmProxyDataServicePropertyHandler
 111                             .updateCmHandleProperties(dmiPluginRegistration.getUpdatedCmHandles()));
 112         }
 113         return dmiPluginRegistrationResponse;
 114     }
 115
 116     @Override
 117     public Object getResourceDataOperationalForCmHandle(final String cmHandleId,
 118                                                         final String resourceIdentifier,
 119                                                         final String optionsParamInQuery,
 120                                                         final String topicParamInQuery,
 121                                                         final String requestId) {
 122         final ResponseEntity<?> responseEntity = dmiDataOperations.getResourceDataFromDmi(cmHandleId,
 123                 resourceIdentifier,
 124                 optionsParamInQuery,
 125                 DmiOperations.DataStoreEnum.PASSTHROUGH_OPERATIONAL,
 126                 requestId, topicParamInQuery);
 127         return responseEntity.getBody();
 128     }
 129
 130     @Override
 131     public Object getResourceDataOperational(final String cmHandleId,
 132                                              final String resourceIdentifier,
 133                                              final FetchDescendantsOption fetchDescendantsOption) {
 134         return cpsDataService.getDataNode(NFP_OPERATIONAL_DATASTORE_DATASPACE_NAME, cmHandleId, resourceIdentifier,
 135                 fetchDescendantsOption);
 136     }
 137
 138     @Override
 139     public Object getResourceDataPassThroughRunningForCmHandle(final String cmHandleId,
 140                                                                final String resourceIdentifier,
 141                                                                final String optionsParamInQuery,
 142                                                                final String topicParamInQuery,
 143                                                                final String requestId) {
 144         final ResponseEntity<?> responseEntity = dmiDataOperations.getResourceDataFromDmi(cmHandleId,
 145                 resourceIdentifier,
 146                 optionsParamInQuery,
 147                 DmiOperations.DataStoreEnum.PASSTHROUGH_RUNNING,
 148                 requestId, topicParamInQuery);
 149         return responseEntity.getBody();
 150     }
 151
 152     @Override
 153     public Object writeResourceDataPassThroughRunningForCmHandle(final String cmHandleId,
 154                                                                  final String resourceIdentifier,
 155                                                                  final OperationEnum operation,
 156                                                                  final String requestData,
 157                                                                  final String dataType) {
 158         return dmiDataOperations.writeResourceDataPassThroughRunningFromDmi(cmHandleId, resourceIdentifier, operation,
 159                 requestData, dataType);
 160     }
 161
 162     @Override
 163     public Collection<ModuleReference> getYangResourcesModuleReferences(final String cmHandleId) {
 164         return inventoryPersistence.getYangResourcesModuleReferences(cmHandleId);
 165     }
 166
 167     @Override
 168     public Collection<ModuleDefinition> getModuleDefinitionsByCmHandleId(final String cmHandleId) {
 169         return inventoryPersistence.getModuleDefinitionsByCmHandleId(cmHandleId);
 170     }
 171
 172     /**
 173      * Retrieve cm handles with details for the given query parameters.
 174      *
 175      * @param cmHandleQueryApiParameters cm handle query parameters
 176      * @return cm handles with details
 177      */
 178     @Override
 179     public Set<NcmpServiceCmHandle> executeCmHandleSearch(final CmHandleQueryApiParameters cmHandleQueryApiParameters) {
 180         final CmHandleQueryServiceParameters cmHandleQueryServiceParameters = jsonObjectMapper.convertToValueType(
 181                 cmHandleQueryApiParameters, CmHandleQueryServiceParameters.class);
 182         validateCmHandleQueryParameters(cmHandleQueryServiceParameters, CmHandleQueryConditions.ALL_CONDITION_NAMES);
 183         return networkCmProxyCmHandlerQueryService.queryCmHandles(cmHandleQueryServiceParameters);
 184     }
 185
 186     /**
 187      * Retrieve cm handle ids for the given query parameters.
 188      *
 189      * @param cmHandleQueryApiParameters cm handle query parameters
 190      * @return cm handle ids
 191      */
 192     @Override
 193     public Set<String> executeCmHandleIdSearch(final CmHandleQueryApiParameters cmHandleQueryApiParameters) {
 194         final CmHandleQueryServiceParameters cmHandleQueryServiceParameters = jsonObjectMapper.convertToValueType(
 195                 cmHandleQueryApiParameters, CmHandleQueryServiceParameters.class);
 196         validateCmHandleQueryParameters(cmHandleQueryServiceParameters, CmHandleQueryConditions.ALL_CONDITION_NAMES);
 197         return networkCmProxyCmHandlerQueryService.queryCmHandleIds(cmHandleQueryServiceParameters);
 198     }
 199
 200     /**
 201      * Set the data sync enabled flag, along with the data sync state
 202      * based on the data sync enabled boolean for the cm handle id provided.
 203      *
 204      * @param cmHandleId      cm handle id
 205      * @param dataSyncEnabled data sync enabled flag
 206      */
 207     @Override
 208     public void setDataSyncEnabled(final String cmHandleId, final boolean dataSyncEnabled) {
 209         final CompositeState compositeState = inventoryPersistence
 210                 .getCmHandleState(cmHandleId);
 211         if (compositeState.getDataSyncEnabled().equals(dataSyncEnabled)) {
 212             log.info("Data-Sync Enabled flag is already: {} ", dataSyncEnabled);
 213         } else if (compositeState.getCmHandleState() != CmHandleState.READY) {
 214             throw new CpsException("State mismatch exception.", "Cm-Handle not in READY state. Cm handle state is: "
 215                     + compositeState.getCmHandleState());
 216         } else {
 217             final DataStoreSyncState dataStoreSyncState = compositeState.getDataStores()
 218                     .getOperationalDataStore().getDataStoreSyncState();
 219             if (!dataSyncEnabled && dataStoreSyncState == DataStoreSyncState.SYNCHRONIZED) {
 220                 cpsDataService.deleteDataNode(NFP_OPERATIONAL_DATASTORE_DATASPACE_NAME, cmHandleId,
 221                         "/netconf-state", OffsetDateTime.now());
 222             }
 223             CompositeStateUtils.setDataSyncEnabledFlagWithDataSyncState(dataSyncEnabled, compositeState);
 224             inventoryPersistence.saveCmHandleState(cmHandleId,
 225                     compositeState);
 226         }
 227     }
 228
 229     /**
 230      * Get all cm handle IDs by DMI plugin identifier.
 231      *
 232      * @param dmiPluginIdentifier DMI plugin identifier
 233      * @return set of cm handle IDs
 234      */
 235     @Override
 236     public Set<String> getAllCmHandleIdsByDmiPluginIdentifier(final String dmiPluginIdentifier) {
 237         final Set<NcmpServiceCmHandle> ncmpServiceCmHandles =
 238                 cmHandleQueries.getCmHandlesByDmiPluginIdentifier(dmiPluginIdentifier);
 239         final Set<String> cmHandleIds = new HashSet<>(ncmpServiceCmHandles.size());
 240         ncmpServiceCmHandles.forEach(cmHandle -> cmHandleIds.add(cmHandle.getCmHandleId()));
 241         return cmHandleIds;
 242     }
 243
 244     /**
 245      * Get all cm handle IDs by various properties.
 246      *
 247      * @param cmHandleQueryServiceParameters cm handle query parameters
 248      * @return set of cm handle IDs
 249      */
 250     @Override
 251     public Set<String> executeCmHandleIdSearchForInventory(
 252             final CmHandleQueryServiceParameters cmHandleQueryServiceParameters) {
 253         validateCmHandleQueryParameters(cmHandleQueryServiceParameters, InventoryQueryConditions.ALL_CONDITION_NAMES);
 254         return networkCmProxyCmHandlerQueryService.queryCmHandleIdsForInventory(cmHandleQueryServiceParameters);
 255     }
 256
 257     /**
 258      * Retrieve cm handle details for a given cm handle.
 259      *
 260      * @param cmHandleId cm handle identifier
 261      * @return cm handle details
 262      */
 263     @Override
 264     public NcmpServiceCmHandle getNcmpServiceCmHandle(final String cmHandleId) {
 265         return YangDataConverter.convertYangModelCmHandleToNcmpServiceCmHandle(
 266                 inventoryPersistence.getYangModelCmHandle(cmHandleId));
 267     }
 268
 269     /**
 270      * Get cm handle public properties for a given cm handle id.
 271      *
 272      * @param cmHandleId cm handle identifier
 273      * @return cm handle public properties
 274      */
 275     @Override
 276     public Map<String, String> getCmHandlePublicProperties(final String cmHandleId) {
 277         final YangModelCmHandle yangModelCmHandle =
 278                 inventoryPersistence.getYangModelCmHandle(cmHandleId);
 279         final List<YangModelCmHandle.Property> yangModelPublicProperties = yangModelCmHandle.getPublicProperties();
 280         final Map<String, String> cmHandlePublicProperties = new HashMap<>();
 281         YangDataConverter.asPropertiesMap(yangModelPublicProperties, cmHandlePublicProperties);
 282         return cmHandlePublicProperties;
 283     }
 284
 285     /**
 286      * Get cm handle composite state for a given cm handle id.
 287      *
 288      * @param cmHandleId cm handle identifier
 289      * @return cm handle state
 290      */
 291     @Override
 292     public CompositeState getCmHandleCompositeState(final String cmHandleId) {
 293         return inventoryPersistence.getYangModelCmHandle(cmHandleId).getCompositeState();
 294     }
 295
 296     /**
 297      * THis method registers a cm handle and initiates modules sync.
 298      *
 299      * @param dmiPluginRegistration dmi plugin registration information.
 300      * @return cm-handle registration response for create cm-handle requests.
 301      */
 302     public List<CmHandleRegistrationResponse> parseAndCreateCmHandlesInDmiRegistrationAndSyncModules(
 303             final DmiPluginRegistration dmiPluginRegistration) {
 304         List<CmHandleRegistrationResponse> cmHandleRegistrationResponses = new ArrayList<>();
 305         final Map<YangModelCmHandle, CmHandleState> cmHandleStatePerCmHandle = new HashMap<>();
 306         try {
 307             dmiPluginRegistration.getCreatedCmHandles()
 308                     .forEach(cmHandle -> {
 309                         final YangModelCmHandle yangModelCmHandle = YangModelCmHandle.toYangModelCmHandle(
 310                                 dmiPluginRegistration.getDmiPlugin(),
 311                                 dmiPluginRegistration.getDmiDataPlugin(),
 312                                 dmiPluginRegistration.getDmiModelPlugin(),
 313                                 cmHandle);
 314                         cmHandleStatePerCmHandle.put(yangModelCmHandle, CmHandleState.ADVISED);
 315                     });
 316             cmHandleRegistrationResponses = registerNewCmHandles(cmHandleStatePerCmHandle);
 317         } catch (final DataValidationException dataValidationException) {
 318             cmHandleRegistrationResponses.add(CmHandleRegistrationResponse.createFailureResponse(dmiPluginRegistration
 319                             .getCreatedCmHandles().stream()
 320                             .map(NcmpServiceCmHandle::getCmHandleId).findFirst().orElse(null),
 321                     RegistrationError.CM_HANDLE_INVALID_ID));
 322         }
 323         return cmHandleRegistrationResponses;
 324     }
 325
 326     protected List<CmHandleRegistrationResponse> parseAndRemoveCmHandlesInDmiRegistration(
 327             final List<String> tobeRemovedCmHandles) {
 328         final List<CmHandleRegistrationResponse> cmHandleRegistrationResponses =
 329                 new ArrayList<>(tobeRemovedCmHandles.size());
 330         final List<YangModelCmHandle> yangModelCmHandles =
 331                 tobeRemovedCmHandles.stream().map(inventoryPersistence::getYangModelCmHandle)
 332                         .collect(Collectors.toList());
 333         updateCmHandleStateBatch(yangModelCmHandles, CmHandleState.DELETING);
 334
 335         for (final List<String> tobeRemovedCmHandleBatch : Lists.partition(tobeRemovedCmHandles, DELETE_BATCH_SIZE)) {
 336             try {
 337                 batchDeleteCmHandlesFromDbAndModuleSyncMap(tobeRemovedCmHandleBatch);
 338                 tobeRemovedCmHandleBatch.forEach(cmHandleId ->
 339                     cmHandleRegistrationResponses.add(CmHandleRegistrationResponse.createSuccessResponse(cmHandleId)));
 340
 341             } catch (final Exception batchException) {
 342                 log.error("Unable to de-register cm-handle batch, retrying on each cm handle");
 343                 for (final String cmHandleId : tobeRemovedCmHandleBatch) {
 344                     final CmHandleRegistrationResponse cmHandleRegistrationResponse =
 345                         deleteCmHandleAndGetCmHandleRegistrationResponse(cmHandleId);
 346                     cmHandleRegistrationResponses.add(cmHandleRegistrationResponse);
 347                 }
 348             }
 349         }
 350
 351         updateCmHandleStateBatch(yangModelCmHandles, CmHandleState.DELETED);
 352         return cmHandleRegistrationResponses;
 353     }
 354
 355     private CmHandleRegistrationResponse deleteCmHandleAndGetCmHandleRegistrationResponse(final String cmHandleId) {
 356         try {
 357             deleteCmHandleFromDbAndModuleSyncMap(cmHandleId);
 358             return CmHandleRegistrationResponse.createSuccessResponse(cmHandleId);
 359         } catch (final DataNodeNotFoundException dataNodeNotFoundException) {
 360             log.error("Unable to find dataNode for cmHandleId : {} , caused by : {}",
 361                 cmHandleId, dataNodeNotFoundException.getMessage());
 362             return CmHandleRegistrationResponse.createFailureResponse(cmHandleId,
 363                 RegistrationError.CM_HANDLE_DOES_NOT_EXIST);
 364         } catch (final DataValidationException dataValidationException) {
 365             log.error("Unable to de-register cm-handle id: {}, caused by: {}",
 366                 cmHandleId, dataValidationException.getMessage());
 367             return CmHandleRegistrationResponse.createFailureResponse(cmHandleId,
 368                 RegistrationError.CM_HANDLE_INVALID_ID);
 369         } catch (final Exception exception) {
 370             log.error("Unable to de-register cm-handle id : {} , caused by : {}", cmHandleId, exception.getMessage());
 371             return CmHandleRegistrationResponse.createFailureResponse(cmHandleId, exception);
 372         }
 373     }
 374
 375     private void updateCmHandleStateBatch(final List<YangModelCmHandle> yangModelCmHandles,
 376                                           final CmHandleState cmHandleState) {
 377         final Map<YangModelCmHandle, CmHandleState> cmHandleIdsToBeRemoved = new HashMap<>();
 378         yangModelCmHandles.stream().forEach(yangModelCmHandle ->
 379                 cmHandleIdsToBeRemoved.put(yangModelCmHandle, cmHandleState));
 380         lcmEventsCmHandleStateHandler.updateCmHandleStateBatch(cmHandleIdsToBeRemoved);
 381     }
 382
 383     private void deleteCmHandleFromDbAndModuleSyncMap(final String cmHandleId) {
 384         inventoryPersistence.deleteSchemaSetWithCascade(cmHandleId);
 385         inventoryPersistence.deleteDataNode("/dmi-registry/cm-handles[@id='" + cmHandleId + "']");
 386         removeDeletedCmHandleFromModuleSyncMap(cmHandleId);
 387     }
 388
 389     private void batchDeleteCmHandlesFromDbAndModuleSyncMap(final Collection<String> tobeRemovedCmHandles) {
 390         tobeRemovedCmHandles.forEach(inventoryPersistence::deleteSchemaSetWithCascade);
 391         inventoryPersistence.deleteDataNodes(mapCmHandleIdsToXpaths(tobeRemovedCmHandles));
 392         tobeRemovedCmHandles.forEach(this::removeDeletedCmHandleFromModuleSyncMap);
 393     }
 394
 395     private Collection<String> mapCmHandleIdsToXpaths(final Collection<String> cmHandles) {
 396         return cmHandles.stream()
 397             .map(cmHandleId -> "/dmi-registry/cm-handles[@id='" + cmHandleId + "']")
 398             .collect(Collectors.toSet());
 399     }
 400
 401     // CPS-1239 Robustness cleaning of in progress cache
 402     private void removeDeletedCmHandleFromModuleSyncMap(final String deletedCmHandleId) {
 403         if (moduleSyncStartedOnCmHandles.remove(deletedCmHandleId) != null) {
 404             log.debug("{} removed from in progress map", deletedCmHandleId);
 405         }
 406     }
 407
 408     private List<CmHandleRegistrationResponse> registerNewCmHandles(final Map<YangModelCmHandle, CmHandleState>
 409                                                                             cmHandleStatePerCmHandle) {
 410         final List<String> cmHandleIds = cmHandleStatePerCmHandle.keySet().stream().map(YangModelCmHandle::getId)
 411                 .collect(Collectors.toList());
 412         try {
 413             lcmEventsCmHandleStateHandler.updateCmHandleStateBatch(cmHandleStatePerCmHandle);
 414             return CmHandleRegistrationResponse.createSuccessResponses(cmHandleIds);
 415         } catch (final AlreadyDefinedExceptionBatch alreadyDefinedExceptionBatch) {
 416             return CmHandleRegistrationResponse.createFailureResponses(
 417                     alreadyDefinedExceptionBatch.getAlreadyDefinedXpaths(),
 418                     RegistrationError.CM_HANDLE_ALREADY_EXIST);
 419         } catch (final Exception exception) {
 420             return CmHandleRegistrationResponse.createFailureResponses(cmHandleIds, exception);
 421         }
 422     }
 423 }