cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/NetworkCmProxyDataServiceImpl.java

   1 /*
   2  *  ============LICENSE_START=======================================================
   3  *  Copyright (C) 2021 highstreet technologies GmbH
   4  *  Modifications Copyright (C) 2021-2023 Nordix Foundation
   5  *  Modifications Copyright (C) 2021 Pantheon.tech
   6  *  Modifications Copyright (C) 2021-2022 Bell Canada
   7  *  Modifications Copyright (C) 2023 TechMahindra Ltd.
   8  *  ================================================================================
   9  *  Licensed under the Apache License, Version 2.0 (the "License");
  10  *  you may not use this file except in compliance with the License.
  11  *  You may obtain a copy of the License at
  12  *
  13  *        http://www.apache.org/licenses/LICENSE-2.0
  14  *
  15  *  Unless required by applicable law or agreed to in writing, software
  16  *  distributed under the License is distributed on an "AS IS" BASIS,
  17  *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  18  *  See the License for the specific language governing permissions and
  19  *  limitations under the License.
  20  *
  21  *  SPDX-License-Identifier: Apache-2.0
  22  *  ============LICENSE_END=========================================================
  23  */
  24
  25 package org.onap.cps.ncmp.api.impl;
  26
  27 import static org.onap.cps.ncmp.api.impl.constants.DmiRegistryConstants.NFP_OPERATIONAL_DATASTORE_DATASPACE_NAME;
  28 import static org.onap.cps.ncmp.api.impl.utils.RestQueryParametersValidator.validateCmHandleQueryParameters;
  29
  30 import com.google.common.collect.Lists;
  31 import com.hazelcast.map.IMap;
  32 import java.time.OffsetDateTime;
  33 import java.util.ArrayList;
  34 import java.util.Collection;
  35 import java.util.HashMap;
  36 import java.util.HashSet;
  37 import java.util.List;
  38 import java.util.Map;
  39 import java.util.Set;
  40 import java.util.stream.Collectors;
  41 import lombok.RequiredArgsConstructor;
  42 import lombok.extern.slf4j.Slf4j;
  43 import org.onap.cps.api.CpsDataService;
  44 import org.onap.cps.ncmp.api.NetworkCmProxyCmHandleQueryService;
  45 import org.onap.cps.ncmp.api.NetworkCmProxyDataService;
  46 import org.onap.cps.ncmp.api.impl.events.lcm.LcmEventsCmHandleStateHandler;
  47 import org.onap.cps.ncmp.api.impl.operations.DmiDataOperations;
  48 import org.onap.cps.ncmp.api.impl.operations.OperationType;
  49 import org.onap.cps.ncmp.api.impl.utils.CmHandleQueryConditions;
  50 import org.onap.cps.ncmp.api.impl.utils.InventoryQueryConditions;
  51 import org.onap.cps.ncmp.api.impl.utils.YangDataConverter;
  52 import org.onap.cps.ncmp.api.impl.yangmodels.YangModelCmHandle;
  53 import org.onap.cps.ncmp.api.inventory.CmHandleQueries;
  54 import org.onap.cps.ncmp.api.inventory.CmHandleState;
  55 import org.onap.cps.ncmp.api.inventory.CompositeState;
  56 import org.onap.cps.ncmp.api.inventory.CompositeStateUtils;
  57 import org.onap.cps.ncmp.api.inventory.DataStoreSyncState;
  58 import org.onap.cps.ncmp.api.inventory.InventoryPersistence;
  59 import org.onap.cps.ncmp.api.models.CmHandleQueryApiParameters;
  60 import org.onap.cps.ncmp.api.models.CmHandleQueryServiceParameters;
  61 import org.onap.cps.ncmp.api.models.CmHandleRegistrationResponse;
  62 import org.onap.cps.ncmp.api.models.CmHandleRegistrationResponse.RegistrationError;
  63 import org.onap.cps.ncmp.api.models.DmiPluginRegistration;
  64 import org.onap.cps.ncmp.api.models.DmiPluginRegistrationResponse;
  65 import org.onap.cps.ncmp.api.models.NcmpServiceCmHandle;
  66 import org.onap.cps.ncmp.api.models.ResourceDataBatchRequest;
  67 import org.onap.cps.spi.FetchDescendantsOption;
  68 import org.onap.cps.spi.exceptions.AlreadyDefinedExceptionBatch;
  69 import org.onap.cps.spi.exceptions.CpsException;
  70 import org.onap.cps.spi.exceptions.DataNodeNotFoundException;
  71 import org.onap.cps.spi.exceptions.DataValidationException;
  72 import org.onap.cps.spi.model.ModuleDefinition;
  73 import org.onap.cps.spi.model.ModuleReference;
  74 import org.onap.cps.utils.JsonObjectMapper;
  75 import org.springframework.http.ResponseEntity;
  76 import org.springframework.stereotype.Service;
  77
  78 @Slf4j
  79 @Service
  80 @RequiredArgsConstructor
  81 public class NetworkCmProxyDataServiceImpl implements NetworkCmProxyDataService {
  82
  83     private static final int DELETE_BATCH_SIZE = 100;
  84     private final JsonObjectMapper jsonObjectMapper;
  85     private final DmiDataOperations dmiDataOperations;
  86     private final NetworkCmProxyDataServicePropertyHandler networkCmProxyDataServicePropertyHandler;
  87     private final InventoryPersistence inventoryPersistence;
  88     private final CmHandleQueries cmHandleQueries;
  89     private final NetworkCmProxyCmHandleQueryService networkCmProxyCmHandleQueryService;
  90     private final LcmEventsCmHandleStateHandler lcmEventsCmHandleStateHandler;
  91     private final CpsDataService cpsDataService;
  92     private final IMap<String, Object> moduleSyncStartedOnCmHandles;
  93
  94     @Override
  95     public DmiPluginRegistrationResponse updateDmiRegistrationAndSyncModule(
  96             final DmiPluginRegistration dmiPluginRegistration) {
  97         dmiPluginRegistration.validateDmiPluginRegistration();
  98         final DmiPluginRegistrationResponse dmiPluginRegistrationResponse = new DmiPluginRegistrationResponse();
  99
 100         if (!dmiPluginRegistration.getRemovedCmHandles().isEmpty()) {
 101             dmiPluginRegistrationResponse.setRemovedCmHandles(
 102                     parseAndRemoveCmHandlesInDmiRegistration(dmiPluginRegistration.getRemovedCmHandles()));
 103         }
 104
 105         if (!dmiPluginRegistration.getCreatedCmHandles().isEmpty()) {
 106             dmiPluginRegistrationResponse.setCreatedCmHandles(
 107                     parseAndCreateCmHandlesInDmiRegistrationAndSyncModules(dmiPluginRegistration));
 108         }
 109         if (!dmiPluginRegistration.getUpdatedCmHandles().isEmpty()) {
 110             dmiPluginRegistrationResponse.setUpdatedCmHandles(
 111                     networkCmProxyDataServicePropertyHandler
 112                             .updateCmHandleProperties(dmiPluginRegistration.getUpdatedCmHandles()));
 113         }
 114         return dmiPluginRegistrationResponse;
 115     }
 116
 117     @Override
 118     public Object getResourceDataForCmHandle(final String datastoreName,
 119                                              final String cmHandleId,
 120                                              final String resourceIdentifier,
 121                                              final String optionsParamInQuery,
 122                                              final String topicParamInQuery,
 123                                              final String requestId) {
 124         final ResponseEntity<?> responseEntity = dmiDataOperations.getResourceDataFromDmi(datastoreName, cmHandleId,
 125                 resourceIdentifier,
 126                 optionsParamInQuery,
 127                 topicParamInQuery,
 128                 requestId);
 129         return responseEntity.getBody();
 130     }
 131
 132     @Override
 133     public Object getResourceDataForCmHandle(final String datastoreName,
 134                                              final String cmHandleId,
 135                                              final String resourceIdentifier,
 136                                              final FetchDescendantsOption fetchDescendantsOption) {
 137         return cpsDataService.getDataNodes(datastoreName, cmHandleId, resourceIdentifier,
 138                 fetchDescendantsOption).iterator().next();
 139     }
 140
 141     @Override
 142     public void requestResourceDataForCmHandleBatch(final String topicParamInQuery,
 143                                                 final ResourceDataBatchRequest
 144                                                         resourceDataBatchRequest,
 145                                                 final String requestId) {
 146         dmiDataOperations.requestResourceDataFromDmi(topicParamInQuery, resourceDataBatchRequest, requestId);
 147     }
 148
 149     @Override
 150     public Object writeResourceDataPassThroughRunningForCmHandle(final String cmHandleId,
 151                                                                  final String resourceIdentifier,
 152                                                                  final OperationType operationType,
 153                                                                  final String requestData,
 154                                                                  final String dataType) {
 155         return dmiDataOperations.writeResourceDataPassThroughRunningFromDmi(cmHandleId, resourceIdentifier,
 156                 operationType, requestData, dataType);
 157     }
 158
 159     @Override
 160     public Collection<ModuleReference> getYangResourcesModuleReferences(final String cmHandleId) {
 161         return inventoryPersistence.getYangResourcesModuleReferences(cmHandleId);
 162     }
 163
 164     @Override
 165     public Collection<ModuleDefinition> getModuleDefinitionsByCmHandleId(final String cmHandleId) {
 166         return inventoryPersistence.getModuleDefinitionsByCmHandleId(cmHandleId);
 167     }
 168
 169     /**
 170      * Retrieve cm handles with details for the given query parameters.
 171      *
 172      * @param cmHandleQueryApiParameters cm handle query parameters
 173      * @return cm handles with details
 174      */
 175     @Override
 176     public Collection<NcmpServiceCmHandle> executeCmHandleSearch(
 177             final CmHandleQueryApiParameters cmHandleQueryApiParameters) {
 178         final CmHandleQueryServiceParameters cmHandleQueryServiceParameters = jsonObjectMapper.convertToValueType(
 179                 cmHandleQueryApiParameters, CmHandleQueryServiceParameters.class);
 180         validateCmHandleQueryParameters(cmHandleQueryServiceParameters, CmHandleQueryConditions.ALL_CONDITION_NAMES);
 181         return networkCmProxyCmHandleQueryService.queryCmHandles(cmHandleQueryServiceParameters);
 182     }
 183
 184     /**
 185      * Retrieve cm handle ids for the given query parameters.
 186      *
 187      * @param cmHandleQueryApiParameters cm handle query parameters
 188      * @return cm handle ids
 189      */
 190     @Override
 191     public Collection<String> executeCmHandleIdSearch(final CmHandleQueryApiParameters cmHandleQueryApiParameters) {
 192         final CmHandleQueryServiceParameters cmHandleQueryServiceParameters = jsonObjectMapper.convertToValueType(
 193                 cmHandleQueryApiParameters, CmHandleQueryServiceParameters.class);
 194         validateCmHandleQueryParameters(cmHandleQueryServiceParameters, CmHandleQueryConditions.ALL_CONDITION_NAMES);
 195         return networkCmProxyCmHandleQueryService.queryCmHandleIds(cmHandleQueryServiceParameters);
 196     }
 197
 198     /**
 199      * Set the data sync enabled flag, along with the data sync state
 200      * based on the data sync enabled boolean for the cm handle id provided.
 201      *
 202      * @param cmHandleId      cm handle id
 203      * @param dataSyncEnabled data sync enabled flag
 204      */
 205     @Override
 206     public void setDataSyncEnabled(final String cmHandleId, final boolean dataSyncEnabled) {
 207         final CompositeState compositeState = inventoryPersistence
 208                 .getCmHandleState(cmHandleId);
 209         if (compositeState.getDataSyncEnabled().equals(dataSyncEnabled)) {
 210             log.info("Data-Sync Enabled flag is already: {} ", dataSyncEnabled);
 211         } else if (compositeState.getCmHandleState() != CmHandleState.READY) {
 212             throw new CpsException("State mismatch exception.", "Cm-Handle not in READY state. Cm handle state is: "
 213                     + compositeState.getCmHandleState());
 214         } else {
 215             final DataStoreSyncState dataStoreSyncState = compositeState.getDataStores()
 216                     .getOperationalDataStore().getDataStoreSyncState();
 217             if (!dataSyncEnabled && dataStoreSyncState == DataStoreSyncState.SYNCHRONIZED) {
 218                 cpsDataService.deleteDataNode(NFP_OPERATIONAL_DATASTORE_DATASPACE_NAME, cmHandleId,
 219                         "/netconf-state", OffsetDateTime.now());
 220             }
 221             CompositeStateUtils.setDataSyncEnabledFlagWithDataSyncState(dataSyncEnabled, compositeState);
 222             inventoryPersistence.saveCmHandleState(cmHandleId,
 223                     compositeState);
 224         }
 225     }
 226
 227     /**
 228      * Get all cm handle IDs by DMI plugin identifier.
 229      *
 230      * @param dmiPluginIdentifier DMI plugin identifier
 231      * @return set of cm handle IDs
 232      */
 233     @Override
 234     public Collection<String> getAllCmHandleIdsByDmiPluginIdentifier(final String dmiPluginIdentifier) {
 235         return cmHandleQueries.getCmHandleIdsByDmiPluginIdentifier(dmiPluginIdentifier);
 236     }
 237
 238     /**
 239      * Get all cm handle IDs by various properties.
 240      *
 241      * @param cmHandleQueryServiceParameters cm handle query parameters
 242      * @return set of cm handle IDs
 243      */
 244     @Override
 245     public Collection<String> executeCmHandleIdSearchForInventory(
 246             final CmHandleQueryServiceParameters cmHandleQueryServiceParameters) {
 247         validateCmHandleQueryParameters(cmHandleQueryServiceParameters, InventoryQueryConditions.ALL_CONDITION_NAMES);
 248         return networkCmProxyCmHandleQueryService.queryCmHandleIdsForInventory(cmHandleQueryServiceParameters);
 249     }
 250
 251     /**
 252      * Retrieve cm handle details for a given cm handle.
 253      *
 254      * @param cmHandleId cm handle identifier
 255      * @return cm handle details
 256      */
 257     @Override
 258     public NcmpServiceCmHandle getNcmpServiceCmHandle(final String cmHandleId) {
 259         return YangDataConverter.convertYangModelCmHandleToNcmpServiceCmHandle(
 260                 inventoryPersistence.getYangModelCmHandle(cmHandleId));
 261     }
 262
 263     /**
 264      * Get cm handle public properties for a given cm handle id.
 265      *
 266      * @param cmHandleId cm handle identifier
 267      * @return cm handle public properties
 268      */
 269     @Override
 270     public Map<String, String> getCmHandlePublicProperties(final String cmHandleId) {
 271         final YangModelCmHandle yangModelCmHandle =
 272                 inventoryPersistence.getYangModelCmHandle(cmHandleId);
 273         final List<YangModelCmHandle.Property> yangModelPublicProperties = yangModelCmHandle.getPublicProperties();
 274         final Map<String, String> cmHandlePublicProperties = new HashMap<>();
 275         YangDataConverter.asPropertiesMap(yangModelPublicProperties, cmHandlePublicProperties);
 276         return cmHandlePublicProperties;
 277     }
 278
 279     /**
 280      * Get cm handle composite state for a given cm handle id.
 281      *
 282      * @param cmHandleId cm handle identifier
 283      * @return cm handle state
 284      */
 285     @Override
 286     public CompositeState getCmHandleCompositeState(final String cmHandleId) {
 287         return inventoryPersistence.getYangModelCmHandle(cmHandleId).getCompositeState();
 288     }
 289
 290     /**
 291      * THis method registers a cm handle and initiates modules sync.
 292      *
 293      * @param dmiPluginRegistration dmi plugin registration information.
 294      * @return cm-handle registration response for create cm-handle requests.
 295      */
 296     public List<CmHandleRegistrationResponse> parseAndCreateCmHandlesInDmiRegistrationAndSyncModules(
 297             final DmiPluginRegistration dmiPluginRegistration) {
 298         List<CmHandleRegistrationResponse> cmHandleRegistrationResponses = new ArrayList<>();
 299         final Map<YangModelCmHandle, CmHandleState> cmHandleStatePerCmHandle = new HashMap<>();
 300         try {
 301             dmiPluginRegistration.getCreatedCmHandles()
 302                     .forEach(cmHandle -> {
 303                         final YangModelCmHandle yangModelCmHandle = YangModelCmHandle.toYangModelCmHandle(
 304                                 dmiPluginRegistration.getDmiPlugin(),
 305                                 dmiPluginRegistration.getDmiDataPlugin(),
 306                                 dmiPluginRegistration.getDmiModelPlugin(),
 307                                 cmHandle);
 308                         cmHandleStatePerCmHandle.put(yangModelCmHandle, CmHandleState.ADVISED);
 309                     });
 310             cmHandleRegistrationResponses = registerNewCmHandles(cmHandleStatePerCmHandle);
 311         } catch (final DataValidationException dataValidationException) {
 312             cmHandleRegistrationResponses.add(CmHandleRegistrationResponse.createFailureResponse(dmiPluginRegistration
 313                             .getCreatedCmHandles().stream()
 314                             .map(NcmpServiceCmHandle::getCmHandleId).findFirst().orElse(null),
 315                     RegistrationError.CM_HANDLE_INVALID_ID));
 316         }
 317         return cmHandleRegistrationResponses;
 318     }
 319
 320     protected List<CmHandleRegistrationResponse> parseAndRemoveCmHandlesInDmiRegistration(
 321             final List<String> tobeRemovedCmHandles) {
 322         final List<CmHandleRegistrationResponse> cmHandleRegistrationResponses =
 323                 new ArrayList<>(tobeRemovedCmHandles.size());
 324         final Collection<YangModelCmHandle> yangModelCmHandles =
 325             inventoryPersistence.getYangModelCmHandles(tobeRemovedCmHandles);
 326
 327         updateCmHandleStateBatch(yangModelCmHandles, CmHandleState.DELETING);
 328
 329         final Set<String> notDeletedCmHandles = new HashSet<>();
 330         for (final List<String> tobeRemovedCmHandleBatch : Lists.partition(tobeRemovedCmHandles, DELETE_BATCH_SIZE)) {
 331             try {
 332                 batchDeleteCmHandlesFromDbAndModuleSyncMap(tobeRemovedCmHandleBatch);
 333                 tobeRemovedCmHandleBatch.forEach(cmHandleId ->
 334                     cmHandleRegistrationResponses.add(CmHandleRegistrationResponse.createSuccessResponse(cmHandleId)));
 335
 336             } catch (final RuntimeException batchException) {
 337                 log.error("Unable to de-register cm-handle batch, retrying on each cm handle");
 338                 for (final String cmHandleId : tobeRemovedCmHandleBatch) {
 339                     final CmHandleRegistrationResponse cmHandleRegistrationResponse =
 340                         deleteCmHandleAndGetCmHandleRegistrationResponse(cmHandleId);
 341                     cmHandleRegistrationResponses.add(cmHandleRegistrationResponse);
 342                     if (cmHandleRegistrationResponse.getStatus() != CmHandleRegistrationResponse.Status.SUCCESS) {
 343                         notDeletedCmHandles.add(cmHandleId);
 344                     }
 345                 }
 346             }
 347         }
 348
 349         yangModelCmHandles.removeIf(yangModelCmHandle -> notDeletedCmHandles.contains(yangModelCmHandle.getId()));
 350         updateCmHandleStateBatch(yangModelCmHandles, CmHandleState.DELETED);
 351
 352         return cmHandleRegistrationResponses;
 353     }
 354
 355     private CmHandleRegistrationResponse deleteCmHandleAndGetCmHandleRegistrationResponse(final String cmHandleId) {
 356         try {
 357             deleteCmHandleFromDbAndModuleSyncMap(cmHandleId);
 358             return CmHandleRegistrationResponse.createSuccessResponse(cmHandleId);
 359         } catch (final DataNodeNotFoundException dataNodeNotFoundException) {
 360             log.error("Unable to find dataNode for cmHandleId : {} , caused by : {}",
 361                 cmHandleId, dataNodeNotFoundException.getMessage());
 362             return CmHandleRegistrationResponse.createFailureResponse(cmHandleId,
 363                 RegistrationError.CM_HANDLE_DOES_NOT_EXIST);
 364         } catch (final DataValidationException dataValidationException) {
 365             log.error("Unable to de-register cm-handle id: {}, caused by: {}",
 366                 cmHandleId, dataValidationException.getMessage());
 367             return CmHandleRegistrationResponse.createFailureResponse(cmHandleId,
 368                 RegistrationError.CM_HANDLE_INVALID_ID);
 369         } catch (final Exception exception) {
 370             log.error("Unable to de-register cm-handle id : {} , caused by : {}", cmHandleId, exception.getMessage());
 371             return CmHandleRegistrationResponse.createFailureResponse(cmHandleId, exception);
 372         }
 373     }
 374
 375     private void updateCmHandleStateBatch(final Collection<YangModelCmHandle> yangModelCmHandles,
 376                                           final CmHandleState cmHandleState) {
 377         final Map<YangModelCmHandle, CmHandleState> cmHandleStatePerCmHandle = new HashMap<>(yangModelCmHandles.size());
 378         yangModelCmHandles.forEach(yangModelCmHandle -> cmHandleStatePerCmHandle.put(yangModelCmHandle, cmHandleState));
 379         lcmEventsCmHandleStateHandler.updateCmHandleStateBatch(cmHandleStatePerCmHandle);
 380     }
 381
 382     private void deleteCmHandleFromDbAndModuleSyncMap(final String cmHandleId) {
 383         inventoryPersistence.deleteSchemaSetWithCascade(cmHandleId);
 384         inventoryPersistence.deleteDataNode("/dmi-registry/cm-handles[@id='" + cmHandleId + "']");
 385         removeDeletedCmHandleFromModuleSyncMap(cmHandleId);
 386     }
 387
 388     private void batchDeleteCmHandlesFromDbAndModuleSyncMap(final Collection<String> tobeRemovedCmHandles) {
 389         inventoryPersistence.deleteSchemaSetsWithCascade(tobeRemovedCmHandles);
 390         inventoryPersistence.deleteDataNodes(mapCmHandleIdsToXpaths(tobeRemovedCmHandles));
 391         tobeRemovedCmHandles.forEach(this::removeDeletedCmHandleFromModuleSyncMap);
 392     }
 393
 394     private Collection<String> mapCmHandleIdsToXpaths(final Collection<String> cmHandles) {
 395         return cmHandles.stream()
 396             .map(cmHandleId -> "/dmi-registry/cm-handles[@id='" + cmHandleId + "']")
 397             .collect(Collectors.toSet());
 398     }
 399
 400     // CPS-1239 Robustness cleaning of in progress cache
 401     private void removeDeletedCmHandleFromModuleSyncMap(final String deletedCmHandleId) {
 402         if (moduleSyncStartedOnCmHandles.remove(deletedCmHandleId) != null) {
 403             log.debug("{} removed from in progress map", deletedCmHandleId);
 404         }
 405     }
 406
 407     private List<CmHandleRegistrationResponse> registerNewCmHandles(final Map<YangModelCmHandle, CmHandleState>
 408                                                                             cmHandleStatePerCmHandle) {
 409         final List<String> cmHandleIds = cmHandleStatePerCmHandle.keySet().stream().map(YangModelCmHandle::getId)
 410                 .collect(Collectors.toList());
 411         try {
 412             lcmEventsCmHandleStateHandler.updateCmHandleStateBatch(cmHandleStatePerCmHandle);
 413             return CmHandleRegistrationResponse.createSuccessResponses(cmHandleIds);
 414         } catch (final AlreadyDefinedExceptionBatch alreadyDefinedExceptionBatch) {
 415             return CmHandleRegistrationResponse.createFailureResponses(
 416                     alreadyDefinedExceptionBatch.getAlreadyDefinedXpaths(),
 417                     RegistrationError.CM_HANDLE_ALREADY_EXIST);
 418         } catch (final Exception exception) {
 419             return CmHandleRegistrationResponse.createFailureResponses(cmHandleIds, exception);
 420         }
 421     }
 422
 423 }