cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/NetworkCmProxyDataServiceImpl.java

   1 /*
   2  *  ============LICENSE_START=======================================================
   3  *  Copyright (C) 2021 highstreet technologies GmbH
   4  *  Modifications Copyright (C) 2021-2023 Nordix Foundation
   5  *  Modifications Copyright (C) 2021 Pantheon.tech
   6  *  Modifications Copyright (C) 2021-2022 Bell Canada
   7  *  Modifications Copyright (C) 2023 TechMahindra Ltd.
   8  *  ================================================================================
   9  *  Licensed under the Apache License, Version 2.0 (the "License");
  10  *  you may not use this file except in compliance with the License.
  11  *  You may obtain a copy of the License at
  12  *
  13  *        http://www.apache.org/licenses/LICENSE-2.0
  14  *
  15  *  Unless required by applicable law or agreed to in writing, software
  16  *  distributed under the License is distributed on an "AS IS" BASIS,
  17  *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  18  *  See the License for the specific language governing permissions and
  19  *  limitations under the License.
  20  *
  21  *  SPDX-License-Identifier: Apache-2.0
  22  *  ============LICENSE_END=========================================================
  23  */
  24
  25 package org.onap.cps.ncmp.api.impl;
  26
  27 import static org.onap.cps.ncmp.api.impl.constants.DmiRegistryConstants.NFP_OPERATIONAL_DATASTORE_DATASPACE_NAME;
  28 import static org.onap.cps.ncmp.api.impl.operations.DmiRequestBody.OperationEnum;
  29 import static org.onap.cps.ncmp.api.impl.utils.RestQueryParametersValidator.validateCmHandleQueryParameters;
  30
  31 import com.google.common.collect.Lists;
  32 import com.hazelcast.map.IMap;
  33 import java.time.OffsetDateTime;
  34 import java.util.ArrayList;
  35 import java.util.Collection;
  36 import java.util.HashMap;
  37 import java.util.HashSet;
  38 import java.util.List;
  39 import java.util.Map;
  40 import java.util.Set;
  41 import java.util.stream.Collectors;
  42 import lombok.RequiredArgsConstructor;
  43 import lombok.extern.slf4j.Slf4j;
  44 import org.onap.cps.api.CpsDataService;
  45 import org.onap.cps.ncmp.api.NetworkCmProxyCmHandlerQueryService;
  46 import org.onap.cps.ncmp.api.NetworkCmProxyDataService;
  47 import org.onap.cps.ncmp.api.impl.event.lcm.LcmEventsCmHandleStateHandler;
  48 import org.onap.cps.ncmp.api.impl.operations.DmiDataOperations;
  49 import org.onap.cps.ncmp.api.impl.operations.DmiOperations;
  50 import org.onap.cps.ncmp.api.impl.utils.CmHandleQueryConditions;
  51 import org.onap.cps.ncmp.api.impl.utils.InventoryQueryConditions;
  52 import org.onap.cps.ncmp.api.impl.utils.YangDataConverter;
  53 import org.onap.cps.ncmp.api.impl.yangmodels.YangModelCmHandle;
  54 import org.onap.cps.ncmp.api.inventory.CmHandleQueries;
  55 import org.onap.cps.ncmp.api.inventory.CmHandleState;
  56 import org.onap.cps.ncmp.api.inventory.CompositeState;
  57 import org.onap.cps.ncmp.api.inventory.CompositeStateUtils;
  58 import org.onap.cps.ncmp.api.inventory.DataStoreSyncState;
  59 import org.onap.cps.ncmp.api.inventory.InventoryPersistence;
  60 import org.onap.cps.ncmp.api.models.CmHandleQueryApiParameters;
  61 import org.onap.cps.ncmp.api.models.CmHandleQueryServiceParameters;
  62 import org.onap.cps.ncmp.api.models.CmHandleRegistrationResponse;
  63 import org.onap.cps.ncmp.api.models.CmHandleRegistrationResponse.RegistrationError;
  64 import org.onap.cps.ncmp.api.models.DmiPluginRegistration;
  65 import org.onap.cps.ncmp.api.models.DmiPluginRegistrationResponse;
  66 import org.onap.cps.ncmp.api.models.NcmpServiceCmHandle;
  67 import org.onap.cps.spi.FetchDescendantsOption;
  68 import org.onap.cps.spi.exceptions.AlreadyDefinedExceptionBatch;
  69 import org.onap.cps.spi.exceptions.CpsException;
  70 import org.onap.cps.spi.exceptions.DataNodeNotFoundException;
  71 import org.onap.cps.spi.exceptions.DataValidationException;
  72 import org.onap.cps.spi.model.ModuleDefinition;
  73 import org.onap.cps.spi.model.ModuleReference;
  74 import org.onap.cps.utils.JsonObjectMapper;
  75 import org.springframework.http.ResponseEntity;
  76 import org.springframework.stereotype.Service;
  77
  78 @Slf4j
  79 @Service
  80 @RequiredArgsConstructor
  81 public class NetworkCmProxyDataServiceImpl implements NetworkCmProxyDataService {
  82
  83     private static final int DELETE_BATCH_SIZE = 100;
  84     private final JsonObjectMapper jsonObjectMapper;
  85     private final DmiDataOperations dmiDataOperations;
  86     private final NetworkCmProxyDataServicePropertyHandler networkCmProxyDataServicePropertyHandler;
  87     private final InventoryPersistence inventoryPersistence;
  88     private final CmHandleQueries cmHandleQueries;
  89     private final NetworkCmProxyCmHandlerQueryService networkCmProxyCmHandlerQueryService;
  90     private final LcmEventsCmHandleStateHandler lcmEventsCmHandleStateHandler;
  91     private final CpsDataService cpsDataService;
  92     private final IMap<String, Object> moduleSyncStartedOnCmHandles;
  93
  94     @Override
  95     public DmiPluginRegistrationResponse updateDmiRegistrationAndSyncModule(
  96             final DmiPluginRegistration dmiPluginRegistration) {
  97         dmiPluginRegistration.validateDmiPluginRegistration();
  98         final DmiPluginRegistrationResponse dmiPluginRegistrationResponse = new DmiPluginRegistrationResponse();
  99
 100         if (!dmiPluginRegistration.getRemovedCmHandles().isEmpty()) {
 101             dmiPluginRegistrationResponse.setRemovedCmHandles(
 102                     parseAndRemoveCmHandlesInDmiRegistration(dmiPluginRegistration.getRemovedCmHandles()));
 103         }
 104
 105         if (!dmiPluginRegistration.getCreatedCmHandles().isEmpty()) {
 106             dmiPluginRegistrationResponse.setCreatedCmHandles(
 107                     parseAndCreateCmHandlesInDmiRegistrationAndSyncModules(dmiPluginRegistration));
 108         }
 109         if (!dmiPluginRegistration.getUpdatedCmHandles().isEmpty()) {
 110             dmiPluginRegistrationResponse.setUpdatedCmHandles(
 111                     networkCmProxyDataServicePropertyHandler
 112                             .updateCmHandleProperties(dmiPluginRegistration.getUpdatedCmHandles()));
 113         }
 114         return dmiPluginRegistrationResponse;
 115     }
 116
 117     @Override
 118     public Object getResourceDataOperationalForCmHandle(final String cmHandleId,
 119                                                         final String resourceIdentifier,
 120                                                         final String optionsParamInQuery,
 121                                                         final String topicParamInQuery,
 122                                                         final String requestId) {
 123         final ResponseEntity<?> responseEntity = dmiDataOperations.getResourceDataFromDmi(cmHandleId,
 124                 resourceIdentifier,
 125                 optionsParamInQuery,
 126                 DmiOperations.DataStoreEnum.PASSTHROUGH_OPERATIONAL,
 127                 requestId, topicParamInQuery);
 128         return responseEntity.getBody();
 129     }
 130
 131     @Override
 132     public Object getResourceDataOperational(final String cmHandleId,
 133                                              final String resourceIdentifier,
 134                                              final FetchDescendantsOption fetchDescendantsOption) {
 135         return cpsDataService.getDataNodes(NFP_OPERATIONAL_DATASTORE_DATASPACE_NAME, cmHandleId, resourceIdentifier,
 136                 fetchDescendantsOption).iterator().next();
 137     }
 138
 139     @Override
 140     public Object getResourceDataPassThroughRunningForCmHandle(final String cmHandleId,
 141                                                                final String resourceIdentifier,
 142                                                                final String optionsParamInQuery,
 143                                                                final String topicParamInQuery,
 144                                                                final String requestId) {
 145         final ResponseEntity<?> responseEntity = dmiDataOperations.getResourceDataFromDmi(cmHandleId,
 146                 resourceIdentifier,
 147                 optionsParamInQuery,
 148                 DmiOperations.DataStoreEnum.PASSTHROUGH_RUNNING,
 149                 requestId, topicParamInQuery);
 150         return responseEntity.getBody();
 151     }
 152
 153     @Override
 154     public Object writeResourceDataPassThroughRunningForCmHandle(final String cmHandleId,
 155                                                                  final String resourceIdentifier,
 156                                                                  final OperationEnum operation,
 157                                                                  final String requestData,
 158                                                                  final String dataType) {
 159         return dmiDataOperations.writeResourceDataPassThroughRunningFromDmi(cmHandleId, resourceIdentifier, operation,
 160                 requestData, dataType);
 161     }
 162
 163     @Override
 164     public Collection<ModuleReference> getYangResourcesModuleReferences(final String cmHandleId) {
 165         return inventoryPersistence.getYangResourcesModuleReferences(cmHandleId);
 166     }
 167
 168     @Override
 169     public Collection<ModuleDefinition> getModuleDefinitionsByCmHandleId(final String cmHandleId) {
 170         return inventoryPersistence.getModuleDefinitionsByCmHandleId(cmHandleId);
 171     }
 172
 173     /**
 174      * Retrieve cm handles with details for the given query parameters.
 175      *
 176      * @param cmHandleQueryApiParameters cm handle query parameters
 177      * @return cm handles with details
 178      */
 179     @Override
 180     public Set<NcmpServiceCmHandle> executeCmHandleSearch(final CmHandleQueryApiParameters cmHandleQueryApiParameters) {
 181         final CmHandleQueryServiceParameters cmHandleQueryServiceParameters = jsonObjectMapper.convertToValueType(
 182                 cmHandleQueryApiParameters, CmHandleQueryServiceParameters.class);
 183         validateCmHandleQueryParameters(cmHandleQueryServiceParameters, CmHandleQueryConditions.ALL_CONDITION_NAMES);
 184         return networkCmProxyCmHandlerQueryService.queryCmHandles(cmHandleQueryServiceParameters);
 185     }
 186
 187     /**
 188      * Retrieve cm handle ids for the given query parameters.
 189      *
 190      * @param cmHandleQueryApiParameters cm handle query parameters
 191      * @return cm handle ids
 192      */
 193     @Override
 194     public Set<String> executeCmHandleIdSearch(final CmHandleQueryApiParameters cmHandleQueryApiParameters) {
 195         final CmHandleQueryServiceParameters cmHandleQueryServiceParameters = jsonObjectMapper.convertToValueType(
 196                 cmHandleQueryApiParameters, CmHandleQueryServiceParameters.class);
 197         validateCmHandleQueryParameters(cmHandleQueryServiceParameters, CmHandleQueryConditions.ALL_CONDITION_NAMES);
 198         return networkCmProxyCmHandlerQueryService.queryCmHandleIds(cmHandleQueryServiceParameters);
 199     }
 200
 201     /**
 202      * Set the data sync enabled flag, along with the data sync state
 203      * based on the data sync enabled boolean for the cm handle id provided.
 204      *
 205      * @param cmHandleId      cm handle id
 206      * @param dataSyncEnabled data sync enabled flag
 207      */
 208     @Override
 209     public void setDataSyncEnabled(final String cmHandleId, final boolean dataSyncEnabled) {
 210         final CompositeState compositeState = inventoryPersistence
 211                 .getCmHandleState(cmHandleId);
 212         if (compositeState.getDataSyncEnabled().equals(dataSyncEnabled)) {
 213             log.info("Data-Sync Enabled flag is already: {} ", dataSyncEnabled);
 214         } else if (compositeState.getCmHandleState() != CmHandleState.READY) {
 215             throw new CpsException("State mismatch exception.", "Cm-Handle not in READY state. Cm handle state is: "
 216                     + compositeState.getCmHandleState());
 217         } else {
 218             final DataStoreSyncState dataStoreSyncState = compositeState.getDataStores()
 219                     .getOperationalDataStore().getDataStoreSyncState();
 220             if (!dataSyncEnabled && dataStoreSyncState == DataStoreSyncState.SYNCHRONIZED) {
 221                 cpsDataService.deleteDataNode(NFP_OPERATIONAL_DATASTORE_DATASPACE_NAME, cmHandleId,
 222                         "/netconf-state", OffsetDateTime.now());
 223             }
 224             CompositeStateUtils.setDataSyncEnabledFlagWithDataSyncState(dataSyncEnabled, compositeState);
 225             inventoryPersistence.saveCmHandleState(cmHandleId,
 226                     compositeState);
 227         }
 228     }
 229
 230     /**
 231      * Get all cm handle IDs by DMI plugin identifier.
 232      *
 233      * @param dmiPluginIdentifier DMI plugin identifier
 234      * @return set of cm handle IDs
 235      */
 236     @Override
 237     public Set<String> getAllCmHandleIdsByDmiPluginIdentifier(final String dmiPluginIdentifier) {
 238         final Set<NcmpServiceCmHandle> ncmpServiceCmHandles =
 239                 cmHandleQueries.getCmHandlesByDmiPluginIdentifier(dmiPluginIdentifier);
 240         final Set<String> cmHandleIds = new HashSet<>(ncmpServiceCmHandles.size());
 241         ncmpServiceCmHandles.forEach(cmHandle -> cmHandleIds.add(cmHandle.getCmHandleId()));
 242         return cmHandleIds;
 243     }
 244
 245     /**
 246      * Get all cm handle IDs by various properties.
 247      *
 248      * @param cmHandleQueryServiceParameters cm handle query parameters
 249      * @return set of cm handle IDs
 250      */
 251     @Override
 252     public Set<String> executeCmHandleIdSearchForInventory(
 253             final CmHandleQueryServiceParameters cmHandleQueryServiceParameters) {
 254         validateCmHandleQueryParameters(cmHandleQueryServiceParameters, InventoryQueryConditions.ALL_CONDITION_NAMES);
 255         return networkCmProxyCmHandlerQueryService.queryCmHandleIdsForInventory(cmHandleQueryServiceParameters);
 256     }
 257
 258     /**
 259      * Retrieve cm handle details for a given cm handle.
 260      *
 261      * @param cmHandleId cm handle identifier
 262      * @return cm handle details
 263      */
 264     @Override
 265     public NcmpServiceCmHandle getNcmpServiceCmHandle(final String cmHandleId) {
 266         return YangDataConverter.convertYangModelCmHandleToNcmpServiceCmHandle(
 267                 inventoryPersistence.getYangModelCmHandle(cmHandleId));
 268     }
 269
 270     /**
 271      * Get cm handle public properties for a given cm handle id.
 272      *
 273      * @param cmHandleId cm handle identifier
 274      * @return cm handle public properties
 275      */
 276     @Override
 277     public Map<String, String> getCmHandlePublicProperties(final String cmHandleId) {
 278         final YangModelCmHandle yangModelCmHandle =
 279                 inventoryPersistence.getYangModelCmHandle(cmHandleId);
 280         final List<YangModelCmHandle.Property> yangModelPublicProperties = yangModelCmHandle.getPublicProperties();
 281         final Map<String, String> cmHandlePublicProperties = new HashMap<>();
 282         YangDataConverter.asPropertiesMap(yangModelPublicProperties, cmHandlePublicProperties);
 283         return cmHandlePublicProperties;
 284     }
 285
 286     /**
 287      * Get cm handle composite state for a given cm handle id.
 288      *
 289      * @param cmHandleId cm handle identifier
 290      * @return cm handle state
 291      */
 292     @Override
 293     public CompositeState getCmHandleCompositeState(final String cmHandleId) {
 294         return inventoryPersistence.getYangModelCmHandle(cmHandleId).getCompositeState();
 295     }
 296
 297     /**
 298      * THis method registers a cm handle and initiates modules sync.
 299      *
 300      * @param dmiPluginRegistration dmi plugin registration information.
 301      * @return cm-handle registration response for create cm-handle requests.
 302      */
 303     public List<CmHandleRegistrationResponse> parseAndCreateCmHandlesInDmiRegistrationAndSyncModules(
 304             final DmiPluginRegistration dmiPluginRegistration) {
 305         List<CmHandleRegistrationResponse> cmHandleRegistrationResponses = new ArrayList<>();
 306         final Map<YangModelCmHandle, CmHandleState> cmHandleStatePerCmHandle = new HashMap<>();
 307         try {
 308             dmiPluginRegistration.getCreatedCmHandles()
 309                     .forEach(cmHandle -> {
 310                         final YangModelCmHandle yangModelCmHandle = YangModelCmHandle.toYangModelCmHandle(
 311                                 dmiPluginRegistration.getDmiPlugin(),
 312                                 dmiPluginRegistration.getDmiDataPlugin(),
 313                                 dmiPluginRegistration.getDmiModelPlugin(),
 314                                 cmHandle);
 315                         cmHandleStatePerCmHandle.put(yangModelCmHandle, CmHandleState.ADVISED);
 316                     });
 317             cmHandleRegistrationResponses = registerNewCmHandles(cmHandleStatePerCmHandle);
 318         } catch (final DataValidationException dataValidationException) {
 319             cmHandleRegistrationResponses.add(CmHandleRegistrationResponse.createFailureResponse(dmiPluginRegistration
 320                             .getCreatedCmHandles().stream()
 321                             .map(NcmpServiceCmHandle::getCmHandleId).findFirst().orElse(null),
 322                     RegistrationError.CM_HANDLE_INVALID_ID));
 323         }
 324         return cmHandleRegistrationResponses;
 325     }
 326
 327     protected List<CmHandleRegistrationResponse> parseAndRemoveCmHandlesInDmiRegistration(
 328             final List<String> tobeRemovedCmHandles) {
 329         final List<CmHandleRegistrationResponse> cmHandleRegistrationResponses =
 330                 new ArrayList<>(tobeRemovedCmHandles.size());
 331         final Collection<YangModelCmHandle> yangModelCmHandles =
 332             inventoryPersistence.getYangModelCmHandles(tobeRemovedCmHandles);
 333
 334         updateCmHandleStateBatch(yangModelCmHandles, CmHandleState.DELETING);
 335
 336         final Set<String> notDeletedCmHandles = new HashSet<>();
 337         for (final List<String> tobeRemovedCmHandleBatch : Lists.partition(tobeRemovedCmHandles, DELETE_BATCH_SIZE)) {
 338             try {
 339                 batchDeleteCmHandlesFromDbAndModuleSyncMap(tobeRemovedCmHandleBatch);
 340                 tobeRemovedCmHandleBatch.forEach(cmHandleId ->
 341                     cmHandleRegistrationResponses.add(CmHandleRegistrationResponse.createSuccessResponse(cmHandleId)));
 342
 343             } catch (final RuntimeException batchException) {
 344                 log.error("Unable to de-register cm-handle batch, retrying on each cm handle");
 345                 for (final String cmHandleId : tobeRemovedCmHandleBatch) {
 346                     final CmHandleRegistrationResponse cmHandleRegistrationResponse =
 347                         deleteCmHandleAndGetCmHandleRegistrationResponse(cmHandleId);
 348                     cmHandleRegistrationResponses.add(cmHandleRegistrationResponse);
 349                     if (cmHandleRegistrationResponse.getStatus() != CmHandleRegistrationResponse.Status.SUCCESS) {
 350                         notDeletedCmHandles.add(cmHandleId);
 351                     }
 352                 }
 353             }
 354         }
 355
 356         yangModelCmHandles.removeIf(yangModelCmHandle -> notDeletedCmHandles.contains(yangModelCmHandle.getId()));
 357         updateCmHandleStateBatch(yangModelCmHandles, CmHandleState.DELETED);
 358
 359         return cmHandleRegistrationResponses;
 360     }
 361
 362     private CmHandleRegistrationResponse deleteCmHandleAndGetCmHandleRegistrationResponse(final String cmHandleId) {
 363         try {
 364             deleteCmHandleFromDbAndModuleSyncMap(cmHandleId);
 365             return CmHandleRegistrationResponse.createSuccessResponse(cmHandleId);
 366         } catch (final DataNodeNotFoundException dataNodeNotFoundException) {
 367             log.error("Unable to find dataNode for cmHandleId : {} , caused by : {}",
 368                 cmHandleId, dataNodeNotFoundException.getMessage());
 369             return CmHandleRegistrationResponse.createFailureResponse(cmHandleId,
 370                 RegistrationError.CM_HANDLE_DOES_NOT_EXIST);
 371         } catch (final DataValidationException dataValidationException) {
 372             log.error("Unable to de-register cm-handle id: {}, caused by: {}",
 373                 cmHandleId, dataValidationException.getMessage());
 374             return CmHandleRegistrationResponse.createFailureResponse(cmHandleId,
 375                 RegistrationError.CM_HANDLE_INVALID_ID);
 376         } catch (final Exception exception) {
 377             log.error("Unable to de-register cm-handle id : {} , caused by : {}", cmHandleId, exception.getMessage());
 378             return CmHandleRegistrationResponse.createFailureResponse(cmHandleId, exception);
 379         }
 380     }
 381
 382     private void updateCmHandleStateBatch(final Collection<YangModelCmHandle> yangModelCmHandles,
 383                                           final CmHandleState cmHandleState) {
 384         final Map<YangModelCmHandle, CmHandleState> cmHandleStatePerCmHandle = new HashMap<>(yangModelCmHandles.size());
 385         yangModelCmHandles.forEach(yangModelCmHandle -> cmHandleStatePerCmHandle.put(yangModelCmHandle, cmHandleState));
 386         lcmEventsCmHandleStateHandler.updateCmHandleStateBatch(cmHandleStatePerCmHandle);
 387     }
 388
 389     private void deleteCmHandleFromDbAndModuleSyncMap(final String cmHandleId) {
 390         inventoryPersistence.deleteSchemaSetWithCascade(cmHandleId);
 391         inventoryPersistence.deleteDataNode("/dmi-registry/cm-handles[@id='" + cmHandleId + "']");
 392         removeDeletedCmHandleFromModuleSyncMap(cmHandleId);
 393     }
 394
 395     private void batchDeleteCmHandlesFromDbAndModuleSyncMap(final Collection<String> tobeRemovedCmHandles) {
 396         inventoryPersistence.deleteSchemaSetsWithCascade(tobeRemovedCmHandles);
 397         inventoryPersistence.deleteDataNodes(mapCmHandleIdsToXpaths(tobeRemovedCmHandles));
 398         tobeRemovedCmHandles.forEach(this::removeDeletedCmHandleFromModuleSyncMap);
 399     }
 400
 401     private Collection<String> mapCmHandleIdsToXpaths(final Collection<String> cmHandles) {
 402         return cmHandles.stream()
 403             .map(cmHandleId -> "/dmi-registry/cm-handles[@id='" + cmHandleId + "']")
 404             .collect(Collectors.toSet());
 405     }
 406
 407     // CPS-1239 Robustness cleaning of in progress cache
 408     private void removeDeletedCmHandleFromModuleSyncMap(final String deletedCmHandleId) {
 409         if (moduleSyncStartedOnCmHandles.remove(deletedCmHandleId) != null) {
 410             log.debug("{} removed from in progress map", deletedCmHandleId);
 411         }
 412     }
 413
 414     private List<CmHandleRegistrationResponse> registerNewCmHandles(final Map<YangModelCmHandle, CmHandleState>
 415                                                                             cmHandleStatePerCmHandle) {
 416         final List<String> cmHandleIds = cmHandleStatePerCmHandle.keySet().stream().map(YangModelCmHandle::getId)
 417                 .collect(Collectors.toList());
 418         try {
 419             lcmEventsCmHandleStateHandler.updateCmHandleStateBatch(cmHandleStatePerCmHandle);
 420             return CmHandleRegistrationResponse.createSuccessResponses(cmHandleIds);
 421         } catch (final AlreadyDefinedExceptionBatch alreadyDefinedExceptionBatch) {
 422             return CmHandleRegistrationResponse.createFailureResponses(
 423                     alreadyDefinedExceptionBatch.getAlreadyDefinedXpaths(),
 424                     RegistrationError.CM_HANDLE_ALREADY_EXIST);
 425         } catch (final Exception exception) {
 426             return CmHandleRegistrationResponse.createFailureResponses(cmHandleIds, exception);
 427         }
 428     }
 429 }