cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/NetworkCmProxyDataServiceImpl.java

   1 /*
   2  *  ============LICENSE_START=======================================================
   3  *  Copyright (C) 2021 highstreet technologies GmbH
   4  *  Modifications Copyright (C) 2021 Nordix Foundation
   5  *  Modifications Copyright (C) 2021 Pantheon.tech
   6  *  Modifications Copyright (C) 2021 Bell Canada
   7  *  ================================================================================
   8  *  Licensed under the Apache License, Version 2.0 (the "License");
   9  *  you may not use this file except in compliance with the License.
  10  *  You may obtain a copy of the License at
  11  *
  12  *        http://www.apache.org/licenses/LICENSE-2.0
  13  *
  14  *  Unless required by applicable law or agreed to in writing, software
  15  *  distributed under the License is distributed on an "AS IS" BASIS,
  16  *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  17  *  See the License for the specific language governing permissions and
  18  *  limitations under the License.
  19  *
  20  *  SPDX-License-Identifier: Apache-2.0
  21  *  ============LICENSE_END=========================================================
  22  */
  23
  24 package org.onap.cps.ncmp.api.impl;
  25
  26 import com.fasterxml.jackson.core.JsonProcessingException;
  27 import com.fasterxml.jackson.databind.ObjectMapper;
  28 import com.google.gson.Gson;
  29 import com.google.gson.JsonArray;
  30 import com.google.gson.JsonElement;
  31 import com.google.gson.JsonObject;
  32 import java.time.OffsetDateTime;
  33 import java.util.ArrayList;
  34 import java.util.Collection;
  35 import java.util.Collections;
  36 import java.util.HashMap;
  37 import java.util.LinkedHashMap;
  38 import java.util.List;
  39 import java.util.Map;
  40 import javax.validation.constraints.NotNull;
  41 import lombok.extern.slf4j.Slf4j;
  42 import org.onap.cps.api.CpsAdminService;
  43 import org.onap.cps.api.CpsDataService;
  44 import org.onap.cps.api.CpsModuleService;
  45 import org.onap.cps.api.CpsQueryService;
  46 import org.onap.cps.ncmp.api.NetworkCmProxyDataService;
  47 import org.onap.cps.ncmp.api.impl.exception.NcmpException;
  48 import org.onap.cps.ncmp.api.impl.operation.DmiOperations;
  49 import org.onap.cps.ncmp.api.models.CmHandle;
  50 import org.onap.cps.ncmp.api.models.DmiPluginRegistration;
  51 import org.onap.cps.ncmp.api.models.GenericRequestBody;
  52 import org.onap.cps.ncmp.api.models.GenericRequestBody.OperationEnum;
  53 import org.onap.cps.ncmp.api.models.PersistenceCmHandle;
  54 import org.onap.cps.ncmp.api.models.PersistenceCmHandle.AdditionalProperty;
  55 import org.onap.cps.ncmp.api.models.PersistenceCmHandlesList;
  56 import org.onap.cps.ncmp.api.models.YangResource;
  57 import org.onap.cps.spi.FetchDescendantsOption;
  58 import org.onap.cps.spi.exceptions.DataNodeNotFoundException;
  59 import org.onap.cps.spi.exceptions.DataValidationException;
  60 import org.onap.cps.spi.model.DataNode;
  61 import org.onap.cps.spi.model.ModuleReference;
  62 import org.springframework.http.HttpStatus;
  63 import org.springframework.http.ResponseEntity;
  64 import org.springframework.stereotype.Service;
  65 import org.springframework.util.StringUtils;
  66
  67 @Slf4j
  68 @Service
  69 public class NetworkCmProxyDataServiceImpl implements NetworkCmProxyDataService {
  70
  71     private static final String NFP_OPERATIONAL_DATASTORE_DATASPACE_NAME = "NFP-Operational";
  72
  73     private static final String NCMP_DATASPACE_NAME = "NCMP-Admin";
  74
  75     private static final String NCMP_DMI_REGISTRY_ANCHOR = "ncmp-dmi-registry";
  76
  77     private static final OffsetDateTime NO_TIMESTAMP = null;
  78
  79     private static  final String NCMP_DMI_SERVICE_NAME = "dmi-service-name";
  80
  81     private  static final String REVISION = "revision";
  82
  83     private CpsDataService cpsDataService;
  84
  85     private ObjectMapper objectMapper;
  86
  87     private CpsQueryService cpsQueryService;
  88
  89     private DmiOperations dmiOperations;
  90
  91     private CpsModuleService cpsModuleService;
  92
  93     private CpsAdminService cpsAdminService;
  94
  95     /**
  96      * Constructor Injection for Dependencies.
  97      * @param dmiOperations DMI operation
  98      * @param cpsDataService Data Service Interface
  99      * @param cpsQueryService Query Service Interface
 100      * @param objectMapper Object Mapper
 101      */
 102     public NetworkCmProxyDataServiceImpl(final DmiOperations dmiOperations,
 103         final CpsModuleService cpsModuleService,
 104         final CpsDataService cpsDataService,
 105         final CpsQueryService cpsQueryService,
 106         final CpsAdminService cpsAdminService,
 107         final ObjectMapper objectMapper) {
 108         this.dmiOperations = dmiOperations;
 109         this.cpsModuleService = cpsModuleService;
 110         this.cpsDataService = cpsDataService;
 111         this.cpsQueryService = cpsQueryService;
 112         this.cpsAdminService = cpsAdminService;
 113         this.objectMapper = objectMapper;
 114     }
 115
 116     @Override
 117     public DataNode getDataNode(final String cmHandle, final String xpath,
 118         final FetchDescendantsOption fetchDescendantsOption) {
 119         return cpsDataService
 120             .getDataNode(NFP_OPERATIONAL_DATASTORE_DATASPACE_NAME, cmHandle, xpath, fetchDescendantsOption);
 121     }
 122
 123     @Override
 124     public Collection<DataNode> queryDataNodes(final String cmHandle, final String cpsPath,
 125         final FetchDescendantsOption fetchDescendantsOption) {
 126         return cpsQueryService
 127             .queryDataNodes(NFP_OPERATIONAL_DATASTORE_DATASPACE_NAME, cmHandle, cpsPath, fetchDescendantsOption);
 128     }
 129
 130     @Override
 131     public void createDataNode(final String cmHandle, final String parentNodeXpath, final String jsonData) {
 132         if (!StringUtils.hasText(parentNodeXpath) || "/".equals(parentNodeXpath)) {
 133             cpsDataService.saveData(NFP_OPERATIONAL_DATASTORE_DATASPACE_NAME, cmHandle, jsonData, NO_TIMESTAMP);
 134         } else {
 135             cpsDataService
 136                 .saveData(NFP_OPERATIONAL_DATASTORE_DATASPACE_NAME, cmHandle, parentNodeXpath, jsonData, NO_TIMESTAMP);
 137         }
 138     }
 139
 140     @Override
 141     public void addListNodeElements(final String cmHandle, final String parentNodeXpath, final String jsonData) {
 142         cpsDataService.saveListElements(NFP_OPERATIONAL_DATASTORE_DATASPACE_NAME, cmHandle, parentNodeXpath, jsonData,
 143             NO_TIMESTAMP);
 144     }
 145
 146     @Override
 147     public void updateNodeLeaves(final String cmHandle, final String parentNodeXpath, final String jsonData) {
 148         cpsDataService
 149             .updateNodeLeaves(NFP_OPERATIONAL_DATASTORE_DATASPACE_NAME, cmHandle, parentNodeXpath, jsonData,
 150                 NO_TIMESTAMP);
 151     }
 152
 153     @Override
 154     public void replaceNodeTree(final String cmHandle, final String parentNodeXpath, final String jsonData) {
 155         cpsDataService.replaceNodeTree(NFP_OPERATIONAL_DATASTORE_DATASPACE_NAME, cmHandle, parentNodeXpath, jsonData,
 156             NO_TIMESTAMP);
 157     }
 158
 159     @Override
 160     public void updateDmiRegistrationAndSyncModule(final DmiPluginRegistration dmiPluginRegistration) {
 161         try {
 162             if (dmiPluginRegistration.getCreatedCmHandles() != null) {
 163                 parseAndCreateCmHandlesInDmiRegistrationAndSyncModule(dmiPluginRegistration);
 164             }
 165             if (dmiPluginRegistration.getUpdatedCmHandles() != null) {
 166                 parseAndUpdateCmHandlesInDmiRegistration(dmiPluginRegistration);
 167             }
 168             if (dmiPluginRegistration.getRemovedCmHandles() != null) {
 169                 parseAndRemoveCmHandlesInDmiRegistration(dmiPluginRegistration);
 170             }
 171         } catch (final JsonProcessingException e) {
 172             handleJsonProcessingException(dmiPluginRegistration, e);
 173         }
 174     }
 175
 176     @Override
 177     public Object getResourceDataOperationalForCmHandle(final @NotNull String cmHandle,
 178                                                         final @NotNull String resourceIdentifier,
 179                                                         final String acceptParamInHeader,
 180                                                         final String optionsParamInQuery) {
 181
 182         final DataNode cmHandleDataNode = fetchDataNodeFromDmiRegistryForCmHandle(cmHandle);
 183         final String dmiServiceName = String.valueOf(cmHandleDataNode.getLeaves().get(NCMP_DMI_SERVICE_NAME));
 184         final String dmiRequestBody = getGenericRequestBody(cmHandleDataNode);
 185         final ResponseEntity<Object> response = dmiOperations.getResourceDataOperationalFromDmi(dmiServiceName,
 186                 cmHandle,
 187                 resourceIdentifier,
 188                 optionsParamInQuery,
 189                 acceptParamInHeader,
 190                 dmiRequestBody);
 191         return handleResponse(response);
 192     }
 193
 194     @Override
 195     public Object getResourceDataPassThroughRunningForCmHandle(final @NotNull String cmHandle,
 196                                                                final @NotNull String resourceIdentifier,
 197                                                                final String acceptParamInHeader,
 198                                                                final String optionsParamInQuery) {
 199         final DataNode cmHandleDataNode = fetchDataNodeFromDmiRegistryForCmHandle(cmHandle);
 200         final String dmiServiceName = String.valueOf(cmHandleDataNode.getLeaves().get(NCMP_DMI_SERVICE_NAME));
 201         final String dmiRequestBody = getGenericRequestBody(cmHandleDataNode);
 202         final ResponseEntity<Object> response = dmiOperations.getResourceDataPassThroughRunningFromDmi(dmiServiceName,
 203                 cmHandle,
 204                 resourceIdentifier,
 205                 optionsParamInQuery,
 206                 acceptParamInHeader,
 207                 dmiRequestBody);
 208         return handleResponse(response);
 209     }
 210
 211     @Override
 212     public void createResourceDataPassThroughRunningForCmHandle(final @NotNull String cmHandle,
 213                                                                 final @NotNull String resourceIdentifier,
 214                                                                 final @NotNull String requestBody,
 215                                                                 final String contentType) {
 216         final DataNode cmHandleDataNode = fetchDataNodeFromDmiRegistryForCmHandle(cmHandle);
 217         final String dmiServiceName = String.valueOf(cmHandleDataNode.getLeaves().get(NCMP_DMI_SERVICE_NAME));
 218         final Collection<DataNode> cmHandlePropertiesAsDataNodes     = cmHandleDataNode.getChildDataNodes();
 219         final Map<String, String> cmHandlePropertiesAsMap = getCmHandlePropertiesAsMap(cmHandlePropertiesAsDataNodes);
 220         final GenericRequestBody dmiRequestBodyObject = GenericRequestBody.builder()
 221                 .operation(GenericRequestBody.OperationEnum.CREATE)
 222                 .dataType(contentType)
 223                 .data(requestBody)
 224                 .cmHandleProperties(cmHandlePropertiesAsMap)
 225                 .build();
 226         final String dmiRequestBody = prepareOperationBody(dmiRequestBodyObject);
 227         final ResponseEntity<String> responseEntity = dmiOperations
 228                 .createResourceDataPassThroughRunningFromDmi(dmiServiceName,
 229                         cmHandle,
 230                         resourceIdentifier,
 231                         dmiRequestBody);
 232         handleResponseFromDmi(responseEntity, "Not able to create resource data.");
 233     }
 234
 235     @Override
 236     public Collection<ModuleReference> getYangResourcesModuleReferences(final String cmHandle) {
 237         return cpsModuleService.getYangResourcesModuleReferences(NFP_OPERATIONAL_DATASTORE_DATASPACE_NAME, cmHandle);
 238     }
 239
 240     /**
 241      * Retrieve cm handle identifiers for the given list of module names.
 242      *
 243      * @param moduleNames module names.
 244      * @return a collection of anchor identifiers
 245      */
 246     @Override
 247     public Collection<String> executeCmHandleHasAllModulesSearch(final Collection<String> moduleNames) {
 248         return cpsAdminService.queryAnchorNames(NFP_OPERATIONAL_DATASTORE_DATASPACE_NAME, moduleNames);
 249     }
 250
 251     /**
 252      * Update resource data for data store pass-through running using dmi for given cm-handle.
 253      *
 254      * @param cmHandle           cm handle
 255      * @param resourceIdentifier resource identifier
 256      * @param requestBody        request body to create resource
 257      * @param contentType        content type in body
 258      */
 259     @Override
 260     public void updateResourceDataPassThroughRunningForCmHandle(final String cmHandle, final String resourceIdentifier,
 261         final String requestBody, final String contentType) {
 262         final DataNode cmHandleDataNode = fetchDataNodeFromDmiRegistryForCmHandle(cmHandle);
 263         final String dmiServiceName = String.valueOf(cmHandleDataNode.getLeaves().get(NCMP_DMI_SERVICE_NAME));
 264         final Collection<DataNode> cmHandlePropertiesAsDataNodes = cmHandleDataNode.getChildDataNodes();
 265         final Map<String, String> cmHandlePropertiesAsMap = getCmHandlePropertiesAsMap(cmHandlePropertiesAsDataNodes);
 266         final GenericRequestBody dmiRequestBodyObject = GenericRequestBody.builder()
 267             .operation(OperationEnum.UPDATE)
 268             .dataType(contentType)
 269             .data(requestBody)
 270             .cmHandleProperties(cmHandlePropertiesAsMap)
 271             .build();
 272         final String dmiRequestBody = prepareOperationBody(dmiRequestBodyObject);
 273         final ResponseEntity<String> responseEntity = dmiOperations
 274             .updateResourceDataPassThroughRunningFromDmi(dmiServiceName,
 275                 cmHandle,
 276                 resourceIdentifier,
 277                 dmiRequestBody);
 278         handleResponseFromDmi(responseEntity, "Unable to replace resource data.");
 279     }
 280
 281     private DataNode fetchDataNodeFromDmiRegistryForCmHandle(final String cmHandle) {
 282         final String xpathForDmiRegistryToFetchCmHandle = "/dmi-registry/cm-handles[@id='" + cmHandle + "']";
 283         return cpsDataService.getDataNode(NCMP_DATASPACE_NAME,
 284                 NCMP_DMI_REGISTRY_ANCHOR,
 285                 xpathForDmiRegistryToFetchCmHandle,
 286                 FetchDescendantsOption.INCLUDE_ALL_DESCENDANTS);
 287     }
 288
 289     private String prepareOperationBody(final GenericRequestBody requestBodyObject) {
 290         try {
 291             return objectMapper.writeValueAsString(requestBodyObject);
 292         } catch (final JsonProcessingException e) {
 293             log.error("Parsing error occurred while converting Object to JSON.");
 294             throw new NcmpException("Parsing error occurred while converting given object to JSON.",
 295                 e.getMessage());
 296         }
 297     }
 298
 299     private static Map<String, String> getCmHandlePropertiesAsMap(
 300             final Collection<DataNode> cmHandlePropertiesAsDataNode) {
 301         if (cmHandlePropertiesAsDataNode.isEmpty()) {
 302             return Collections.emptyMap();
 303         }
 304         final Map<String, String> cmHandlePropertiesAsMap = new LinkedHashMap<>();
 305         for (final DataNode dataNode: cmHandlePropertiesAsDataNode) {
 306             cmHandlePropertiesAsMap.put(String.valueOf(dataNode.getLeaves().get("name")),
 307                     String.valueOf(dataNode.getLeaves().get("value")));
 308         }
 309         return cmHandlePropertiesAsMap;
 310     }
 311
 312     private static Map<String, String> getCmHandlePropertiesAsMap(
 313             final List<AdditionalProperty> cmHandlePropertiesAsList) {
 314         if (cmHandlePropertiesAsList == null || cmHandlePropertiesAsList.isEmpty()) {
 315             return Collections.emptyMap();
 316         }
 317         final Map<String, String> cmHandlePropertiesAsMap = new LinkedHashMap<>();
 318         for (final AdditionalProperty additionalProperty: cmHandlePropertiesAsList) {
 319             cmHandlePropertiesAsMap.put(additionalProperty.getName(),
 320                     additionalProperty.getValue());
 321         }
 322         return cmHandlePropertiesAsMap;
 323     }
 324
 325     private static Object handleResponse(final @NotNull ResponseEntity<Object> responseEntity) {
 326         if (responseEntity.getStatusCode() == HttpStatus.OK) {
 327             return responseEntity.getBody();
 328         } else {
 329             throw new NcmpException("Not able to get resource data.",
 330                     "DMI status code: " + responseEntity.getStatusCodeValue()
 331                             + ", DMI response body: " + responseEntity.getBody());
 332         }
 333     }
 334
 335     private static void handleResponseFromDmi(final @NotNull ResponseEntity<String> responseEntity,
 336         final String exceptionMessage) {
 337         if (!HttpStatus.valueOf(responseEntity.getStatusCodeValue()).is2xxSuccessful()) {
 338             throw new NcmpException(exceptionMessage,
 339                 "DMI status code: " + responseEntity.getStatusCodeValue()
 340                     + ", DMI response body: " + responseEntity.getBody());
 341         }
 342     }
 343
 344     private String getGenericRequestBody(final DataNode cmHandleDataNode) {
 345         final Collection<DataNode> cmHandlePropertiesAsDataNodes = cmHandleDataNode.getChildDataNodes();
 346         final Map<String, String> cmHandlePropertiesAsMap = getCmHandlePropertiesAsMap(cmHandlePropertiesAsDataNodes);
 347         final GenericRequestBody requestBodyObject = GenericRequestBody.builder()
 348                 .operation(GenericRequestBody.OperationEnum.READ)
 349                 .cmHandleProperties(cmHandlePropertiesAsMap)
 350                 .build();
 351         return prepareOperationBody(requestBodyObject);
 352     }
 353
 354     private void parseAndUpdateCmHandlesInDmiRegistration(final DmiPluginRegistration dmiPluginRegistration)
 355         throws JsonProcessingException {
 356         final PersistenceCmHandlesList updatedPersistenceCmHandlesList = toPersistenceCmHandlesList(
 357             dmiPluginRegistration.getDmiPlugin(),
 358             dmiPluginRegistration.getUpdatedCmHandles());
 359         final String cmHandlesAsJson = objectMapper.writeValueAsString(updatedPersistenceCmHandlesList);
 360         cpsDataService.updateNodeLeavesAndExistingDescendantLeaves(NCMP_DATASPACE_NAME, NCMP_DMI_REGISTRY_ANCHOR,
 361                 "/dmi-registry", cmHandlesAsJson, NO_TIMESTAMP);
 362     }
 363
 364     private void parseAndCreateCmHandlesInDmiRegistrationAndSyncModule(
 365         final DmiPluginRegistration dmiPluginRegistration) throws JsonProcessingException {
 366         final PersistenceCmHandlesList createdPersistenceCmHandlesList = toPersistenceCmHandlesList(
 367             dmiPluginRegistration.getDmiPlugin(),
 368             dmiPluginRegistration.getCreatedCmHandles());
 369         registerAndSyncNewCmHandles(createdPersistenceCmHandlesList);
 370     }
 371
 372     private static PersistenceCmHandlesList toPersistenceCmHandlesList(final String dmiPlugin,
 373                                                                        final Collection<CmHandle> cmHandles) {
 374         final PersistenceCmHandlesList persistenceCmHandlesList = new PersistenceCmHandlesList();
 375         for (final CmHandle cmHandle : cmHandles) {
 376             final PersistenceCmHandle persistenceCmHandle = toPersistenceCmHandle(dmiPlugin, cmHandle);
 377             persistenceCmHandlesList.add(persistenceCmHandle);
 378         }
 379         return persistenceCmHandlesList;
 380     }
 381
 382     private static void handleJsonProcessingException(final DmiPluginRegistration dmiPluginRegistration,
 383                                                       final JsonProcessingException e) {
 384         final String message = "Parsing error occurred while processing DMI Plugin Registration"
 385             + dmiPluginRegistration;
 386         log.error(message);
 387         throw new DataValidationException(message, e.getMessage(), e);
 388     }
 389
 390     private void registerAndSyncNewCmHandles(final PersistenceCmHandlesList persistenceCmHandlesList)
 391         throws JsonProcessingException  {
 392         final String cmHandleJsonData = objectMapper.writeValueAsString(persistenceCmHandlesList);
 393         cpsDataService.saveListElements(NCMP_DATASPACE_NAME, NCMP_DMI_REGISTRY_ANCHOR, "/dmi-registry",
 394             cmHandleJsonData, NO_TIMESTAMP);
 395
 396         for (final PersistenceCmHandle persistenceCmHandle : persistenceCmHandlesList.getPersistenceCmHandles()) {
 397             syncModulesAndCreateAnchor(persistenceCmHandle);
 398         }
 399     }
 400
 401     protected void syncModulesAndCreateAnchor(final PersistenceCmHandle persistenceCmHandle) {
 402         fetchAndSyncModules(persistenceCmHandle);
 403         createAnchor(persistenceCmHandle);
 404     }
 405
 406     private static PersistenceCmHandle toPersistenceCmHandle(final String dmiPluginService,
 407                                                              final CmHandle cmHandle) {
 408         final PersistenceCmHandle persistenceCmHandle = new PersistenceCmHandle();
 409         persistenceCmHandle.setDmiServiceName(dmiPluginService);
 410         persistenceCmHandle.setId(cmHandle.getCmHandleID());
 411         if (cmHandle.getCmHandleProperties() == null) {
 412             persistenceCmHandle.setAdditionalProperties(Collections.emptyMap());
 413         } else {
 414             persistenceCmHandle.setAdditionalProperties(cmHandle.getCmHandleProperties());
 415         }
 416         return persistenceCmHandle;
 417     }
 418
 419     private void parseAndRemoveCmHandlesInDmiRegistration(final DmiPluginRegistration dmiPluginRegistration) {
 420         for (final String cmHandle : dmiPluginRegistration.getRemovedCmHandles()) {
 421             try {
 422                 cpsDataService.deleteListOrListElement(NCMP_DATASPACE_NAME, NCMP_DMI_REGISTRY_ANCHOR,
 423                     "/dmi-registry/cm-handles[@id='" + cmHandle + "']", NO_TIMESTAMP);
 424             } catch (final DataNodeNotFoundException e) {
 425                 log.warn("Datanode {} not deleted message {}", cmHandle, e.getMessage());
 426             }
 427         }
 428     }
 429
 430     private void fetchAndSyncModules(final PersistenceCmHandle persistenceCmHandle) {
 431         final Map<String, String> cmHandlePropertiesAsMap = getCmHandlePropertiesAsMap(
 432             persistenceCmHandle.getAdditionalProperties());
 433
 434         final List<ModuleReference> moduleReferencesFromCmHandle =
 435             fetchModuleReferencesFromDmi(persistenceCmHandle, cmHandlePropertiesAsMap);
 436         final List<ModuleReference> existingModuleReferences = new ArrayList<>();
 437         final List<ModuleReference> unknownModuleReferences = new ArrayList<>();
 438         prepareModuleSubsets(moduleReferencesFromCmHandle, existingModuleReferences, unknownModuleReferences);
 439
 440         final Map<String, String> newYangResourcesModuleNameToContentMap;
 441         if (unknownModuleReferences.isEmpty()) {
 442             newYangResourcesModuleNameToContentMap = new HashMap<>();
 443         } else {
 444             newYangResourcesModuleNameToContentMap = getNewYangResourcesFromDmi(persistenceCmHandle,
 445                 unknownModuleReferences, cmHandlePropertiesAsMap);
 446         }
 447         cpsModuleService
 448             .createSchemaSetFromModules(NFP_OPERATIONAL_DATASTORE_DATASPACE_NAME, persistenceCmHandle.getId(),
 449                 newYangResourcesModuleNameToContentMap, existingModuleReferences);
 450     }
 451
 452     private void prepareModuleSubsets(final List<ModuleReference> moduleReferencesFromCmHandle,
 453                                       final List<ModuleReference> existingModuleReferences,
 454                                       final List<ModuleReference> unknownModuleReferences) {
 455
 456         final Collection<ModuleReference> knownModuleReferencesInCps =
 457             cpsModuleService.getYangResourceModuleReferences(NFP_OPERATIONAL_DATASTORE_DATASPACE_NAME);
 458
 459         for (final ModuleReference moduleReferenceFromDmiForCmHandle : moduleReferencesFromCmHandle) {
 460             if (knownModuleReferencesInCps.contains(moduleReferenceFromDmiForCmHandle)) {
 461                 existingModuleReferences.add(moduleReferenceFromDmiForCmHandle);
 462             } else {
 463                 unknownModuleReferences.add(moduleReferenceFromDmiForCmHandle);
 464             }
 465         }
 466     }
 467
 468     private List<ModuleReference> fetchModuleReferencesFromDmi(final PersistenceCmHandle persistenceCmHandle,
 469                                                                final Map<String, String> cmHandlePropertiesAsMap) {
 470         final GenericRequestBody genericRequestBody = GenericRequestBody.builder()
 471                 .cmHandleProperties(cmHandlePropertiesAsMap)
 472                 .build();
 473         final String jsonBodyWithOnlyCmHandleProperties = prepareOperationBody(genericRequestBody);
 474         final ResponseEntity<String> dmiFetchModulesResponseEntity =
 475             dmiOperations.getResourceFromDmiWithJsonData(persistenceCmHandle.getDmiServiceName(),
 476                     jsonBodyWithOnlyCmHandleProperties, persistenceCmHandle.getId(), "modules");
 477         return toModuleReferences(dmiFetchModulesResponseEntity);
 478     }
 479
 480     private void createAnchor(final PersistenceCmHandle persistenceCmHandle) {
 481         cpsAdminService.createAnchor(NFP_OPERATIONAL_DATASTORE_DATASPACE_NAME, persistenceCmHandle.getId(),
 482             persistenceCmHandle.getId());
 483     }
 484
 485     private String getRequestBodyToFetchYangResourceFromDmi(final List<ModuleReference> unknownModuleReferences,
 486                                                             final Map<String, String> cmHandlePropertiesAsMap) {
 487         final JsonArray moduleReferencesAsJson = getModuleReferencesAsJson(unknownModuleReferences);
 488         final JsonObject data = new JsonObject();
 489         data.add("modules", moduleReferencesAsJson);
 490         final JsonObject jsonRequestObject = new JsonObject();
 491         jsonRequestObject.add("data", data);
 492         final Gson gson = new Gson();
 493         jsonRequestObject.add("cmHandleProperties", gson.toJsonTree(cmHandlePropertiesAsMap));
 494         return jsonRequestObject.toString();
 495     }
 496
 497     private static JsonArray getModuleReferencesAsJson(final List<ModuleReference> unknownModuleReferences) {
 498         final JsonArray moduleReferences = new JsonArray();
 499
 500         for (final ModuleReference moduleReference : unknownModuleReferences) {
 501             final JsonObject moduleReferenceAsJson = new JsonObject();
 502             moduleReferenceAsJson.addProperty("name", moduleReference.getModuleName());
 503             moduleReferenceAsJson.addProperty(REVISION, moduleReference.getRevision());
 504             moduleReferences.add(moduleReferenceAsJson);
 505         }
 506         return moduleReferences;
 507     }
 508
 509     private Map<String, String> getNewYangResourcesFromDmi(final PersistenceCmHandle persistenceCmHandle,
 510                                                            final List<ModuleReference> unknownModuleReferences,
 511                                                            final Map<String, String> cmHandlePropertiesAsMap) {
 512         final String jsonDataWithDataAndCmHandleProperties = getRequestBodyToFetchYangResourceFromDmi(
 513                 unknownModuleReferences, cmHandlePropertiesAsMap);
 514
 515         final ResponseEntity<String> moduleResourcesAsJsonString =  dmiOperations.getResourceFromDmiWithJsonData(
 516                 persistenceCmHandle.getDmiServiceName(),
 517                 jsonDataWithDataAndCmHandleProperties,
 518                 persistenceCmHandle.getId(),
 519                 "moduleResources");
 520
 521         final JsonArray moduleResources = new Gson().fromJson(moduleResourcesAsJsonString.getBody(),
 522             JsonArray.class);
 523         final Map<String, String> newYangResourcesModuleNameToContentMap = new HashMap<>();
 524
 525         for (final JsonElement moduleResource : moduleResources) {
 526             final YangResource yangResource = toYangResource((JsonObject) moduleResource);
 527             newYangResourcesModuleNameToContentMap.put(yangResource.getModuleName(), yangResource.getYangSource());
 528         }
 529         return newYangResourcesModuleNameToContentMap;
 530     }
 531
 532     private static YangResource toYangResource(final JsonObject yangResourceAsJson) {
 533         final YangResource yangResource = new YangResource();
 534         yangResource.setModuleName(yangResourceAsJson.get("moduleName").getAsString());
 535         yangResource.setRevision(yangResourceAsJson.get(REVISION).getAsString());
 536         final String yangSourceJson = yangResourceAsJson.get("yangSource").getAsString();
 537
 538         String yangSource = JsonUtils.removeWrappingTokens(yangSourceJson);
 539         yangSource = JsonUtils.removeRedundantEscapeCharacters(yangSource);
 540         yangResource.setYangSource(yangSource);
 541
 542         return yangResource;
 543     }
 544
 545     private static List<ModuleReference> toModuleReferences(
 546             final ResponseEntity<String> dmiFetchModulesResponseEntity) {
 547         final List<ModuleReference> moduleReferences = new ArrayList<>();
 548         final JsonObject bodyAsJsonObject = new Gson().fromJson(dmiFetchModulesResponseEntity.getBody(),
 549             JsonObject.class);
 550         final JsonArray moduleReferencesAsJson = bodyAsJsonObject.getAsJsonArray("schemas");
 551         for (final JsonElement moduleReferenceAsJson : moduleReferencesAsJson) {
 552             final ModuleReference moduleReference = toModuleReference((JsonObject) moduleReferenceAsJson);
 553             moduleReferences.add(moduleReference);
 554         }
 555         return moduleReferences;
 556     }
 557
 558     private static ModuleReference toModuleReference(final JsonObject moduleReferenceAsJson) {
 559         final ModuleReference moduleReference = new ModuleReference();
 560         moduleReference.setModuleName(moduleReferenceAsJson.get("moduleName").getAsString());
 561         moduleReference.setRevision(moduleReferenceAsJson.get(REVISION).getAsString());
 562         return moduleReference;
 563     }
 564 }