2 * ============LICENSE_START=======================================================
3 * Copyright (C) 2022-2023 Nordix Foundation
4 * Modifications Copyright (C) 2022 Bell Canada
5 * ================================================================================
6 * Licensed under the Apache License, Version 2.0 (the "License");
7 * you may not use this file except in compliance with the License.
8 * You may obtain a copy of the License at
10 * http://www.apache.org/licenses/LICENSE-2.0
12 * Unless required by applicable law or agreed to in writing, software
13 * distributed under the License is distributed on an "AS IS" BASIS,
14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 * See the License for the specific language governing permissions and
16 * limitations under the License.
18 * SPDX-License-Identifier: Apache-2.0
19 * ============LICENSE_END=========================================================
22 package org.onap.cps.ncmp.api.impl.inventory.sync;
24 import com.hazelcast.map.IMap;
25 import java.util.Collection;
26 import java.util.HashSet;
27 import java.util.List;
29 import java.util.concurrent.BlockingQueue;
30 import java.util.concurrent.TimeUnit;
31 import java.util.concurrent.atomic.AtomicInteger;
33 import lombok.RequiredArgsConstructor;
34 import lombok.extern.slf4j.Slf4j;
35 import org.onap.cps.ncmp.api.impl.config.embeddedcache.SynchronizationCacheConfig;
36 import org.onap.cps.ncmp.api.impl.inventory.sync.executor.AsyncTaskExecutor;
37 import org.onap.cps.ncmp.api.impl.yangmodels.YangModelCmHandle;
38 import org.onap.cps.spi.model.DataNode;
39 import org.springframework.scheduling.annotation.Scheduled;
40 import org.springframework.stereotype.Service;
43 @RequiredArgsConstructor
45 public class ModuleSyncWatchdog {
47 private final SyncUtils syncUtils;
48 private final BlockingQueue<DataNode> moduleSyncWorkQueue;
49 private final IMap<String, Object> moduleSyncStartedOnCmHandles;
50 private final ModuleSyncTasks moduleSyncTasks;
51 private final AsyncTaskExecutor asyncTaskExecutor;
52 private final IMap<String, Set<String>> moduleSetTagCache;
53 private static final int MODULE_SYNC_BATCH_SIZE = 100;
54 private static final long PREVENT_CPU_BURN_WAIT_TIME_MILLIS = 10;
55 private static final String VALUE_FOR_HAZELCAST_IN_PROGRESS_MAP = "Started";
56 private static final long ASYNC_TASK_TIMEOUT_IN_MILLISECONDS = TimeUnit.MINUTES.toMillis(5);
58 private AtomicInteger batchCounter = new AtomicInteger(1);
61 * Check DB for any cm handles in 'ADVISED' state.
62 * Queue and create batches to process them asynchronously.
63 * This method will only finish when there are no more 'ADVISED' cm handles in the DB.
64 * This method wil be triggered on a configurable interval
66 @Scheduled(fixedDelayString = "${ncmp.timers.advised-modules-sync.sleep-time-ms:5000}")
67 public void moduleSyncAdvisedCmHandles() {
68 log.info("Processing module sync watchdog waking up.");
69 populateWorkQueueIfNeeded();
70 while (!moduleSyncWorkQueue.isEmpty()) {
71 if (batchCounter.get() <= asyncTaskExecutor.getAsyncTaskParallelismLevel()) {
72 final Collection<DataNode> nextBatch = prepareNextBatch();
73 log.info("Processing module sync batch of {}. {} batch(es) active.",
74 nextBatch.size(), batchCounter.get());
75 if (!nextBatch.isEmpty()) {
76 asyncTaskExecutor.executeTask(() ->
77 moduleSyncTasks.performModuleSync(nextBatch, batchCounter),
78 ASYNC_TASK_TIMEOUT_IN_MILLISECONDS);
79 batchCounter.getAndIncrement();
88 * Find any failed (locked) cm handles and change state back to 'ADVISED'.
90 @Scheduled(fixedDelayString = "${ncmp.timers.locked-modules-sync.sleep-time-ms:300000}")
91 public void resetPreviouslyFailedCmHandles() {
92 log.info("Processing module sync retry-watchdog waking up.");
93 final List<YangModelCmHandle> failedCmHandles = syncUtils.getModuleSyncFailedCmHandles();
94 log.info("Retrying {} cmHandles", failedCmHandles.size());
95 moduleSyncTasks.resetFailedCmHandles(failedCmHandles);
98 private void preventBusyWait() {
100 log.info("Busy waiting now");
101 TimeUnit.MILLISECONDS.sleep(PREVENT_CPU_BURN_WAIT_TIME_MILLIS);
102 } catch (final InterruptedException e) {
103 Thread.currentThread().interrupt();
107 private void populateWorkQueueIfNeeded() {
108 if (moduleSyncWorkQueue.isEmpty()) {
109 final List<DataNode> advisedCmHandles = syncUtils.getAdvisedCmHandles();
110 log.info("Processing module sync fetched {} advised cm handles from DB", advisedCmHandles.size());
111 for (final DataNode advisedCmHandle : advisedCmHandles) {
112 if (!moduleSyncWorkQueue.offer(advisedCmHandle)) {
113 log.warn("Unable to add cm handle {} to the work queue", advisedCmHandle.getLeaves().get("id"));
116 log.info("Work Queue Size : {}", moduleSyncWorkQueue.size());
120 private Collection<DataNode> prepareNextBatch() {
121 final Collection<DataNode> nextBatchCandidates = new HashSet<>(MODULE_SYNC_BATCH_SIZE);
122 final Collection<DataNode> nextBatch = new HashSet<>(MODULE_SYNC_BATCH_SIZE);
123 moduleSyncWorkQueue.drainTo(nextBatchCandidates, MODULE_SYNC_BATCH_SIZE);
124 log.info("nextBatchCandidates size : {}", nextBatchCandidates.size());
125 for (final DataNode batchCandidate : nextBatchCandidates) {
126 final String cmHandleId = String.valueOf(batchCandidate.getLeaves().get("id"));
127 final boolean alreadyAddedToInProgressMap = VALUE_FOR_HAZELCAST_IN_PROGRESS_MAP.equals(
128 moduleSyncStartedOnCmHandles.putIfAbsent(cmHandleId, VALUE_FOR_HAZELCAST_IN_PROGRESS_MAP,
129 SynchronizationCacheConfig.MODULE_SYNC_STARTED_TTL_SECS, TimeUnit.SECONDS));
130 if (alreadyAddedToInProgressMap) {
131 log.info("module sync for {} already in progress by other instance", cmHandleId);
133 log.info("Adding cmHandle : {} to current batch", cmHandleId);
134 nextBatch.add(batchCandidate);
137 log.debug("nextBatch size : {}", nextBatch.size());