import org.openecomp.sdc.be.datatypes.enums.ComponentTypeEnum;
import org.openecomp.sdc.be.datatypes.enums.NodeTypeEnum;
import org.openecomp.sdc.be.datatypes.enums.ResourceTypeEnum;
-import org.openecomp.sdc.be.model.Component;
-import org.openecomp.sdc.be.model.LifecycleStateEnum;
-import org.openecomp.sdc.be.model.Product;
-import org.openecomp.sdc.be.model.Resource;
-import org.openecomp.sdc.be.model.Service;
+import org.openecomp.sdc.be.model.*;
import org.openecomp.sdc.be.model.jsontitan.operations.ToscaOperationFacade;
-import org.openecomp.sdc.be.model.operations.api.StorageOperationStatus;
import org.openecomp.sdc.be.resources.data.ComponentCacheData;
+import org.openecomp.sdc.common.log.wrappers.Logger;
import org.openecomp.sdc.common.util.SerializationUtils;
import org.openecomp.sdc.common.util.ZipUtil;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import javax.annotation.PostConstruct;
import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.LinkedList;
+import java.util.*;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+import java.util.function.Function;
+import java.util.stream.Collectors;
import java.util.List;
import java.util.Map;
import java.util.Set;
@org.springframework.stereotype.Component("component-cache")
public class ComponentCache {
- private static Logger logger = LoggerFactory.getLogger(ComponentCache.class.getName());
-
- @Autowired
- ComponentCassandraDao componentCassandraDao;
-
- @Autowired
- ToscaOperationFacade toscaOperationFacade;
-
- private Map<ComponentTypeEnum, Map<String, Component>> catalogInMemoryCache = new HashMap<>();
- private final ReentrantReadWriteLock rwCatalogLock = new ReentrantReadWriteLock();
- private final Lock rCatalogLock = rwCatalogLock.readLock();
- private final Lock wCatalogLock = rwCatalogLock.writeLock();
-
- boolean enabled = true;
- int catalogInMemorySizePerResource = 300;
- int catalogInMemorySizePerService = 200;
- int catalogInMemorySizePerProduct = 100;
- boolean catalogInMemoryEnabled = true;
- Map<ComponentTypeEnum, Integer> limitMemoryCatalogSizePerType = new HashMap<>();
-
- @PostConstruct
- public void init() {
-
- Configuration configuration = ConfigurationManager.getConfigurationManager().getConfiguration();
- if (configuration != null) {
- ApplicationL2CacheConfig applicationL2Cache = configuration.getApplicationL2Cache();
- if (applicationL2Cache != null) {
- boolean isEnabled = applicationL2Cache.isEnabled();
- this.enabled = isEnabled;
-
- ApplicationL1CacheCatalogInfo catalog = applicationL2Cache.getCatalogL1Cache();
- if (catalog != null) {
- catalogInMemoryEnabled = catalog.getEnabled();
- catalogInMemorySizePerResource = catalog.getResourcesSizeInCache();
- catalogInMemorySizePerService = catalog.getServicesSizeInCache();
- catalogInMemorySizePerProduct = catalog.getProductsSizeInCache();
- }
- }
- }
-
- ComponentTypeEnum[] typesForCache = { ComponentTypeEnum.RESOURCE, ComponentTypeEnum.SERVICE,
- ComponentTypeEnum.PRODUCT };
- for (ComponentTypeEnum typeEnum : typesForCache) {
- Map<String, Component> map = new HashMap<>();
- catalogInMemoryCache.put(typeEnum, map);
- }
-
- limitMemoryCatalogSizePerType.put(ComponentTypeEnum.RESOURCE, catalogInMemorySizePerResource);
- limitMemoryCatalogSizePerType.put(ComponentTypeEnum.SERVICE, catalogInMemorySizePerService);
- limitMemoryCatalogSizePerType.put(ComponentTypeEnum.PRODUCT, catalogInMemorySizePerProduct);
- }
-
- public boolean isEnabled() {
- return enabled;
- }
-
- public void setEnabled(boolean enabled) {
- this.enabled = enabled;
- }
-
- public Either<Component, ActionStatus> getComponent(String componentUid, Long lastModificationTime,
- Function<Component, Component> filterFieldsFunc) {
-
- Either<ImmutablePair<Component, ComponentCacheData>, ActionStatus> componentFromCache = getComponentFromCache(
- componentUid, lastModificationTime, filterFieldsFunc);
-
- if (componentFromCache.isRight()) {
- return Either.right(componentFromCache.right().value());
- }
-
- return Either.left(componentFromCache.left().value().left);
-
- }
-
- public Either<List<ComponentCacheData>, ActionStatus> getAllComponentIdTimeAndType() {
- if (false == isEnabled()) {
- return Either.right(ActionStatus.NOT_ALLOWED);
- }
-
- Either<List<ComponentCacheData>, ActionStatus> componentRes = componentCassandraDao
- .getAllComponentIdTimeAndType();
-
- return componentRes;
-
- }
-
- /**
- * @param foundComponents
- * @param componentTypeEnum
- */
- private void updateCatalogInMemoryCacheWithCertified(List<Component> foundComponents,
- ComponentTypeEnum componentTypeEnum) {
-
- try {
- wCatalogLock.lock();
-
- long start = System.currentTimeMillis();
- Map<String, Component> map = catalogInMemoryCache.get(componentTypeEnum);
- int mapSizeBefore = map.size();
- map.clear();
- Map<String, Component> collect = foundComponents.stream()
- .filter(p -> p.getLifecycleState() == LifecycleStateEnum.CERTIFIED)
- .limit(limitMemoryCatalogSizePerType.get(componentTypeEnum))
- .collect(Collectors.toMap(p -> p.getUniqueId(), p -> p));
- map.putAll(collect);
- logger.debug(
- "Size of in memory cache for catalog {}(certified only): Before {}, After {}. Replacement Time is {} ms.",
- componentTypeEnum.name().toLowerCase(), mapSizeBefore, map.size(),
- System.currentTimeMillis() - start);
- } finally {
- wCatalogLock.unlock();
- }
-
- }
-
- private List<Component> getDataFromInMemoryCache(Set<String> components, ComponentTypeEnum componentTypeEnum) {
- List<Component> foundComponents = new ArrayList<>();
-
- try {
-
- rCatalogLock.lock();
-
- Map<String, Component> map = catalogInMemoryCache.get(componentTypeEnum);
- for (String compUid : components) {
- Component component = map.get(compUid);
- if (component != null) {
- foundComponents.add(component);
- }
- }
-
- } finally {
- rCatalogLock.unlock();
- }
-
- return foundComponents;
- }
-
- /**
- *
- * get full components from cassandra. On each component apply filter function
- * in order to remove unused members
- *
- * @param components
- * @param filterFieldsFunc
- * @return <found components, found dirty components, not found components list>
- * or Error
- */
- public Either<ImmutableTriple<List<Component>, List<Component>, Set<String>>, ActionStatus> getComponents(
- Set<String> components, Function<List<Component>, List<Component>> filterFieldsFunc) {
-
- if (false == isEnabled()) {
- logger.debug("Component Cache is disabled");
- return Either.right(ActionStatus.NOT_ALLOWED);
- }
-
- Either<ImmutableTriple<List<Component>, List<Component>, Set<String>>, ActionStatus> componentsFull = getComponentsFull(
- components);
-
- if (componentsFull.isRight()) {
- return Either.right(componentsFull.right().value());
- }
-
- ImmutableTriple<List<Component>, List<Component>, Set<String>> immutableTriple = componentsFull.left().value();
- List<Component> foundResources = immutableTriple.left;
- List<Component> foundDirtyResources = immutableTriple.middle;
- Set<String> notFoundResources = immutableTriple.right;
-
- List<Component> filterdFoundResources = filterFieldsFunc.apply(foundResources);
- List<Component> filterdFoundDirtyResources = filterFieldsFunc.apply(foundDirtyResources);
-
- ImmutableTriple<List<Component>, List<Component>, Set<String>> result = new ImmutableTriple<List<Component>, List<Component>, Set<String>>(
- filterdFoundResources, filterdFoundDirtyResources, notFoundResources);
-
- return Either.left(result);
-
- }
-
- public Either<ImmutableTriple<List<Component>, List<Component>, Set<String>>, ActionStatus> getComponentsForLeftPanel(
- ComponentTypeEnum componentTypeEnum, String internalComponentType, Set<String> filteredResources) {
-
- logger.debug("In getComponentsForLeftPanel componentTypeEnum = {}, internalComponentType = {}",
- componentTypeEnum, internalComponentType);
-
- Function<List<Component>, List<Component>> filterFieldsFunc = x -> filterForLeftPanel(x);
-
- return getComponents(filteredResources, filterFieldsFunc);
-
- }
-
- private List<Component> filterForLeftPanel(List<Component> components) {
-
- List<Component> result = new ArrayList<>();
- if (components != null) {
- components.forEach(p -> result.add(filterFieldsForLeftPanel(p)));
- }
-
- return result;
- }
-
- private List<Component> filterForCatalog(List<Component> components) {
-
- List<Component> result = new ArrayList<>();
- if (components != null) {
- components.forEach(p -> result.add(filterFieldsForCatalog(p)));
- }
-
- return result;
- }
-
- private Component filterFieldsForLeftPanel(Component component) {
-
- Component result = null;
- ComponentTypeEnum componentTypeEnum = component.getComponentType();
- switch (componentTypeEnum) {
- case RESOURCE:
- result = new Resource();
- copyFieldsForLeftPanel(component, result);
- break;
- case SERVICE:
- result = new Service();
- copyFieldsForLeftPanel(component, result);
- break;
- default:
- break;
- }
-
- return result;
- }
-
- private Component filterFieldsForCatalog(Component component) {
-
- Component result = null;
- ComponentTypeEnum componentTypeEnum = component.getComponentType();
- switch (componentTypeEnum) {
- case RESOURCE:
- result = new Resource();
- copyFieldsForCatalog(component, result);
- break;
- case SERVICE:
- result = new Service();
- copyFieldsForCatalog(component, result);
- break;
- case PRODUCT:
- result = new Product();
- copyFieldsForCatalog(component, result);
- default:
- break;
- }
-
- return result;
- }
-
- /**
- * Copy relevant fields to the filtered component for left panel
- *
- * @param component
- * @param filteredComponent
- */
- private void copyFieldsForLeftPanel(Component component, Component filteredComponent) {
-
- ComponentTypeEnum componentTypeEnum = component.getComponentType();
- filteredComponent.setCategories(component.getCategories());
- filteredComponent.setComponentType(component.getComponentType());
- if (ComponentTypeEnum.RESOURCE.equals(component.getComponentType())
- && ResourceTypeEnum.VL.equals(((ResourceMetadataDataDefinition) component
- .getComponentMetadataDefinition().getMetadataDataDefinition()).getResourceType())) {
- filteredComponent.setCapabilities(component.getCapabilities());
- filteredComponent.setRequirements(component.getRequirements());
- }
- filteredComponent.setVersion(component.getVersion());
- filteredComponent.setDescription(component.getDescription());
- filteredComponent.setUniqueId(component.getUniqueId());
- filteredComponent.setIcon(component.getIcon());
- filteredComponent.setTags(component.getTags());
- filteredComponent.setLifecycleState(component.getLifecycleState());
- filteredComponent.setInvariantUUID(component.getInvariantUUID());
- filteredComponent.setUUID(component.getUUID());
- filteredComponent.setSystemName(component.getSystemName());
- filteredComponent.setName(component.getName());
-
- if (componentTypeEnum == ComponentTypeEnum.RESOURCE) {
- Resource resource = (Resource) component;
- Resource filteredResource = (Resource) filteredComponent;
- filteredResource.setToscaResourceName(resource.getToscaResourceName());
- filteredResource.setResourceType(resource.getResourceType());
- }
- }
-
- private void copyFieldsForCatalog(Component component, Component filteredComponent) {
-
- ComponentTypeEnum componentTypeEnum = component.getComponentType();
- filteredComponent.setCategories(component.getCategories());
- filteredComponent.setComponentType(component.getComponentType());
- filteredComponent.setVersion(component.getVersion());
- filteredComponent.setDescription(component.getDescription());
- filteredComponent.setUniqueId(component.getUniqueId());
- filteredComponent.setIcon(component.getIcon());
- filteredComponent.setTags(component.getTags());
- filteredComponent.setLifecycleState(component.getLifecycleState());
- filteredComponent.setSystemName(component.getSystemName());
- filteredComponent.setName(component.getName());
- filteredComponent.setLastUpdateDate(component.getLastUpdateDate());
-
- if (componentTypeEnum == ComponentTypeEnum.RESOURCE) {
- Resource resource = (Resource) component;
- Resource filteredResource = (Resource) filteredComponent;
- filteredResource.setToscaResourceName(resource.getToscaResourceName());
- filteredResource.setResourceType(resource.getResourceType());
- } else if (componentTypeEnum == ComponentTypeEnum.SERVICE) {
- Service service = (Service) component;
- Service filteredService = (Service) filteredComponent;
- filteredService.setDistributionStatus(service.getDistributionStatus());
- }
- }
-
- /**
- * get components from cache of a given list ou unique ids.
- *
- * for each component data from cassandra, unzip the data if needed and
- * deserialize the unzipped data to java object(Component).
- *
- * @param filteredResources
- * @return ImmutableTripple or ActionStatus. | |-- components |-- dirty
- * components - components with dirty flag = true. |-- set of non cached
- * components
- *
- */
- private Either<ImmutableTriple<List<Component>, List<Component>, Set<String>>, ActionStatus> getComponentsFull(
- Set<String> filteredResources) {
-
- if (false == isEnabled()) {
- logger.debug("Component Cache is disabled");
- return Either.right(ActionStatus.NOT_ALLOWED);
- }
-
- List<Component> foundResources = new LinkedList<>();
- List<Component> foundDirtyResources = new LinkedList<>();
- Set<String> notFoundResources = new HashSet<>();
- ImmutableTriple<List<Component>, List<Component>, Set<String>> result = new ImmutableTriple<List<Component>, List<Component>, Set<String>>(
- foundResources, foundDirtyResources, notFoundResources);
-
- long cassandraFetchStart = System.currentTimeMillis();
- List<String> uidsList = new ArrayList<>();
- uidsList.addAll(filteredResources);
- Either<List<ComponentCacheData>, ActionStatus> componentsFromCache = componentCassandraDao
- .getComponents(uidsList);
-
- long cassandraFetchEnd = System.currentTimeMillis();
- logger.debug("Fetch time from cassandara of all components took {} ms",
- (cassandraFetchEnd - cassandraFetchStart));
- if (componentsFromCache.isRight()) {
- BeEcompErrorManager.getInstance().logInternalFlowError("FetchFromCache",
- "Failed to fetch components from cache", ErrorSeverity.ERROR);
- return Either.right(componentsFromCache.right().value());
- }
-
- List<ComponentCacheData> list = componentsFromCache.left().value();
- logger.debug("Number of components fetched from cassandra is {}", (list == null ? 0 : list.size()));
- if (list != null && false == list.isEmpty()) {
-
- List<ComponentCacheData> filteredData = list.stream().filter(p -> filteredResources.contains(p.getId()))
- .collect(Collectors.toList());
- logger.debug("Number of components filterd is {}", filteredData == null ? 0 : filteredData.size());
-
- if (filteredData != null) {
- long desStart = System.currentTimeMillis();
-
- for (ComponentCacheData componentCacheData : filteredData) {
-
- logger.debug("Process uid {} from cache", componentCacheData.getId());
-
- String compUid = componentCacheData.getId();
-
- Either<? extends Component, Boolean> deserializeExt = convertComponentCacheToComponent(
- componentCacheData);
-
- if (deserializeExt.isLeft()) {
- Component component = deserializeExt.left().value();
- if (false == componentCacheData.getIsDirty()) {
- foundResources.add(component);
- } else {
- foundDirtyResources.add(component);
- }
- } else {
- notFoundResources.add(compUid);
- }
-
- }
- long desEnd = System.currentTimeMillis();
- logger.debug("Deserialization and unzip of {} components took {} ms", filteredData.size(),
- (desEnd - desStart));
- }
- }
- List<String> foundResourcesUid = foundResources.stream().map(p -> p.getUniqueId()).collect(Collectors.toList());
- List<String> foundDirtyResourcesUid = foundDirtyResources.stream().map(p -> p.getUniqueId())
- .collect(Collectors.toList());
- logger.debug("Number of processed components from cache is {}",
- (foundResourcesUid.size() + foundDirtyResourcesUid.size()));
- Set<String> notCachedResources = filteredResources.stream()
- .filter(p -> false == foundResourcesUid.contains(p) && false == foundDirtyResourcesUid.contains(p))
- .collect(Collectors.toSet());
- notFoundResources.addAll(notCachedResources);
-
- logger.debug("Number of components fetched is {}", foundResources.size());
- logger.debug("Number of components fetched dirty is {}", foundDirtyResources.size());
- logger.debug("Number of components non cached is {}", notCachedResources.size());
-
- return Either.left(result);
- }
-
- private Either<? extends Component, Boolean> convertComponentCacheToComponent(
- ComponentCacheData componentCacheData) {
-
- String compUid = componentCacheData.getId();
-
- byte[] dataAsArray = componentCacheData.getDataAsArray();
-
- if (true == componentCacheData.getIsZipped()) {
- long startUnzip = System.nanoTime();
- dataAsArray = ZipUtil.unzip(dataAsArray);
- long endUnzip = System.nanoTime();
- logger.trace("Unzip component {} took {} microsecond", compUid, (endUnzip - startUnzip) / 1000);
- }
-
- long startDes = System.nanoTime();
-
- Either<? extends Component, Boolean> deserializeExt = deserializeComponent(componentCacheData, dataAsArray);
-
- long endDes = System.nanoTime();
- logger.trace("Deserialize component {} took {} microsecond", compUid, (endDes - startDes) / 1000);
- return deserializeExt;
- }
-
- private Either<? extends Component, Boolean> deserializeComponent(ComponentCacheData componentCacheData,
- byte[] dataAsArray) {
- String type = componentCacheData.getType();
- NodeTypeEnum typeEnum = NodeTypeEnum.getByNameIgnoreCase(type);
-
- Either<? extends Component, Boolean> deserializeExt = Either.right(false);
- switch (typeEnum) {
- case Resource:
- deserializeExt = SerializationUtils.deserializeExt(dataAsArray, Resource.class, componentCacheData.getId());
- break;
- case Service:
- deserializeExt = SerializationUtils.deserializeExt(dataAsArray, Service.class, componentCacheData.getId());
- break;
- case Product:
- deserializeExt = SerializationUtils.deserializeExt(dataAsArray, Product.class, componentCacheData.getId());
- break;
- default:
- break;
- }
- return deserializeExt;
- }
-
- public Either<Component, ActionStatus> getComponent(String componentUid) {
-
- return getComponent(componentUid, null, Function.identity());
- }
-
- public Either<Component, ActionStatus> getComponent(String componentUid, Long lastModificationTime) {
-
- return getComponent(componentUid, lastModificationTime, Function.identity());
-
- }
-
- public boolean setComponent(String componentUid, Long lastModificationTime, NodeTypeEnum nodeTypeEnum) {
-
- boolean result = false;
-
- if (false == isEnabled()) {
- logger.debug("Component Cache is disabled");
- return false;
- }
-
- Either<Component, StorageOperationStatus> either = toscaOperationFacade.getToscaElement(componentUid);
- if (either.isLeft()) {
- Component component = either.left().value();
- result = saveComponent(componentUid, lastModificationTime, nodeTypeEnum, component);
- } else {
- logger.debug("Failed to get component {} of type {} from graph. Status is {}", componentUid,
- nodeTypeEnum.name().toLowerCase(), either.right().value());
- }
-
- return result;
-
- }
-
- private boolean saveComponent(String componentUid, Long lastModificationTime, NodeTypeEnum nodeTypeEnum,
- Component component) {
-
- logger.trace("Going to save component {} of type {} in cache", componentUid, nodeTypeEnum.name().toLowerCase());
-
- boolean result = false;
-
- Either<byte[], Boolean> serializeExt = SerializationUtils.serializeExt(component);
- if (serializeExt.isLeft()) {
- byte[] serializedData = serializeExt.left().value();
- byte[] zipBytes;
- try {
- zipBytes = ZipUtil.zipBytes(serializedData);
- ComponentCacheData componentCacheData = new ComponentCacheData();
- componentCacheData.setDataAsArray(zipBytes);
- componentCacheData.setIsZipped(true);
- componentCacheData.setId(componentUid);
- componentCacheData.setModificationTime(new Date(lastModificationTime));
- componentCacheData.setType(component.getComponentType().name().toLowerCase());
-
- CassandraOperationStatus status = componentCassandraDao.saveComponent(componentCacheData);
-
- if (status == CassandraOperationStatus.OK) {
- result = true;
- }
-
- } catch (IOException e) {
- logger.debug("Failed to prepare component {} of type {} for cache", componentUid,
- nodeTypeEnum.name().toLowerCase());
- }
- } else {
- logger.debug("Failed to serialize component {} of type {} for cache", componentUid,
- nodeTypeEnum.name().toLowerCase());
- }
- return result;
- }
-
- public boolean setComponent(Component component, NodeTypeEnum nodeTypeEnum) {
-
- boolean result = false;
-
- if (false == isEnabled()) {
- logger.debug("Component Cache is disabled");
- return false;
- }
-
- String componentUid = component.getUniqueId();
- Long lastUpdateDate = component.getLastUpdateDate();
-
- result = saveComponent(componentUid, lastUpdateDate, nodeTypeEnum, component);
-
- return result;
-
- }
-
- /**
- * get components from cache of a given list ou unique ids.
- *
- * for each component data from cassandra, unzip the data if needed and
- * deserialize the unzipped data to java object(Component).
- *
- * @param filteredResources
- * @return ImmutableTripple or ActionStatus. | |-- components |-- set of non
- * cached components
- *
- */
- private Either<ImmutablePair<List<Component>, Set<String>>, ActionStatus> getComponentsFull(
- Map<String, Long> filteredResources) {
-
- if (false == isEnabled()) {
- logger.debug("Component Cache is disabled");
- return Either.right(ActionStatus.NOT_ALLOWED);
- }
-
- List<Component> foundResources = new LinkedList<>();
- Set<String> notFoundResources = new HashSet<>();
- ImmutablePair<List<Component>, Set<String>> result = new ImmutablePair<List<Component>, Set<String>>(
- foundResources, notFoundResources);
-
- long cassandraFetchStart = System.currentTimeMillis();
-
- Either<ImmutablePair<List<ComponentCacheData>, Set<String>>, ActionStatus> componentsFromCache = componentCassandraDao
- .getComponents(filteredResources);
-
- long cassandraFetchEnd = System.currentTimeMillis();
- logger.debug("Fetch time from cassandara of all components took {} ms",
- (cassandraFetchEnd - cassandraFetchStart));
- if (componentsFromCache.isRight()) {
- BeEcompErrorManager.getInstance().logInternalFlowError("FetchFromCache",
- "Failed to fetch components from cache", ErrorSeverity.ERROR);
- return Either.right(componentsFromCache.right().value());
- }
-
- ImmutablePair<List<ComponentCacheData>, Set<String>> immutablePair = componentsFromCache.left().value();
- List<ComponentCacheData> list = immutablePair.getLeft();
- logger.debug("Number of components fetched from cassandra is {}", (list == null ? 0 : list.size()));
- if (list != null && false == list.isEmpty()) {
-
- // List<ComponentCacheData> filteredData = list.stream().filter(p ->
- // filteredResources.contains(p.getId())).collect(Collectors.toList());
- logger.debug("Number of components filterd is {}", list == null ? 0 : list.size());
-
- if (list != null) {
- long desStart = System.currentTimeMillis();
-
- for (ComponentCacheData componentCacheData : list) {
-
- logger.debug("Process uid {} from cache", componentCacheData.getId());
-
- String compUid = componentCacheData.getId();
-
- Either<? extends Component, Boolean> deserializeExt = convertComponentCacheToComponent(
- componentCacheData);
-
- if (deserializeExt.isLeft()) {
- Component component = deserializeExt.left().value();
- foundResources.add(component);
- } else {
- notFoundResources.add(compUid);
- }
-
- }
- long desEnd = System.currentTimeMillis();
- logger.debug("Deserialization and unzip of {} components took {} ms", list.size(), (desEnd - desStart));
- }
- }
- logger.debug("Number of processed components from cache is {}", foundResources.size());
-
- Set<String> notFoundInCache = immutablePair.getRight();
- notFoundResources.addAll(notFoundInCache);
-
- logger.debug("Number of components fetched is {}", foundResources.size());
- logger.debug("Number of components non cached is {}", notFoundResources.size());
-
- return Either.left(result);
- }
-
- /**
- * get components for catalog
- *
- * @param components
- * @param componentTypeEnum
- * @return
- */
- public Either<ImmutablePair<List<Component>, Set<String>>, ActionStatus> getComponentsForCatalog(
- Map<String, Long> components, ComponentTypeEnum componentTypeEnum) {
-
- if (false == isEnabled()) {
- logger.debug("In getComponentsForCatalog for type {}. Cache is disabled.",
- componentTypeEnum.name().toLowerCase());
- return Either.right(ActionStatus.NOT_ALLOWED);
- }
- logger.debug("In getComponentsForCatalog for type {}", componentTypeEnum.name().toLowerCase());
-
- Function<List<Component>, List<Component>> filterFieldsFunc = x -> filterForCatalog(x);
-
- Map<String, Long> leftComponentsForSearch = new HashMap<>();
- leftComponentsForSearch.putAll(components);
-
- // get components from inmemory cache
- List<Component> componentsFromMemory = null;
- if (true == catalogInMemoryEnabled) {
- componentsFromMemory = getDataFromInMemoryCache(components.keySet(), componentTypeEnum);
- logger.debug("The number of components of type {} fetched from memory is {}",
- componentTypeEnum.name().toLowerCase(),
- componentsFromMemory == null ? 0 : componentsFromMemory.size());
- if (componentsFromMemory != null) {
- List<String> ignoredComponents = new ArrayList<>();
- for (Component componentFromMem : componentsFromMemory) {
- if (componentFromMem.getLastUpdateDate().longValue() != components
- .get(componentFromMem.getUniqueId()).longValue()) {
- // Ignore the component from memory
- ignoredComponents.add(componentFromMem.getUniqueId());
- }
- }
-
- logger.debug("Number of components from type {} ignored from memory cache is {}",
- componentTypeEnum.name().toLowerCase(), ignoredComponents.size());
- // remove from memory result the components which are not valid
- componentsFromMemory = componentsFromMemory.stream()
- .filter(p -> false == ignoredComponents.contains(p.getUniqueId())).collect(Collectors.toList());
- // Remove from leftComponentsForSearch the valid components from
- // memory
- componentsFromMemory.forEach(p -> leftComponentsForSearch.remove(p.getUniqueId()));
-
- }
- } else {
- logger.debug("Catalog InMemory cache is disabled");
- }
-
- logger.debug("Number of components from type {} needed to fetch is {}", componentTypeEnum.name().toLowerCase(),
- leftComponentsForSearch.size());
-
- // get components from cassandra cache and filter each component
- Either<ImmutablePair<List<Component>, Set<String>>, ActionStatus> result = getComponents(
- leftComponentsForSearch, filterFieldsFunc);
-
- if (result.isLeft()) {
- // add inmemory components to the valid components(not dirty)
- List<Component> foundComponents = result.left().value().getLeft();
- if (componentsFromMemory != null) {
- foundComponents.addAll(componentsFromMemory);
- }
- if (true == catalogInMemoryEnabled) {
- updateCatalogInMemoryCacheWithCertified(foundComponents, componentTypeEnum);
- }
- }
-
- return result;
- }
-
- /**
- * @param components
- * - Map of <componentUniqueId, last update date>
- * @param filterFieldsFunc
- * @return
- */
- public Either<ImmutablePair<List<Component>, Set<String>>, ActionStatus> getComponents(Map<String, Long> components,
- Function<List<Component>, List<Component>> filterFieldsFunc) {
-
- if (false == isEnabled()) {
- logger.debug("Component Cache is disabled");
- return Either.right(ActionStatus.NOT_ALLOWED);
- }
-
- Either<ImmutablePair<List<Component>, Set<String>>, ActionStatus> componentsFull = getComponentsFull(
- components);
-
- if (componentsFull.isRight()) {
- return Either.right(componentsFull.right().value());
- }
-
- ImmutablePair<List<Component>, Set<String>> immutablePair = componentsFull.left().value();
- List<Component> foundResources = immutablePair.left;
- Set<String> notFoundResources = immutablePair.right;
-
- List<Component> filterdFoundResources = filterFieldsFunc.apply(foundResources);
-
- ImmutablePair<List<Component>, Set<String>> result = new ImmutablePair<List<Component>, Set<String>>(
- filterdFoundResources, notFoundResources);
-
- return Either.left(result);
-
- }
-
- /**
- * get the component and its modification time from cache
- *
- * @param componentUid
- * @param filterFieldsFunc
- * @return
- */
- public Either<ImmutablePair<Component, Long>, ActionStatus> getComponentAndTime(String componentUid,
- Function<Component, Component> filterFieldsFunc) {
-
- Either<ImmutablePair<Component, ComponentCacheData>, ActionStatus> componentFromCache = getComponentFromCache(
- componentUid, null, filterFieldsFunc);
-
- if (componentFromCache.isRight()) {
- return Either.right(componentFromCache.right().value());
- }
-
- ImmutablePair<Component, ComponentCacheData> immutablePair = componentFromCache.left().value();
-
- ImmutablePair<Component, Long> result = new ImmutablePair<Component, Long>(immutablePair.left,
- immutablePair.right.getModificationTime().getTime());
-
- return Either.left(result);
- }
-
- private Either<ImmutablePair<Component, ComponentCacheData>, ActionStatus> getComponentFromCache(
- String componentUid, Long lastModificationTime, Function<Component, Component> filterFieldsFunc) {
- if (false == isEnabled()) {
- return Either.right(ActionStatus.NOT_ALLOWED);
- }
-
- Either<ComponentCacheData, ActionStatus> componentRes = componentCassandraDao.getComponent(componentUid);
-
- if (componentRes.isRight()) {
- return Either.right(componentRes.right().value());
- }
-
- ComponentCacheData componentCacheData = componentRes.left().value();
-
- if (lastModificationTime != null) {
- long cacheCompModificationTime = componentCacheData.getModificationTime().getTime();
- if (lastModificationTime != cacheCompModificationTime) {
- logger.debug(
- "Component {} found in cache but its modification time {} does not match to the timestamp in cache {}.",
- componentUid, lastModificationTime, cacheCompModificationTime);
- return Either.right(ActionStatus.INVALID_CONTENT);
- }
- }
-
- Either<? extends Component, Boolean> convertRes = convertComponentCacheToComponent(componentCacheData);
- if (convertRes.isRight()) {
- return Either.right(ActionStatus.CONVERT_COMPONENT_ERROR);
- }
-
- Component component = convertRes.left().value();
-
- Component filteredComponent = component;
- if (filterFieldsFunc != null) {
- filteredComponent = filterFieldsFunc.apply(component);
- }
-
- ImmutablePair<Component, ComponentCacheData> result = new ImmutablePair<Component, ComponentCacheData>(
- filteredComponent, componentCacheData);
-
- return Either.left(result);
- }
-
- public ActionStatus deleteComponentFromCache(String id) {
- if (false == isEnabled()) {
- return ActionStatus.NOT_ALLOWED;
- }
- CassandraOperationStatus status = componentCassandraDao.deleteComponent(id);
- if (CassandraOperationStatus.OK.equals(status)) {
- return ActionStatus.OK;
- } else {
- logger.debug("delete component failed with error {}", status);
- return ActionStatus.GENERAL_ERROR;
- }
- }
+ private static final String COMPONENT_CACHE_IS_DISABLED = "Component Cache is disabled";
+
+ private static final Logger log = Logger.getLogger(ComponentCache.class);
+
+ @Autowired
+ ComponentCassandraDao componentCassandraDao;
+
+ @Autowired
+ ToscaOperationFacade toscaOperationFacade;
+
+ private Map<ComponentTypeEnum, Map<String, Component>> catalogInMemoryCache = new HashMap<>();
+ private final ReentrantReadWriteLock rwCatalogLock = new ReentrantReadWriteLock();
+ private final Lock rCatalogLock = rwCatalogLock.readLock();
+ private final Lock wCatalogLock = rwCatalogLock.writeLock();
+
+ boolean enabled = true;
+ int catalogInMemorySizePerResource = 300;
+ int catalogInMemorySizePerService = 200;
+ int catalogInMemorySizePerProduct = 100;
+ boolean catalogInMemoryEnabled = true;
+ Map<ComponentTypeEnum, Integer> limitMemoryCatalogSizePerType = new HashMap<>();
+
+ @PostConstruct
+ public void init() {
+
+ Configuration configuration = ConfigurationManager.getConfigurationManager().getConfiguration();
+ if (configuration != null) {
+ ApplicationL2CacheConfig applicationL2Cache = configuration.getApplicationL2Cache();
+ if (applicationL2Cache != null) {
+ this.enabled = applicationL2Cache.isEnabled();
+
+ ApplicationL1CacheCatalogInfo catalog = applicationL2Cache.getCatalogL1Cache();
+ if (catalog != null) {
+ catalogInMemoryEnabled = catalog.getEnabled();
+ catalogInMemorySizePerResource = catalog.getResourcesSizeInCache();
+ catalogInMemorySizePerService = catalog.getServicesSizeInCache();
+ catalogInMemorySizePerProduct = catalog.getProductsSizeInCache();
+ }
+ }
+ }
+
+ ComponentTypeEnum[] typesForCache = { ComponentTypeEnum.RESOURCE, ComponentTypeEnum.SERVICE,
+ ComponentTypeEnum.PRODUCT };
+ for (ComponentTypeEnum typeEnum : typesForCache) {
+ Map<String, Component> map = new HashMap<>();
+ catalogInMemoryCache.put(typeEnum, map);
+ }
+
+ limitMemoryCatalogSizePerType.put(ComponentTypeEnum.RESOURCE, catalogInMemorySizePerResource);
+ limitMemoryCatalogSizePerType.put(ComponentTypeEnum.SERVICE, catalogInMemorySizePerService);
+ limitMemoryCatalogSizePerType.put(ComponentTypeEnum.PRODUCT, catalogInMemorySizePerProduct);
+ }
+
+ public boolean isEnabled() {
+ return enabled;
+ }
+
+ public void setEnabled(boolean enabled) {
+ this.enabled = enabled;
+ }
+
+ public Either<Component, ActionStatus> getComponent(String componentUid, Long lastModificationTime,
+ Function<Component, Component> filterFieldsFunc) {
+
+ Either<ImmutablePair<Component, ComponentCacheData>, ActionStatus> componentFromCache = getComponentFromCache(
+ componentUid, lastModificationTime, filterFieldsFunc);
+
+ if (componentFromCache.isRight()) {
+ return Either.right(componentFromCache.right().value());
+ }
+
+ return Either.left(componentFromCache.left().value().left);
+
+ }
+
+ public Either<List<ComponentCacheData>, ActionStatus> getAllComponentIdTimeAndType() {
+ if (!isEnabled()) {
+ return Either.right(ActionStatus.NOT_ALLOWED);
+ }
+
+ return componentCassandraDao
+ .getAllComponentIdTimeAndType();
+
+ }
+
+ /**
+ * get components for catalog
+ *
+ * @param components
+ * @param componentTypeEnum
+ * @return
+ */
+ @Deprecated
+ public Either<ImmutableTriple<List<Component>, List<Component>, Set<String>>, ActionStatus> getComponentsForCatalog(
+ Set<String> components, ComponentTypeEnum componentTypeEnum) {
+
+ if (!isEnabled()) {
+ log.debug("In getComponentsForCatalog for type {}. Cache is disabled.",
+ componentTypeEnum.name().toLowerCase());
+ return Either.right(ActionStatus.NOT_ALLOWED);
+ }
+ log.debug("In getComponentsForCatalog for type {}", componentTypeEnum.name().toLowerCase());
+
+ Function<List<Component>, List<Component>> filterFieldsFunc = this::filterForCatalog;
+
+ Set<String> leftComponentsForSearch = new HashSet<>();
+ leftComponentsForSearch.addAll(components);
+
+ // get components from inmemory cache
+ List<Component> componentsFromMemory = null;
+ if (catalogInMemoryEnabled) {
+ componentsFromMemory = getDataFromInMemoryCache(components, componentTypeEnum);
+ log.debug("The number of components of type {} fetched from memory is {}",
+ componentTypeEnum.name().toLowerCase(),
+ componentsFromMemory == null ? 0 : componentsFromMemory.size());
+ if (componentsFromMemory != null) {
+ componentsFromMemory.forEach(p -> leftComponentsForSearch.remove(p.getUniqueId()));
+ }
+ } else {
+ log.debug("Catalog InMemory cache is disabled");
+ }
+
+ log.debug("Number of components from type {} needed to fetch is {}", componentTypeEnum.name().toLowerCase(),
+ leftComponentsForSearch.size());
+
+ // get components from cassandra cache and filter each component
+ Either<ImmutableTriple<List<Component>, List<Component>, Set<String>>, ActionStatus> result = getComponents(
+ leftComponentsForSearch, filterFieldsFunc);
+
+ if (result.isLeft()) {
+ // add inmemory components to the valid components(not dirty)
+ List<Component> foundComponents = result.left().value().getLeft();
+ if (componentsFromMemory != null) {
+ foundComponents.addAll(componentsFromMemory);
+ }
+ if (catalogInMemoryEnabled) {
+ updateCatalogInMemoryCacheWithCertified(foundComponents, componentTypeEnum);
+ }
+ }
+
+ return result;
+ }
+
+ /**
+ * @param foundComponents
+ * @param componentTypeEnum
+ */
+ private void updateCatalogInMemoryCacheWithCertified(List<Component> foundComponents,
+ ComponentTypeEnum componentTypeEnum) {
+
+ try {
+ wCatalogLock.lock();
+
+ long start = System.currentTimeMillis();
+ Map<String, Component> map = catalogInMemoryCache.get(componentTypeEnum);
+ int mapSizeBefore = map.size();
+ map.clear();
+ Map<String, Component> collect = foundComponents.stream()
+ .filter(p -> p.getLifecycleState() == LifecycleStateEnum.CERTIFIED)
+ .limit(limitMemoryCatalogSizePerType.get(componentTypeEnum))
+ .collect(Collectors.toMap(Component::getUniqueId, p -> p));
+ map.putAll(collect);
+ log.debug(
+ "Size of in memory cache for catalog {}(certified only): Before {}, After {}. Replacement Time is {} ms.",
+ componentTypeEnum.name().toLowerCase(), mapSizeBefore, map.size(),
+ System.currentTimeMillis() - start);
+ } finally {
+ wCatalogLock.unlock();
+ }
+
+ }
+
+ private List<Component> getDataFromInMemoryCache(Set<String> components, ComponentTypeEnum componentTypeEnum) {
+ List<Component> foundComponents = new ArrayList<>();
+
+ try {
+
+ rCatalogLock.lock();
+
+ Map<String, Component> map = catalogInMemoryCache.get(componentTypeEnum);
+ for (String compUid : components) {
+ Component component = map.get(compUid);
+ if (component != null) {
+ foundComponents.add(component);
+ }
+ }
+
+ } finally {
+ rCatalogLock.unlock();
+ }
+
+ return foundComponents;
+ }
+
+ /**
+ *
+ * get full components from cassandra. On each component apply filter
+ * function in order to remove unused members
+ *
+ * @param components
+ * @param filterFieldsFunc
+ * @return <found components, found dirty components, not found components
+ * list> or Error
+ */
+ public Either<ImmutableTriple<List<Component>, List<Component>, Set<String>>, ActionStatus> getComponents(
+ Set<String> components, Function<List<Component>, List<Component>> filterFieldsFunc) {
+
+ if (!isEnabled()) {
+ log.debug(COMPONENT_CACHE_IS_DISABLED);
+ return Either.right(ActionStatus.NOT_ALLOWED);
+ }
+
+ Either<ImmutableTriple<List<Component>, List<Component>, Set<String>>, ActionStatus> componentsFull = getComponentsFull(
+ components);
+
+ if (componentsFull.isRight()) {
+ return Either.right(componentsFull.right().value());
+ }
+
+ ImmutableTriple<List<Component>, List<Component>, Set<String>> immutableTriple = componentsFull.left().value();
+ List<Component> foundResources = immutableTriple.left;
+ List<Component> foundDirtyResources = immutableTriple.middle;
+ Set<String> notFoundResources = immutableTriple.right;
+
+ List<Component> filterdFoundResources = filterFieldsFunc.apply(foundResources);
+ List<Component> filterdFoundDirtyResources = filterFieldsFunc.apply(foundDirtyResources);
+
+ ImmutableTriple<List<Component>, List<Component>, Set<String>> result = new ImmutableTriple<>(
+ filterdFoundResources, filterdFoundDirtyResources, notFoundResources);
+
+ return Either.left(result);
+
+ }
+
+ public Either<ImmutableTriple<List<Component>, List<Component>, Set<String>>, ActionStatus> getComponentsForLeftPanel(
+ ComponentTypeEnum componentTypeEnum, String internalComponentType, Set<String> filteredResources) {
+
+ log.debug("In getComponentsForLeftPanel componentTypeEnum = {}, internalComponentType = {}",
+ componentTypeEnum, internalComponentType);
+
+ Function<List<Component>, List<Component>> filterFieldsFunc = this::filterForLeftPanel;
+
+ return getComponents(filteredResources, filterFieldsFunc);
+
+ }
+
+ private List<Component> filterForLeftPanel(List<Component> components) {
+
+ List<Component> result = new ArrayList<>();
+ if (components != null) {
+ components.forEach(p -> result.add(filterFieldsForLeftPanel(p)));
+ }
+
+ return result;
+ }
+
+ private List<Component> filterForCatalog(List<Component> components) {
+
+ List<Component> result = new ArrayList<>();
+ if (components != null) {
+ components.forEach(p -> result.add(filterFieldsForCatalog(p)));
+ }
+
+ return result;
+ }
+
+ private Component filterFieldsForLeftPanel(Component component) {
+
+ Component result = null;
+ ComponentTypeEnum componentTypeEnum = component.getComponentType();
+ switch (componentTypeEnum) {
+ case RESOURCE:
+ result = new Resource();
+ copyFieldsForLeftPanel(component, result);
+ break;
+ case SERVICE:
+ result = new Service();
+ copyFieldsForLeftPanel(component, result);
+ break;
+ default:
+ break;
+ }
+
+ return result;
+ }
+
+ private Component filterFieldsForCatalog(Component component) {
+
+ Component result = null;
+ ComponentTypeEnum componentTypeEnum = component.getComponentType();
+ switch (componentTypeEnum) {
+ case RESOURCE:
+ result = new Resource();
+ copyFieldsForCatalog(component, result);
+ break;
+ case SERVICE:
+ result = new Service();
+ copyFieldsForCatalog(component, result);
+ break;
+ case PRODUCT:
+ result = new Product();
+ copyFieldsForCatalog(component, result);
+ break;
+ default:
+ break;
+ }
+
+ return result;
+ }
+
+ /**
+ * Copy relevant fields to the filtered component for left panel
+ *
+ * @param component
+ * @param filteredComponent
+ */
+ private void copyFieldsForLeftPanel(Component component, Component filteredComponent) {
+
+ ComponentTypeEnum componentTypeEnum = component.getComponentType();
+ filteredComponent.setCategories(component.getCategories());
+ filteredComponent.setComponentType(component.getComponentType());
+ if (ComponentTypeEnum.RESOURCE.equals(component.getComponentType())
+ && ResourceTypeEnum.VL.equals(((ResourceMetadataDataDefinition) component
+ .getComponentMetadataDefinition().getMetadataDataDefinition()).getResourceType())) {
+ filteredComponent.setCapabilities(component.getCapabilities());
+ filteredComponent.setRequirements(component.getRequirements());
+ }
+ filteredComponent.setVersion(component.getVersion());
+ filteredComponent.setDescription(component.getDescription());
+ filteredComponent.setUniqueId(component.getUniqueId());
+ filteredComponent.setIcon(component.getIcon());
+ filteredComponent.setTags(component.getTags());
+ filteredComponent.setLifecycleState(component.getLifecycleState());
+ filteredComponent.setInvariantUUID(component.getInvariantUUID());
+ filteredComponent.setUUID(component.getUUID());
+ filteredComponent.setSystemName(component.getSystemName());
+ filteredComponent.setName(component.getName());
+
+ if (componentTypeEnum == ComponentTypeEnum.RESOURCE) {
+ Resource resource = (Resource) component;
+ Resource filteredResource = (Resource) filteredComponent;
+ filteredResource.setToscaResourceName(resource.getToscaResourceName());
+ filteredResource.setResourceType(resource.getResourceType());
+ }
+ }
+
+ private void copyFieldsForCatalog(Component component, Component filteredComponent) {
+
+ ComponentTypeEnum componentTypeEnum = component.getComponentType();
+ filteredComponent.setCategories(component.getCategories());
+ filteredComponent.setComponentType(component.getComponentType());
+ filteredComponent.setVersion(component.getVersion());
+ filteredComponent.setDescription(component.getDescription());
+ filteredComponent.setUniqueId(component.getUniqueId());
+ filteredComponent.setIcon(component.getIcon());
+ filteredComponent.setTags(component.getTags());
+ filteredComponent.setLifecycleState(component.getLifecycleState());
+ filteredComponent.setSystemName(component.getSystemName());
+ filteredComponent.setName(component.getName());
+ filteredComponent.setLastUpdateDate(component.getLastUpdateDate());
+
+ if (componentTypeEnum == ComponentTypeEnum.RESOURCE) {
+ Resource resource = (Resource) component;
+ Resource filteredResource = (Resource) filteredComponent;
+ filteredResource.setToscaResourceName(resource.getToscaResourceName());
+ filteredResource.setResourceType(resource.getResourceType());
+ } else if (componentTypeEnum == ComponentTypeEnum.SERVICE) {
+ Service service = (Service) component;
+ Service filteredService = (Service) filteredComponent;
+ filteredService.setDistributionStatus(service.getDistributionStatus());
+ }
+ }
+
+ /**
+ * get components from cache of a given list ou unique ids.
+ *
+ * for each component data from cassandra, unzip the data if needed and
+ * deserialize the unzipped data to java object(Component).
+ *
+ * @param filteredResources
+ * @return ImmutableTripple or ActionStatus. | |-- components |-- dirty
+ * components - components with dirty flag = true. |-- set of non
+ * cached components
+ *
+ */
+ private Either<ImmutableTriple<List<Component>, List<Component>, Set<String>>, ActionStatus> getComponentsFull(
+ Set<String> filteredResources) {
+
+ if (!isEnabled()) {
+ log.debug(COMPONENT_CACHE_IS_DISABLED);
+ return Either.right(ActionStatus.NOT_ALLOWED);
+ }
+
+ List<Component> foundResources = new LinkedList<>();
+ List<Component> foundDirtyResources = new LinkedList<>();
+ Set<String> notFoundResources = new HashSet<>();
+ ImmutableTriple<List<Component>, List<Component>, Set<String>> result = new ImmutableTriple<>(
+ foundResources, foundDirtyResources, notFoundResources);
+
+ long cassandraFetchStart = System.currentTimeMillis();
+ List<String> uidsList = new ArrayList<>();
+ uidsList.addAll(filteredResources);
+ Either<List<ComponentCacheData>, ActionStatus> componentsFromCache = componentCassandraDao
+ .getComponents(uidsList);
+
+ long cassandraFetchEnd = System.currentTimeMillis();
+ log.debug("Fetch time from cassandara of all components took {} ms",
+ (cassandraFetchEnd - cassandraFetchStart));
+ if (componentsFromCache.isRight()) {
+ BeEcompErrorManager.getInstance().logInternalFlowError("FetchFromCache",
+ "Failed to fetch components from cache", ErrorSeverity.ERROR);
+ return Either.right(componentsFromCache.right().value());
+ }
+
+ List<ComponentCacheData> list = componentsFromCache.left().value();
+ log.debug("Number of components fetched from cassandra is {}", (list == null ? 0 : list.size()));
+ if (list != null && !list.isEmpty()) {
+
+ List<ComponentCacheData> filteredData = list.stream().filter(p -> filteredResources.contains(p.getId()))
+ .collect(Collectors.toList());
+ log.debug("Number of components filterd is {}", filteredData == null ? 0 : filteredData.size());
+
+ if (filteredData != null) {
+ long desStart = System.currentTimeMillis();
+
+ for (ComponentCacheData componentCacheData : filteredData) {
+
+ log.debug("Process uid {} from cache", componentCacheData.getId());
+
+ String compUid = componentCacheData.getId();
+
+ Either<? extends Component, Boolean> deserializeExt = convertComponentCacheToComponent(
+ componentCacheData);
+
+ if (deserializeExt.isLeft()) {
+ Component component = deserializeExt.left().value();
+ if (!componentCacheData.getIsDirty()) {
+ foundResources.add(component);
+ } else {
+ foundDirtyResources.add(component);
+ }
+ } else {
+ notFoundResources.add(compUid);
+ }
+
+ }
+ long desEnd = System.currentTimeMillis();
+ log.debug("Deserialization and unzip of {} components took {} ms", filteredData.size(),
+ (desEnd - desStart));
+ }
+ }
+ List<String> foundResourcesUid = foundResources.stream().map(Component::getUniqueId).collect(Collectors.toList());
+ List<String> foundDirtyResourcesUid = foundDirtyResources.stream().map(Component::getUniqueId)
+ .collect(Collectors.toList());
+ log.debug("Number of processed components from cache is {}",
+ (foundResourcesUid.size() + foundDirtyResourcesUid.size()));
+ Set<String> notCachedResources = filteredResources.stream()
+ .filter(p -> !foundResourcesUid.contains(p) && !foundDirtyResourcesUid.contains(p))
+ .collect(Collectors.toSet());
+ notFoundResources.addAll(notCachedResources);
+
+ if (log.isDebugEnabled()) {
+ log.debug("Number of components fetched is {}", foundResources.size());
+ log.debug("Number of components fetched dirty is {}", foundDirtyResources.size());
+ log.debug("Number of components non cached is {}", notCachedResources.size());
+ }
+
+ return Either.left(result);
+ }
+
+ private Either<? extends Component, Boolean> convertComponentCacheToComponent(
+ ComponentCacheData componentCacheData) {
+
+ String compUid = componentCacheData.getId();
+
+ byte[] dataAsArray = componentCacheData.getDataAsArray();
+
+ if (componentCacheData.getIsZipped()) {
+ long startUnzip = System.nanoTime();
+ dataAsArray = ZipUtil.unzip(dataAsArray);
+ long endUnzip = System.nanoTime();
+ log.trace("Unzip component {} took {} microsecond", compUid, (endUnzip - startUnzip) / 1000);
+ }
+
+ long startDes = System.nanoTime();
+
+ Either<? extends Component, Boolean> deserializeExt = deserializeComponent(componentCacheData, dataAsArray);
+
+ long endDes = System.nanoTime();
+ log.trace("Deserialize component {} took {} microsecond", compUid, (endDes - startDes) / 1000);
+ return deserializeExt;
+ }
+
+ private Either<? extends Component, Boolean> deserializeComponent(ComponentCacheData componentCacheData,
+ byte[] dataAsArray) {
+ String type = componentCacheData.getType();
+ NodeTypeEnum typeEnum = NodeTypeEnum.getByNameIgnoreCase(type);
+
+ Either<? extends Component, Boolean> deserializeExt = Either.right(false);
+ switch (typeEnum) {
+ case Resource:
+ deserializeExt = SerializationUtils.deserializeExt(dataAsArray, Resource.class, componentCacheData.getId());
+ break;
+ case Service:
+ deserializeExt = SerializationUtils.deserializeExt(dataAsArray, Service.class, componentCacheData.getId());
+ break;
+ case Product:
+ deserializeExt = SerializationUtils.deserializeExt(dataAsArray, Product.class, componentCacheData.getId());
+ break;
+ default:
+ break;
+ }
+ return deserializeExt;
+ }
+
+ public Either<Component, ActionStatus> getComponent(String componentUid) {
+
+ return getComponent(componentUid, null, Function.identity());
+
+ }
+
+ private boolean saveComponent(String componentUid, Long lastModificationTime, NodeTypeEnum nodeTypeEnum,
+ Component component) {
+
+ log.trace("Going to save component {} of type {} in cache", componentUid, nodeTypeEnum.name().toLowerCase());
+
+ boolean result = false;
+
+ Either<byte[], Boolean> serializeExt = SerializationUtils.serializeExt(component);
+ if (serializeExt.isLeft()) {
+ byte[] serializedData = serializeExt.left().value();
+ byte[] zipBytes;
+ try {
+ zipBytes = ZipUtil.zipBytes(serializedData);
+ ComponentCacheData componentCacheData = new ComponentCacheData();
+ componentCacheData.setDataAsArray(zipBytes);
+ componentCacheData.setIsZipped(true);
+ componentCacheData.setId(componentUid);
+ componentCacheData.setModificationTime(new Date(lastModificationTime));
+ componentCacheData.setType(component.getComponentType().name().toLowerCase());
+
+ CassandraOperationStatus status = componentCassandraDao.saveComponent(componentCacheData);
+
+ if (status == CassandraOperationStatus.OK) {
+ result = true;
+ }
+
+ } catch (IOException e) {
+ log.debug("Failed to prepare component {} of type {} for cache", componentUid,
+ nodeTypeEnum.name().toLowerCase());
+ if (log.isTraceEnabled()) {
+ log.trace("Failed to prepare component {} of type {} for cache",componentUid,nodeTypeEnum.name().toLowerCase());
+ }
+ }
+ } else {
+ log.debug("Failed to serialize component {} of type {} for cache", componentUid,
+ nodeTypeEnum.name().toLowerCase());
+ }
+ return result;
+ }
+
+ public boolean setComponent(Component component, NodeTypeEnum nodeTypeEnum) {
+
+ boolean result = false;
+
+ if (!isEnabled()) {
+ log.debug(COMPONENT_CACHE_IS_DISABLED);
+ return false;
+ }
+
+ String componentUid = component.getUniqueId();
+ Long lastUpdateDate = component.getLastUpdateDate();
+
+ result = saveComponent(componentUid, lastUpdateDate, nodeTypeEnum, component);
+
+ return result;
+
+ }
+
+ /**
+ * get components from cache of a given list ou unique ids.
+ *
+ * for each component data from cassandra, unzip the data if needed and
+ * deserialize the unzipped data to java object(Component).
+ *
+ * @param filteredResources
+ * @return ImmutableTripple or ActionStatus. | |-- components |-- set of non
+ * cached components
+ *
+ */
+ private Either<ImmutablePair<List<Component>, Set<String>>, ActionStatus> getComponentsFull(
+ Map<String, Long> filteredResources) {
+
+ if (!isEnabled()) {
+ log.debug(COMPONENT_CACHE_IS_DISABLED);
+ return Either.right(ActionStatus.NOT_ALLOWED);
+ }
+
+ List<Component> foundResources = new LinkedList<>();
+ Set<String> notFoundResources = new HashSet<>();
+ ImmutablePair<List<Component>, Set<String>> result = new ImmutablePair<>(
+ foundResources, notFoundResources);
+
+ long cassandraFetchStart = System.currentTimeMillis();
+
+ Either<ImmutablePair<List<ComponentCacheData>, Set<String>>, ActionStatus> componentsFromCache = componentCassandraDao
+ .getComponents(filteredResources);
+
+ long cassandraFetchEnd = System.currentTimeMillis();
+ log.debug("Fetch time from cassandara of all components took {} ms",
+ (cassandraFetchEnd - cassandraFetchStart));
+ if (componentsFromCache.isRight()) {
+ BeEcompErrorManager.getInstance().logInternalFlowError("FetchFromCache",
+ "Failed to fetch components from cache", ErrorSeverity.ERROR);
+ return Either.right(componentsFromCache.right().value());
+ }
+
+ ImmutablePair<List<ComponentCacheData>, Set<String>> immutablePair = componentsFromCache.left().value();
+ List<ComponentCacheData> list = immutablePair.getLeft();
+ log.debug("Number of components fetched from cassandra is {}", (list == null ? 0 : list.size()));
+ if (list != null && !list.isEmpty()) {
+
+ log.debug("Number of components filterd is {}", list == null ? 0 : list.size());
+
+ if (list != null) {
+ long desStart = System.currentTimeMillis();
+
+ for (ComponentCacheData componentCacheData : list) {
+
+ log.debug("Process uid {} from cache", componentCacheData.getId());
+
+ String compUid = componentCacheData.getId();
+
+ Either<? extends Component, Boolean> deserializeExt = convertComponentCacheToComponent(
+ componentCacheData);
+
+ if (deserializeExt.isLeft()) {
+ Component component = deserializeExt.left().value();
+ foundResources.add(component);
+ } else {
+ notFoundResources.add(compUid);
+ }
+
+ }
+ long desEnd = System.currentTimeMillis();
+ log.debug("Deserialization and unzip of {} components took {} ms", list.size(), (desEnd - desStart));
+ }
+ }
+ log.debug("Number of processed components from cache is {}", foundResources.size());
+
+ Set<String> notFoundInCache = immutablePair.getRight();
+ notFoundResources.addAll(notFoundInCache);
+
+ if (log.isDebugEnabled()) {
+ log.debug("Number of components fetched is {}", foundResources.size());
+ log.debug("Number of components non cached is {}", notFoundResources.size());
+ }
+
+ return Either.left(result);
+ }
+
+ /**
+ * get components for catalog
+ *
+ * @param components
+ * @param componentTypeEnum
+ * @return
+ */
+ public Either<ImmutablePair<List<Component>, Set<String>>, ActionStatus> getComponentsForCatalog(
+ Map<String, Long> components, ComponentTypeEnum componentTypeEnum) {
+
+ if (!isEnabled()) {
+ log.debug("In getComponentsForCatalog for type {}. Cache is disabled.",
+ componentTypeEnum.name().toLowerCase());
+ return Either.right(ActionStatus.NOT_ALLOWED);
+ }
+ log.debug("In getComponentsForCatalog for type {}", componentTypeEnum.name().toLowerCase());
+
+ Function<List<Component>, List<Component>> filterFieldsFunc = this::filterForCatalog;
+
+ Map<String, Long> leftComponentsForSearch = new HashMap<>();
+ leftComponentsForSearch.putAll(components);
+
+ // get components from inmemory cache
+ List<Component> componentsFromMemory = null;
+ if (catalogInMemoryEnabled) {
+ componentsFromMemory = getDataFromInMemoryCache(components.keySet(), componentTypeEnum);
+ log.debug("The number of components of type {} fetched from memory is {}",
+ componentTypeEnum.name().toLowerCase(),
+ componentsFromMemory == null ? 0 : componentsFromMemory.size());
+ if (componentsFromMemory != null) {
+ List<String> ignoredComponents = new ArrayList<>();
+ for (Component componentFromMem : componentsFromMemory) {
+ if (componentFromMem.getLastUpdateDate().longValue() != components
+ .get(componentFromMem.getUniqueId()).longValue()) {
+ // Ignore the component from memory
+ ignoredComponents.add(componentFromMem.getUniqueId());
+ }
+ }
+
+ log.debug("Number of components from type {} ignored from memory cache is {}",
+ componentTypeEnum.name().toLowerCase(), ignoredComponents.size());
+ // remove from memory result the components which are not valid
+ componentsFromMemory = componentsFromMemory.stream()
+ .filter(p -> !ignoredComponents.contains(p.getUniqueId())).collect(Collectors.toList());
+ // Remove from leftComponentsForSearch the valid components from
+ // memory
+ componentsFromMemory.forEach(p -> leftComponentsForSearch.remove(p.getUniqueId()));
+
+ }
+ } else {
+ log.debug("Catalog InMemory cache is disabled");
+ }
+
+ log.debug("Number of components from type {} needed to fetch is {}", componentTypeEnum.name().toLowerCase(),
+ leftComponentsForSearch.size());
+
+ // get components from cassandra cache and filter each component
+ Either<ImmutablePair<List<Component>, Set<String>>, ActionStatus> result = getComponents(
+ leftComponentsForSearch, filterFieldsFunc);
+
+ if (result.isLeft()) {
+ // add inmemory components to the valid components(not dirty)
+ List<Component> foundComponents = result.left().value().getLeft();
+ if (componentsFromMemory != null) {
+ foundComponents.addAll(componentsFromMemory);
+ }
+ if (catalogInMemoryEnabled) {
+ updateCatalogInMemoryCacheWithCertified(foundComponents, componentTypeEnum);
+ }
+ }
+
+ return result;
+ }
+
+ /**
+ * @param components
+ * - Map of <componentUniqueId, last update date>
+ * @param filterFieldsFunc
+ * @return
+ */
+ public Either<ImmutablePair<List<Component>, Set<String>>, ActionStatus> getComponents(Map<String, Long> components,
+ Function<List<Component>, List<Component>> filterFieldsFunc) {
+
+ if (!isEnabled()) {
+ log.debug(COMPONENT_CACHE_IS_DISABLED);
+ return Either.right(ActionStatus.NOT_ALLOWED);
+ }
+
+ Either<ImmutablePair<List<Component>, Set<String>>, ActionStatus> componentsFull = getComponentsFull(
+ components);
+
+ if (componentsFull.isRight()) {
+ return Either.right(componentsFull.right().value());
+ }
+
+ ImmutablePair<List<Component>, Set<String>> immutablePair = componentsFull.left().value();
+ List<Component> foundResources = immutablePair.left;
+ Set<String> notFoundResources = immutablePair.right;
+
+ List<Component> filterdFoundResources = filterFieldsFunc.apply(foundResources);
+
+ ImmutablePair<List<Component>, Set<String>> result = new ImmutablePair<>(
+ filterdFoundResources, notFoundResources);
+
+ return Either.left(result);
+
+ }
+
+ /**
+ * get the component and its modification time from cache
+ *
+ * @param componentUid
+ * @param filterFieldsFunc
+ * @return
+ */
+ public Either<ImmutablePair<Component, Long>, ActionStatus> getComponentAndTime(String componentUid,
+ Function<Component, Component> filterFieldsFunc) {
+
+ Either<ImmutablePair<Component, ComponentCacheData>, ActionStatus> componentFromCache = getComponentFromCache(
+ componentUid, null, filterFieldsFunc);
+
+ if (componentFromCache.isRight()) {
+ return Either.right(componentFromCache.right().value());
+ }
+
+ ImmutablePair<Component, ComponentCacheData> immutablePair = componentFromCache.left().value();
+
+ ImmutablePair<Component, Long> result = new ImmutablePair<>(immutablePair.left,
+ immutablePair.right.getModificationTime().getTime());
+
+ return Either.left(result);
+ }
+
+ private Either<ImmutablePair<Component, ComponentCacheData>, ActionStatus> getComponentFromCache(
+ String componentUid, Long lastModificationTime, Function<Component, Component> filterFieldsFunc) {
+ if (!isEnabled()) {
+ return Either.right(ActionStatus.NOT_ALLOWED);
+ }
+
+ Either<ComponentCacheData, ActionStatus> componentRes = componentCassandraDao.getComponent(componentUid);
+
+ if (componentRes.isRight()) {
+ return Either.right(componentRes.right().value());
+ }
+
+ ComponentCacheData componentCacheData = componentRes.left().value();
+
+ if (lastModificationTime != null) {
+ long cacheCompModificationTime = componentCacheData.getModificationTime().getTime();
+ if (lastModificationTime != cacheCompModificationTime) {
+ log.debug(
+ "Component {} found in cache but its modification time {} does not match to the timestamp in cache {}.",
+ componentUid, lastModificationTime, cacheCompModificationTime);
+ return Either.right(ActionStatus.INVALID_CONTENT);
+ }
+ }
+
+ Either<? extends Component, Boolean> convertRes = convertComponentCacheToComponent(componentCacheData);
+ if (convertRes.isRight()) {
+ return Either.right(ActionStatus.CONVERT_COMPONENT_ERROR);
+ }
+
+ Component component = convertRes.left().value();
+
+ Component filteredComponent = component;
+ if (filterFieldsFunc != null) {
+ filteredComponent = filterFieldsFunc.apply(component);
+ }
+
+ ImmutablePair<Component, ComponentCacheData> result = new ImmutablePair<>(
+ filteredComponent, componentCacheData);
+
+ return Either.left(result);
+ }
+
+ public ActionStatus deleteComponentFromCache(String id) {
+ if (!isEnabled()) {
+ return ActionStatus.NOT_ALLOWED;
+ }
+ CassandraOperationStatus status = this.componentCassandraDao.deleteComponent(id);
+ if (CassandraOperationStatus.OK.equals(status)) {
+ return ActionStatus.OK;
+ } else {
+ log.debug("delete component failed with error {}", status);
+ return ActionStatus.GENERAL_ERROR;
+ }
+ }
}
Code Review / sdc.git / blobdiff