/** * ============LICENSE_START======================================================= * org.onap.aai * ================================================================================ * Copyright © 2017-18 AT&T Intellectual Property. All rights reserved. * ================================================================================ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. * ============LICENSE_END========================================================= */ package org.onap.aai.edges; import com.att.eelf.configuration.EELFLogger; import com.att.eelf.configuration.EELFManager; import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; import com.google.common.collect.ArrayListMultimap; import com.google.common.collect.Multimap; import com.jayway.jsonpath.Criteria; import com.jayway.jsonpath.DocumentContext; import com.jayway.jsonpath.Filter; import org.apache.tinkerpop.gremlin.structure.Direction; import org.onap.aai.edges.enums.DirectionNotation; import org.onap.aai.edges.enums.EdgeField; import org.onap.aai.edges.enums.EdgeType; import org.onap.aai.edges.exceptions.AmbiguousRuleChoiceException; import org.onap.aai.edges.exceptions.EdgeRuleNotFoundException; import org.onap.aai.setup.SchemaVersion; import org.onap.aai.setup.SchemaVersions; import org.onap.aai.setup.Translator; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import javax.annotation.PostConstruct; import java.io.IOException; import java.util.*; import java.util.Map.Entry; import java.util.concurrent.ExecutionException; import java.util.stream.Collectors; import static com.jayway.jsonpath.Criteria.where; import static com.jayway.jsonpath.Filter.filter; /** * EdgeIngestor - ingests A&AI edge rule schema files per given config, serves that edge rule * information, including allowing various filters to extract particular rules. */ @Component public class EdgeIngestor { private static final EELFLogger LOGGER = EELFManager.getInstance().getLogger(EdgeIngestor.class); private Map> versionJsonFilesMap; private static final String READ_START = "$.rules.[?]"; private static final String READ_ALL_START = "$.rules.*"; private SchemaVersions schemaVersions; Map> filesToIngest; private Set multipleLabelKeys; private LoadingCache> cacheFilterStore; private LoadingCache cousinLabelStore; private Set translators; @Autowired public EdgeIngestor(Set translatorSet) { LOGGER.debug("Local Schema files will be fetched"); this.translators = translatorSet; } @PostConstruct public void initialize() { for (Translator translator : translators) { try { LOGGER.debug("Processing the translator"); translateAll(translator); } catch (Exception e) { LOGGER.debug("Error while Processing the translator" + e.getMessage()); continue; } } if (versionJsonFilesMap.isEmpty() || schemaVersions==null ) { throw new ExceptionInInitializerError(); } } public void translateAll(Translator translator) { /* Use SchemaVersions from the Translator */ this.schemaVersions = translator.getSchemaVersions(); List schemaVersionList = this.schemaVersions.getVersions(); List jsonPayloads = null; JsonIngestor ji = new JsonIngestor(); Map> edgeRulesToIngest = new HashMap<>(); // Obtain a map of schema versions to a list of strings. One List per key // Add to the map the JSON file per version. for (SchemaVersion version : schemaVersionList) { LOGGER.debug("Version being processed" + version); // If the flag is set to not use the local files, obtain the Json from the service. try { jsonPayloads = translator.getJsonPayload(version); // need to change this - need to receive the json files. } catch (IOException e) { LOGGER.info("Exception in retrieving the JSON Payload"+e.getMessage()); } if (jsonPayloads == null || jsonPayloads.isEmpty()) { continue; } LOGGER.debug("Retrieved json from SchemaService"); edgeRulesToIngest.put(version, jsonPayloads); } versionJsonFilesMap = ji.ingestContent(edgeRulesToIngest); this.cacheFilterStore = CacheBuilder.newBuilder() .maximumSize(2000) .build( new CacheLoader>() { @Override public Multimap load(SchemaFilter key) { return extractRules(key); } } ); this.cousinLabelStore = CacheBuilder.newBuilder() .maximumSize(50) .build( new CacheLoader() { @Override public String[] load(String key) throws Exception { return retrieveCousinLabels(key); } } ); } // //-----methods for getting rule info-----// // /** * Gets list of all edge rules defined in the latest version's schema * * @return Multimap of node names keys to the EdgeRules associated with those types * where the key takes the form of * {alphabetically first nodetype}|{alphabetically second nodetype}. Map will be empty if * no rules are found. * ex: buildAlphabetizedKey("l-interface", "logical-link") -> "l-interface|logical-link" * buildAlphabetizedKey("logical-link", "l-interface") -> "l-interface|logical-link" * * This is alphabetical order to normalize the keys, as sometimes there will be multiple * rules for a pair of node types but the from/to value in the json is flipped for some of them. * @throws EdgeRuleNotFoundException if none found */ public Multimap getAllCurrentRules() throws EdgeRuleNotFoundException { return getAllRules(schemaVersions.getDefaultVersion()); } /** * Retrieves all the nodes that contain multiple edge labels * * A lazy instantiation to retrieve all this info on first call * * @return a set containing a list of strings where each string is * concatenated by a pipe (|) character such as aNodeType|bNodeType */ public Set getMultipleLabelKeys(){ if(multipleLabelKeys == null){ multipleLabelKeys = new HashSet<>(); try { final Multimap edges = this.getAllCurrentRules(); if(edges == null || edges.isEmpty()){ LOGGER.warn("Unable to find any edge rules for the latest version"); return multipleLabelKeys; } edges.keySet().forEach(key -> { Collection rules = edges.get(key); if(rules.size() > 1){ multipleLabelKeys.add(key); } }); } catch (EdgeRuleNotFoundException e) { LOGGER.info("For the latest schema version, unable to find any edges with multiple keys"); } } return multipleLabelKeys; } /** * Gets list of all edge rules defined in the given version's schema * * @return Multimap of node names keys to the EdgeRules associated with those types * where the key takes the form of * {alphabetically first nodetype}|{alphabetically second nodetype}. Map will be empty if * no rules are found. * ex: buildAlphabetizedKey("l-interface", "logical-link") -> "l-interface|logical-link" * buildAlphabetizedKey("logical-link", "l-interface") -> "l-interface|logical-link" * * This is alphabetical order to normalize the keys, as sometimes there will be multiple * rules for a pair of node types but the from/to value in the json is flipped for some of them. * @throws EdgeRuleNotFoundException if none found */ public Multimap getAllRules(SchemaVersion v) throws EdgeRuleNotFoundException { Multimap found = extractRules(null, v); if (found.isEmpty()) { throw new EdgeRuleNotFoundException("No rules found for version " + v.toString() + "."); } else { return found; } } /** * Finds the rules (if any) matching the given query criteria. If none, the returned Multimap * will be empty. * * @param q - EdgeRuleQuery with filter criteria set * * @return Multimap of node names keys to the EdgeRules where the key takes the form of * {alphabetically first nodetype}|{alphabetically second nodetype}. Map will be empty if * no rules are found. * ex: buildAlphabetizedKey("l-interface", "logical-link") -> "l-interface|logical-link" * buildAlphabetizedKey("logical-link", "l-interface") -> "l-interface|logical-link" * * This is alphabetical order to normalize the keys, as sometimes there will be multiple * rules for a pair of node types but the from/to value in the json is flipped for some of them. * @throws EdgeRuleNotFoundException if none found */ public Multimap getRules(EdgeRuleQuery q) throws EdgeRuleNotFoundException { Multimap found = null; if(q.getVersion().isPresent()){ found = extractRules(q.getFilter(), q.getVersion().get()); } else { found = extractRules(q.getFilter(), schemaVersions.getDefaultVersion()); } if (found.isEmpty()) { throw new EdgeRuleNotFoundException("No rules found for " + q.toString()); } else { Multimap copy = ArrayListMultimap.create(); found.entries().stream().forEach((entry) -> { EdgeRule rule = new EdgeRule(entry.getValue()); if (!q.getFromType().equals(rule.getFrom())) { /* To maintain backwards compatibility with old EdgeRules API, * where the direction of the returned EdgeRule would be * flipped (if necessary) to match the directionality of * the input params. * ie, If the rule is from=A,to=B,direction=OUT, * if the user asked (A,B) the direction would be OUT, * if they asked (B,A), it would be IN to match. */ rule.flipDirection(); } copy.put(entry.getKey(), rule); }); return copy; } } /** * Gets the rule satisfying the given filter criteria. If there are more than one * that match, return the default rule. If there is no clear default to return, or * no rules match at all, error. * * @param q - EdgeRuleQuery with filter criteria set * @return EdgeRule satisfying given criteria * @throws EdgeRuleNotFoundException if none found that match * @throws AmbiguousRuleChoiceException if multiple match but no way to choice one from them * Specifically, if multiple node type pairs come back (ie bar|foo and asdf|foo, * no way to know which is appropriate over the others), * or if there is a mix of Tree and Cousin edges because again there is no way to * know which is "defaulter" than the other. * The default property only clarifies among multiple cousin edges of the same node pair, * ex: which l-interface|logical-link rule to default to. */ public EdgeRule getRule(EdgeRuleQuery q) throws EdgeRuleNotFoundException, AmbiguousRuleChoiceException { Multimap found = null; if(q.getVersion().isPresent()){ found = extractRules(q.getFilter(), q.getVersion().get()); } else { found = extractRules(q.getFilter(), schemaVersions.getDefaultVersion()); } if (found.isEmpty()) { throw new EdgeRuleNotFoundException("No rule found for " + q.toString() + "."); } EdgeRule rule = null; if (found.keys().size() == 1) { //only one found, cool we're done for (Entry e : found.entries()) { rule = e.getValue(); } } else { rule = getDefaultRule(found); } if (rule == null) { //should never get here though throw new EdgeRuleNotFoundException("No rule found for " + q.toString() + "."); } else { rule = new EdgeRule(rule); if (!q.getFromType().equals(rule.getFrom())) { /* To maintain backwards compatibility with old EdgeRules API, * where the direction of the returned EdgeRule would be * flipped (if necessary) to match the directionality of * the input params. * ie, If the rule is from=A,to=B,direction=OUT, * if the user asked (A,B) the direction would be OUT, * if they asked (B,A), it would be IN to match. */ rule.flipDirection(); } return rule; } } private EdgeRule getDefaultRule(Multimap found) throws AmbiguousRuleChoiceException { if (found.keySet().size() > 1) { //ie multiple node pairs (a|c and b|c not just all a|c) case StringBuilder sb = new StringBuilder(); for (String k : found.keySet()) { sb.append(k).append(" "); } throw new AmbiguousRuleChoiceException("No way to select single rule from these pairs: " + sb.toString() + "."); } int defaultCount = 0; EdgeRule defRule = null; for (Entry e : found.entries()) { EdgeRule rule = e.getValue(); if (rule.isDefault()) { defaultCount++; defRule = rule; } } if (defaultCount > 1) { throw new AmbiguousRuleChoiceException("Multiple defaults found."); } else if (defaultCount == 0) { throw new AmbiguousRuleChoiceException("No default found."); } return defRule; } /** * Checks if there exists any rule that satisfies the given filter criteria. * * @param q - EdgeRuleQuery with filter criteria set * @return boolean */ public boolean hasRule(EdgeRuleQuery q) { if(q.getVersion().isPresent()){ return !extractRules(q.getFilter(), q.getVersion().get()).isEmpty(); } else { return !extractRules(q.getFilter(), schemaVersions.getDefaultVersion()).isEmpty(); } } /** * Gets all cousin rules for the given node type in the latest schema version. * * @param nodeType * @return Multimap of node names keys to the EdgeRules where the key takes the form of * {alphabetically first nodetype}|{alphabetically second nodetype}. Map will be empty if * no rules are found. * ex: buildAlphabetizedKey("l-interface", "logical-link") -> "l-interface|logical-link" * buildAlphabetizedKey("logical-link", "l-interface") -> "l-interface|logical-link" * * This is alphabetical order to normalize the keys, as sometimes there will be multiple * rules for a pair of node types but the from/to value in the json is flipped for some of them. */ public Multimap getCousinRules(String nodeType) { return getCousinRules(nodeType, schemaVersions.getDefaultVersion()); //default to latest } public String[] retrieveCousinLabels(String nodeType){ Multimap cousinRules = getCousinRules(nodeType); String[] cousinLabels = new String[cousinRules.size()]; return cousinRules.entries() .stream() .map(entry -> entry.getValue().getLabel()) .collect(Collectors.toList()) .toArray(cousinLabels); } public String[] retrieveCachedCousinLabels(String nodeType) throws ExecutionException { return cousinLabelStore.get(nodeType); } /** * Gets all cousin rules for the given node type in the given schema version. * * @param nodeType * @param v - the version of the edge rules to query * @return Multimap of node names keys to the EdgeRules where the key takes the form of * {alphabetically first nodetype}|{alphabetically second nodetype}. Map will be empty if * no rules are found. * ex: buildAlphabetizedKey("l-interface", "logical-link") -> "l-interface|logical-link" * buildAlphabetizedKey("logical-link", "l-interface") -> "l-interface|logical-link" * * This is alphabetical order to normalize the keys, as sometimes there will be multiple * rules for a pair of node types but the from/to value in the json is flipped for some of them. */ public Multimap getCousinRules(String nodeType, SchemaVersion v) { return extractRules(new EdgeRuleQuery.Builder(nodeType).edgeType(EdgeType.COUSIN).build().getFilter(), v); } /** * Returns if the given node type has any cousin relationships in the current version. * @param nodeType * @return boolean */ public boolean hasCousinRule(String nodeType) { return hasCousinRule(nodeType, schemaVersions.getDefaultVersion()); } /** * Returns if the given node type has any cousin relationships in the given version. * @param nodeType * @return boolean */ public boolean hasCousinRule(String nodeType, SchemaVersion v) { return !getCousinRules(nodeType, v).isEmpty(); } /** * Gets all rules where "{given nodeType} contains {otherType}" in the latest schema version. * * @param nodeType - node type that is the container in the returned relationships * @return Multimap of node names keys to the EdgeRules where the key takes the form of * {alphabetically first nodetype}|{alphabetically second nodetype}. Map will be empty if * no rules are found. * ex: buildAlphabetizedKey("l-interface", "logical-link") -> "l-interface|logical-link" * buildAlphabetizedKey("logical-link", "l-interface") -> "l-interface|logical-link" * * This is alphabetical order to normalize the keys, as sometimes there will be multiple * rules for a pair of node types but the from/to value in the json is flipped for some of them. */ public Multimap getChildRules(String nodeType) { return getChildRules(nodeType, schemaVersions.getDefaultVersion()); } /** * Gets all rules where "{given nodeType} contains {otherType}" in the given schema version. * * @param nodeType - node type that is the container in the returned relationships * @return Multimap of node names keys to the EdgeRules where the key takes the form of * {alphabetically first nodetype}|{alphabetically second nodetype}. Map will be empty if * no rules are found. * ex: buildAlphabetizedKey("l-interface", "logical-link") -> "l-interface|logical-link" * buildAlphabetizedKey("logical-link", "l-interface") -> "l-interface|logical-link" * * This is alphabetical order to normalize the keys, as sometimes there will be multiple * rules for a pair of node types but the from/to value in the json is flipped for some of them. */ public Multimap getChildRules(String nodeType, SchemaVersion v) { Filter from = assembleFilterSegments(where(EdgeField.FROM.toString()).is(nodeType), getSameDirectionContainmentCriteria()); Filter to = assembleFilterSegments(where(EdgeField.TO.toString()).is(nodeType), getOppositeDirectionContainmentCriteria()); Filter total = from.or(to); return extractRules(total, v); } /** * Returns if the given node type has any child relationships (ie it contains another node type) in the current version. * @param nodeType * @return boolean */ public boolean hasChildRule(String nodeType) { return hasChildRule(nodeType, schemaVersions.getDefaultVersion()); } /** * Returns if the given node type has any child relationships (ie it contains another node type) in the given version. * @param nodeType * @return boolean */ public boolean hasChildRule(String nodeType, SchemaVersion v) { return !getChildRules(nodeType, v).isEmpty(); } /** * Gets all rules where "{given nodeType} is contained by {otherType}" in the latest schema version. * * @param nodeType - node type that is the containee in the returned relationships * @return Multimap of node names keys to the EdgeRules where the key takes the form of * {alphabetically first nodetype}|{alphabetically second nodetype}. Map will be empty if * no rules are found. * ex: buildAlphabetizedKey("l-interface", "logical-link") -> "l-interface|logical-link" * buildAlphabetizedKey("logical-link", "l-interface") -> "l-interface|logical-link" * * This is alphabetical order to normalize the keys, as sometimes there will be multiple * rules for a pair of node types but the from/to value in the json is flipped for some of them. */ public Multimap getParentRules(String nodeType) { return getParentRules(nodeType, schemaVersions.getDefaultVersion()); } /** * Gets all rules where "{given nodeType} is contained by {otherType}" in the given schema version. * * @param nodeType - node type that is the containee in the returned relationships * @return Multimap of node names keys to the EdgeRules where the key takes the form of * {alphabetically first nodetype}|{alphabetically second nodetype}. Map will be empty if * no rules are found. * ex: buildAlphabetizedKey("l-interface", "logical-link") -> "l-interface|logical-link" * buildAlphabetizedKey("logical-link", "l-interface") -> "l-interface|logical-link" * * This is alphabetical order to normalize the keys, as sometimes there will be multiple * rules for a pair of node types but the from/to value in the json is flipped for some of them. */ public Multimap getParentRules(String nodeType, SchemaVersion v) { Filter from = assembleFilterSegments(where(EdgeField.FROM.toString()).is(nodeType), getOppositeDirectionContainmentCriteria()); Filter to = assembleFilterSegments(where(EdgeField.TO.toString()).is(nodeType), getSameDirectionContainmentCriteria()); Filter total = from.or(to); return extractRules(total, v); } /** * Returns if the given node type has any parent relationships (ie it is contained by another node type) in the current version. * @param nodeType * @return boolean */ public boolean hasParentRule(String nodeType) { return hasParentRule(nodeType, schemaVersions.getDefaultVersion()); } /** * Returns if the given node type has any parent relationships (ie it is contained by another node type) in the given version. * @param nodeType * @return boolean */ public boolean hasParentRule(String nodeType, SchemaVersion v) { return !getParentRules(nodeType, v).isEmpty(); } /** * Applies the given filter to the DocumentContext(s) for the given version to extract * edge rules, and converts this extracted information into the Multimap form * * @param filter - JsonPath filter to read the DocumentContexts with. May be null * to denote no filter, ie get all. * @param v - The schema version to extract from * @return Multimap of node names keys to the EdgeRules where the key takes the form of * {alphabetically first nodetype}|{alphabetically second nodetype}. Map will be empty if * no rules are found. * ex: buildAlphabetizedKey("l-interface", "logical-link") -> "l-interface|logical-link" * buildAlphabetizedKey("logical-link", "l-interface") -> "l-interface|logical-link" * * This is alphabetical order to normalize the keys, as sometimes there will be multiple * rules for a pair of node types but the from/to value in the json is flipped for some of them. */ private Multimap extractRules(Filter filter, SchemaVersion v) { SchemaFilter schemaFilter = new SchemaFilter(filter, v); try { return cacheFilterStore.get(schemaFilter); } catch (ExecutionException e) { LOGGER.info("Encountered exception during the retrieval of the rules"); return ArrayListMultimap.create(); } } public Multimap extractRules(SchemaFilter schemaFilter){ List> foundRules = new ArrayList<>(); List docs = versionJsonFilesMap.get(schemaFilter.getSchemaVersion()); if (docs != null) { for (DocumentContext doc : docs) { if (schemaFilter.getFilter() == null) { foundRules.addAll(doc.read(READ_ALL_START)); } else { foundRules.addAll(doc.read(READ_START, Filter.parse(schemaFilter.getFilter()))); } } } return convertToEdgeRules(foundRules); } //-----filter building helpers-----// /** * ANDs together the given start criteria with each criteria in the pieces list, and * then ORs together these segments into one filter. * * JsonPath doesn't have an OR method on Criteria, only on Filters, so assembling * a complete filter requires this sort of roundabout construction. * * @param start - Criteria of the form where(from/to).is(nodeType) * (ie the start of any A&AI edge rule query) * @param pieces - Other Criteria to be applied * @return Filter constructed from the given Criteria */ private Filter assembleFilterSegments(Criteria start, List pieces) { List segments = new ArrayList<>(); for (Criteria c : pieces) { segments.add(filter(start).and(c)); } Filter assembled = segments.remove(0); for (Filter f : segments) { assembled = assembled.or(f); } return assembled; } /** * Builds the sub-Criteria for a containment edge rule query where the direction * and containment fields must match. * * Used for getChildRules() where the container node type is in the "from" position and * for getParentRules() where the containee type is in the "to" position. * * @return List covering property permutations defined with either notation or explicit direction */ private List getSameDirectionContainmentCriteria() { List crits = new ArrayList<>(); crits.add(where(EdgeField.CONTAINS.toString()).is(DirectionNotation.DIRECTION.toString())); crits.add(where(EdgeField.DIRECTION.toString()).is(Direction.OUT.toString()) .and(EdgeField.CONTAINS.toString()).is(Direction.OUT.toString())); crits.add(where(EdgeField.DIRECTION.toString()).is(Direction.IN.toString()) .and(EdgeField.CONTAINS.toString()).is(Direction.IN.toString())); return crits; } /** * Builds the sub-Criteria for a containment edge rule query where the direction * and containment fields must not match. * * Used for getChildRules() where the container node type is in the "to" position and * for getParentRules() where the containee type is in the "from" position. * * @return List covering property permutations defined with either notation or explicit direction */ private List getOppositeDirectionContainmentCriteria() { List crits = new ArrayList<>(); crits.add(where(EdgeField.CONTAINS.toString()).is(DirectionNotation.OPPOSITE.toString())); crits.add(where(EdgeField.DIRECTION.toString()).is(Direction.OUT.toString()) .and(EdgeField.CONTAINS.toString()).is(Direction.IN.toString())); crits.add(where(EdgeField.DIRECTION.toString()).is(Direction.IN.toString()) .and(EdgeField.CONTAINS.toString()).is(Direction.OUT.toString())); return crits; } //-----rule packaging helpers-----// /** * Converts the raw output from reading the json file to the Multimap format * * @param allFound - raw edge rule output read from json file(s) * (could be empty if none found) * @return Multimap of node names keys to the EdgeRules where the key takes the form of * {alphabetically first nodetype}|{alphabetically second nodetype}. Will be empty if input * was empty. * ex: buildAlphabetizedKey("l-interface", "logical-link") -> "l-interface|logical-link" * buildAlphabetizedKey("logical-link", "l-interface") -> "l-interface|logical-link" * * This is alphabetical order to normalize the keys, as sometimes there will be multiple * rules for a pair of node types but the from/to value in the json is flipped for some of them. */ private Multimap convertToEdgeRules(List> allFound) { Multimap rules = ArrayListMultimap.create(); TypeAlphabetizer alpher = new TypeAlphabetizer(); for (Map raw : allFound) { EdgeRule converted = new EdgeRule(raw); if (converted.getFrom().equals(converted.getTo())) { /* the way the code worked in the past was with outs and * when we switched it to in the same-node-type to * same-node-type parent child edges were failing because all * of the calling code would pass the parent as the left argument, * so it was either in that method swap the parent/child, * flip the edge rule or make all callers swap. the last seemed * like a bad idea. and felt like the edge flip was the better * of the remaining 2 */ converted.flipDirection(); } String alphabetizedKey = alpher.buildAlphabetizedKey(raw.get(EdgeField.FROM.toString()), raw.get(EdgeField.TO.toString())); rules.put(alphabetizedKey, converted); } return rules; } }