src/main/java/org/onap/aai/migration/v14/PserverDedupWithDifferentSourcesOfTruth.java

   1 /**
   2  * ============LICENSE_START=======================================================
   3  * org.onap.aai
   4  * ================================================================================
   5  * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
   6  * ================================================================================
   7  * Licensed under the Apache License, Version 2.0 (the "License");
   8  * you may not use this file except in compliance with the License.
   9  * You may obtain a copy of the License at
  10  *
  11  *    http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  * ============LICENSE_END=========================================================
  19  */
  20 package org.onap.aai.migration.v14;
  21
  22 import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
  23 import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__;
  24 import org.apache.tinkerpop.gremlin.process.traversal.P;
  25 import org.apache.tinkerpop.gremlin.structure.*;
  26 import org.janusgraph.core.attribute.Text;
  27 import org.javatuples.Pair;
  28 import org.onap.aai.db.props.AAIProperties;
  29 import org.onap.aai.edges.EdgeIngestor;
  30 import org.onap.aai.edges.enums.AAIDirection;
  31 import org.onap.aai.edges.enums.EdgeProperty;
  32 import org.onap.aai.exceptions.AAIException;
  33 import org.onap.aai.introspection.LoaderFactory;
  34 import org.onap.aai.migration.*;
  35 import org.onap.aai.serialization.engines.TransactionalGraphEngine;
  36 import org.springframework.web.util.UriUtils;
  37
  38 import javax.ws.rs.core.UriBuilder;
  39
  40 import java.io.UnsupportedEncodingException;
  41 import java.net.URI;
  42 import java.util.*;
  43 import java.util.stream.Collectors;
  44
  45 import org.onap.aai.serialization.db.EdgeSerializer;
  46 import org.onap.aai.setup.SchemaVersions;
  47 import org.onap.aai.introspection.Introspector;
  48
  49 //@Enabled
  50 @MigrationPriority(10)
  51 @MigrationDangerRating(100)
  52 public class PserverDedupWithDifferentSourcesOfTruth extends EdgeSwingMigrator {
  53     /**
  54      * Instantiates a new migrator.
  55      *
  56      * @param engine
  57      */
  58     private final String PARENT_NODE_TYPE = "pserver";
  59     private boolean success = true;
  60     protected Set<Object> seen = new HashSet<>();
  61     private Map<String, UriBuilder> nodeTypeToUri;
  62     private Map<String, Set<String>> nodeTypeToKeys;
  63     private static List<String> dmaapMsgList = new ArrayList<String>();
  64     private static List<Introspector> dmaapDeleteList = new ArrayList<Introspector>();
  65     private static int pserversUpdatedCount = 0;
  66     private static int pserversDeletedCount = 0;
  67
  68
  69     private static String[] rctSourceOfTruth = new String[]{"AAIRctFeed", "RCT"};
  70     private static String[] roSourceOfTruth = new String[]{"AAI-EXTENSIONS", "RO"};
  71
  72     List<Vertex> RemoveROList = new ArrayList<>();
  73
  74     public PserverDedupWithDifferentSourcesOfTruth(TransactionalGraphEngine engine , LoaderFactory loaderFactory, EdgeIngestor edgeIngestor, EdgeSerializer edgeSerializer, SchemaVersions schemaVersions) {
  75         super(engine, loaderFactory, edgeIngestor, edgeSerializer, schemaVersions);
  76     }
  77     @Override
  78     public void commit() {
  79         engine.commit();
  80         createDmaapFiles(dmaapMsgList);
  81         createDmaapFilesForDelete(dmaapDeleteList);
  82
  83     }
  84
  85     @Override
  86     public Status getStatus() {
  87         if (success) {
  88             return Status.SUCCESS;
  89         }
  90         else {
  91             return Status.FAILURE;
  92         }
  93     }
  94
  95     @Override
  96     public List<Pair<Vertex, Vertex>> getAffectedNodePairs() {
  97         return null;
  98     }
  99
 100     @Override
 101     public String getNodeTypeRestriction() {
 102         return null;
 103     }
 104
 105     @Override
 106     public String getEdgeLabelRestriction() {
 107         return null;
 108     }
 109
 110     @Override
 111     public String getEdgeDirRestriction() {
 112         return null;
 113     }
 114
 115     @Override
 116     public void cleanupAsAppropriate(List<Pair<Vertex, Vertex>> nodePairL) {
 117
 118     }
 119
 120     @Override
 121     public Optional<String[]> getAffectedNodeTypes() {
 122         return null;
 123     }
 124
 125     @Override
 126     public String getMigrationName() {
 127         return "PserverDedupWithDifferentSourcesOfTruth";
 128     }
 129
 130     @Override
 131     public void run() {
 132
 133         int dupCount = 0;
 134         nodeTypeToUri = loader.getAllObjects().entrySet().stream().filter(e -> e.getValue().getGenericURI().contains("{")).collect(
 135                 Collectors.toMap(
 136                         e -> e.getKey(),
 137                         e -> UriBuilder.fromPath(e.getValue().getFullGenericURI().replaceAll("\\{"+ e.getKey() + "-", "{"))
 138                 ));
 139
 140         nodeTypeToKeys = loader.getAllObjects().entrySet().stream().filter(e -> e.getValue().getGenericURI().contains("{")).collect(
 141                 Collectors.toMap(
 142                         e -> e.getKey(),
 143                         e -> e.getValue().getKeys()
 144                 ));
 145
 146         List<Vertex> rctList = graphTraversalSource().V().has("aai-node-type", "pserver").has("source-of-truth", P.within(rctSourceOfTruth)).toList();
 147         List<Vertex> roList =  graphTraversalSource().V().has("aai-node-type", "pserver").has("source-of-truth", P.within(roSourceOfTruth)).toList();
 148
 149         logger.info("Total number of RCT sourced pservers in A&AI :" +rctList.size());
 150         logger.info("Total number of RO sourced pservers in A&AI :" +roList.size());
 151
 152         for(int i=0;i<rctList.size();i++){
 153             Vertex currRct = rctList.get(i);
 154             Object currRctFqdn = null;
 155             if (currRct.property("fqdn").isPresent() && (currRct.property("fqdn").value() != null)){
 156                 currRctFqdn = currRct.property("fqdn").value();
 157                 logger.info("\n");
 158                 logger.info("Current RCT Pserver hostname: " + currRct.property("hostname").value().toString() + " fqdn: " +currRct.property("fqdn").value().toString());
 159                     for(int j=0;j<roList.size();j++){
 160                         Vertex currRo = roList.get(j);
 161                         Object currRoHostname = null;
 162                         if (currRo.property("hostname").isPresent()){
 163                                 currRoHostname = currRo.property("hostname").value();
 164                         }
 165                         if (currRoHostname != null){
 166                                 String[] rctFqdnSplit = currRctFqdn.toString().split("\\.");
 167                                 String[] roHostnameSplit = currRoHostname.toString().split("\\.");
 168                                 if (rctFqdnSplit.length >0 && roHostnameSplit.length > 0){
 169                                         if(!rctFqdnSplit[0].isEmpty() && !roHostnameSplit[0].isEmpty() && rctFqdnSplit[0].equals(roHostnameSplit[0])){
 170                                                 logger.info("\tPserver match found - RO Pserver with hostname: "+currRo.property("hostname").value().toString());
 171                                                 dupCount++;
 172                                             try {
 173                                                 mergePservers(currRct,currRo);
 174                                                 break;
 175                                             } catch (UnsupportedEncodingException e) {
 176                                                 success = false;
 177                                             } catch (AAIException e) {
 178                                                 success = false;
 179                                             }
 180                                         }
 181                                 }
 182                         }
 183                     }
 184             }
 185         }
 186         RemoveROList.forEach(v ->v.remove());
 187         logger.info ("\n \n ******* Migration Summary Counts for Dedup of RCT and RO sourced pservers ********* \n");
 188         logger.info(this.MIGRATION_SUMMARY_COUNT + "Total number of RCT: " +rctList.size());
 189         logger.info(this.MIGRATION_SUMMARY_COUNT + "Total number of RO: " +roList.size());
 190         logger.info(this.MIGRATION_SUMMARY_COUNT + "Duplicate pserver count: "+ dupCount);
 191         logger.info(this.MIGRATION_SUMMARY_COUNT + "Number of RCT updated: "+pserversUpdatedCount);
 192         logger.info(this.MIGRATION_SUMMARY_COUNT + "Number of RO deleted: "+ pserversDeletedCount +"\n");
 193     }
 194         private GraphTraversalSource graphTraversalSource() {
 195                 return this.engine.asAdmin().getTraversalSource();
 196         }
 197
 198
 199     public void mergePservers(Vertex rct, Vertex ro) throws UnsupportedEncodingException, AAIException {
 200         Introspector obj = serializer.getLatestVersionView(ro);
 201         dmaapDeleteList.add(obj);
 202         rct.property("fqdn",ro.property("hostname").value().toString());
 203         dropComplexEdge(ro);
 204         dropMatchingROPInterfaces(ro, rct);
 205         dropMatchingROLagInterfaces(ro, rct);
 206         swingEdges(ro, rct, null, null, "BOTH");
 207         modifyChildrenUri(rct);
 208         if(!(rct.property("pserver-id").isPresent())){
 209             rct.property("pserver-id",UUID.randomUUID().toString());
 210         }
 211         String dmaapMsg = System.nanoTime() + "_" + rct.id().toString() + "_"   + rct.value("resource-version").toString();
 212         dmaapMsgList.add(dmaapMsg);
 213         pserversUpdatedCount++;
 214         logger.info("\tAdding RO pserver to the delete list....");
 215         RemoveROList.add(ro);
 216         pserversDeletedCount++;
 217     }
 218
 219     private void dropMatchingROPInterfaces(Vertex ro, Vertex rct) {
 220         Map<String, Vertex> removeROPIntMap = new HashMap<String, Vertex>();
 221         List<Vertex> pIntList = graphTraversalSource().V(ro).in("tosca.relationships.network.BindsTo").has("aai-node-type","p-interface").toList();
 222         if (pIntList != null && !pIntList.isEmpty()) {
 223                 Iterator<Vertex> pIntListItr = pIntList.iterator();
 224                 while(pIntListItr.hasNext()){
 225                         Vertex pInt = pIntListItr.next();
 226
 227                         removeROPIntMap.put(pInt.property("interface-name").value().toString(), pInt);
 228                 }
 229                 Set<String> interfaceNameSet = removeROPIntMap.keySet();
 230                 List<Vertex> rctPIntList = graphTraversalSource().V(rct).in("tosca.relationships.network.BindsTo").has("aai-node-type","p-interface").toList();
 231                 if (rctPIntList != null && !rctPIntList.isEmpty()){
 232                         Iterator<Vertex> rctPIntListItr = rctPIntList.iterator();
 233                         while(rctPIntListItr.hasNext()){
 234                                 Vertex rctPInt = rctPIntListItr.next();
 235                                 String rctIntfName = rctPInt.property("interface-name").value().toString();
 236                                 if (interfaceNameSet.contains(rctIntfName)){
 237                                         Vertex pIntToRemoveFromROPserver = removeROPIntMap.get(rctIntfName);
 238                                         String roPIntUri = "roPIntUri";
 239                                         if (pIntToRemoveFromROPserver.property("aai-uri").isPresent()){
 240                                                 roPIntUri = pIntToRemoveFromROPserver.property("aai-uri").value().toString();
 241                                         }
 242                                         Edge roPIntToPserverEdge = pIntToRemoveFromROPserver.edges(Direction.OUT, "tosca.relationships.network.BindsTo").next();
 243                                         roPIntToPserverEdge.remove();
 244                                         pIntToRemoveFromROPserver.remove();
 245                                         logger.info("\tRemoved p-interface "+roPIntUri + " and its edge to RO pserver, not swinging the p-interface to RCT pserver");
 246                                 }
 247                         }
 248                 }
 249         }
 250         }
 251
 252     private void dropMatchingROLagInterfaces(Vertex ro, Vertex rct) {
 253         Map<String, Vertex> removeROLagIntMap = new HashMap<String, Vertex>();
 254         List<Vertex> lagIntList = graphTraversalSource().V(ro).in("tosca.relationships.network.BindsTo").has("aai-node-type","lag-interface").toList();
 255         if (lagIntList != null && !lagIntList.isEmpty()) {
 256                 Iterator<Vertex> lagIntListItr = lagIntList.iterator();
 257                 while(lagIntListItr.hasNext()){
 258                         Vertex lagInt = lagIntListItr.next();
 259
 260                         removeROLagIntMap.put(lagInt.property("interface-name").value().toString(), lagInt);
 261                 }
 262                 Set<String> interfaceNameSet = removeROLagIntMap.keySet();
 263                 List<Vertex> rctLagIntList = graphTraversalSource().V(rct).in("tosca.relationships.network.BindsTo").has("aai-node-type","lag-interface").toList();
 264                 if (rctLagIntList != null && !rctLagIntList.isEmpty()){
 265                         Iterator<Vertex> rctLagIntListItr = rctLagIntList.iterator();
 266                         while(rctLagIntListItr.hasNext()){
 267                                 Vertex rctPInt = rctLagIntListItr.next();
 268                                 String rctIntfName = rctPInt.property("interface-name").value().toString();
 269                                 if (interfaceNameSet.contains(rctIntfName)){
 270                                         Vertex lagIntToRemoveFromROPserver = removeROLagIntMap.get(rctIntfName);
 271                                         String roLagIntUri = "roPIntUri";
 272                                         if (lagIntToRemoveFromROPserver.property("aai-uri").isPresent()){
 273                                                 roLagIntUri = lagIntToRemoveFromROPserver.property("aai-uri").value().toString();
 274                                         }
 275                                         Edge roLagIntToPserverEdge = lagIntToRemoveFromROPserver.edges(Direction.OUT, "tosca.relationships.network.BindsTo").next();
 276                                         roLagIntToPserverEdge.remove();
 277                                         lagIntToRemoveFromROPserver.remove();
 278                                         logger.info("\tRemoved lag-interface "+roLagIntUri + " and its edge to RO pserver, not swinging the lag-interface to RCT pserver");
 279                                 }
 280                         }
 281                 }
 282         }
 283         }
 284
 285         public void dropComplexEdge(Vertex ro){
 286         List<Vertex> locatedInEdgeVertexList = graphTraversalSource().V(ro).has("aai-node-type", "pserver").out("org.onap.relationships.inventory.LocatedIn").has("aai-node-type","complex").toList();
 287         if (locatedInEdgeVertexList != null && !locatedInEdgeVertexList.isEmpty()){
 288                 Iterator<Vertex> locatedInEdgeVertexListItr = locatedInEdgeVertexList.iterator();
 289                 while (locatedInEdgeVertexListItr.hasNext()){
 290                         Vertex v = locatedInEdgeVertexListItr.next();
 291                         if ("complex".equalsIgnoreCase(v.property("aai-node-type").value().toString())){
 292                                 Edge pserverToComplexEdge = v.edges(Direction.IN, "org.onap.relationships.inventory.LocatedIn").next();
 293                                 pserverToComplexEdge.remove();
 294                         }
 295                 }
 296         }
 297     }
 298
 299
 300     private void modifyChildrenUri(Vertex v) throws UnsupportedEncodingException, AAIException {
 301         Set<Vertex> parentSet = new HashSet<>();
 302         parentSet.add(v);
 303         verifyOrAddUri("", parentSet);
 304     }
 305
 306
 307     protected void verifyOrAddUri(String parentUri, Set<Vertex> vertexSet) throws UnsupportedEncodingException, AAIException {
 308
 309
 310         String correctUri;
 311         for (Vertex v : vertexSet) {
 312             seen.add(v.id());
 313             //if there is an issue generating the uri catch, log and move on;
 314             try {
 315                 correctUri = parentUri + this.getUriForVertex(v);
 316             } catch (Exception e) {
 317                 logger.error("Vertex has issue generating uri " + e.getMessage() + "\n\t" + this.asString(v));
 318                 continue;
 319             }
 320             try {
 321                 v.property(AAIProperties.AAI_URI, correctUri);
 322             } catch (Exception e) {
 323                 logger.info("\t" + e.getMessage() + "\n\t" + this.asString(v));
 324             }
 325             if (!v.property(AAIProperties.AAI_UUID).isPresent()) {
 326                 v.property(AAIProperties.AAI_UUID, UUID.randomUUID().toString());
 327             }
 328             this.verifyOrAddUri(correctUri, getChildren(v));
 329         }
 330     }
 331
 332     protected Set<Vertex> getChildren(Vertex v) {
 333
 334         Set<Vertex> children = graphTraversalSource().V(v).bothE().not(__.has(EdgeProperty.CONTAINS.toString(), AAIDirection.NONE.toString())).otherV().toSet();
 335
 336         return children.stream().filter(child -> !seen.contains(child.id())).collect(Collectors.toSet());
 337     }
 338
 339     protected String getUriForVertex(Vertex v) {
 340         String aaiNodeType = v.property(AAIProperties.NODE_TYPE).value().toString();
 341
 342
 343         Map<String, String> parameters = this.nodeTypeToKeys.get(aaiNodeType).stream().collect(Collectors.toMap(
 344                 key -> key,
 345                 key -> encodeProp(v.property(key).value().toString())
 346         ));
 347
 348         return this.nodeTypeToUri.get(aaiNodeType).buildFromEncodedMap(parameters).toString();
 349     }
 350     private static String encodeProp(String s) {
 351         try {
 352             return UriUtils.encode(s, "UTF-8");
 353         } catch (UnsupportedEncodingException e) {
 354             return "";
 355         }
 356     }
 357
 358 }