src/test/java/org/onap/aai/datasnapshot/DataSnapshotTest4HistInit.java

   1 /**
   2  * ============LICENSE_START=======================================================
   3  * org.onap.aai
   4  * ================================================================================
   5  * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
   6  * ================================================================================
   7  * Licensed under the Apache License, Version 2.0 (the "License");
   8  * you may not use this file except in compliance with the License.
   9  * You may obtain a copy of the License at
  10  *
  11  *    http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  * ============LICENSE_END=========================================================
  19  */
  20
  21 package org.onap.aai.datasnapshot;
  22
  23 import org.apache.commons.io.FileUtils;
  24 import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
  25 import org.apache.tinkerpop.gremlin.structure.Property;
  26 import org.apache.tinkerpop.gremlin.structure.Vertex;
  27 import org.apache.tinkerpop.gremlin.structure.VertexProperty;
  28
  29 import org.janusgraph.core.JanusGraph;
  30 import org.janusgraph.core.JanusGraphTransaction;
  31 import org.junit.After;
  32 import org.junit.Before;
  33 import org.junit.Rule;
  34 import org.junit.Test;
  35 import org.onap.aai.AAISetup;
  36 import org.onap.aai.datagrooming.DataGrooming;
  37 import org.onap.aai.dbmap.AAIGraph;
  38 import org.onap.aai.exceptions.AAIException;
  39
  40 import org.onap.aai.logging.LogFormatTools;
  41 import org.onap.aai.util.AAISystemExitUtil;
  42 import org.springframework.boot.test.rule.OutputCapture;
  43
  44 import com.beust.jcommander.ParameterException;
  45
  46 import java.lang.NumberFormatException;
  47 import java.io.File;
  48 import java.io.IOException;
  49 import java.nio.file.Files;
  50 import java.nio.file.Path;
  51 import java.nio.file.Paths;
  52 import java.util.ArrayList;
  53 import java.util.HashMap;
  54 import java.util.Iterator;
  55 import java.util.List;
  56 import java.util.Map;
  57 import java.util.Set;
  58 import java.util.stream.Collectors;
  59
  60 import static org.hamcrest.CoreMatchers.is;
  61 import static org.hamcrest.Matchers.containsString;
  62 import static org.junit.Assert.*;
  63
  64
  65 public class DataSnapshotTest4HistInit extends AAISetup {
  66
  67     private GraphTraversalSource g;
  68
  69     private JanusGraphTransaction currentTransaction;
  70
  71     private List<Vertex> vertexes;
  72
  73     private DataSnapshot4HistInit dataSnapshot4HistInit;
  74
  75     @Rule
  76     public OutputCapture outputCapture = new OutputCapture();
  77
  78     @Before
  79     public void setup() throws AAIException {
  80         dataSnapshot4HistInit = new DataSnapshot4HistInit(loaderFactory, schemaVersions);
  81
  82         JanusGraph graph = AAIGraph.getInstance().getGraph();
  83         currentTransaction = graph.newTransaction();
  84         g = currentTransaction.traversal();
  85
  86         // Setup the graph so it has one pserver vertex
  87         vertexes = setupPserverData(g);
  88         currentTransaction.commit();
  89     }
  90
  91     @After
  92     public void tearDown(){
  93
  94         JanusGraph graph = AAIGraph.getInstance().getGraph();
  95         currentTransaction = graph.newTransaction();
  96         g = currentTransaction.traversal();
  97
  98         vertexes.stream().forEach((v) -> g.V(v).next().remove());
  99         currentTransaction.commit();
 100     }
 101
 102     @Test
 103     public void testClearEntireDatabaseAndVerifyDataIsRemoved() throws IOException {
 104
 105         // Copy the pserver.graphson file from src/test/resoures to ${AJSC_HOME}/logs/data/dataSnapshots/ folder
 106         String sourceFileName = "src/test/resources/pserver.graphson";
 107         String destFileName   = System.getProperty("AJSC_HOME") + "/logs/data/dataSnapshots/pserver.graphson";
 108         copySnapshotFile(sourceFileName,destFileName);
 109
 110
 111         // Run the dataSnapshot to clear the graph
 112         String [] args = {"-c", "CLEAR_ENTIRE_DATABASE", "-f", "pserver.graphson"};
 113         dataSnapshot4HistInit.executeCommand(args);
 114
 115         // Since the code doesn't clear the graph using AAIGraph.getInstance().getGraph(), its creating a second inmemory graph
 116         // so we can't verify this with by counting the vertexes and edges in the graph
 117         // In the future we could do that but for now we will depend on the following string "All done clearing DB"
 118
 119         // Capture the standard output and see if the following text is there
 120         assertThat(outputCapture.toString(), containsString("All done clearing DB"));
 121     }
 122
 123
 124     @Test
 125     public void testClearEntireDatabaseWithEmptyGraphSONFileAndItShouldNotClearDatabase() throws IOException {
 126
 127         // Create a empty file called empty.graphson in src/test/resources/
 128
 129         // Copy that file to ${AJSC_HOME}/logs/data/dataSnapshots/
 130         String sourceFileName = "src/test/resources/empty.graphson";
 131         String destFileName   = System.getProperty("AJSC_HOME") + "/logs/data/dataSnapshots/empty.graphson";
 132         copySnapshotFile(sourceFileName,destFileName);
 133
 134         // Run the clear dataSnapshot and this time it should fail
 135         String [] args = {"-c","CLEAR_ENTIRE_DATABASE", "-f","empty.graphson"};
 136         dataSnapshot4HistInit.executeCommand(args);
 137
 138         // Capture the standard output and see if the following text had no data is there
 139         // Since the graphson is empty it should output that and not clear the graph
 140         // Uncomment the following line after the test changes are done
 141          assertThat(outputCapture.toString(), containsString("graphson had no data."));
 142     }
 143
 144
 145     @Test
 146     public void testTakeSnapshotAndItShouldCreateASnapshotFileWithOneVertex() throws IOException, InterruptedException {
 147
 148         String logsFolder     = System.getProperty("AJSC_HOME") + "/logs/data/dataSnapshots/";
 149
 150         Set<Path> preSnapshotFiles = Files.walk(Paths.get(logsFolder)).collect(Collectors.toSet());
 151
 152         // Run the clear dataSnapshot and this time it should fail
 153         //String [] args = {"JUST_TAKE_SNAPSHOT"};  >> default behavior is now to use 15 threads
 154         // To just get one file, you have to tell it to just use one.
 155         String [] args = {"-c","THREADED_SNAPSHOT", "-threadCount" ,"1"};
 156
 157         dataSnapshot4HistInit.executeCommand(args);
 158
 159         // Add sleep so the file actually gets created with the data
 160
 161         Set<Path> postSnapshotFiles = Files.walk(Paths.get(logsFolder)).collect(Collectors.toSet());
 162
 163         assertThat(postSnapshotFiles.size(), is(preSnapshotFiles.size()+1));
 164         postSnapshotFiles.removeAll(preSnapshotFiles);
 165         List<Path> snapshotPathList = postSnapshotFiles.stream().collect(Collectors.toList());
 166
 167         assertThat(snapshotPathList.size(), is(1));
 168
 169         List<String> fileContents = Files.readAllLines(snapshotPathList.get(0));
 170         assertThat(fileContents.get(0), containsString("id"));
 171     }
 172
 173
 174     @Test
 175     public void testTakeSnapshotMultiAndItShouldCreateMultipleSnapshotFiles() throws IOException {
 176
 177         String logsFolder     = System.getProperty("AJSC_HOME") + "/logs/data/dataSnapshots/";
 178
 179         // Run the clear dataSnapshot and this time it should fail
 180         String [] args = {"-c","THREADED_SNAPSHOT", "-threadCount","2"};
 181
 182         dataSnapshot4HistInit.executeCommand(args);
 183
 184         // For this test if there is only one vertex in the graph, not sure if it will create multiple files
 185         // would need to add more data to the janusgraph
 186     }
 187
 188
 189     @Test
 190     public void testTakeSnapshotMultiWithDebugAndItShouldCreateMultipleSnapshotFiles() throws IOException {
 191
 192         String logsFolder     = System.getProperty("AJSC_HOME") + "/logs/data/dataSnapshots/";
 193
 194         // Run the clear dataSnapshot and this time it should fail
 195         String [] args = {"-c","THREADED_SNAPSHOT", "-threadCount","2", "-debugFlag","DEBUG"};
 196
 197         dataSnapshot4HistInit.executeCommand(args);
 198
 199         // For this test if there is only one vertex in the graph, not sure if it will create multiple files
 200         // would need to add more data to the janusgraph
 201     }
 202
 203
 204     @Test
 205     public void testTakeSnapshotMultiWithDebugAndInvalidNumberAndItShouldFail() throws IOException {
 206
 207         boolean thrown = false;
 208         String logsFolder     = System.getProperty("AJSC_HOME") + "/logs/data/dataSnapshots/";
 209
 210         // Run the clear dataSnapshot and this time it should fail
 211         String [] args = {"-c","THREADED_SNAPSHOT", "-threadCount","foo","-debugFlag", "DEBUG"};
 212
 213         dataSnapshot4HistInit.executeCommand(args);
 214
 215         // For this test if there is only one vertex in the graph, not sure if it will create multiple files
 216         // would need to add more data to the janusgraph
 217     }
 218
 219     @Test
 220     public void testTakeSnapshotMultiWithDebugAndTimeDelayAndInvalidNumberAndItShouldFail() throws IOException {
 221
 222         String logsFolder     = System.getProperty("AJSC_HOME") + "/logs/data/dataSnapshots/";
 223
 224         // Run the clear dataSnapshot and this time it should fail
 225         String [] args = {"-c","THREADED_SNAPSHOT","-threadCount", "foo", "-debugFlag","DEBUG","-debugAddDelayTime", "100"};
 226
 227         dataSnapshot4HistInit.executeCommand(args);
 228
 229         // For this test if there is only one vertex in the graph, not sure if it will create multiple files
 230         // would need to add more data to the janusgraph
 231     }
 232
 233     @Test
 234     public void testTakeSnapshotMultiWithDebugAndTimeDelayAndZeroThreadsAndItShouldFail() throws IOException {
 235
 236         String logsFolder     = System.getProperty("AJSC_HOME") + "/logs/data/dataSnapshots/";
 237
 238         // Run the clear dataSnapshot and this time it should fail
 239         String [] args = {"-c","THREADED_SNAPSHOT", "-threadCount","0", "-debugFlag","DEBUG", "-debugAddDelayTime","100"};
 240
 241         dataSnapshot4HistInit.executeCommand(args);
 242
 243         // For this test if there is only one vertex in the graph, not sure if it will create multiple files
 244         // would need to add more data to the janusgraph
 245     }
 246
 247     @Test
 248     public void testTakeSnapshotMultiWithDebugAndTimeDelayIsInvalidNumberAndItShouldFail() throws IOException {
 249
 250         String logsFolder     = System.getProperty("AJSC_HOME") + "/logs/data/dataSnapshots/";
 251
 252         // Run the clear dataSnapshot and this time it should fail
 253         String [] args = {"-c","THREADED_SNAPSHOT","-threadCount", "0","-debugFlag","DEBUG", "-debugAddDelayTime","foo"};
 254
 255         dataSnapshot4HistInit.executeCommand(args);
 256
 257         // For this test if there is only one vertex in the graph, not sure if it will create multiple files
 258         // would need to add more data to the janusgraph
 259     }
 260
 261 //    @Test
 262     public void testTakeSnapshotMultiWithMoreParametersThanAllowedAndItShouldFail() throws IOException {
 263
 264         String logsFolder     = System.getProperty("AJSC_HOME") + "/logs/data/dataSnapshots/";
 265
 266         // Run the clear dataSnapshot and this time it should fail
 267         String [] args = {"-c","THREADED_SNAPSHOT", "-threadCount", "0", "-debugFlag","DEBUG",  "-debugAddDelayTime","foo", "bar"};
 268
 269         dataSnapshot4HistInit.executeCommand(args);
 270
 271         // For this test if there is only one vertex in the graph, not sure if it will create multiple files
 272         // would need to add more data to the janusgraph
 273     }
 274
 275     @Test
 276     public void testTakeSnapshotMultiWithZeroThreadsAndItShouldFail(){
 277
 278         // For this test if there is only one vertex in the graph, not sure if it will create multiple files
 279         // would need to add more data to the janusgraph
 280         String logsFolder     = System.getProperty("AJSC_HOME") + "/logs/data/dataSnapshots/";
 281
 282         // Run the clear dataSnapshot and this time it should fail
 283         String [] args = {"-c","THREADED_SNAPSHOT", "-threadCount","0"};
 284
 285         dataSnapshot4HistInit.executeCommand(args);
 286     }
 287
 288     @Test
 289     public void testTakeSnapshotMultiWithInvalidNumberForThreadsAndItShouldFail(){
 290
 291         // For this test if there is only one vertex in the graph, not sure if it will create multiple files
 292         // would need to add more data to the janusgraph
 293         String logsFolder     = System.getProperty("AJSC_HOME") + "/logs/data/dataSnapshots/";
 294
 295         // Run the clear dataSnapshot and this time it should fail
 296         String [] args = {"-c","THREADED_SNAPSHOT","-threadCount", "foo"};
 297
 298         dataSnapshot4HistInit.executeCommand(args);
 299     }
 300
 301
 302     @Test
 303     public void testReloadDataAndVerifyDataInGraphMatchesGraphson() throws IOException {
 304
 305         // Create a graphson file that contains a couple of vertexes in src/test/resources
 306         // Copy that file to ${AJSC_HOME}/logs/data/dataSnasphots/
 307         // Run the reload arguments and ensure that the graph was recreated by checking vertexes in graph
 308
 309         // After reload remove the added vertexes in the graph
 310         // The reason for this so each test is independent
 311         // as there shouldn't be dependencies and cause weird issues
 312         String sourceFileName = "src/test/resources/pserver.graphson";
 313         String destFileName   = System.getProperty("AJSC_HOME") + "/logs/data/dataSnapshots/pserver.graphson";
 314         copySnapshotFile(sourceFileName,destFileName);
 315
 316         String [] args = {"-c","RELOAD_DATA", "-f","pserver.graphson"};
 317
 318         dataSnapshot4HistInit.executeCommand(args);
 319     }
 320
 321
 322     @Test
 323     public void testMultiReloadDataAndVerifyDataInGraphMatchesGraphson() throws IOException, AAIException {
 324
 325         // Create multiple graphson files that contains a couple of vertexes in src/test/resources
 326         // Copy those files to ${AJSC_HOME}/logs/data/dataSnasphots/
 327         // Run the reload arguments and ensure that the graph was recreated by checking vertexes in graph
 328         String sourceFileName = "src/test/resources/pserver2.graphson.P0";
 329         String destFileName   = System.getProperty("AJSC_HOME") + "/logs/data/dataSnapshots/pserver2.graphson.P0";
 330         copySnapshotFile(sourceFileName,destFileName);
 331
 332         sourceFileName = "src/test/resources/pserver2.graphson.P1";
 333         destFileName   = System.getProperty("AJSC_HOME") + "/logs/data/dataSnapshots/pserver2.graphson.P1";
 334         copySnapshotFile(sourceFileName,destFileName);
 335
 336         // After reload remove the added vertexes in the graph
 337         // The reason for this so each test is independent
 338         // as there shouldn't be dependencies and cause weird issues
 339
 340         String [] args = {"-c","MULTITHREAD_RELOAD","-f", "pserver2.graphson"};
 341         dataSnapshot4HistInit.executeCommand(args);
 342
 343     }
 344
 345     @Test
 346     public void testMultiReloadDataWithNonExistentFilesAndItShouldFail() throws IOException {
 347
 348         // After reload remove the added vertexes in the graph
 349         // The reason for this so each test is independent
 350         // as there shouldn't be dependencies and cause weird issues
 351         String [] args = {"-c","MULTITHREAD_RELOAD", "-f","emptyfoo2.graphson"};
 352
 353         dataSnapshot4HistInit.executeCommand(args);
 354     }
 355
 356     @Test
 357     public void testReloadMultiDataAndVerifyDataInGraphMatchesGraphson() throws IOException {
 358
 359         // Create multiple graphson files that contains a couple of vertexes in src/test/resources
 360         // Copy those files to ${AJSC_HOME}/logs/data/dataSnasphots/
 361         // Run the reload arguments and ensure that the graph was recreated by checking vertexes in graph
 362         String sourceFileName = "src/test/resources/pserver2.graphson.P0";
 363         String destFileName   = System.getProperty("AJSC_HOME") + "/logs/data/dataSnapshots/pserver2.graphson.P0";
 364         copySnapshotFile(sourceFileName,destFileName);
 365
 366         sourceFileName = "src/test/resources/pserver2.graphson.P1";
 367         destFileName   = System.getProperty("AJSC_HOME") + "/logs/data/dataSnapshots/pserver2.graphson.P1";
 368         copySnapshotFile(sourceFileName,destFileName);
 369
 370         // After reload remove the added vertexes in the graph
 371         // The reason for this so each test is independent
 372         // as there shouldn't be dependencies and cause weird issues
 373         String [] args = {"-c","RELOAD_DATA_MULTI","-f", "pserver2.graphson"};
 374
 375         dataSnapshot4HistInit.executeCommand(args);
 376     }
 377
 378     @Test
 379     public void testCanRetrieveNamesOfKeyProps() throws IOException {
 380
 381         // Make sure we can get the key names without failing
 382         HashMap <String,ArrayList<String>> keyNamesHash = dataSnapshot4HistInit.getNodeKeyNames();
 383         Iterator  keyItr = keyNamesHash.entrySet().iterator();
 384         while( keyItr.hasNext() ) {
 385                 Map.Entry entry = (Map.Entry) keyItr.next();
 386                 String nodeType = (String)entry.getKey();
 387                 ArrayList<String> keyNames = (ArrayList<String>)entry.getValue();
 388                 keyNamesHash.put(nodeType,keyNames);
 389                 System.out.println("DEBUGjojo === for nType " + nodeType + ", got keys = [" + keyNames + "]");
 390         }
 391
 392         assertTrue(keyNamesHash != null );
 393         assertFalse(keyNamesHash.isEmpty());
 394     }
 395
 396
 397     private void showVertProperties(String propKey, String propVal)  {
 398
 399         Vertex v1 = g.V().has(propKey, propVal).next();
 400         Iterator<VertexProperty<Object>> pI = v1.properties();
 401         while( pI.hasNext() ){
 402                 VertexProperty<Object> tp = pI.next();
 403                 String infStr = " [" + tp.key() + "][" + tp.value() + "] ";
 404                 System.out.println("Regular ole properties are: " + infStr  );
 405                 Iterator<Property<Object>> fullPropI = tp.properties();
 406                 while( fullPropI.hasNext() ){
 407                         // Note - the 'real' key/value of a property are not part of this list, just the
 408                         //    extra stuff beyond those two.
 409                         Property<Object> propOfProp = fullPropI.next();
 410                         String infStr2 = " [" + propOfProp.key() + "][" + propOfProp.value() + "] ";
 411                         System.out.println("For " + infStr + ", got sub-property:" + infStr2 );
 412                 }
 413         }
 414     }
 415
 416
 417     private List<Vertex> setupOneHistoryNode(GraphTraversalSource g) throws AAIException {
 418
 419         Vertex v1 = g.addV().property("aai-node-type", "pserver","start-ts", 9988707,"source-of-truth","N/A")
 420             .property("hostname", "historyHOstGuy--8","start-ts", 9988707,"source-of-truth","N/A")
 421             .property("equip-vendor", "historyVendor","start-ts", 9988707,"source-of-truth","N/A")
 422             .property("role", "historyRole","start-ts", 9988707,"source-of-truth","N/A")
 423             .next();
 424        List<Vertex> list = new ArrayList<>();
 425         list.add(v1);
 426
 427         Iterator<VertexProperty<Object>> pI = v1.properties();
 428         while( pI.hasNext() ){
 429                 VertexProperty<Object> tp = pI.next();
 430                 String infStr = " [" + tp.key() + "|" + tp.value() + "] ";
 431                 System.out.println("Regular ole properties are: " + infStr  );
 432                 Iterator<Property<Object>> fullPropI = tp.properties();
 433                 while( fullPropI.hasNext() ){
 434                         // Note - the 'real' key/value of a property are not part of this list, just the
 435                         //    extra stuff beyond those two.
 436                         Property<Object> propOfProp = fullPropI.next();
 437                         String infStr2 = " [" + propOfProp.key() + "|" + propOfProp.value() + "] ";
 438                         System.out.println("For " + infStr + ", got sub-property:" + infStr2 );
 439                 }
 440         }
 441         return list;
 442     }
 443
 444     private List<Vertex> setupPserverData(GraphTraversalSource g) throws AAIException {
 445         Vertex v1 = g.addV().property("aai-node-type", "pserver")
 446             .property("hostname", "somerandomhostname")
 447             .next();
 448         List<Vertex> list = new ArrayList<>();
 449         list.add(v1);
 450         Vertex v2 = g.addV().property("aai-node-type", "pserver")
 451             .property("hostname", "somerandomhostname2")
 452             .next();
 453         Vertex pinterface = g.addV()
 454                 .property("aai-node-type", "p-interface")
 455                 .property("interface-name", "p-interface-name")
 456                 .property("in-maint", false)
 457                 .property("source-of-truth", "JUNIT")
 458                 .next();
 459         edgeSerializer.addTreeEdge(g, v2, pinterface);
 460         list.add(v2);
 461         return list;
 462     }
 463
 464     private void copySnapshotFile(String sourceFileName, String destFileName) throws IOException {
 465
 466         File inputFile = new File(sourceFileName);
 467         File outputFile = new File(destFileName);
 468
 469         FileUtils.copyFile(inputFile, outputFile);
 470     }
 471 }