src/main/java/org/openecomp/aai/champ/perf/ChampAPIPerformanceTest.java

   1 /**
   2  * ============LICENSE_START==========================================
   3  * org.onap.aai
   4  * ===================================================================
   5  * Copyright © 2017 AT&T Intellectual Property. All rights reserved.
   6  * Copyright © 2017 Amdocs
   7  * ===================================================================
   8  * Licensed under the Apache License, Version 2.0 (the "License");
   9  * you may not use this file except in compliance with the License.
  10  * You may obtain a copy of the License at
  11  *
  12  *        http://www.apache.org/licenses/LICENSE-2.0
  13  *
  14  * Unless required by applicable law or agreed to in writing, software
  15  * distributed under the License is distributed on an "AS IS" BASIS,
  16  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  17  * See the License for the specific language governing permissions and
  18  * limitations under the License.
  19  * ============LICENSE_END============================================
  20  * ECOMP is a trademark and service mark of AT&T Intellectual Property.
  21  */
  22 package org.openecomp.aai.champ.perf;
  23
  24 import java.util.Arrays;
  25 import java.util.Collections;
  26 import java.util.HashMap;
  27 import java.util.List;
  28 import java.util.Map;
  29 import java.util.Map.Entry;
  30 import java.util.stream.Collectors;
  31 import java.util.stream.Stream;
  32
  33 import org.openecomp.aai.champ.ChampGraph;
  34 import org.openecomp.aai.champ.exceptions.ChampMarshallingException;
  35 import org.openecomp.aai.champ.exceptions.ChampObjectNotExistsException;
  36 import org.openecomp.aai.champ.exceptions.ChampRelationshipNotExistsException;
  37 import org.openecomp.aai.champ.exceptions.ChampSchemaViolationException;
  38 import org.openecomp.aai.champ.exceptions.ChampUnmarshallingException;
  39 import org.openecomp.aai.champ.graph.impl.InMemoryChampGraphImpl;
  40 import org.openecomp.aai.champ.graph.impl.TitanChampGraphImpl;
  41 import org.openecomp.aai.champ.model.ChampField;
  42 import org.openecomp.aai.champ.model.ChampObject;
  43 import org.openecomp.aai.champ.model.ChampObjectIndex;
  44 import org.openecomp.aai.champ.model.ChampRelationship;
  45 import org.openecomp.aai.champ.model.ChampRelationshipIndex;
  46 import org.openecomp.aai.champ.model.ChampSchema;
  47 import org.openecomp.aai.champ.schema.ChampSchemaEnforcer;
  48 import org.slf4j.Logger;
  49 import org.slf4j.LoggerFactory;
  50
  51 import com.thinkaurelius.titan.core.TitanFactory;
  52 import com.thinkaurelius.titan.core.TitanFactory.Builder;
  53 import com.thinkaurelius.titan.core.TitanGraph;
  54 import com.thinkaurelius.titan.core.util.TitanCleanup;
  55
  56 public class ChampAPIPerformanceTest {
  57
  58         private static final Logger LOGGER = LoggerFactory.getLogger(ChampAPIPerformanceTest.class);
  59
  60         private static final int NUM_OBJECTS = 1000;
  61         private static final int NUM_RELATIONSHIPS = 1000;
  62         private static final String GRAPH_NAME = ChampAPIPerformanceTest.class.getSimpleName();
  63
  64         private static final String getGraphName() {
  65                 return GRAPH_NAME;
  66         }
  67
  68         private static void cleanUp(String graphName, Map<String, String> settings) {
  69                 LOGGER.debug("Cleaning up graph {}", graphName);
  70
  71                 try {
  72                         final Builder graphBuilder = TitanFactory.build();
  73
  74                         for (Entry<String, String> setting : settings.entrySet()) {
  75                                 graphBuilder.set(setting.getKey(), setting.getValue());
  76                         }
  77
  78                         final String storageBackend = settings.getOrDefault("storage.backend", "inmemory");
  79
  80                         if (storageBackend.equals("cassandra") ||
  81                                 storageBackend.equals("cassandrathrift") ||
  82                                 storageBackend.equals("astyanax") ||
  83                                 storageBackend.equals("embeddedcassandra")) {
  84                                 graphBuilder.set("storage.cassandra.keyspace", graphName);
  85                         } else if (storageBackend.equals("hbase")) {
  86                                 graphBuilder.set("storage.hbase.table", graphName);
  87                         }
  88
  89                         final TitanGraph graph = graphBuilder.open();
  90
  91                         graph.close();
  92                         TitanCleanup.clear(graph);
  93                 } catch (IllegalArgumentException e) {
  94                         LOGGER.warn("Could not clean up graph - unable to instantiate");
  95                 }
  96         }
  97
  98         public static void main(String[] args) {
  99
 100                 if (args.length < 1 || !args[0].startsWith("--champ.graph.type=")) {
 101                         throw new RuntimeException("Must provide --champ.graph.type=" + ChampGraph.Type.values() + " as first parameter");
 102                 }
 103
 104                 final ChampGraph.Type graphType = ChampGraph.Type.valueOf(args[0].split("=")[1]);
 105
 106                 final Map<String, String> settings = new HashMap<String, String> ();
 107
 108                 for (int i = 1; i < args.length; i++) {
 109                         if (!args[i].startsWith("--")) throw new RuntimeException("Bad command line argument: " + args[i]);
 110
 111                         final String[] keyValue = args[i].replaceFirst("--", "").split("=");
 112
 113                         if (keyValue.length != 2) throw new RuntimeException("Bad command line argument: " + args[i]);
 114
 115                         settings.put(keyValue[0], keyValue[1]);
 116                 }
 117
 118                 LOGGER.info("Provided graph settings: " + settings);
 119
 120                 if (graphType == ChampGraph.Type.TITAN) cleanUp(getGraphName(), settings);
 121
 122                 LOGGER.info("Graph cleaned, instantiating ChampGraph");
 123
 124                 final ChampGraph graph;
 125
 126                 switch (graphType) {
 127                 case IN_MEMORY:
 128                         final InMemoryChampGraphImpl.Builder inMemGraphBuilder = new InMemoryChampGraphImpl.Builder();
 129
 130                         if (settings.containsKey("champ.schema.enforcer")) {
 131                                 final String schemaEnforcerClassStr = settings.get("champ.schema.enforcer");
 132
 133                                 try {
 134                                         final Class<?> schemaEnforcer = Class.forName(schemaEnforcerClassStr);
 135
 136                                         if (!schemaEnforcer.isAssignableFrom(ChampSchemaEnforcer.class)) throw new RuntimeException("Unknown ChampSchemaEnforcer " + schemaEnforcer);
 137
 138                                         inMemGraphBuilder.schemaEnforcer((ChampSchemaEnforcer) schemaEnforcer.newInstance());
 139                                 } catch (ClassNotFoundException e) {
 140                                         throw new RuntimeException(e);
 141                                 } catch (InstantiationException e) {
 142                                         throw new RuntimeException(e);
 143                                 } catch (IllegalAccessException e) {
 144                                         throw new RuntimeException(e);
 145                                 }
 146                         }
 147
 148                         graph = inMemGraphBuilder.build();
 149                 break;
 150                 case TITAN:
 151                         final TitanChampGraphImpl.Builder graphBuilder = new TitanChampGraphImpl.Builder(getGraphName());
 152
 153                         for (Entry<String, String> setting : settings.entrySet()) {
 154                                 graphBuilder.property(setting.getKey(), setting.getValue());
 155                         }
 156
 157                         graph = graphBuilder.build();
 158                 break;
 159                 default:
 160                         throw new RuntimeException("Unknown ChampGraph.Type " + graphType);
 161                 }
 162
 163                 if (graph.queryObjects(Collections.emptyMap()).limit(1).count() > 0) {
 164                         graph.shutdown();
 165                         throw new RuntimeException("Expected empty graph");
 166                 }
 167
 168                 LOGGER.info("Graph instantiated, warming up JVM");
 169                 warmUp(graph);
 170
 171                 LOGGER.info("Warm up complete, starting to record performance measurements");
 172
 173                 LOGGER.info("Performance without indexing/schema");
 174
 175                 storeObjects(graph, false);
 176                 storeRelationships(graph, false);
 177                 retrieveIndividualObjects(graph, false);
 178                 retrieveBulkRelationships(graph, false);
 179                 retrieveIndividualRelationships(graph, false);
 180
 181                 LOGGER.info("Storing indices + schema");
 182
 183                 storeIndices(graph, false);
 184                 storeSchema(graph, false);
 185
 186                 LOGGER.info("Stored indices + schema");
 187
 188                 LOGGER.info("Performance with indexing + schema");
 189
 190                 storeObjects(graph, false);
 191                 storeRelationships(graph, false);
 192                 retrieveIndividualObjects(graph, false);
 193                 retrieveBulkRelationships(graph, false);
 194                 retrieveIndividualRelationships(graph, false);
 195
 196                 LOGGER.info("Performance test complete, shutting down graph");
 197
 198                 graph.shutdown();
 199
 200                 LOGGER.info("Graph shutdown, JVM exiting");
 201         }
 202
 203         private static void storeSchema(ChampGraph graph, boolean warmUp) {
 204                 try {
 205                         graph.storeSchema(
 206                                 ChampSchema.create()
 207                                                         .withObjectConstraint()
 208                                                                 .onType("foo")
 209                                                                 .withPropertyConstraint()
 210                                                                         .onField("fooObjectNumber")
 211                                                                         .optional()
 212                                                                         .build()
 213                                                                 .build()
 214                                                         .withRelationshipConstraint()
 215                                                                 .onType("bar")
 216                                                                 .withPropertyConstraint()
 217                                                                         .onField("barObjectNumber")
 218                                                                         .ofType(ChampField.Type.INTEGER)
 219                                                                         .optional()
 220                                                                         .build()
 221                                                                 .build()
 222                                                         .build()
 223                         );
 224                 } catch (ChampSchemaViolationException e) {
 225                         throw new AssertionError(e);
 226                 }
 227         }
 228
 229         private static void storeIndices(ChampGraph graph, boolean warmUp) {
 230                 graph.storeObjectIndex(
 231                         ChampObjectIndex.create()
 232                                                         .ofName("objectNumberIndex")
 233                                                         .onType("foo")
 234                                                         .forField("objectNumber")
 235                                                         .build()
 236                 );
 237
 238                 graph.storeRelationshipIndex(ChampRelationshipIndex.create()
 239                                                                                                                         .ofName("relationshipNumberIndex")
 240                                                                                                                         .onType("bazz")
 241                                                                                                                         .forField("relationshipNumber")
 242                                                                                                                         .build()
 243                 );
 244         }
 245
 246         private static void warmUp(ChampGraph graph) {
 247                 storeObjects(graph, false);
 248                 storeRelationships(graph, false);
 249                 retrieveIndividualObjects(graph, false);
 250                 retrieveBulkRelationships(graph, false);
 251                 retrieveIndividualRelationships(graph, false);
 252         }
 253
 254         private static void retrieveIndividualRelationships(ChampGraph graph, boolean warmUp) {
 255                 final double[] latencies = new double[NUM_RELATIONSHIPS];
 256                 final long totalStartTime = System.nanoTime();
 257
 258                 for (int i = 0; i < NUM_RELATIONSHIPS; i++) {
 259                         final long startTime = System.nanoTime();
 260
 261                         final Stream<ChampRelationship> objects = graph.queryRelationships(Collections.singletonMap("relationshipNumber", i));
 262                         objects.findFirst().get();
 263                         final double elapsedMs = (System.nanoTime() - startTime) / 1000.0 / 1000.0;
 264                         latencies[i] = elapsedMs;
 265                 }
 266
 267                 final double totalElapsedTimeSecs = (System.nanoTime() - totalStartTime) / 1000.0 / 1000.0 / 1000.0;
 268                 LOGGER.info("Individually read " + NUM_RELATIONSHIPS + " relationships in " + totalElapsedTimeSecs + "s (" + NUM_RELATIONSHIPS / totalElapsedTimeSecs + " relationships/s)");
 269
 270                 Arrays.sort(latencies);
 271
 272                 if (!warmUp) {
 273                         LOGGER.info("Retrieve individual relationship latencies");
 274                         LOGGER.info("\t50th percentile: " + latencies[(int) (NUM_RELATIONSHIPS * 0.50)]);
 275                         LOGGER.info("\t75th percentile: " + latencies[(int) (NUM_RELATIONSHIPS * 0.75)]);
 276                         LOGGER.info("\t90th percentile: " + latencies[(int) (NUM_RELATIONSHIPS * 0.90)]);
 277                         LOGGER.info("\t99th percentile: " + latencies[(int) (NUM_RELATIONSHIPS * 0.99)]);
 278                 }
 279         }
 280
 281         private static void retrieveIndividualObjects(ChampGraph graph, boolean warmUp) {
 282
 283                 final double[] latencies = new double[NUM_OBJECTS];
 284                 final long totalStartTime = System.nanoTime();
 285
 286                 for (int i = 0; i < NUM_OBJECTS; i++) {
 287                         final long startTime = System.nanoTime();
 288                         final Stream<ChampObject> objects = graph.queryObjects(Collections.singletonMap("objectNumber", i));
 289
 290                         objects.findFirst().get();
 291
 292                         final double elapsedMs = (System.nanoTime() - startTime) / 1000.0 / 1000.0;
 293
 294                         latencies[i] = elapsedMs;
 295                 }
 296
 297                 final double totalElapsedTimeSecs = (System.nanoTime() - totalStartTime) / 1000.0 / 1000.0 / 1000.0;
 298
 299                 LOGGER.info("Individually read " + NUM_OBJECTS + " objects in " + totalElapsedTimeSecs + "s (" + NUM_OBJECTS / totalElapsedTimeSecs + " objects/s)");
 300                 Arrays.sort(latencies);
 301
 302                 if (!warmUp) {
 303                         LOGGER.info("Retrieve individual object latencies");
 304                         LOGGER.info("\t50th percentile: " + latencies[(int) (NUM_OBJECTS * 0.50)]);
 305                         LOGGER.info("\t75th percentile: " + latencies[(int) (NUM_OBJECTS * 0.75)]);
 306                         LOGGER.info("\t90th percentile: " + latencies[(int) (NUM_OBJECTS * 0.90)]);
 307                         LOGGER.info("\t99th percentile: " + latencies[(int) (NUM_OBJECTS * 0.99)]);
 308                 }
 309         }
 310
 311         private static List<ChampObject> retrieveBulkObjects(ChampGraph graph, boolean warmUp) {
 312
 313                 final long startTime = System.nanoTime();
 314                 final Stream<ChampObject> objects = graph.queryObjects(
 315                                                         Collections.singletonMap(
 316                                                                 ChampObject.ReservedPropertyKeys.CHAMP_OBJECT_TYPE.toString(), "foo"
 317                                                         )
 318                                                 );
 319
 320                 final List<ChampObject> objectsAsList = objects.collect(Collectors.toList());
 321                 final double elapsedSecs = (System.nanoTime() - startTime) / 1000.0 / 1000.0 / 1000.0;
 322
 323                 if (!warmUp) LOGGER.info("Bulk read " + objectsAsList.size() + " objects in " + elapsedSecs + "s (" + objectsAsList.size() / elapsedSecs + " objects/s)");
 324
 325                 return objectsAsList;
 326         }
 327
 328         private static List<ChampRelationship> retrieveBulkRelationships(ChampGraph graph, boolean warmUp) {
 329                 final long startTime = System.nanoTime();
 330                 final Stream<ChampRelationship> relationships = graph.queryRelationships(
 331                                                         Collections.singletonMap(
 332                                                                 ChampRelationship.ReservedPropertyKeys.CHAMP_RELATIONSHIP_TYPE.toString(), "bazz"
 333                                                         )
 334                                                 );
 335
 336                 final List<ChampRelationship> relationshipsAsList = relationships.collect(Collectors.toList());
 337                 final double elapsedSecs = (System.nanoTime() - startTime) / 1000.0 / 1000.0 / 1000.0;
 338
 339                 if (!warmUp) LOGGER.info("Bulk read " + relationshipsAsList.size() + " relationships in " + elapsedSecs + "s (" + relationshipsAsList.size() / elapsedSecs + " relationships/s)");
 340
 341                 return relationshipsAsList;
 342         }
 343
 344         private static void storeObjects(ChampGraph graph, boolean warmUp) {
 345                 final double[] latencies = new double[NUM_OBJECTS];
 346                 final long totalStartTime = System.nanoTime();
 347
 348                 for (int i = 0; i < NUM_OBJECTS; i++) {
 349                         try {
 350                                 final long startTime = System.nanoTime();
 351
 352                                 graph.storeObject(
 353                                         ChampObject.create()
 354                                                                 .ofType("foo")
 355                                                                 .withoutKey()
 356                                                                 .withProperty("objectNumber", i)
 357                                                                 .build()
 358                                 );
 359
 360                                 final double elapsedMs = (System.nanoTime() - startTime) / 1000.0 / 1000.0;
 361                                 latencies[i] = elapsedMs;
 362                         } catch (ChampMarshallingException e) {
 363                                 throw new RuntimeException(e);
 364                         } catch (ChampSchemaViolationException e) {
 365                                 //Ignore, no schema set
 366                         } catch (ChampObjectNotExistsException e) {
 367                                 //Ignore, not an update
 368                         }
 369                 }
 370
 371                 final double totalElapsedTimeSecs = (System.nanoTime() - totalStartTime) / 1000.0 / 1000.0 / 1000.0;
 372                 LOGGER.info("Wrote " + NUM_OBJECTS + " objects in " + totalElapsedTimeSecs + "s (" + NUM_OBJECTS / totalElapsedTimeSecs + " objects/s)");
 373
 374                 Arrays.sort(latencies);
 375
 376                 if (!warmUp) {
 377                         LOGGER.info("Store object latencies");
 378                         LOGGER.info("\t50th percentile: " + latencies[(int) (NUM_OBJECTS * 0.50)]);
 379                         LOGGER.info("\t75th percentile: " + latencies[(int) (NUM_OBJECTS * 0.75)]);
 380                         LOGGER.info("\t90th percentile: " + latencies[(int) (NUM_OBJECTS * 0.90)]);
 381                         LOGGER.info("\t99th percentile: " + latencies[(int) (NUM_OBJECTS * 0.99)]);
 382                 }
 383         }
 384
 385         private static void storeRelationships(ChampGraph graph, boolean warmUp) {
 386                 final List<ChampObject> objects = retrieveBulkObjects(graph, warmUp);
 387                 final double[] latencies = new double[NUM_RELATIONSHIPS];
 388                 final long totalStartTime = System.nanoTime();
 389
 390                 for (int i = 0; i < NUM_RELATIONSHIPS; i++) {
 391                         try {
 392                                 final long startTime = System.nanoTime();
 393
 394                                 graph.storeRelationship(
 395                                         new ChampRelationship.Builder(
 396                                                 objects.get(i % objects.size()), objects.get((i + 1) % objects.size()), "bazz"
 397                                         ).property("relationshipNumber", i)
 398                                         .build()
 399                                 );
 400
 401                                 final double elapsedMs = (System.nanoTime() - startTime) / 1000.0 / 1000.0;
 402
 403                                 latencies[i] = elapsedMs;
 404                         } catch (ChampMarshallingException e) {
 405                                 throw new RuntimeException(e);
 406                         } catch (ChampObjectNotExistsException e) {
 407                                 throw new RuntimeException(e);
 408                         } catch (ChampSchemaViolationException e) {
 409                                 throw new RuntimeException(e);
 410                         } catch (ChampRelationshipNotExistsException e) {
 411                                 throw new RuntimeException(e);
 412                         } catch (ChampUnmarshallingException e) {
 413                                 throw new RuntimeException(e);
 414                         }
 415                 }
 416
 417                 final double totalElapsedTimeSecs = (System.nanoTime() - totalStartTime) / 1000.0 / 1000.0 / 1000.0;
 418                 LOGGER.info("Wrote " + NUM_RELATIONSHIPS + " relationships in " + totalElapsedTimeSecs + "s (" + NUM_RELATIONSHIPS / totalElapsedTimeSecs + " relationships/s)");
 419
 420                 Arrays.sort(latencies);
 421
 422                 if (!warmUp) {
 423                         LOGGER.info("Store relationship latencies");
 424                         LOGGER.info("\t50th percentile: " + latencies[(int) (NUM_RELATIONSHIPS * 0.50)]);
 425                         LOGGER.info("\t75th percentile: " + latencies[(int) (NUM_RELATIONSHIPS * 0.75)]);
 426                         LOGGER.info("\t90th percentile: " + latencies[(int) (NUM_RELATIONSHIPS * 0.90)]);
 427                         LOGGER.info("\t99th percentile: " + latencies[(int) (NUM_RELATIONSHIPS * 0.99)]);
 428                 }
 429         }
 430 }