champ-lib/champ-titan/src/main/java/org/onap/aai/champtitan/perf/ChampAPIPerformanceTest.java

   1 /**
   2  * ============LICENSE_START==========================================
   3  * org.onap.aai
   4  * ===================================================================
   5  * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
   6  * Copyright © 2017-2018 Amdocs
   7  * ===================================================================
   8  * Licensed under the Apache License, Version 2.0 (the "License");
   9  * you may not use this file except in compliance with the License.
  10  * You may obtain a copy of the License at
  11  *
  12  *        http://www.apache.org/licenses/LICENSE-2.0
  13  *
  14  * Unless required by applicable law or agreed to in writing, software
  15  * distributed under the License is distributed on an "AS IS" BASIS,
  16  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  17  * See the License for the specific language governing permissions and
  18  * limitations under the License.
  19  * ============LICENSE_END============================================
  20  */
  21 package org.onap.aai.champtitan.perf;
  22
  23 import com.thinkaurelius.titan.core.TitanFactory;
  24 import com.thinkaurelius.titan.core.TitanFactory.Builder;
  25 import com.thinkaurelius.titan.core.TitanGraph;
  26 import com.thinkaurelius.titan.core.util.TitanCleanup;
  27
  28 import org.onap.aai.champcore.ChampGraph;
  29 import org.onap.aai.champcore.exceptions.*;
  30 import org.onap.aai.champcore.graph.impl.InMemoryChampGraphImpl;
  31 import org.onap.aai.champcore.model.*;
  32 import org.onap.aai.champcore.schema.ChampSchemaEnforcer;
  33 import org.onap.aai.champtitan.graph.impl.TitanChampGraphImpl;
  34 import org.slf4j.Logger;
  35 import org.slf4j.LoggerFactory;
  36
  37 import java.util.*;
  38 import java.util.Map.Entry;
  39 import java.util.stream.Collectors;
  40 import java.util.stream.Stream;
  41
  42 public class ChampAPIPerformanceTest {
  43
  44         private static final Logger LOGGER = LoggerFactory.getLogger(ChampAPIPerformanceTest.class);
  45
  46         private static final int NUM_OBJECTS = 1000;
  47         private static final int NUM_RELATIONSHIPS = 1000;
  48         private static final String GRAPH_NAME = ChampAPIPerformanceTest.class.getSimpleName();
  49
  50         private static final String getGraphName() {
  51                 return GRAPH_NAME;
  52         }
  53
  54         private static void cleanUp(String graphName, Map<String, String> settings) {
  55                 LOGGER.debug("Cleaning up graph {}", graphName);
  56
  57                 try {
  58                         final Builder graphBuilder = TitanFactory.build();
  59
  60                         for (Entry<String, String> setting : settings.entrySet()) {
  61                                 graphBuilder.set(setting.getKey(), setting.getValue());
  62                         }
  63
  64                         final String storageBackend = settings.getOrDefault("storage.backend", "inmemory");
  65
  66                         if (storageBackend.equals("cassandra") ||
  67                                 storageBackend.equals("cassandrathrift") ||
  68                                 storageBackend.equals("astyanax") ||
  69                                 storageBackend.equals("embeddedcassandra")) {
  70                                 graphBuilder.set("storage.cassandra.keyspace", graphName);
  71                         } else if (storageBackend.equals("hbase")) {
  72                                 graphBuilder.set("storage.hbase.table", graphName);
  73                         }
  74
  75                         final TitanGraph graph = graphBuilder.open();
  76
  77                         graph.close();
  78                         TitanCleanup.clear(graph);
  79                 } catch (IllegalArgumentException e) {
  80                         LOGGER.warn("Could not clean up graph - unable to instantiate");
  81                 }
  82         }
  83
  84         public static void main(String[] args) {
  85
  86                 if (args.length < 1 || !args[0].startsWith("--champcore.graph.type=")) {
  87                         throw new RuntimeException("Must provide --champcore.graph.type=" + " as first parameter");
  88                 }
  89
  90                 final String graphType = args[0].split("=")[1];
  91
  92                 final Map<String, String> settings = new HashMap<String, String> ();
  93
  94                 for (int i = 1; i < args.length; i++) {
  95                         if (!args[i].startsWith("--")) {
  96                                 throw new RuntimeException("Bad command line argument: " + args[i]);
  97                         }
  98
  99                         final String[] keyValue = args[i].replaceFirst("--", "").split("=");
 100
 101                         if (keyValue.length != 2) {
 102                                 throw new RuntimeException("Bad command line argument: " + args[i]);
 103                         }
 104
 105                         settings.put(keyValue[0], keyValue[1]);
 106                 }
 107
 108                 LOGGER.info("Provided graph settings: " + settings);
 109
 110                 if (graphType.equals("TITAN")) {
 111                         cleanUp(getGraphName(), settings);
 112                 }
 113
 114                 LOGGER.info("Graph cleaned, instantiating ChampGraph");
 115
 116                 final ChampGraph graph;
 117
 118                 switch (graphType) {
 119                 case "IN_MEMORY":
 120                         final InMemoryChampGraphImpl.Builder inMemGraphBuilder = new InMemoryChampGraphImpl.Builder();
 121
 122                         if (settings.containsKey("champcore.schema.enforcer")) {
 123                                 final String schemaEnforcerClassStr = settings.get("champcore.schema.enforcer");
 124
 125                                 try {
 126                                         final Class<?> schemaEnforcer = Class.forName(schemaEnforcerClassStr);
 127
 128                                         if (!schemaEnforcer.isAssignableFrom(ChampSchemaEnforcer.class)) {
 129                                                 throw new RuntimeException("Unknown ChampSchemaEnforcer " + schemaEnforcer);
 130                                         }
 131
 132                                         inMemGraphBuilder.schemaEnforcer((ChampSchemaEnforcer) schemaEnforcer.newInstance());
 133                                 } catch (ClassNotFoundException e) {
 134                                         throw new RuntimeException(e);
 135                                 } catch (InstantiationException e) {
 136                                         throw new RuntimeException(e);
 137                                 } catch (IllegalAccessException e) {
 138                                         throw new RuntimeException(e);
 139                                 }
 140                         }
 141
 142                         graph = inMemGraphBuilder.build();
 143                 break;
 144                 case "TITAN":
 145                         final TitanChampGraphImpl.Builder graphBuilder = new TitanChampGraphImpl.Builder(getGraphName());
 146
 147                         for (Entry<String, String> setting : settings.entrySet()) {
 148                                 graphBuilder.property(setting.getKey(), setting.getValue());
 149                         }
 150
 151                         graph = graphBuilder.build();
 152                 break;
 153                 default:
 154                         throw new RuntimeException("Unknown ChampGraph.Type " + graphType);
 155                 }
 156
 157                 try {
 158           if (graph.queryObjects(Collections.emptyMap(), Optional.empty()).limit(1).count() > 0) {
 159                 graph.shutdown();
 160                 throw new RuntimeException("Expected empty graph");
 161           }
 162         } catch (ChampTransactionException e) {
 163           throw new RuntimeException("Transaction failure");
 164         }
 165
 166                 LOGGER.info("Graph instantiated, warming up JVM");
 167                 warmUp(graph);
 168
 169                 LOGGER.info("Warm up complete, starting to record performance measurements");
 170
 171                 LOGGER.info("Performance without indexing/schema");
 172
 173                 storeObjects(graph, false);
 174                 storeRelationships(graph, false);
 175                 retrieveIndividualObjects(graph, false);
 176                 retrieveBulkRelationships(graph, false);
 177                 retrieveIndividualRelationships(graph, false);
 178
 179                 LOGGER.info("Storing indices + schema");
 180
 181                 storeIndices(graph, false);
 182                 storeSchema(graph, false);
 183
 184                 LOGGER.info("Stored indices + schema");
 185
 186                 LOGGER.info("Performance with indexing + schema");
 187
 188                 storeObjects(graph, false);
 189                 storeRelationships(graph, false);
 190                 retrieveIndividualObjects(graph, false);
 191                 retrieveBulkRelationships(graph, false);
 192                 retrieveIndividualRelationships(graph, false);
 193
 194                 LOGGER.info("Performance test complete, shutting down graph");
 195
 196                 graph.shutdown();
 197
 198                 LOGGER.info("Graph shutdown, JVM exiting");
 199         }
 200
 201         private static void storeSchema(ChampGraph graph, boolean warmUp) {
 202                 try {
 203                         graph.storeSchema(
 204                                         ChampSchema.create()
 205                                                         .withObjectConstraint()
 206                                                         .onType("foo")
 207                                                         .withPropertyConstraint()
 208                                                         .onField("fooObjectNumber")
 209                                                         .optional()
 210                                                         .build()
 211                                                         .build()
 212                                                         .withRelationshipConstraint()
 213                                                         .onType("bar")
 214                                                         .withPropertyConstraint()
 215                                                         .onField("barObjectNumber")
 216                                                         .ofType(ChampField.Type.INTEGER)
 217                                                         .optional()
 218                                                         .build()
 219                                                         .build()
 220                                                         .build()
 221                         );
 222                 } catch (ChampSchemaViolationException e) {
 223                         throw new AssertionError(e);
 224                 }
 225         }
 226
 227         private static void storeIndices(ChampGraph graph, boolean warmUp) {
 228                 graph.storeObjectIndex(
 229                         ChampObjectIndex.create()
 230                                                         .ofName("objectNumberIndex")
 231                                                         .onType("foo")
 232                                                         .forField("objectNumber")
 233                                                         .build()
 234                 );
 235
 236                 graph.storeRelationshipIndex(ChampRelationshipIndex.create()
 237                                                                                                                         .ofName("relationshipNumberIndex")
 238                                                                                                                         .onType("bazz")
 239                                                                                                                         .forField("relationshipNumber")
 240                                                                                                                         .build()
 241                 );
 242         }
 243
 244         private static void warmUp(ChampGraph graph) {
 245                 storeObjects(graph, false);
 246                 storeRelationships(graph, false);
 247                 retrieveIndividualObjects(graph, false);
 248                 retrieveBulkRelationships(graph, false);
 249                 retrieveIndividualRelationships(graph, false);
 250         }
 251
 252         private static void retrieveIndividualRelationships(ChampGraph graph, boolean warmUp) {
 253                 final double[] latencies = new double[NUM_RELATIONSHIPS];
 254                 final long totalStartTime = System.nanoTime();
 255
 256                 for (int i = 0; i < NUM_RELATIONSHIPS; i++) {
 257                         final long startTime = System.nanoTime();
 258
 259                         Stream<ChampRelationship> objects;
 260             try {
 261               objects = graph.queryRelationships(Collections.singletonMap("relationshipNumber", i), Optional.empty());
 262             } catch (ChampTransactionException e) {
 263               throw new RuntimeException(e);
 264             }
 265
 266                         objects.findFirst().get();
 267                         final double elapsedMs = (System.nanoTime() - startTime) / 1000.0 / 1000.0;
 268                         latencies[i] = elapsedMs;
 269                 }
 270
 271                 final double totalElapsedTimeSecs = (System.nanoTime() - totalStartTime) / 1000.0 / 1000.0 / 1000.0;
 272                 LOGGER.info("Individually read " + NUM_RELATIONSHIPS + " relationships in " + totalElapsedTimeSecs + "s (" + NUM_RELATIONSHIPS / totalElapsedTimeSecs + " relationships/s)");
 273
 274                 Arrays.sort(latencies);
 275
 276                 if (!warmUp) {
 277                         LOGGER.info("Retrieve individual relationship latencies");
 278                         LOGGER.info("\t50th percentile: " + latencies[(int) (NUM_RELATIONSHIPS * 0.50)]);
 279                         LOGGER.info("\t75th percentile: " + latencies[(int) (NUM_RELATIONSHIPS * 0.75)]);
 280                         LOGGER.info("\t90th percentile: " + latencies[(int) (NUM_RELATIONSHIPS * 0.90)]);
 281                         LOGGER.info("\t99th percentile: " + latencies[(int) (NUM_RELATIONSHIPS * 0.99)]);
 282                 }
 283         }
 284
 285         private static void retrieveIndividualObjects(ChampGraph graph, boolean warmUp) {
 286
 287                 final double[] latencies = new double[NUM_OBJECTS];
 288                 final long totalStartTime = System.nanoTime();
 289
 290                 for (int i = 0; i < NUM_OBJECTS; i++) {
 291                         final long startTime = System.nanoTime();
 292                         Stream<ChampObject> objects;
 293             try {
 294               objects = graph.queryObjects(Collections.singletonMap("objectNumber", i), Optional.empty());
 295             } catch (ChampTransactionException e) {
 296                 throw new RuntimeException(e);
 297             }
 298
 299                         objects.findFirst().get();
 300
 301                         final double elapsedMs = (System.nanoTime() - startTime) / 1000.0 / 1000.0;
 302
 303                         latencies[i] = elapsedMs;
 304                 }
 305
 306                 final double totalElapsedTimeSecs = (System.nanoTime() - totalStartTime) / 1000.0 / 1000.0 / 1000.0;
 307
 308                 LOGGER.info("Individually read " + NUM_OBJECTS + " objects in " + totalElapsedTimeSecs + "s (" + NUM_OBJECTS / totalElapsedTimeSecs + " objects/s)");
 309                 Arrays.sort(latencies);
 310
 311                 if (!warmUp) {
 312                         LOGGER.info("Retrieve individual object latencies");
 313                         LOGGER.info("\t50th percentile: " + latencies[(int) (NUM_OBJECTS * 0.50)]);
 314                         LOGGER.info("\t75th percentile: " + latencies[(int) (NUM_OBJECTS * 0.75)]);
 315                         LOGGER.info("\t90th percentile: " + latencies[(int) (NUM_OBJECTS * 0.90)]);
 316                         LOGGER.info("\t99th percentile: " + latencies[(int) (NUM_OBJECTS * 0.99)]);
 317                 }
 318         }
 319
 320         private static List<ChampObject> retrieveBulkObjects(ChampGraph graph, boolean warmUp) {
 321
 322                 final long startTime = System.nanoTime();
 323                 Stream<ChampObject> objects;
 324         try {
 325           objects = graph.queryObjects(
 326                                         Collections.singletonMap(
 327                                                 ChampObject.ReservedPropertyKeys.CHAMP_OBJECT_TYPE.toString(), "foo"
 328                                         ), Optional.empty()
 329                                 );
 330         } catch (ChampTransactionException e) {
 331           throw new RuntimeException(e);
 332         }
 333
 334                 final List<ChampObject> objectsAsList = objects.collect(Collectors.toList());
 335                 final double elapsedSecs = (System.nanoTime() - startTime) / 1000.0 / 1000.0 / 1000.0;
 336
 337                 if (!warmUp) {
 338                         LOGGER.info("Bulk read " + objectsAsList.size() + " objects in " + elapsedSecs + "s (" + objectsAsList.size() / elapsedSecs + " objects/s)");
 339                 }
 340
 341                 return objectsAsList;
 342         }
 343
 344         private static List<ChampRelationship> retrieveBulkRelationships(ChampGraph graph, boolean warmUp) {
 345                 final long startTime = System.nanoTime();
 346                 Stream<ChampRelationship> relationships;
 347         try {
 348           relationships = graph.queryRelationships(
 349                                         Collections.singletonMap(
 350                                                 ChampRelationship.ReservedPropertyKeys.CHAMP_RELATIONSHIP_TYPE.toString(), "bazz"
 351                                         ), Optional.empty()
 352                                 );
 353         } catch (ChampTransactionException e) {
 354             throw new RuntimeException(e);
 355         }
 356
 357                 final List<ChampRelationship> relationshipsAsList = relationships.collect(Collectors.toList());
 358                 final double elapsedSecs = (System.nanoTime() - startTime) / 1000.0 / 1000.0 / 1000.0;
 359
 360                 if (!warmUp) {
 361                         LOGGER.info("Bulk read " + relationshipsAsList.size() + " relationships in " + elapsedSecs + "s (" + relationshipsAsList.size() / elapsedSecs + " relationships/s)");
 362                 }
 363
 364                 return relationshipsAsList;
 365         }
 366
 367         private static void storeObjects(ChampGraph graph, boolean warmUp) {
 368                 final double[] latencies = new double[NUM_OBJECTS];
 369                 final long totalStartTime = System.nanoTime();
 370
 371                 for (int i = 0; i < NUM_OBJECTS; i++) {
 372                         try {
 373                                 final long startTime = System.nanoTime();
 374
 375                                 graph.storeObject(
 376                                         ChampObject.create()
 377                                                                 .ofType("foo")
 378                                                                 .withoutKey()
 379                                                                 .withProperty("objectNumber", i)
 380                                                                 .build(), Optional.empty()
 381                                 );
 382
 383                                 final double elapsedMs = (System.nanoTime() - startTime) / 1000.0 / 1000.0;
 384                                 latencies[i] = elapsedMs;
 385                         } catch (ChampMarshallingException e) {
 386                                 throw new RuntimeException(e);
 387                         } catch (ChampSchemaViolationException e) {
 388                                 //Ignore, no schema set
 389                         } catch (ChampObjectNotExistsException e) {
 390                                 //Ignore, not an update
 391                         } catch (ChampTransactionException e) {
 392               throw new RuntimeException(e);
 393             }
 394                 }
 395
 396                 final double totalElapsedTimeSecs = (System.nanoTime() - totalStartTime) / 1000.0 / 1000.0 / 1000.0;
 397                 LOGGER.info("Wrote " + NUM_OBJECTS + " objects in " + totalElapsedTimeSecs + "s (" + NUM_OBJECTS / totalElapsedTimeSecs + " objects/s)");
 398
 399                 Arrays.sort(latencies);
 400
 401                 if (!warmUp) {
 402                         LOGGER.info("Store object latencies");
 403                         LOGGER.info("\t50th percentile: " + latencies[(int) (NUM_OBJECTS * 0.50)]);
 404                         LOGGER.info("\t75th percentile: " + latencies[(int) (NUM_OBJECTS * 0.75)]);
 405                         LOGGER.info("\t90th percentile: " + latencies[(int) (NUM_OBJECTS * 0.90)]);
 406                         LOGGER.info("\t99th percentile: " + latencies[(int) (NUM_OBJECTS * 0.99)]);
 407                 }
 408         }
 409
 410         private static void storeRelationships(ChampGraph graph, boolean warmUp) {
 411                 final List<ChampObject> objects = retrieveBulkObjects(graph, warmUp);
 412                 final double[] latencies = new double[NUM_RELATIONSHIPS];
 413                 final long totalStartTime = System.nanoTime();
 414
 415                 for (int i = 0; i < NUM_RELATIONSHIPS; i++) {
 416                         try {
 417                                 final long startTime = System.nanoTime();
 418
 419                                 graph.storeRelationship(
 420                                         new ChampRelationship.Builder(
 421                                                 objects.get(i % objects.size()), objects.get((i + 1) % objects.size()), "bazz"
 422                                         ).property("relationshipNumber", i)
 423                                         .build()
 424                                         , Optional.empty());
 425
 426                                 final double elapsedMs = (System.nanoTime() - startTime) / 1000.0 / 1000.0;
 427
 428                                 latencies[i] = elapsedMs;
 429                         } catch (ChampMarshallingException e) {
 430                                 throw new RuntimeException(e);
 431                         } catch (ChampObjectNotExistsException e) {
 432                                 throw new RuntimeException(e);
 433                         } catch (ChampSchemaViolationException e) {
 434                                 throw new RuntimeException(e);
 435                         } catch (ChampRelationshipNotExistsException e) {
 436                                 throw new RuntimeException(e);
 437                         } catch (ChampUnmarshallingException e) {
 438                                 throw new RuntimeException(e);
 439                         } catch (ChampTransactionException e) {
 440               throw new RuntimeException(e);
 441                         }
 442                 }
 443
 444                 final double totalElapsedTimeSecs = (System.nanoTime() - totalStartTime) / 1000.0 / 1000.0 / 1000.0;
 445                 LOGGER.info("Wrote " + NUM_RELATIONSHIPS + " relationships in " + totalElapsedTimeSecs + "s (" + NUM_RELATIONSHIPS / totalElapsedTimeSecs + " relationships/s)");
 446
 447                 Arrays.sort(latencies);
 448
 449                 if (!warmUp) {
 450                         LOGGER.info("Store relationship latencies");
 451                         LOGGER.info("\t50th percentile: " + latencies[(int) (NUM_RELATIONSHIPS * 0.50)]);
 452                         LOGGER.info("\t75th percentile: " + latencies[(int) (NUM_RELATIONSHIPS * 0.75)]);
 453                         LOGGER.info("\t90th percentile: " + latencies[(int) (NUM_RELATIONSHIPS * 0.90)]);
 454                         LOGGER.info("\t99th percentile: " + latencies[(int) (NUM_RELATIONSHIPS * 0.99)]);
 455                 }
 456         }
 457 }