champ-lib/champ-titan/src/test/java/org/onap/aai/champtitan/perf/ChampAPIPerformanceTest.java

   1 /**
   2  * ============LICENSE_START==========================================
   3  * org.onap.aai
   4  * ===================================================================
   5  * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
   6  * Copyright © 2017-2018 Amdocs
   7  * ===================================================================
   8  * Licensed under the Apache License, Version 2.0 (the "License");
   9  * you may not use this file except in compliance with the License.
  10  * You may obtain a copy of the License at
  11  *
  12  *        http://www.apache.org/licenses/LICENSE-2.0
  13  *
  14  * Unless required by applicable law or agreed to in writing, software
  15  * distributed under the License is distributed on an "AS IS" BASIS,
  16  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  17  * See the License for the specific language governing permissions and
  18  * limitations under the License.
  19  * ============LICENSE_END============================================
  20  */
  21 package org.onap.aai.champtitan.perf;
  22
  23 import com.thinkaurelius.titan.core.TitanFactory;
  24 import com.thinkaurelius.titan.core.TitanFactory.Builder;
  25 import com.thinkaurelius.titan.core.TitanGraph;
  26 import com.thinkaurelius.titan.core.util.TitanCleanup;
  27
  28 import org.onap.aai.champcore.ChampGraph;
  29 import org.onap.aai.champcore.exceptions.*;
  30 import org.onap.aai.champcore.graph.impl.InMemoryChampGraphImpl;
  31 import org.onap.aai.champcore.model.*;
  32 import org.onap.aai.champcore.schema.ChampSchemaEnforcer;
  33 import org.onap.aai.champtitan.graph.impl.TitanChampGraphImpl;
  34 import org.slf4j.Logger;
  35 import org.slf4j.LoggerFactory;
  36
  37 import java.util.*;
  38 import java.util.Map.Entry;
  39 import java.util.stream.Collectors;
  40 import java.util.stream.Stream;
  41
  42 public class ChampAPIPerformanceTest {
  43
  44         private static final Logger LOGGER = LoggerFactory.getLogger(ChampAPIPerformanceTest.class);
  45
  46         private static final int NUM_OBJECTS = 1000;
  47         private static final int NUM_RELATIONSHIPS = 1000;
  48         private static final String GRAPH_NAME = ChampAPIPerformanceTest.class.getSimpleName();
  49
  50         private static final String getGraphName() {
  51                 return GRAPH_NAME;
  52         }
  53
  54         private static void cleanUp(String graphName, Map<String, String> settings) {
  55                 LOGGER.debug("Cleaning up graph {}", graphName);
  56
  57                 try {
  58                         final Builder graphBuilder = TitanFactory.build();
  59
  60                         for (Entry<String, String> setting : settings.entrySet()) {
  61                                 graphBuilder.set(setting.getKey(), setting.getValue());
  62                         }
  63
  64                         final String storageBackend = settings.getOrDefault("storage.backend", "inmemory");
  65
  66                         if (storageBackend.equals("cassandra") ||
  67                                 storageBackend.equals("cassandrathrift") ||
  68                                 storageBackend.equals("astyanax") ||
  69                                 storageBackend.equals("embeddedcassandra")) {
  70                                 graphBuilder.set("storage.cassandra.keyspace", graphName);
  71                         } else if (storageBackend.equals("hbase")) {
  72                                 graphBuilder.set("storage.hbase.table", graphName);
  73                         }
  74
  75                         final TitanGraph graph = graphBuilder.open();
  76
  77                         graph.close();
  78                         TitanCleanup.clear(graph);
  79                 } catch (IllegalArgumentException e) {
  80                         LOGGER.warn("Could not clean up graph - unable to instantiate");
  81                 }
  82         }
  83
  84         public static void main(String[] args) {
  85
  86                 if (args.length < 1 || !args[0].startsWith("--champcore.graph.type=")) {
  87                         throw new RuntimeException("Must provide --champcore.graph.type=" + " as first parameter");
  88                 }
  89
  90                 final String graphType = args[0].split("=")[1];
  91
  92                 final Map<String, String> settings = new HashMap<String, String> ();
  93
  94                 for (int i = 1; i < args.length; i++) {
  95                         if (!args[i].startsWith("--")) {
  96                                 throw new RuntimeException("Bad command line argument: " + args[i]);
  97                         }
  98
  99                         final String[] keyValue = args[i].replaceFirst("--", "").split("=");
 100
 101                         if (keyValue.length != 2) {
 102                                 throw new RuntimeException("Bad command line argument: " + args[i]);
 103                         }
 104
 105                         settings.put(keyValue[0], keyValue[1]);
 106                 }
 107
 108                 LOGGER.info("Provided graph settings: " + settings);
 109
 110                 if (graphType.equals("TITAN")) {
 111                         cleanUp(getGraphName(), settings);
 112                 }
 113
 114                 LOGGER.info("Graph cleaned, instantiating ChampGraph");
 115
 116                 final ChampGraph graph;
 117
 118                 switch (graphType) {
 119                 case "IN_MEMORY":
 120                         final InMemoryChampGraphImpl.Builder inMemGraphBuilder = new InMemoryChampGraphImpl.Builder();
 121
 122                         if (settings.containsKey("champcore.schema.enforcer")) {
 123                                 final String schemaEnforcerClassStr = settings.get("champcore.schema.enforcer");
 124
 125                                 try {
 126                                         final Class<?> schemaEnforcer = Class.forName(schemaEnforcerClassStr);
 127
 128                                         if (!schemaEnforcer.isAssignableFrom(ChampSchemaEnforcer.class)) {
 129                                                 throw new RuntimeException("Unknown ChampSchemaEnforcer " + schemaEnforcer);
 130                                         }
 131
 132                                         inMemGraphBuilder.schemaEnforcer((ChampSchemaEnforcer) schemaEnforcer.newInstance());
 133                                 } catch (ClassNotFoundException e) {
 134                                         throw new RuntimeException(e);
 135                                 } catch (InstantiationException e) {
 136                                         throw new RuntimeException(e);
 137                                 } catch (IllegalAccessException e) {
 138                                         throw new RuntimeException(e);
 139                                 }
 140                         }
 141
 142                         graph = inMemGraphBuilder.build();
 143                 break;
 144                 case "TITAN":
 145                         final TitanChampGraphImpl.Builder graphBuilder = new TitanChampGraphImpl.Builder(getGraphName());
 146
 147                         for (Entry<String, String> setting : settings.entrySet()) {
 148                                 graphBuilder.property(setting.getKey(), setting.getValue());
 149                         }
 150
 151                         graph = graphBuilder.build();
 152                 break;
 153                 default:
 154                         throw new RuntimeException("Unknown ChampGraph.Type " + graphType);
 155                 }
 156
 157                 try {
 158           if (graph.queryObjects(Collections.emptyMap(), Optional.empty()).limit(1).count() > 0) {
 159                 graph.shutdown();
 160                 throw new RuntimeException("Expected empty graph");
 161           }
 162         } catch (ChampTransactionException e) {
 163           throw new RuntimeException("Transaction failure");
 164         }
 165
 166                 LOGGER.info("Graph instantiated, warming up JVM");
 167                 warmUp(graph);
 168
 169                 LOGGER.info("Warm up complete, starting to record performance measurements");
 170
 171                 LOGGER.info("Performance without indexing/schema");
 172
 173                 storeObjects(graph, false);
 174                 storeRelationships(graph, false);
 175                 retrieveIndividualObjects(graph, false);
 176                 retrieveBulkRelationships(graph, false);
 177                 retrieveIndividualRelationships(graph, false);
 178
 179                 LOGGER.info("Storing indices + schema");
 180
 181                 storeIndices(graph, false);
 182                 storeSchema(graph, false);
 183
 184                 LOGGER.info("Stored indices + schema");
 185
 186                 LOGGER.info("Performance with indexing + schema");
 187
 188                 storeObjects(graph, false);
 189                 storeRelationships(graph, false);
 190                 retrieveIndividualObjects(graph, false);
 191                 retrieveBulkRelationships(graph, false);
 192                 retrieveIndividualRelationships(graph, false);
 193
 194                 LOGGER.info("Performance test complete, shutting down graph");
 195
 196                 graph.shutdown();
 197
 198                 LOGGER.info("Graph shutdown, JVM exiting");
 199         }
 200
 201         private static void storeSchema(ChampGraph graph, boolean warmUp) {
 202                 try {
 203                         graph.storeSchema(
 204                                         ChampSchema.create()
 205                                                         .withObjectConstraint()
 206                                                         .onType("foo")
 207                                                         .withPropertyConstraint()
 208                                                         .onField("fooObjectNumber")
 209                                                         .optional()
 210                                                         .build()
 211                                                         .build()
 212                                                         .withRelationshipConstraint()
 213                                                         .onType("bar")
 214                                                         .withPropertyConstraint()
 215                                                         .onField("barObjectNumber")
 216                                                         .ofType(ChampField.Type.INTEGER)
 217                                                         .optional()
 218                                                         .build()
 219                                                         .build()
 220                                                         .build()
 221                         );
 222                 } catch (ChampSchemaViolationException e) {
 223                         throw new AssertionError(e);
 224                 }
 225         }
 226
 227         private static void storeIndices(ChampGraph graph, boolean warmUp) {
 228         List<String> fields = new ArrayList<String>();
 229         fields.add("objectNumber");
 230                 graph.storeObjectIndex(
 231                         ChampObjectIndex.create()
 232                                                         .ofName("objectNumberIndex")
 233                                                         .onType("foo")
 234                                                         .forFields(fields)
 235                                                         .build()
 236                 );
 237
 238                 graph.storeRelationshipIndex(ChampRelationshipIndex.create()
 239                                                                                                                         .ofName("relationshipNumberIndex")
 240                                                                                                                         .onType("bazz")
 241                                                                                                                         .forField("relationshipNumber")
 242                                                                                                                         .build()
 243                 );
 244         }
 245
 246         private static void warmUp(ChampGraph graph) {
 247                 storeObjects(graph, false);
 248                 storeRelationships(graph, false);
 249                 retrieveIndividualObjects(graph, false);
 250                 retrieveBulkRelationships(graph, false);
 251                 retrieveIndividualRelationships(graph, false);
 252         }
 253
 254         private static void retrieveIndividualRelationships(ChampGraph graph, boolean warmUp) {
 255                 final double[] latencies = new double[NUM_RELATIONSHIPS];
 256                 final long totalStartTime = System.nanoTime();
 257
 258                 for (int i = 0; i < NUM_RELATIONSHIPS; i++) {
 259                         final long startTime = System.nanoTime();
 260
 261                         Stream<ChampRelationship> objects;
 262             try {
 263               objects = graph.queryRelationships(Collections.singletonMap("relationshipNumber", i), Optional.empty());
 264             } catch (ChampTransactionException e) {
 265               throw new RuntimeException(e);
 266             }
 267
 268                         objects.findFirst().get();
 269                         final double elapsedMs = (System.nanoTime() - startTime) / 1000.0 / 1000.0;
 270                         latencies[i] = elapsedMs;
 271                 }
 272
 273                 final double totalElapsedTimeSecs = (System.nanoTime() - totalStartTime) / 1000.0 / 1000.0 / 1000.0;
 274                 LOGGER.info("Individually read " + NUM_RELATIONSHIPS + " relationships in " + totalElapsedTimeSecs + "s (" + NUM_RELATIONSHIPS / totalElapsedTimeSecs + " relationships/s)");
 275
 276                 Arrays.sort(latencies);
 277
 278                 if (!warmUp) {
 279                         LOGGER.info("Retrieve individual relationship latencies");
 280                         LOGGER.info("\t50th percentile: " + latencies[(int) (NUM_RELATIONSHIPS * 0.50)]);
 281                         LOGGER.info("\t75th percentile: " + latencies[(int) (NUM_RELATIONSHIPS * 0.75)]);
 282                         LOGGER.info("\t90th percentile: " + latencies[(int) (NUM_RELATIONSHIPS * 0.90)]);
 283                         LOGGER.info("\t99th percentile: " + latencies[(int) (NUM_RELATIONSHIPS * 0.99)]);
 284                 }
 285         }
 286
 287         private static void retrieveIndividualObjects(ChampGraph graph, boolean warmUp) {
 288
 289                 final double[] latencies = new double[NUM_OBJECTS];
 290                 final long totalStartTime = System.nanoTime();
 291
 292                 for (int i = 0; i < NUM_OBJECTS; i++) {
 293                         final long startTime = System.nanoTime();
 294                         Stream<ChampObject> objects;
 295             try {
 296               objects = graph.queryObjects(Collections.singletonMap("objectNumber", i), Optional.empty());
 297             } catch (ChampTransactionException e) {
 298                 throw new RuntimeException(e);
 299             }
 300
 301                         objects.findFirst().get();
 302
 303                         final double elapsedMs = (System.nanoTime() - startTime) / 1000.0 / 1000.0;
 304
 305                         latencies[i] = elapsedMs;
 306                 }
 307
 308                 final double totalElapsedTimeSecs = (System.nanoTime() - totalStartTime) / 1000.0 / 1000.0 / 1000.0;
 309
 310                 LOGGER.info("Individually read " + NUM_OBJECTS + " objects in " + totalElapsedTimeSecs + "s (" + NUM_OBJECTS / totalElapsedTimeSecs + " objects/s)");
 311                 Arrays.sort(latencies);
 312
 313                 if (!warmUp) {
 314                         LOGGER.info("Retrieve individual object latencies");
 315                         LOGGER.info("\t50th percentile: " + latencies[(int) (NUM_OBJECTS * 0.50)]);
 316                         LOGGER.info("\t75th percentile: " + latencies[(int) (NUM_OBJECTS * 0.75)]);
 317                         LOGGER.info("\t90th percentile: " + latencies[(int) (NUM_OBJECTS * 0.90)]);
 318                         LOGGER.info("\t99th percentile: " + latencies[(int) (NUM_OBJECTS * 0.99)]);
 319                 }
 320         }
 321
 322         private static List<ChampObject> retrieveBulkObjects(ChampGraph graph, boolean warmUp) {
 323
 324                 final long startTime = System.nanoTime();
 325                 Stream<ChampObject> objects;
 326         try {
 327           objects = graph.queryObjects(
 328                                         Collections.singletonMap(
 329                                                 ChampObject.ReservedPropertyKeys.CHAMP_OBJECT_TYPE.toString(), "foo"
 330                                         ), Optional.empty()
 331                                 );
 332         } catch (ChampTransactionException e) {
 333           throw new RuntimeException(e);
 334         }
 335
 336                 final List<ChampObject> objectsAsList = objects.collect(Collectors.toList());
 337                 final double elapsedSecs = (System.nanoTime() - startTime) / 1000.0 / 1000.0 / 1000.0;
 338
 339                 if (!warmUp) {
 340                         LOGGER.info("Bulk read " + objectsAsList.size() + " objects in " + elapsedSecs + "s (" + objectsAsList.size() / elapsedSecs + " objects/s)");
 341                 }
 342
 343                 return objectsAsList;
 344         }
 345
 346         private static List<ChampRelationship> retrieveBulkRelationships(ChampGraph graph, boolean warmUp) {
 347                 final long startTime = System.nanoTime();
 348                 Stream<ChampRelationship> relationships;
 349         try {
 350           relationships = graph.queryRelationships(
 351                                         Collections.singletonMap(
 352                                                 ChampRelationship.ReservedPropertyKeys.CHAMP_RELATIONSHIP_TYPE.toString(), "bazz"
 353                                         ), Optional.empty()
 354                                 );
 355         } catch (ChampTransactionException e) {
 356             throw new RuntimeException(e);
 357         }
 358
 359                 final List<ChampRelationship> relationshipsAsList = relationships.collect(Collectors.toList());
 360                 final double elapsedSecs = (System.nanoTime() - startTime) / 1000.0 / 1000.0 / 1000.0;
 361
 362                 if (!warmUp) {
 363                         LOGGER.info("Bulk read " + relationshipsAsList.size() + " relationships in " + elapsedSecs + "s (" + relationshipsAsList.size() / elapsedSecs + " relationships/s)");
 364                 }
 365
 366                 return relationshipsAsList;
 367         }
 368
 369         private static void storeObjects(ChampGraph graph, boolean warmUp) {
 370                 final double[] latencies = new double[NUM_OBJECTS];
 371                 final long totalStartTime = System.nanoTime();
 372
 373                 for (int i = 0; i < NUM_OBJECTS; i++) {
 374                         try {
 375                                 final long startTime = System.nanoTime();
 376
 377                                 graph.storeObject(
 378                                         ChampObject.create()
 379                                                                 .ofType("foo")
 380                                                                 .withoutKey()
 381                                                                 .withProperty("objectNumber", i)
 382                                                                 .build(), Optional.empty()
 383                                 );
 384
 385                                 final double elapsedMs = (System.nanoTime() - startTime) / 1000.0 / 1000.0;
 386                                 latencies[i] = elapsedMs;
 387                         } catch (ChampMarshallingException e) {
 388                                 throw new RuntimeException(e);
 389                         } catch (ChampSchemaViolationException e) {
 390                                 //Ignore, no schema set
 391                         } catch (ChampObjectNotExistsException e) {
 392                                 //Ignore, not an update
 393                         } catch (ChampTransactionException e) {
 394               throw new RuntimeException(e);
 395             }
 396                 }
 397
 398                 final double totalElapsedTimeSecs = (System.nanoTime() - totalStartTime) / 1000.0 / 1000.0 / 1000.0;
 399                 LOGGER.info("Wrote " + NUM_OBJECTS + " objects in " + totalElapsedTimeSecs + "s (" + NUM_OBJECTS / totalElapsedTimeSecs + " objects/s)");
 400
 401                 Arrays.sort(latencies);
 402
 403                 if (!warmUp) {
 404                         LOGGER.info("Store object latencies");
 405                         LOGGER.info("\t50th percentile: " + latencies[(int) (NUM_OBJECTS * 0.50)]);
 406                         LOGGER.info("\t75th percentile: " + latencies[(int) (NUM_OBJECTS * 0.75)]);
 407                         LOGGER.info("\t90th percentile: " + latencies[(int) (NUM_OBJECTS * 0.90)]);
 408                         LOGGER.info("\t99th percentile: " + latencies[(int) (NUM_OBJECTS * 0.99)]);
 409                 }
 410         }
 411
 412         private static void storeRelationships(ChampGraph graph, boolean warmUp) {
 413                 final List<ChampObject> objects = retrieveBulkObjects(graph, warmUp);
 414                 final double[] latencies = new double[NUM_RELATIONSHIPS];
 415                 final long totalStartTime = System.nanoTime();
 416
 417                 for (int i = 0; i < NUM_RELATIONSHIPS; i++) {
 418                         try {
 419                                 final long startTime = System.nanoTime();
 420
 421                                 graph.storeRelationship(
 422                                         new ChampRelationship.Builder(
 423                                                 objects.get(i % objects.size()), objects.get((i + 1) % objects.size()), "bazz"
 424                                         ).property("relationshipNumber", i)
 425                                         .build()
 426                                         , Optional.empty());
 427
 428                                 final double elapsedMs = (System.nanoTime() - startTime) / 1000.0 / 1000.0;
 429
 430                                 latencies[i] = elapsedMs;
 431                         } catch (ChampMarshallingException e) {
 432                                 throw new RuntimeException(e);
 433                         } catch (ChampObjectNotExistsException e) {
 434                                 throw new RuntimeException(e);
 435                         } catch (ChampSchemaViolationException e) {
 436                                 throw new RuntimeException(e);
 437                         } catch (ChampRelationshipNotExistsException e) {
 438                                 throw new RuntimeException(e);
 439                         } catch (ChampUnmarshallingException e) {
 440                                 throw new RuntimeException(e);
 441                         } catch (ChampTransactionException e) {
 442               throw new RuntimeException(e);
 443                         }
 444                 }
 445
 446                 final double totalElapsedTimeSecs = (System.nanoTime() - totalStartTime) / 1000.0 / 1000.0 / 1000.0;
 447                 LOGGER.info("Wrote " + NUM_RELATIONSHIPS + " relationships in " + totalElapsedTimeSecs + "s (" + NUM_RELATIONSHIPS / totalElapsedTimeSecs + " relationships/s)");
 448
 449                 Arrays.sort(latencies);
 450
 451                 if (!warmUp) {
 452                         LOGGER.info("Store relationship latencies");
 453                         LOGGER.info("\t50th percentile: " + latencies[(int) (NUM_RELATIONSHIPS * 0.50)]);
 454                         LOGGER.info("\t75th percentile: " + latencies[(int) (NUM_RELATIONSHIPS * 0.75)]);
 455                         LOGGER.info("\t90th percentile: " + latencies[(int) (NUM_RELATIONSHIPS * 0.90)]);
 456                         LOGGER.info("\t99th percentile: " + latencies[(int) (NUM_RELATIONSHIPS * 0.99)]);
 457                 }
 458         }
 459 }