champ-lib/champ-titan/src/main/java/org/onap/aai/champtitan/perf/ChampAPIPerformanceTest.java

   1 /**
   2  * ============LICENSE_START==========================================
   3  * org.onap.aai
   4  * ===================================================================
   5  * Copyright © 2017 AT&T Intellectual Property. All rights reserved.
   6  * Copyright © 2017 Amdocs
   7  * ===================================================================
   8  * Licensed under the Apache License, Version 2.0 (the "License");
   9  * you may not use this file except in compliance with the License.
  10  * You may obtain a copy of the License at
  11  *
  12  *        http://www.apache.org/licenses/LICENSE-2.0
  13  *
  14  * Unless required by applicable law or agreed to in writing, software
  15  * distributed under the License is distributed on an "AS IS" BASIS,
  16  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  17  * See the License for the specific language governing permissions and
  18  * limitations under the License.
  19  * ============LICENSE_END============================================
  20  * ECOMP is a trademark and service mark of AT&T Intellectual Property.
  21  */
  22 package org.onap.aai.champtitan.perf;
  23
  24 import com.thinkaurelius.titan.core.TitanFactory;
  25 import com.thinkaurelius.titan.core.TitanFactory.Builder;
  26 import com.thinkaurelius.titan.core.TitanGraph;
  27 import com.thinkaurelius.titan.core.util.TitanCleanup;
  28
  29 import org.onap.aai.champcore.ChampGraph;
  30 import org.onap.aai.champcore.exceptions.*;
  31 import org.onap.aai.champcore.graph.impl.InMemoryChampGraphImpl;
  32 import org.onap.aai.champcore.model.*;
  33 import org.onap.aai.champcore.schema.ChampSchemaEnforcer;
  34 import org.onap.aai.champtitan.graph.impl.TitanChampGraphImpl;
  35 import org.slf4j.Logger;
  36 import org.slf4j.LoggerFactory;
  37
  38 import java.util.*;
  39 import java.util.Map.Entry;
  40 import java.util.stream.Collectors;
  41 import java.util.stream.Stream;
  42
  43 public class ChampAPIPerformanceTest {
  44
  45         private static final Logger LOGGER = LoggerFactory.getLogger(ChampAPIPerformanceTest.class);
  46
  47         private static final int NUM_OBJECTS = 1000;
  48         private static final int NUM_RELATIONSHIPS = 1000;
  49         private static final String GRAPH_NAME = ChampAPIPerformanceTest.class.getSimpleName();
  50
  51         private static final String getGraphName() {
  52                 return GRAPH_NAME;
  53         }
  54
  55         private static void cleanUp(String graphName, Map<String, String> settings) {
  56                 LOGGER.debug("Cleaning up graph {}", graphName);
  57
  58                 try {
  59                         final Builder graphBuilder = TitanFactory.build();
  60
  61                         for (Entry<String, String> setting : settings.entrySet()) {
  62                                 graphBuilder.set(setting.getKey(), setting.getValue());
  63                         }
  64
  65                         final String storageBackend = settings.getOrDefault("storage.backend", "inmemory");
  66
  67                         if (storageBackend.equals("cassandra") ||
  68                                 storageBackend.equals("cassandrathrift") ||
  69                                 storageBackend.equals("astyanax") ||
  70                                 storageBackend.equals("embeddedcassandra")) {
  71                                 graphBuilder.set("storage.cassandra.keyspace", graphName);
  72                         } else if (storageBackend.equals("hbase")) {
  73                                 graphBuilder.set("storage.hbase.table", graphName);
  74                         }
  75
  76                         final TitanGraph graph = graphBuilder.open();
  77
  78                         graph.close();
  79                         TitanCleanup.clear(graph);
  80                 } catch (IllegalArgumentException e) {
  81                         LOGGER.warn("Could not clean up graph - unable to instantiate");
  82                 }
  83         }
  84
  85         public static void main(String[] args) {
  86
  87                 if (args.length < 1 || !args[0].startsWith("--champcore.graph.type=")) {
  88                         throw new RuntimeException("Must provide --champcore.graph.type=" + " as first parameter");
  89                 }
  90
  91                 final String graphType = args[0].split("=")[1];
  92
  93                 final Map<String, String> settings = new HashMap<String, String> ();
  94
  95                 for (int i = 1; i < args.length; i++) {
  96                         if (!args[i].startsWith("--")) {
  97                                 throw new RuntimeException("Bad command line argument: " + args[i]);
  98                         }
  99
 100                         final String[] keyValue = args[i].replaceFirst("--", "").split("=");
 101
 102                         if (keyValue.length != 2) {
 103                                 throw new RuntimeException("Bad command line argument: " + args[i]);
 104                         }
 105
 106                         settings.put(keyValue[0], keyValue[1]);
 107                 }
 108
 109                 LOGGER.info("Provided graph settings: " + settings);
 110
 111                 if (graphType.equals("TITAN")) {
 112                         cleanUp(getGraphName(), settings);
 113                 }
 114
 115                 LOGGER.info("Graph cleaned, instantiating ChampGraph");
 116
 117                 final ChampGraph graph;
 118
 119                 switch (graphType) {
 120                 case "IN_MEMORY":
 121                         final InMemoryChampGraphImpl.Builder inMemGraphBuilder = new InMemoryChampGraphImpl.Builder();
 122
 123                         if (settings.containsKey("champcore.schema.enforcer")) {
 124                                 final String schemaEnforcerClassStr = settings.get("champcore.schema.enforcer");
 125
 126                                 try {
 127                                         final Class<?> schemaEnforcer = Class.forName(schemaEnforcerClassStr);
 128
 129                                         if (!schemaEnforcer.isAssignableFrom(ChampSchemaEnforcer.class)) {
 130                                                 throw new RuntimeException("Unknown ChampSchemaEnforcer " + schemaEnforcer);
 131                                         }
 132
 133                                         inMemGraphBuilder.schemaEnforcer((ChampSchemaEnforcer) schemaEnforcer.newInstance());
 134                                 } catch (ClassNotFoundException e) {
 135                                         throw new RuntimeException(e);
 136                                 } catch (InstantiationException e) {
 137                                         throw new RuntimeException(e);
 138                                 } catch (IllegalAccessException e) {
 139                                         throw new RuntimeException(e);
 140                                 }
 141                         }
 142
 143                         graph = inMemGraphBuilder.build();
 144                 break;
 145                 case "TITAN":
 146                         final TitanChampGraphImpl.Builder graphBuilder = new TitanChampGraphImpl.Builder(getGraphName());
 147
 148                         for (Entry<String, String> setting : settings.entrySet()) {
 149                                 graphBuilder.property(setting.getKey(), setting.getValue());
 150                         }
 151
 152                         graph = graphBuilder.build();
 153                 break;
 154                 default:
 155                         throw new RuntimeException("Unknown ChampGraph.Type " + graphType);
 156                 }
 157
 158                 try {
 159           if (graph.queryObjects(Collections.emptyMap(), Optional.empty()).limit(1).count() > 0) {
 160                 graph.shutdown();
 161                 throw new RuntimeException("Expected empty graph");
 162           }
 163         } catch (ChampTransactionException e) {
 164           throw new RuntimeException("Transaction failure");
 165         }
 166
 167                 LOGGER.info("Graph instantiated, warming up JVM");
 168                 warmUp(graph);
 169
 170                 LOGGER.info("Warm up complete, starting to record performance measurements");
 171
 172                 LOGGER.info("Performance without indexing/schema");
 173
 174                 storeObjects(graph, false);
 175                 storeRelationships(graph, false);
 176                 retrieveIndividualObjects(graph, false);
 177                 retrieveBulkRelationships(graph, false);
 178                 retrieveIndividualRelationships(graph, false);
 179
 180                 LOGGER.info("Storing indices + schema");
 181
 182                 storeIndices(graph, false);
 183                 storeSchema(graph, false);
 184
 185                 LOGGER.info("Stored indices + schema");
 186
 187                 LOGGER.info("Performance with indexing + schema");
 188
 189                 storeObjects(graph, false);
 190                 storeRelationships(graph, false);
 191                 retrieveIndividualObjects(graph, false);
 192                 retrieveBulkRelationships(graph, false);
 193                 retrieveIndividualRelationships(graph, false);
 194
 195                 LOGGER.info("Performance test complete, shutting down graph");
 196
 197                 graph.shutdown();
 198
 199                 LOGGER.info("Graph shutdown, JVM exiting");
 200         }
 201
 202         private static void storeSchema(ChampGraph graph, boolean warmUp) {
 203                 try {
 204                         graph.storeSchema(
 205                                         ChampSchema.create()
 206                                                         .withObjectConstraint()
 207                                                         .onType("foo")
 208                                                         .withPropertyConstraint()
 209                                                         .onField("fooObjectNumber")
 210                                                         .optional()
 211                                                         .build()
 212                                                         .build()
 213                                                         .withRelationshipConstraint()
 214                                                         .onType("bar")
 215                                                         .withPropertyConstraint()
 216                                                         .onField("barObjectNumber")
 217                                                         .ofType(ChampField.Type.INTEGER)
 218                                                         .optional()
 219                                                         .build()
 220                                                         .build()
 221                                                         .build()
 222                         );
 223                 } catch (ChampSchemaViolationException e) {
 224                         throw new AssertionError(e);
 225                 }
 226         }
 227
 228         private static void storeIndices(ChampGraph graph, boolean warmUp) {
 229                 graph.storeObjectIndex(
 230                         ChampObjectIndex.create()
 231                                                         .ofName("objectNumberIndex")
 232                                                         .onType("foo")
 233                                                         .forField("objectNumber")
 234                                                         .build()
 235                 );
 236
 237                 graph.storeRelationshipIndex(ChampRelationshipIndex.create()
 238                                                                                                                         .ofName("relationshipNumberIndex")
 239                                                                                                                         .onType("bazz")
 240                                                                                                                         .forField("relationshipNumber")
 241                                                                                                                         .build()
 242                 );
 243         }
 244
 245         private static void warmUp(ChampGraph graph) {
 246                 storeObjects(graph, false);
 247                 storeRelationships(graph, false);
 248                 retrieveIndividualObjects(graph, false);
 249                 retrieveBulkRelationships(graph, false);
 250                 retrieveIndividualRelationships(graph, false);
 251         }
 252
 253         private static void retrieveIndividualRelationships(ChampGraph graph, boolean warmUp) {
 254                 final double[] latencies = new double[NUM_RELATIONSHIPS];
 255                 final long totalStartTime = System.nanoTime();
 256
 257                 for (int i = 0; i < NUM_RELATIONSHIPS; i++) {
 258                         final long startTime = System.nanoTime();
 259
 260                         Stream<ChampRelationship> objects;
 261             try {
 262               objects = graph.queryRelationships(Collections.singletonMap("relationshipNumber", i), Optional.empty());
 263             } catch (ChampTransactionException e) {
 264               throw new RuntimeException(e);
 265             }
 266
 267                         objects.findFirst().get();
 268                         final double elapsedMs = (System.nanoTime() - startTime) / 1000.0 / 1000.0;
 269                         latencies[i] = elapsedMs;
 270                 }
 271
 272                 final double totalElapsedTimeSecs = (System.nanoTime() - totalStartTime) / 1000.0 / 1000.0 / 1000.0;
 273                 LOGGER.info("Individually read " + NUM_RELATIONSHIPS + " relationships in " + totalElapsedTimeSecs + "s (" + NUM_RELATIONSHIPS / totalElapsedTimeSecs + " relationships/s)");
 274
 275                 Arrays.sort(latencies);
 276
 277                 if (!warmUp) {
 278                         LOGGER.info("Retrieve individual relationship latencies");
 279                         LOGGER.info("\t50th percentile: " + latencies[(int) (NUM_RELATIONSHIPS * 0.50)]);
 280                         LOGGER.info("\t75th percentile: " + latencies[(int) (NUM_RELATIONSHIPS * 0.75)]);
 281                         LOGGER.info("\t90th percentile: " + latencies[(int) (NUM_RELATIONSHIPS * 0.90)]);
 282                         LOGGER.info("\t99th percentile: " + latencies[(int) (NUM_RELATIONSHIPS * 0.99)]);
 283                 }
 284         }
 285
 286         private static void retrieveIndividualObjects(ChampGraph graph, boolean warmUp) {
 287
 288                 final double[] latencies = new double[NUM_OBJECTS];
 289                 final long totalStartTime = System.nanoTime();
 290
 291                 for (int i = 0; i < NUM_OBJECTS; i++) {
 292                         final long startTime = System.nanoTime();
 293                         Stream<ChampObject> objects;
 294             try {
 295               objects = graph.queryObjects(Collections.singletonMap("objectNumber", i), Optional.empty());
 296             } catch (ChampTransactionException e) {
 297                 throw new RuntimeException(e);
 298             }
 299
 300                         objects.findFirst().get();
 301
 302                         final double elapsedMs = (System.nanoTime() - startTime) / 1000.0 / 1000.0;
 303
 304                         latencies[i] = elapsedMs;
 305                 }
 306
 307                 final double totalElapsedTimeSecs = (System.nanoTime() - totalStartTime) / 1000.0 / 1000.0 / 1000.0;
 308
 309                 LOGGER.info("Individually read " + NUM_OBJECTS + " objects in " + totalElapsedTimeSecs + "s (" + NUM_OBJECTS / totalElapsedTimeSecs + " objects/s)");
 310                 Arrays.sort(latencies);
 311
 312                 if (!warmUp) {
 313                         LOGGER.info("Retrieve individual object latencies");
 314                         LOGGER.info("\t50th percentile: " + latencies[(int) (NUM_OBJECTS * 0.50)]);
 315                         LOGGER.info("\t75th percentile: " + latencies[(int) (NUM_OBJECTS * 0.75)]);
 316                         LOGGER.info("\t90th percentile: " + latencies[(int) (NUM_OBJECTS * 0.90)]);
 317                         LOGGER.info("\t99th percentile: " + latencies[(int) (NUM_OBJECTS * 0.99)]);
 318                 }
 319         }
 320
 321         private static List<ChampObject> retrieveBulkObjects(ChampGraph graph, boolean warmUp) {
 322
 323                 final long startTime = System.nanoTime();
 324                 Stream<ChampObject> objects;
 325         try {
 326           objects = graph.queryObjects(
 327                                         Collections.singletonMap(
 328                                                 ChampObject.ReservedPropertyKeys.CHAMP_OBJECT_TYPE.toString(), "foo"
 329                                         ), Optional.empty()
 330                                 );
 331         } catch (ChampTransactionException e) {
 332           throw new RuntimeException(e);
 333         }
 334
 335                 final List<ChampObject> objectsAsList = objects.collect(Collectors.toList());
 336                 final double elapsedSecs = (System.nanoTime() - startTime) / 1000.0 / 1000.0 / 1000.0;
 337
 338                 if (!warmUp) {
 339                         LOGGER.info("Bulk read " + objectsAsList.size() + " objects in " + elapsedSecs + "s (" + objectsAsList.size() / elapsedSecs + " objects/s)");
 340                 }
 341
 342                 return objectsAsList;
 343         }
 344
 345         private static List<ChampRelationship> retrieveBulkRelationships(ChampGraph graph, boolean warmUp) {
 346                 final long startTime = System.nanoTime();
 347                 Stream<ChampRelationship> relationships;
 348         try {
 349           relationships = graph.queryRelationships(
 350                                         Collections.singletonMap(
 351                                                 ChampRelationship.ReservedPropertyKeys.CHAMP_RELATIONSHIP_TYPE.toString(), "bazz"
 352                                         ), Optional.empty()
 353                                 );
 354         } catch (ChampTransactionException e) {
 355             throw new RuntimeException(e);
 356         }
 357
 358                 final List<ChampRelationship> relationshipsAsList = relationships.collect(Collectors.toList());
 359                 final double elapsedSecs = (System.nanoTime() - startTime) / 1000.0 / 1000.0 / 1000.0;
 360
 361                 if (!warmUp) {
 362                         LOGGER.info("Bulk read " + relationshipsAsList.size() + " relationships in " + elapsedSecs + "s (" + relationshipsAsList.size() / elapsedSecs + " relationships/s)");
 363                 }
 364
 365                 return relationshipsAsList;
 366         }
 367
 368         private static void storeObjects(ChampGraph graph, boolean warmUp) {
 369                 final double[] latencies = new double[NUM_OBJECTS];
 370                 final long totalStartTime = System.nanoTime();
 371
 372                 for (int i = 0; i < NUM_OBJECTS; i++) {
 373                         try {
 374                                 final long startTime = System.nanoTime();
 375
 376                                 graph.storeObject(
 377                                         ChampObject.create()
 378                                                                 .ofType("foo")
 379                                                                 .withoutKey()
 380                                                                 .withProperty("objectNumber", i)
 381                                                                 .build(), Optional.empty()
 382                                 );
 383
 384                                 final double elapsedMs = (System.nanoTime() - startTime) / 1000.0 / 1000.0;
 385                                 latencies[i] = elapsedMs;
 386                         } catch (ChampMarshallingException e) {
 387                                 throw new RuntimeException(e);
 388                         } catch (ChampSchemaViolationException e) {
 389                                 //Ignore, no schema set
 390                         } catch (ChampObjectNotExistsException e) {
 391                                 //Ignore, not an update
 392                         } catch (ChampTransactionException e) {
 393               throw new RuntimeException(e);
 394             }
 395                 }
 396
 397                 final double totalElapsedTimeSecs = (System.nanoTime() - totalStartTime) / 1000.0 / 1000.0 / 1000.0;
 398                 LOGGER.info("Wrote " + NUM_OBJECTS + " objects in " + totalElapsedTimeSecs + "s (" + NUM_OBJECTS / totalElapsedTimeSecs + " objects/s)");
 399
 400                 Arrays.sort(latencies);
 401
 402                 if (!warmUp) {
 403                         LOGGER.info("Store object latencies");
 404                         LOGGER.info("\t50th percentile: " + latencies[(int) (NUM_OBJECTS * 0.50)]);
 405                         LOGGER.info("\t75th percentile: " + latencies[(int) (NUM_OBJECTS * 0.75)]);
 406                         LOGGER.info("\t90th percentile: " + latencies[(int) (NUM_OBJECTS * 0.90)]);
 407                         LOGGER.info("\t99th percentile: " + latencies[(int) (NUM_OBJECTS * 0.99)]);
 408                 }
 409         }
 410
 411         private static void storeRelationships(ChampGraph graph, boolean warmUp) {
 412                 final List<ChampObject> objects = retrieveBulkObjects(graph, warmUp);
 413                 final double[] latencies = new double[NUM_RELATIONSHIPS];
 414                 final long totalStartTime = System.nanoTime();
 415
 416                 for (int i = 0; i < NUM_RELATIONSHIPS; i++) {
 417                         try {
 418                                 final long startTime = System.nanoTime();
 419
 420                                 graph.storeRelationship(
 421                                         new ChampRelationship.Builder(
 422                                                 objects.get(i % objects.size()), objects.get((i + 1) % objects.size()), "bazz"
 423                                         ).property("relationshipNumber", i)
 424                                         .build()
 425                                         , Optional.empty());
 426
 427                                 final double elapsedMs = (System.nanoTime() - startTime) / 1000.0 / 1000.0;
 428
 429                                 latencies[i] = elapsedMs;
 430                         } catch (ChampMarshallingException e) {
 431                                 throw new RuntimeException(e);
 432                         } catch (ChampObjectNotExistsException e) {
 433                                 throw new RuntimeException(e);
 434                         } catch (ChampSchemaViolationException e) {
 435                                 throw new RuntimeException(e);
 436                         } catch (ChampRelationshipNotExistsException e) {
 437                                 throw new RuntimeException(e);
 438                         } catch (ChampUnmarshallingException e) {
 439                                 throw new RuntimeException(e);
 440                         } catch (ChampTransactionException e) {
 441               throw new RuntimeException(e);
 442                         }
 443                 }
 444
 445                 final double totalElapsedTimeSecs = (System.nanoTime() - totalStartTime) / 1000.0 / 1000.0 / 1000.0;
 446                 LOGGER.info("Wrote " + NUM_RELATIONSHIPS + " relationships in " + totalElapsedTimeSecs + "s (" + NUM_RELATIONSHIPS / totalElapsedTimeSecs + " relationships/s)");
 447
 448                 Arrays.sort(latencies);
 449
 450                 if (!warmUp) {
 451                         LOGGER.info("Store relationship latencies");
 452                         LOGGER.info("\t50th percentile: " + latencies[(int) (NUM_RELATIONSHIPS * 0.50)]);
 453                         LOGGER.info("\t75th percentile: " + latencies[(int) (NUM_RELATIONSHIPS * 0.75)]);
 454                         LOGGER.info("\t90th percentile: " + latencies[(int) (NUM_RELATIONSHIPS * 0.90)]);
 455                         LOGGER.info("\t99th percentile: " + latencies[(int) (NUM_RELATIONSHIPS * 0.99)]);
 456                 }
 457         }
 458 }