dcaedt_catalog/db/src/main/java/org/onap/sdc/dcae/db/neo4j/Modeled.java

   1 /*
   2  *                        AT&T - PROPRIETARY
   3  *          THIS FILE CONTAINS PROPRIETARY INFORMATION OF
   4  *        AT&T AND IS NOT TO BE DISCLOSED OR USED EXCEPT IN
   5  *             ACCORDANCE WITH APPLICABLE AGREEMENTS.
   6  *
   7  *          Copyright (c) 2014 AT&T Knowledge Ventures
   8  *              Unpublished and Not for Publication
   9  *                     All Rights Reserved
  10  */
  11 package org.onap.sdc.dcae.db.neo4j;
  12
  13 import java.io.FileInputStream;
  14 import java.io.IOException;
  15 import java.nio.charset.Charset;
  16 import java.net.URI;
  17 import java.net.URISyntaxException;
  18 import java.util.HashMap;
  19 import java.util.Iterator;
  20 import java.util.Map;
  21 import java.util.List;
  22 import java.util.LinkedList;
  23 import java.util.Collections;
  24
  25 import org.apache.commons.cli.BasicParser;
  26 import org.apache.commons.cli.CommandLine;
  27 import org.apache.commons.cli.CommandLineParser;
  28 import org.apache.commons.cli.HelpFormatter;
  29 import org.apache.commons.cli.OptionBuilder;
  30 import org.apache.commons.cli.Options;
  31 import org.apache.commons.cli.ParseException;
  32 import org.apache.commons.io.IOUtils;
  33 import org.apache.commons.codec.binary.Base64;
  34
  35 import org.apache.commons.jxpath.JXPathContext;
  36 import org.apache.commons.jxpath.JXPathException;
  37
  38 import org.apache.http.Header;
  39 import org.apache.http.HttpHeaders;
  40 import org.apache.http.HttpResponse;
  41 import org.apache.http.client.HttpClient;
  42 import org.apache.http.client.methods.HttpUriRequest;
  43 import org.apache.http.client.methods.HttpPost;
  44 import org.apache.http.client.methods.HttpDelete;
  45 import org.apache.http.entity.ContentType;
  46 import org.apache.http.entity.StringEntity;
  47 import org.apache.http.impl.client.HttpClientBuilder;
  48 import org.json.JSONException;
  49 import org.json.JSONObject;
  50 import org.json.JSONArray;
  51
  52 import org.onap.sdc.common.onaplog.OnapLoggerDebug;
  53 import org.onap.sdc.common.onaplog.OnapLoggerError;
  54 import org.onap.sdc.common.onaplog.Enums.LogLevel;
  55 import org.yaml.snakeyaml.Yaml;
  56
  57 import com.google.common.collect.Table;
  58 import com.google.common.collect.HashBasedTable;
  59
  60 /* A few less obvious design choices:
  61  *   - representing properties across type hierarchies (same for requirements
  62  * and capabilities, and will be for attributes and interfaces when we'll
  63  * add them): we attach to each type only those properties it declares (such a
  64  * declaration might be the re-definition of a property defined by a supertype).
  65  * Calculating the set of properties for a type (i.e. the one it declares plus
  66  * the ones it inherits, with respect to re-defintions) is a 2 step process:
  67  *       1. run a query matching all properties acrosss the type's hierarchy, from
  68  * leaf to root type (neo's job)
  69  *       2. collecting them in a set that accumulates them with respect to
  70  * re-definition (model catalog client library job)
  71  * A (viable) alternative would have been to calculate the entire property set
  72  * at model import time and associate them it the type node. It would simplify
  73  * the query and processing in the catalog API. It has the drawback of making
  74  * the reverse process (exporting a yaml model from neo) tedious.
  75  * As we get a better sense of were the optimizations are needed this might
  76  * be a change to be made ..
  77  *
  78  *
  79  *   - representing requirements and capability as nodes. At first glance
  80  * both can be represented as edges pointing from a Type Node or Template Node
  81  * to another Type Node or Template Node. While this is true for capabilities
  82  * it is not so for requirements: a requirement could point to a capability
  83  * of a Type Node, i.e. it is a hyperedge between a Type Node (or Tempate Node), * another Type Node (the target) and a capability of the target. As such, the
  84  * requirements ands up being represented as a node and the capability will need
  85  * to do the same in order to be able to be pointed at (and for the sake of
  86  * uniformity ..).
  87  *
  88  *
  89  */
  90 public class Modeled {
  91
  92         private static OnapLoggerError errLogger = OnapLoggerError.getInstance();
  93         private static OnapLoggerDebug debugLogger = OnapLoggerDebug.getInstance();
  94
  95         private static HttpClientBuilder httpClientBuilder =
  96                         HttpClientBuilder.create();
  97         private static String USAGE = "oil oil_stylesheet_path | bigdata | aws | awsdata input_file customer";
  98
  99         private static List<String> ignoreMissing = new LinkedList<String>();
 100
 101         static {
 102                 Collections.addAll(ignoreMissing,
 103                                 "tosca.datatypes",
 104                                 "tosca.capabilities",
 105                                 "tosca.relationships",
 106                                 "tosca.interfaces",
 107                                 "tosca.nodes",
 108                                 "tosca.artifacts",
 109                                 "tosca.policies",
 110                                 "tosca.groups");
 111         }
 112
 113         public static void main(String[] theArgs) {
 114
 115                 CommandLineParser parser = new BasicParser();
 116
 117                 // create the Options
 118                 Options options = new Options();
 119                 options.addOption(OptionBuilder.
 120                                 withArgName("target")
 121                                 .withLongOpt("target")
 122                                 .withDescription("target ice4j database uri")
 123                                 .hasArg()
 124                                 .isRequired()
 125                                 .create('t'));
 126
 127                 options.addOption(OptionBuilder.
 128                                 withArgName("action")
 129                                 .withLongOpt("action")
 130                                 .withDescription("one of import, annotate, list, remove")
 131                                 .hasArg()
 132                                 .isRequired()
 133                                 .create('a'));
 134
 135                 options.addOption(
 136                                 OptionBuilder.withArgName("input")
 137                                                 .withLongOpt("input")
 138                                                 .withDescription(
 139                                                                 "for import/annotate: the tosca template file, " +
 140                                                                                 "for list: an optional json filter, " +
 141                                                                                 "for remove: the template id")
 142                                                 .hasArgs()
 143                                                 .create('i')).addOption(
 144                                 OptionBuilder.withArgName("labels")
 145                                                 .withLongOpt("labels")
 146                                                 .withDescription(
 147                                                                 "for annotate: the ':' sepatated list of annotation labels")
 148                                                 .hasArgs()
 149                                                 .create('l'));
 150
 151                 options.addOption(OptionBuilder.
 152                                 withArgName("ignore")
 153                                 .withLongOpt("ignore")
 154                                 .isRequired(false)
 155                                 .withDescription(
 156                                                 "for annotate: the ':' sepatated list of namespaces who's missing constructs can be ignored")
 157                                 .hasArgs()
 158                                 .create());
 159
 160
 161                 CommandLine line;
 162                 try {
 163                         line = parser.parse(options, theArgs);
 164                 } catch (ParseException exp) {
 165                         errLogger.log(LogLevel.ERROR, Modeled.class.getName(), exp.getMessage());
 166                         HelpFormatter formatter = new HelpFormatter();
 167                         formatter.printHelp("import", options);
 168                         return;
 169                 }
 170
 171                 String ignores = line.getOptionValue("ignore");
 172                 if (ignores != null)
 173                         Collections.addAll(ignoreMissing, ignores.split(":"));
 174
 175                 Modeled modeled = new Modeled();
 176                 try {
 177                         modeled.setNeoUri(new URI(line.getOptionValue("target")));
 178                 } catch (URISyntaxException urisx) {
 179                         errLogger.log(LogLevel.ERROR, Modeled.class.getName(), "Invalid target specification: {}", urisx);
 180                         return;
 181                 }
 182
 183                 try {
 184                         loadStorageSpec();
 185
 186                         String action = line.getOptionValue("action");
 187                         if ("import".equals(action)) {
 188                                 modeled.importTemplate(line.getOptionValue("input"));
 189                         } else if ("annotate".equals(action)) {
 190                                 modeled.annotateItem(line.getOptionValue("input"), line.getOptionValue("labels"));
 191                         } else if ("list".equals(action)) {
 192                                 modeled.listTemplates(line.getOptionValue("input"));
 193                         } else if ("remove".equals(action)) {
 194                                 modeled.removeTemplate(line.getOptionValue("input"));
 195                         } else {
 196                                 HelpFormatter formatter = new HelpFormatter();
 197                                 formatter.printHelp("import", options);
 198                         }
 199                 } catch (Exception x) {
 200                         errLogger.log(LogLevel.ERROR, Modeled.class.getName(), x.getMessage());
 201                 }
 202         }
 203
 204         private static Tracker<String> tracker = new Tracker<String>();
 205         private static Map toscaStorageSpec;
 206
 207         private static void loadStorageSpec() {
 208                 toscaStorageSpec = (Map) new Yaml().load(
 209                                 Modeled.class.getClassLoader().getResourceAsStream("tosca-schema.yaml"));
 210
 211                 Map storageSpec = (Map) new Yaml().load(
 212                                 Modeled.class.getClassLoader().getResourceAsStream("tosca-storage-schema.yaml"));
 213
 214                 JXPathContext jxPath = JXPathContext.newContext(toscaStorageSpec);
 215                 for (Iterator<Map.Entry<String, Object>> ces =
 216                          storageSpec.entrySet().iterator();
 217                          ces.hasNext(); ) {
 218                         Map.Entry<String, Object> ce = ces.next();
 219                         try {
 220                                 Map m = (Map) jxPath.getValue(ce.getKey());
 221                                 if (m == null) {
 222                                         debugLogger.log(LogLevel.DEBUG, Modeled.class.getName(), "No schema entry '{}'", ce.getKey());
 223                                         continue;
 224                                 }
 225
 226                                 m.putAll((Map) ce.getValue());
 227                         } catch (JXPathException jxpx) {
 228                                 errLogger.log(LogLevel.WARN, Modeled.class.getName(), "Failed to apply storage info {}", jxpx);
 229                         }
 230                 }
 231         }
 232
 233
 234         private static JSONObject EMPTY_JSON_OBJECT = new JSONObject();
 235
 236         private URI neoUri = null;
 237
 238         private Modeled() {
 239         }
 240
 241         private void setNeoUri(URI theUri) {
 242                 this.neoUri = theUri;
 243         }
 244
 245         public URI getNeoUri() {
 246                 return this.neoUri;
 247         }
 248
 249         /* Experimental in nature. I was reluctant creating another node to represent
 250          * the set of constraints as they're integral part of the property (or other
 251          * artifact) they're related to. I was also looking for a representation
 252          * that would easily be processable into a TOSCA abstraction in the
 253          * Catalog API. So ... we pack all the constraints as a JSON string and store
 254          * them as a single property of the TOSCA artifact they belog to.
 255          * Highs: easily un-winds in an object
 256          * Lows: can't write query selectors based on constraints values ..
 257                                 //the TOSCA/yaml spec exposes constraints as a List .. where each
 258                                 //entry is a Map .. why??
 259          */
 260         private static String yamlEncodeConstraints(List theConstraints) {
 261                 Map allConstraints = new HashMap();
 262                 for (Object c : theConstraints) {
 263                         allConstraints.putAll((Map) c);
 264                         //this would be the place to add dedicate processing of those
 265                         //constraints with 'special' values, i.e. in_range: dual scalar,
 266                         //valid_values: list
 267                 }
 268                 return JSONObject.valueToString(allConstraints);
 269         }
 270
 271         /* TODO: attributes handling to be added, similar to properties.
 272          */
 273         private void yamlNodeProperties(String theNodeId,
 274                                                                         Map<String, Object> theProperties,
 275                                                                         NeoTransaction theTrx)
 276                         throws IOException {
 277
 278                 for (Map.Entry<String, Object> propertyEntry : theProperties.entrySet()) {
 279                         String propName = propertyEntry.getKey();
 280                         Object propObject = propertyEntry.getValue();
 281
 282                         Map propValues;
 283                         if (propObject instanceof Map) {
 284                                 propValues = (Map) propObject;
 285                         } else {
 286                                 //valuation, not of interest here
 287                                 debugLogger.log(LogLevel.DEBUG, this.getClass().getName(), "neoNode, unknown property representation {} for {}, node {}", propObject.getClass(), propObject, theNodeId);
 288                                 continue;
 289                         }
 290
 291                         String constraintsValue = null;
 292                         if (propValues.containsKey("constraints")) {
 293                                 constraintsValue = yamlEncodeConstraints(
 294                                                 (List) propValues.get("constraints"));
 295                         }
 296
 297                         String neoPropId = neoCreateNode(
 298                                         theTrx, false,
 299                                         new JSONObject()
 300                                                         .put("name", propName)
 301                                                         .put("type", propValues.getOrDefault("type", "string"))
 302                                                         .put("required", propValues.getOrDefault("required", Boolean.TRUE))
 303                                                         .putOpt("default", propValues.get("default"))
 304                                                         .putOpt("description", propValues.get("description"))
 305                                                         .putOpt("status", propValues.get("status"))
 306                                                         .putOpt("constraints", constraintsValue),
 307                                         "TOSCA", "Property");
 308
 309                         neoEdge(theTrx, false,
 310                                         neoPropId,
 311                                         theNodeId,
 312                                         EMPTY_JSON_OBJECT,
 313                                         "PROPERTY_OF");
 314                 }
 315
 316         }
 317
 318         private void yamlNodeTypeCapabilities(String theNodeId,
 319                                                                                   Map<String, Object> theCapabilities,
 320                                                                                   NeoTransaction theTrx)
 321                         throws IOException {
 322
 323                 for (Map.Entry<String, Object> capability : theCapabilities.entrySet()) {
 324                         String capabilityName = capability.getKey();
 325                         Object capabilityValue = capability.getValue();
 326
 327                         String capabilityType = null,
 328                                         capabilityDesc = null;
 329                         Map<String, Object> capabilitySpec = null;
 330
 331                         if (capabilityValue instanceof String) {
 332                                 //short notation was used, we get the name of a capability type
 333                                 capabilityType = (String) capabilityValue;
 334
 335                                 capabilitySpec = Collections.singletonMap("type", capabilityType);
 336                         } else if (capabilityValue instanceof Map) {
 337                                 //extended notation
 338                                 capabilitySpec = (Map<String, Object>) capabilityValue;
 339
 340                                 capabilityType = (String) capabilitySpec.get("type");
 341                                 //cannot be missing
 342                                 if (capabilityType == null) {
 343                                         //ERROR!!
 344                                         errLogger.log(LogLevel.WARN, this.getClass().getName(), "neoNode, missing capability type in {} for node {}", capabilitySpec, theNodeId);
 345                                         continue; //rollback ..
 346                                 }
 347                                 capabilityDesc = (String) capabilitySpec.get("description");
 348                         }
 349
 350                         //
 351                         String anonCapabilityTypeId = null;
 352                         if (capabilitySpec.containsKey("properties")) {
 353                                 //we need an anonymous capability type (augmentation)
 354                                 //or they could be added to the 'Capabillity' node but anonymous
 355                                 //types make processing more uniform
 356                                 anonCapabilityTypeId =
 357                                                 yamlAnonymousType(capabilitySpec,
 358                                                                 capabilityType,
 359 //not a very nice owner string as theNodeId is cryptic (we should use
 360 //node name but do not have it here ..
 361                                                                 theNodeId + "#" + capabilityName,
 362                                                                 true,
 363                                                                 false,
 364                                                                 theTrx);
 365                         }
 366
 367                         JSONObject capabilityDef = new JSONObject()
 368                                         .put("name", capabilityName)
 369                                         .putOpt("description", capabilityDesc);
 370                         if (capabilitySpec != null) {
 371                                 List occurrences = (List) capabilitySpec.get("occurrences");
 372                                 if (occurrences != null) {
 373                                         capabilityDef.put("occurrences", encodeRange(occurrences));
 374                                 }
 375                                 List valid_source_types = (List) capabilitySpec.get("valid_source_types");
 376                                 if (valid_source_types != null) {
 377                                         capabilityDef.put("validSourceTypes",
 378                                                         new JSONArray(valid_source_types));
 379                                 }
 380                         }
 381
 382                         String capabilityId = neoCreateNode(
 383                                         theTrx, false,
 384                                         capabilityDef,
 385                                         "TOSCA", "Capability");
 386                         neoEdge(theTrx, false,
 387                                         capabilityId,
 388                                         theNodeId,
 389                                         EMPTY_JSON_OBJECT,
 390                                         "CAPABILITY_OF");
 391
 392                         if (anonCapabilityTypeId != null) {
 393                                 neoEdge(theTrx, false,
 394                                                 capabilityId,
 395                                                 anonCapabilityTypeId,
 396                                                 new JSONObject()
 397                                                                 .put("name", capabilityName)
 398                                                                 .putOpt("description", capabilityDesc),
 399                                                 "FEATURES"/* TARGETS */);
 400                                 //no reason this one would point to a non-existing capability as we just created one
 401                         } else {
 402                                 if (null == neoEdge(theTrx, false,
 403                                                 capabilityId,
 404                                                 "Type",
 405                                                 new JSONObject()
 406                                                                 .put("name", capabilityType),
 407                                                 new JSONObject()
 408                                                                 .put("name", capabilityName)
 409                                                                 .putOpt("description", capabilityDesc),
 410                                                 "FEATURES"/* TARGETS */)) {
 411                                         errLogger.log(LogLevel.WARN, this.getClass().getName(), "yamlNodeTypeCapabilities, Node {}, capability {} (id: {}) seems to point to invalid capability type: {}", theNodeId, capabilityName, capabilityId, capabilityType);
 412                                         ignoreMissing(capabilityType);
 413                                 }
 414                         }
 415
 416                 }
 417
 418         }
 419
 420         private void yamlNodeTypeRequirements(
 421                         String theNodeTypeId,
 422                         List<Map<String, Object>> theRequirements,
 423                         NeoTransaction theTrx)
 424                         throws IOException {
 425
 426                 for (Map<String, Object> arequirement : theRequirements) {
 427                         //supposed to have only one entry
 428                         Map.Entry<String, Object> requirement =
 429                                         arequirement.entrySet().iterator().next();
 430
 431                         String requirementName = requirement.getKey();
 432                         Object requirementValue = requirement.getValue();
 433
 434                         String targetNode = null,
 435                                         targetCapability = null,
 436                                         targetRelationship = null;
 437                         Map<String, Object> requirementSpec = null;
 438
 439                         if (requirementValue instanceof String) {
 440                                 //short form, points to a capability type
 441                                 targetCapability = (String) requirementValue;
 442                         } else if (requirementValue instanceof Map) {
 443                                 //extended notation
 444                                 requirementSpec = (Map<String, Object>) requirementValue;
 445
 446                                 targetCapability = (String) requirementSpec.get("capability");
 447                                 targetNode = (String) requirementSpec.get("node");
 448                                 //this assumes a short form for the relationship specification
 449                                 //it can actually be a map (indicating the relationship type and the
 450                                 //additional interface definitions).
 451                                 targetRelationship = (String) requirementSpec.get("relationship");
 452                         }
 453
 454                         if (targetCapability == null) {
 455                                 throw new IOException(theNodeTypeId + "missing capability type");
 456                         }
 457
 458                         JSONObject requirementDef = new JSONObject()
 459                                         .put("name", requirementName);
 460                         if (requirementSpec != null) {
 461                                 List occurrences = (List) requirementSpec.get("occurrences");
 462                                 if (occurrences != null) {
 463                                         requirementDef.put("occurrences", encodeRange(occurrences));
 464                                 }
 465                         }
 466
 467                         String requirementId = neoCreateNode(
 468                                         requirementDef,
 469                                         "TOSCA", "Requirement");
 470                         neoEdge(theTrx, false,
 471                                         requirementId,
 472                                         theNodeTypeId,
 473                                         EMPTY_JSON_OBJECT,
 474                                         "REQUIREMENT_OF");
 475
 476                         //we're not verifying here that this a capability type .. just a type
 477                         if (null == neoEdge(theTrx, false,
 478                                         requirementId,
 479                                         "Type",
 480                                         new JSONObject()
 481                                                         .put("name", targetCapability),
 482                                         EMPTY_JSON_OBJECT,
 483                                         "CAPABILITY")) {
 484                                 errLogger.log(LogLevel.WARN, this.getClass().getName(), "yamlNodeTypeRequirements, Node {}, requirement {} (id: {}) seems to point to invalid capability type: {}", theNodeTypeId, requirementName, requirementId, targetCapability);
 485                         }
 486
 487                         if (targetNode != null) {
 488                                 //points to a node type
 489                                 if (null == neoEdge(theTrx, false,
 490                                                 requirementId,
 491                                                 "Type",
 492                                                 new JSONObject()
 493                                                                 .put("name", targetNode),
 494                                                 EMPTY_JSON_OBJECT,
 495                                                 "REQUIRES")) {
 496                                         errLogger.log(LogLevel.WARN, this.getClass().getName(), "yamlNodeTypeRequirements, Node {}, requirement {} (id: {}) seems to point to invalid capability type: {}", theNodeTypeId, requirementName, requirementId, targetCapability);
 497                                 }
 498                         }
 499
 500                         if (targetRelationship != null) {
 501                                 //points to a relationship type
 502                                 if (null == neoEdge(theTrx, false,
 503                                                 requirementId,
 504                                                 "Type",
 505                                                 new JSONObject()
 506                                                                 .put("name", targetRelationship),
 507                                                 EMPTY_JSON_OBJECT,
 508                                                 "RELATIONSHIP")) {
 509                                         errLogger.log(LogLevel.WARN, this.getClass().getName(), "yamlNodeTypeRequirements, Node {}, requirement {} (id: {}) seems to point to invalid relationship type: {}", theNodeTypeId, requirementName, requirementId, targetRelationship);
 510                                 }
 511                         }
 512                 }
 513         }
 514
 515         /*
 516          * handles the requirement assignments
 517          */
 518         private void toscaRequirementsAssignment(
 519                         String theNodeId,
 520                         List<Map<String, Object>> theRequirements,
 521                         NeoTransaction theTrx)
 522                         throws IOException {
 523
 524                 for (Map<String, Object> arequirement : theRequirements) {
 525                         //supposed to have only one entry
 526                         Map.Entry<String, Object> requirement =
 527                                         arequirement.entrySet().iterator().next();
 528
 529                         String requirementName = requirement.getKey();
 530                         Object requirementValue = requirement.getValue();
 531
 532                         String targetNode = null,
 533                                         targetCapability = null,
 534                                         targetRelationship = null;
 535                         //TODO: targetFilter
 536
 537                         Map<String, Object> requirementSpec = null;
 538
 539                         if (requirementValue instanceof String) {
 540                                 //short notation was used, we get the name of a local node
 541                                 targetNode = (String) requirementValue;
 542                         } else if (requirementValue instanceof Map) {
 543                                 //extended notation
 544                                 requirementSpec = (Map<String, Object>) requirementValue;
 545
 546                                 targetNode = (String) requirementSpec.get("node");
 547                                 targetCapability = (String) requirementSpec.get("capability");
 548                                 targetRelationship = (String) requirementSpec.get("relationship");
 549                         }
 550
 551                         /* TODO: add targetFilter definition in here (most likely place)
 552                          */
 553                         String requirementId = neoCreateNode(
 554                                         theTrx, false,
 555                                         new JSONObject()
 556                                                         .put("name", requirementName),
 557                                         "TOSCA", "Requirement");
 558
 559                         neoEdge(theTrx, false,
 560                                         requirementId,
 561                                         theNodeId,
 562                                         EMPTY_JSON_OBJECT,
 563                                         "REQUIREMENT_OF");
 564
 565                         String targetNodeTemplate = null;
 566                         if (targetNode != null) {
 567                                 //check if the target is a node within the template (in which case the
 568                                 //requirement is really defined by that node type. i.e. its type's
 569                                 //capabilities
 570                                 targetNodeTemplate = tracker.lookupTemplate("Node", targetNode);
 571                                 if (targetNodeTemplate != null) {
 572                                         neoEdge(theTrx, false,
 573                                                         requirementId,
 574                                                         targetNodeTemplate,
 575                                                         new JSONObject()
 576                                                                         .put("name", requirementName),
 577                                                         "REQUIRES" /* TARGETS */);
 578                                 } else {
 579                                         //if not a local node template then it must be node type
 580                                         if (null == neoEdge(theTrx, false,
 581                                                         requirementId,
 582                                                         "Type",
 583                                                         new JSONObject()
 584                                                                         .put("name", targetNode),
 585                                                         EMPTY_JSON_OBJECT,
 586                                                         "REQUIRES")) {
 587                                                 errLogger.log(LogLevel.WARN, this.getClass().getName(), "yamlNodeTypeRequirements, Node {}, requirement {} (id: {}) seems to point to invalid node type: {}", theNodeId, requirementName, requirementId, targetNode);
 588                                         }
 589                                 }
 590                         }
 591
 592                         if (targetCapability != null) {
 593                                 /*
 594                                  * Can point to a capability of the targetNode (template or type,
 595                                  * whatever was specified) or to a capability type;
 596                                  */
 597                                 if (targetNode != null) {
 598                                         String stmt = null;
 599                                         if (targetNodeTemplate != null) {
 600                                                 //a capability of a local node template
 601                                                 //TODO: could be a capability type of a local node (and is up to the
 602                                                 //orchestrator to pick) given that the target node has at least one                                             //capability of that type
 603                                                 stmt =
 604                                                                 "MATCH (c:Capability)-[:CAPABILITY_OF]->(n:Node), (r:Requirement) " +
 605                                                                                 "WHERE id(n)=" + targetNodeTemplate + " " +
 606                                                                                 "AND c.name = \"" + targetCapability + "\" " +
 607                                                                                 "AND id(r)=" + requirementId + "  " +
 608                                                                                 "MERGE (r)-[rq:REQUIRES_CAPABILITY]->(c) " +
 609                                                                                 "RETURN id(rq)";
 610                                         } else {
 611                                                 //a capability of the node type
 612                                                 stmt =
 613                                                                 "MATCH (c:Type:Capability)-[:CAPABILITY_OF]->(t:Type), (r:Requirement) " +
 614                                                                                 "WHERE t.name = \"" + targetNode + "\" " +
 615                                                                                 "AND c.name = \"" + targetCapability + "\" " +
 616                                                                                 "AND id(r)=" + requirementId + "  " +
 617                                                                                 "MERGE (r)-[rq:REQUIRES_CAPABILITY]->(c) " +
 618                                                                                 "RETURN id(rq)";
 619                                         }
 620                                         if (null == neoId(theTrx
 621                                                         .statement(
 622                                                                         new JSONObject()
 623                                                                                         .put("statement", stmt))
 624                                                         .execute()
 625                                                         .result())) {
 626                                                 errLogger.log(LogLevel.WARN, this.getClass().getName(), "toscaRequirementsAssignment, Node {}, requirement {} (id: {}) seems to point to invalid node capability: {}", theNodeId, requirementName, requirementId, targetCapability);
 627                                         }
 628                                 } else {
 629                                         if (null == neoEdge(theTrx, false,
 630                                                         requirementId,
 631                                                         "Type",
 632                                                         new JSONObject()
 633                                                                         .put("name", targetCapability),
 634                                                         EMPTY_JSON_OBJECT,
 635                                                         "REQUIRES_CAPABILITY")) {
 636                                                 errLogger.log(LogLevel.WARN, this.getClass().getName(), "toscaRequirementsAssignment, Node {}, requirement {} (id: {}) seems to point to invalid capability type: {}", theNodeId, requirementName, requirementId, targetCapability);
 637                                         }
 638                                 }
 639                         }
 640
 641                         if (targetRelationship != null) {
 642                                 if (null == neoEdge(theTrx, false,
 643                                                 requirementId,
 644                                                 "Type",
 645                                                 new JSONObject()
 646                                                                 .put("name", targetRelationship),
 647                                                 EMPTY_JSON_OBJECT,
 648                                                 "RELATIONSHIP")) {
 649                                         errLogger.log(LogLevel.WARN, this.getClass().getName(), "toscaRequirementsAssignment, Node {}, requirement {} (id: {}) seems to point to invalid relationship type: {}", theNodeId, requirementName, requirementId, targetRelationship);
 650                                 }
 651                         } else {
 652                                 //TODO: does the presence of properties/attributes/interfaces in the
 653                                 //requirement definition trigger the defintion of an anonymous
 654                                 //relationship type?? (maybe derived from the one under the
 655                                 //'relationship_type' key, if present?)
 656                         }
 657                 }
 658         }
 659
 660         /* an anonymous type is created from a node specification (type,template)
 661          */
 662         private String yamlAnonymousType(Map<String, Object> theInfo,
 663                                                                          String theType,
 664                                                                          String theOwner,
 665                                                                          boolean doProperties,
 666                                                                          boolean doCapabilities,
 667                                                                          NeoTransaction theTrx)
 668                         throws IOException {
 669
 670                 //is this naming scheme capable enough??NO!
 671                 String anonTypeId = theOwner + "#" + (theType == null ? "" : theType);
 672
 673                 String neoAnonTypeId = neoMergeNode(
 674                                 theTrx, false,
 675                                 new JSONObject()
 676                                                 .put("name", anonTypeId)
 677                                                 .put("id", anonTypeId),
 678                                 "TOSCA", "Type");
 679
 680                 if (theType != null) {
 681                         neoEdge(theTrx, false,
 682                                         neoAnonTypeId,
 683                                         "Type",
 684                                         new JSONObject()
 685                                                         .put("name", theType),
 686                                         EMPTY_JSON_OBJECT,
 687                                         "DERIVED_FROM");
 688                 }
 689
 690                 //shoudl the properties spec be passed explcitly??
 691                 if (doProperties) {
 692                         Map<String, Object> props = (Map<String, Object>) theInfo.get("properties");
 693                         if (props != null) {
 694                                 yamlNodeProperties(neoAnonTypeId, props, theTrx);
 695                         }
 696                 }
 697
 698                 return neoAnonTypeId;
 699         }
 700
 701         /*
 702          * A first pass over a type spec provisions each type individually
 703          * and its properties.
 704          * We process here types for all constructs: data, capability, relationship,
 705          * node, [interface, artifact]
 706          */
 707         private void toscaTypeSpec(String theConstruct,
 708                                                            Map<String, Map> theTypes,
 709                                                            NeoTransaction theTrx)
 710                         throws IOException {
 711                 //first pass, provision each type individually (and their properties)
 712                 String rule = "_" + theConstruct.toLowerCase() + "_type_definition";
 713                 Map storageSpec = (Map) toscaStorageSpec.get(rule);
 714
 715                 for (Map.Entry<String, Map> toscaType : theTypes.entrySet()) {
 716                         String typeName = toscaType.getKey();
 717                         Map<String, Map> typeValue = (Map<String, Map>) toscaType.getValue();
 718
 719                         debugLogger.log(LogLevel.DEBUG, this.getClass().getName(), "Type: {}", typeName);
 720
 721                         JSONObject data = pack(storageSpec, typeValue)
 722                                         .put("name", typeName)
 723                                         .put("id", typeName);
 724
 725                         String neoTypeId = neoMergeNode(theTrx, false, data, "TOSCA", "Type", theConstruct);
 726
 727                         tracker.trackType(theConstruct, typeName, neoTypeId);
 728
 729                         Map<String, Object> toscaTypeProps = (Map<String, Object>) typeValue.get("properties");
 730                         if (toscaTypeProps != null) {
 731                                 yamlNodeProperties(neoTypeId, toscaTypeProps, theTrx);
 732                         } //type props
 733                 } //types
 734
 735                 toscaTypePostProc(theConstruct, theTypes, theTrx);
 736         }
 737
 738         /*
 739          * A second pass to process the derived_from relationship and
 740          * the capabilities (now that the capabilities types have been provisioned)
 741          */
 742         private void toscaTypePostProc(String theConstruct,
 743                                                                    Map<String, Map> theTypes,
 744                                                                    NeoTransaction theTrx)
 745                         throws IOException {
 746                 for (Map.Entry<String, Map> typeEntry : theTypes.entrySet()) {
 747                         Map typeValue = typeEntry.getValue();
 748                         String typeName = typeEntry.getKey();
 749
 750                         //supertype and description: all types
 751                         String superTypeName = (String) typeValue.get("derived_from");
 752                         if (superTypeName != null) {
 753                                 debugLogger.log(LogLevel.DEBUG, this.getClass().getName(), "{}-DERIVED_FROM->{}", typeName, superTypeName);
 754
 755                                 if (tracker.tracksType(theConstruct, superTypeName)) {
 756                                         if (null == neoEdge(theTrx, false,
 757                                                         tracker.lookupType(theConstruct, typeName),
 758                                                         tracker.lookupType(theConstruct, superTypeName),
 759                                                         EMPTY_JSON_OBJECT,
 760                                                         "DERIVED_FROM")) {
 761                                                 errLogger.log(LogLevel.WARN, this.getClass().getName(), "yamlTypePostProc, missing parent type {}, id {} for type {}, id {}", superTypeName, tracker.lookupType(theConstruct, superTypeName), typeName, tracker.lookupType(theConstruct, typeName));
 762                                         }
 763                                 } else {
 764                                         if (null == neoEdge(theTrx, false,
 765                                                         tracker.lookupType(theConstruct, typeName),
 766                                                         "Type",
 767                                                         new JSONObject()
 768                                                                         .put("name", superTypeName),
 769                                                         new JSONObject(),
 770                                                         "DERIVED_FROM")) {
 771                                                 errLogger.log(LogLevel.WARN, this.getClass().getName(), "yamlTypePostProc, missing parent type {} for type {}", superTypeName, typeName);
 772                                         }
 773                                 }
 774                         }
 775
 776                         //requirements/capabilities: for node types
 777                         Map<String, Object> capabilities =
 778                                         (Map<String, Object>) typeValue.get("capabilities");
 779                         if (capabilities != null) {
 780                                 debugLogger.log(LogLevel.DEBUG, this.getClass().getName(), "Processing: {}", capabilities);
 781                                 yamlNodeTypeCapabilities(
 782                                                 tracker.lookupType(theConstruct, typeName), capabilities, theTrx);
 783                         }
 784
 785                         List<Map<String, Object>> requirements =
 786                                         (List<Map<String, Object>>) typeValue.get("requirements");
 787                         if (requirements != null) {
 788                                 yamlNodeTypeRequirements(
 789                                                 tracker.lookupType(theConstruct, typeName), requirements, theTrx);
 790                         }
 791
 792                         //interfaces: for node types or relationship types
 793                         Object interfaces = typeValue.get("interfaces");
 794                         if (interfaces != null) {
 795                                 errLogger.log(LogLevel.WARN, this.getClass().getName(), "yamlTypePostProc, Type {}: interfaces section declared but not handled", typeName);
 796                                 if (interfaces instanceof List) {
 797                                         //expect a list of interface types
 798                                 }
 799                         }
 800
 801                         //valid targets: for relationship types
 802                         List valid_targets = (List) typeValue.get("valid_targets");
 803                         if (valid_targets != null) {
 804                                 //add as a property to the type node, can be used for validation
 805                                 //whereever this type is used
 806                                 //the list should contain node type names and we should check that we
 807                                 //have those types
 808                                 errLogger.log(LogLevel.WARN, this.getClass().getName(), "yamlTypePostProc, Type {}: valid_targets section declared but not handled", typeName);
 809
 810                         }
 811
 812                         List artifacts = (List) typeValue.get("artifacts");
 813                         if (artifacts != null) {
 814                                 errLogger.log(LogLevel.WARN, this.getClass().getName(), "yamlTypePostProc, Type {}: artifacts section declared but not handled", typeName);
 815                         }
 816
 817                         /* Artifact types can have "mime_type" and "file_ext" sections
 818                          */
 819                 }
 820         }
 821
 822         private void toscaTemplate(String theTopologyTemplateId,
 823                                                            String theConstruct,
 824                                                            Map<String, Object> theTemplates,
 825                                                            NeoTransaction theTrx)
 826                         throws IOException {
 827
 828                 String rule = "_" + theConstruct.toLowerCase() + "_template_definition";
 829                 Map storageSpec = (Map) toscaStorageSpec.get(rule);
 830                 if (storageSpec == null) {
 831                         debugLogger.log(LogLevel.DEBUG, this.getClass().getName(), "No rule '{}', can't make up the storage specification for {}", rule, theConstruct);
 832                 }
 833
 834                 for (Map.Entry<String, Object> template : theTemplates.entrySet()) {
 835
 836                         String templateName = template.getKey();
 837                         Map<String, Object> templateSpec = (Map<String, Object>) template.getValue();
 838
 839                         String templateType = (String) templateSpec.get("type");
 840                         if (templateType == null) {
 841                                 errLogger.log(LogLevel.WARN, this.getClass().getName(), "neoNode, template {}'{}', does not have a type specification .. skipping", theConstruct, templateName);
 842                                 continue;
 843                         }
 844
 845                         try {
 846                                 //we use create here as node names are not unique across templates
 847                                 JSONObject neoTemplateNode =
 848                                                 pack(storageSpec, templateSpec)
 849                                                                 .put("name", templateName);
 850
 851                                 String templateNodeId = neoCreateNode(
 852                                                 theTrx, false, neoTemplateNode, "TOSCA", theConstruct);
 853
 854                                 tracker.trackTemplate(theConstruct, templateName, templateNodeId);
 855
 856                                 neoEdge(theTrx, false,
 857                                                 templateNodeId,
 858                                                 theTopologyTemplateId,
 859                                                 new JSONObject(),
 860                                                 theConstruct.toUpperCase() + "_OF");
 861
 862                                 if (null == neoEdge(theTrx, false,
 863                                                 templateNodeId,
 864                                                 "Type",
 865                                                 new JSONObject()
 866                                                                 .put("name", templateType),
 867                                                 new JSONObject(),
 868                                                 "OF_TYPE")) {
 869                                         errLogger.log(LogLevel.WARN, this.getClass().getName(), "yamlSpec, Template {}, {} {}: failed to identify type {}", theTopologyTemplateId, theConstruct, templateName, templateType);
 870                                 }
 871
 872                                 //facets
 873
 874                                 //we handle properties for all constructs (as they all have them)
 875                                 Map<String, Object> templateProps =
 876                                                 (Map<String, Object>) templateSpec.get("properties");
 877                                 if (templateProps != null) {
 878                                         for (Map.Entry<String, Object> templateProp :
 879                                                         templateProps.entrySet()) {
 880                                                 String templatePropName = templateProp.getKey();
 881                                                 Object templatePropObject = templateProp.getValue();
 882
 883                                                 final Map templatePropValues;
 884                                                 if (templatePropObject instanceof Map) {
 885                                                         templatePropValues = (Map) templatePropObject;
 886                                                 } else {
 887
 888                                                         //this is dealing with short form, if we ran the first 2 stages of the checker  //we'd always be working on a canonical form ..
 889                                                         //
 890                                                         templatePropValues = new HashMap();
 891                                                         templatePropValues.put("value", templatePropObject);
 892                                                 }
 893
 894                                                 //a node will contain the means for property valuation:
 895                                                 //straight value or a call to get_input/get_property/get_attribute
 896
 897                                                 //find the property node (in the type) this valuation belongs to
 898                                                 if (templatePropValues != null) {
 899
 900                                                         String propertyId =
 901                                                                         neoId(
 902                                                                                         theTrx.statement(
 903                                                                                                         new JSONObject()
 904                                                                                                                         .put("statement",
 905                                                                                                                                         "MATCH (t:Type)-[:DERIVED_FROM*0..5]->(:Type)<-[:PROPERTY_OF]-(p:Property) " +
 906                                                                                                                                                         "WHERE t.name='" + templateType + "' " +
 907                                                                                                                                                         "AND p.name='" + templatePropName + "' " +
 908                                                                                                                                                         "RETURN id(p)"))
 909                                                                                                         .execute()
 910                                                                                                         .result()
 911                                                                         );
 912
 913                                                         if (propertyId == null) {
 914                                                                 errLogger.log(LogLevel.WARN, this.getClass().getName(), "yamlSpec, Template {},  {} template {}, property {} does not match the node type spec, skipping property", templateName, theConstruct, templateName, templatePropName);
 915                                                                 continue;
 916                                                         }
 917
 918                                                         //remove valuation by function: for now handle only get_input
 919                                                         String propInput = (String) templatePropValues.remove("get_input");
 920
 921                                                         List constraints = (List) templatePropValues.remove("constraints");
 922                                                         if (constraints != null) {
 923                                                                 //flattening
 924                                                                 templatePropValues.put("constraints",
 925                                                                                 yamlEncodeConstraints(constraints));
 926                                                         }
 927
 928                                                         Object val = templatePropValues.remove("value");
 929                                                         //check if the value is a collection or user defined data type, the cheap way
 930                                                         if (val instanceof List ||
 931                                                                         val instanceof Map) {
 932                                                                 /* An interesting option here:
 933                                                                  *      1. store the whole flatten value under the 'value' property
 934                                                                                                         templatePropValues.put("value", JsonFlattener.flatten(JsonObject.valueToString(val)));
 935                                                                                          Simpler but almost impossible to write queries based on property value
 936                                                                  *      2. store each entry in the flatten map as a separate property (we prefix it with 'value' for
 937                                                                  *               clarity).
 938                                                                  *       see below
 939                                                                  */
 940                                         /*
 941                                                                 JsonFlattener.flattenAsMap(JSONObject.valueToString(Collections.singletonMap("value",val)))
 942                                                                         .entrySet()
 943                                                                                 .stream()
 944                                                                                         .forEach(e -> templatePropValues.put(e.getKey(), e.getValue()));
 945                                         */
 946                                                                 //simply stores a collection in its (json) string representation. Cannot be used if
 947                                                                 //queries are necessary based on the value (on one of its elements).
 948                                                                 templatePropValues.put("value", JSONObject.valueToString(val));
 949                                                         } else {
 950                                                                 /* scalar, store as such */
 951                                                                 templatePropValues.put("value", val);
 952                                                         }
 953
 954                                                         String templatePropValueId =
 955                                                                         neoCreateNode(
 956                                                                                         theTrx, false,
 957                                                                                         new JSONObject(templatePropValues),
 958                                                                                         "TOSCA", /*"Property",*/ "Assignment");
 959
 960                                                         neoEdge(theTrx, false,
 961                                                                         templatePropValueId,
 962                                                                         templateNodeId,
 963                                                                         new JSONObject(),
 964                                                                         "OF_TEMPLATE");
 965
 966                                                         neoEdge(theTrx, false,
 967                                                                         templatePropValueId,
 968                                                                         propertyId,
 969                                                                         new JSONObject(),
 970                                                                         "OF_" + theConstruct.toUpperCase() + "_PROPERTY");
 971
 972                                                         if (propInput != null) {
 973                                                                 String inputId = tracker.lookupTemplate("Input", propInput);
 974                                                                 if (inputId == null) {
 975                                                                         errLogger.log(LogLevel.WARN, this.getClass().getName(), "neoNode, Template {},node {}, property {} input {} not found", theTopologyTemplateId, templateName, templatePropName, propInput);
 976                                                                 }
 977
 978                                                                 neoEdge(theTrx, false,
 979                                                                                 templatePropValueId,
 980                                                                                 inputId,
 981                                                                                 new JSONObject(),
 982                                                                                 "GET_INPUT");
 983                                                         }
 984                                                 }
 985                                         }
 986                                 }
 987                                 tracker.trackTemplate(theConstruct, templateName, templateNodeId);
 988                                 debugLogger.log(LogLevel.DEBUG, this.getClass().getName(), "{} template {} of type {}", theConstruct, templateName, templateType);
 989                         } catch (IOException iox) {
 990                                 errLogger.log(LogLevel.WARN, this.getClass().getName(), "toscaTemplate, Failed to persist template {}", iox);
 991                                 throw iox;
 992                         }
 993                 }
 994         }
 995
 996         /* while we persist basic type values inline (in the assigment node) we store complex values
 997          * in a graph of their own.
 998          * We handle the neo4j 'limitation' stated below
 999          *      Neo4j can only store collections (map, list) of basic types.
1000          *
1001          * User defined data types can created undefinitely nested strctures of collections.
1002          * We could store collections of basic types inline but it would make for a less uniform structure.
1003          */
1004         private void toscaPropertyAssignment(
1005                         String theAssignmentId,
1006                         Object theValue,
1007                         NeoTransaction theTrx)
1008                         throws IOException {
1009                 //look the grammar rules to see if we inline (stringify) or not
1010
1011                 if (theValue instanceof Map) {
1012                         //a map type property or a user-defined datatype
1013                         Map<String, Object> elements = (Map<String, Object>) theValue;
1014                         for (Map.Entry element : elements.entrySet()) {
1015
1016                                 String elementId = neoCreateNode(theTrx, false,
1017                                                 new JSONObject().
1018                                                                 put("name", element.getKey()),
1019                                                 "TOSCA", "Data", "Element");
1020
1021                                 neoEdge(theTrx, false,
1022                                                 elementId,
1023                                                 theAssignmentId,
1024                                                 EMPTY_JSON_OBJECT,
1025                                                 "ELEMENT_OF");
1026
1027                                 toscaPropertyAssignment(elementId, element.getValue(), theTrx);
1028                         }
1029                 } else if (theValue instanceof List) {
1030                         //a list type property
1031                         for (int i = 0; i < ((List) theValue).size(); i++) {
1032
1033                                 String elementId = neoCreateNode(theTrx, false,
1034                                                 new JSONObject().
1035                                                                 put("pos", i),
1036                                                 "TOSCA", "Data", "Element");
1037
1038                                 neoEdge(theTrx, false,
1039                                                 elementId,
1040                                                 theAssignmentId,
1041                                                 EMPTY_JSON_OBJECT,
1042                                                 "ELEMENT_OF");
1043
1044                                 toscaPropertyAssignment(elementId, ((List) theValue).get(i), theTrx);
1045                         }
1046
1047                         //update theAssignment with a length property
1048                         neoNodeProperties(theTrx, false, theAssignmentId,
1049                                         new JSONObject().
1050                                                         put("length", ((List) theValue).size()));
1051                 } else {
1052                         //update the assignment with a 'value' attribute
1053                         neoNodeProperties(theTrx, false, theAssignmentId,
1054                                         new JSONObject().
1055                                                         put("value", theValue));
1056                 }
1057         }
1058
1059         /*
1060          * We only handle properties for now so we assume these are properties
1061          * assignments
1062          */
1063         private void toscaCapabilityAssignment(
1064                         String theNodeTemplateId,
1065                         String theCapabilityName,
1066                         Map<String, Object> theValuations,
1067                         NeoTransaction theTrx)
1068                         throws IOException {
1069
1070                 for (Map.Entry<String, Object> valuation : theValuations.entrySet()) {
1071                         String propertyName = valuation.getKey();
1072                         Object propertyValueSpec = valuation.getValue();
1073
1074                         Map propertyValue = null;
1075                         if (propertyValueSpec instanceof Map) {
1076                                 propertyValue = (Map) propertyValueSpec;
1077                         } else {
1078                                 //this is dealing with short form, if we ran the first 2 stages of
1079                                 //the checker we'd always be working on a canonical form ..
1080                                 propertyValue = new HashMap();
1081                                 propertyValue.put("value", propertyValueSpec);
1082                         }
1083
1084                         //we need to link the assignment to the node template, the capability
1085                         //and the property of the capability type (a node can have multiple
1086                         //capabilities of the same type).
1087                         String[] ids =
1088                                         neoIds(
1089                                                         theTrx.statement(
1090                                                                         new JSONObject()
1091                                                                                         .put("statement",
1092                                                                                                         "MATCH (n:Node)-[:OF_TYPE]->(:Node:Type)<-[:CAPABILITY_OF]-(c:Capability)-[:FEATURES]->(:Capability:Type)-[:DERIVED_FROM*0..5]->(:Capability:Type)<-[:PROPERTY_OF]-(p:Property) " +
1093                                                                                                                         "WHERE id(n) = " + theNodeTemplateId + " " +
1094                                                                                                                         "AND c.name = '" + theCapabilityName + "' " +
1095                                                                                                                         "AND p.name = '" + propertyName + "' " +
1096                                                                                                                         "RETURN id(p), id(c)"))
1097                                                                         .execute()
1098                                                                         .result());
1099
1100                         if (ids == null) {
1101                                 throw new IOException("toscaCapabilityAssignment: " +
1102                                                 "node template " + theNodeTemplateId + ",  " +
1103                                                 "capability " + theCapabilityName + ", " +
1104                                                 "property " + propertyName +
1105                                                 " does not match the node type spec");
1106                         }
1107
1108                         /* this node represents the assignment of a value to a capability property
1109                          * hence my doubts about hoe to label it ['Assignment', 'Property'] or ['Assignment','Capability']
1110                          * I am inclined towards the second option as there is no other capability assignment in itself.
1111                          */
1112                         String assignmentId =
1113                                         neoCreateNode(
1114                                                         theTrx, false,
1115                                                         new JSONObject(propertyValue),
1116                                                         "TOSCA", /*Capability,*/"Assignment");
1117
1118                         neoEdge(theTrx, false,
1119                                         assignmentId,
1120                                         theNodeTemplateId,
1121                                         new JSONObject(),
1122                                         "OF_TEMPLATE");
1123
1124                         neoEdge(theTrx, false,
1125                                         assignmentId,
1126                                         ids[1],
1127                                         new JSONObject(),
1128                                         "OF_CAPABILITY");
1129
1130                         neoEdge(theTrx, false,
1131                                         assignmentId,
1132                                         ids[0],
1133                                         new JSONObject(),
1134                                         "OF_CAPABILITY_PROPERTY");
1135                 }
1136         }
1137
1138         /*
1139          *
1140          * */
1141         private void importTemplate(String thePath) throws IOException {
1142                 try (FileInputStream input = new FileInputStream(thePath)){
1143                         for (Object yaml : new Yaml().loadAll(input)) {
1144                                 toscaSpec((Map) yaml);
1145                         }
1146                 }
1147         }
1148
1149         private void toscaSpec(Map theSpec) throws IOException {
1150
1151                 // type specifications
1152                 // at this time we do not record the relation between a type and the
1153                 // template it was defined in.
1154
1155                 NeoTransaction trx = new NeoTransaction(this.neoUri);
1156                 try {
1157                         {
1158                                 Map<String, Map> types = (Map<String, Map>) theSpec.get("data_types");
1159                                 if (types != null) {
1160                                         toscaTypeSpec("Data", types, trx);
1161                                 }
1162
1163                                 types = (Map<String, Map>) theSpec.get("capability_types");
1164                                 if (types != null) {
1165                                         toscaTypeSpec("Capability", types, trx);
1166                                 }
1167
1168                                 types = (Map<String, Map>) theSpec.get("relationship_types");
1169                                 if (types != null) {
1170                                         toscaTypeSpec("Relationship", types, trx);
1171                                 }
1172
1173                                 types = (Map<String, Map>) theSpec.get("node_types");
1174                                 if (types != null) {
1175                                         toscaTypeSpec("Node", types, trx);
1176                                 }
1177
1178                                 types = (Map<String, Map>) theSpec.get("policy_types");
1179                                 if (types != null) {
1180                                         toscaTypeSpec("Policy", types, trx);
1181                                 }
1182                         }
1183
1184                         Map<String, Map> topologyTemplate = (Map<String, Map>)
1185                                         theSpec.get("topology_template");
1186                         if (topologyTemplate != null) {
1187
1188                                 Map<String, Object> metadata = (Map<String, Object>) theSpec.get("metadata");
1189                                 if (metadata == null) {
1190                                         throw new IOException("Missing metadata, cannot register template");
1191                                 }
1192                                 String templateName = (String) metadata.get("template_name");
1193                                 String templateId = neoMergeNode(
1194                                                 trx, false,
1195                                                 new JSONObject()
1196                                                                 .put("name", templateName)
1197                                                                 .putOpt("description", (String) theSpec.get("description"))
1198                                                                 .putOpt("version", (String) metadata.get("template_version"))
1199                                                                 .putOpt("author", (String) metadata.get("template_author"))
1200                                                                 .putOpt("scope", (String) metadata.get("scope")),
1201                                                 "TOSCA", "Template");
1202
1203                                 /* inputs */
1204                                 Map<String, Map> toscaInputs = (Map) topologyTemplate.get("inputs");
1205                                 if (toscaInputs != null) {
1206                                         for (Map.Entry<String, Map> toscaInput : toscaInputs.entrySet()) {
1207                                                 //we use create here as input names are not unique across templates
1208                                                 //also, constraints require special encoding
1209                                                 Map toscaInputSpec = toscaInput.getValue();
1210
1211                                                 List constraints = (List) toscaInputSpec.remove("constraints");
1212                                                 if (constraints != null) {
1213                                                         //flattening
1214                                                         toscaInputSpec.put("constraints",
1215                                                                         yamlEncodeConstraints(constraints));
1216                                                 }
1217                                                 String neoInputNodeId =
1218                                                                 neoCreateNode(
1219                                                                                 trx, false,
1220                                                                                 new JSONObject(toscaInputSpec)
1221                                                                                                 .put("name", toscaInput.getKey())
1222                                                                                                 .putOpt("type", toscaInputSpec.get("type")),
1223                                                                                 "TOSCA", "Input");
1224
1225                                                 tracker.trackTemplate(
1226                                                                 "Input", (String) toscaInput.getKey(), neoInputNodeId);
1227
1228                                                 neoEdge(trx, false,
1229                                                                 neoInputNodeId,
1230                                                                 templateId,
1231                                                                 new JSONObject(),
1232                                                                 "INPUT_OF");
1233                                         }
1234                                 }
1235
1236                                 /*
1237                                  * The main issue that I have here is with the defintion given to each
1238                                  * section (properties, capabilities, requirements ..) of a Node template:
1239                                  * they are said to 'augment' the information provided in its Node Type but
1240                                  * without specifying the semantics of 'augment'. Can new properties be
1241                                  * added? can interface specification contain new operations?
1242                                  */
1243                                 Map<String, Object> toscaNodes = (Map) topologyTemplate.get("node_templates");
1244                                 if (toscaNodes != null) {
1245                                         toscaTemplate(templateId, "Node", toscaNodes, trx);
1246
1247                                         //now that all nodes are in we need a second path over the nodes set in
1248                                         //order to handle the capabilities, requirements ..
1249
1250                                         for (Map.Entry<String, Object> toscaNode : toscaNodes.entrySet()) {
1251
1252                                                 String toscaNodeName = toscaNode.getKey();
1253                                                 Map<String, Object> toscaNodeValues = (Map<String, Object>) toscaNode.getValue();
1254
1255                                                 Map<String, Map> capabilities =
1256                                                                 (Map<String, Map>) toscaNodeValues.get("capabilities");
1257                                                 if (capabilities != null) {
1258                                                         for (Map.Entry<String, Map> capability : capabilities.entrySet()) {
1259                                                                 Map<String, Map> assignments = (Map<String, Map>) capability.getValue();
1260                                                                 Map<String, Object> propertiesAssignments =
1261                                                                                 assignments.get("properties");
1262                                                                 if (propertiesAssignments != null) {
1263                                                                         toscaCapabilityAssignment(
1264                                                                                         tracker.lookupTemplate("Node", toscaNodeName),
1265                                                                                         capability.getKey(),
1266                                                                                         propertiesAssignments,
1267                                                                                         trx);
1268                                                                 }
1269                                                         }
1270                                                 }
1271
1272                                                 List<Map<String, Object>> requirements = (List<Map<String, Object>>)
1273                                                                 toscaNodeValues.get("requirements");
1274                                                 if (requirements != null) {
1275                                                         toscaRequirementsAssignment(
1276                                                                         tracker.lookupTemplate("Node", toscaNodeName), requirements, trx);
1277                                                 }
1278
1279                                                 //interfaces
1280                                         }
1281                                 }
1282
1283                                 List toscaPolicies = (List) topologyTemplate.get("policies");
1284                                 if (toscaPolicies != null) {
1285                                         for (Object toscaPolicy : toscaPolicies) {
1286                                                 toscaTemplate(templateId, "Policy", (Map<String, Object>) toscaPolicy, trx);
1287                                         }
1288                                 }
1289
1290                                 Map<String, Map> toscaOutputs = (Map) topologyTemplate.get("outputs");
1291                                 if (toscaOutputs != null) {
1292                                         for (Map.Entry<String, Map> toscaOutput : toscaOutputs.entrySet()) {
1293                                                 Object outputValue = toscaOutput.getValue().get("value");
1294                                                 if (outputValue instanceof Map) { //shouldn't I be doing this in all cases??
1295                                                         outputValue = JSONObject.valueToString((Map) outputValue);
1296                                                 }
1297
1298                                                 String neoOutputNodeId = neoCreateNode(
1299                                                                 trx, false,
1300                                                                 new JSONObject()
1301                                                                                 .put("name", (String) toscaOutput.getKey())
1302                                                                                 .putOpt("description", (String) toscaOutput.getValue().get("description"))
1303                                                                                 .put("value", outputValue.toString()),
1304                                                                 "TOSCA", "Output");
1305
1306                                                 neoEdge(trx, false,
1307                                                                 neoOutputNodeId,
1308                                                                 templateId,
1309                                                                 new JSONObject(),
1310                                                                 "OUTPUT_OF");
1311                                         }
1312                                 }
1313
1314                                 //if this is a service template look for its type mapping specification
1315                                 Map<String, Object> substitutionSpec =
1316                                                 (Map<String, Object>) theSpec.get("substitution_mappings");
1317                                 if (substitutionSpec != null) {
1318
1319                                         String nodeType = (String) substitutionSpec.get("node_type");
1320                                         if (nodeType != null) {
1321                                                 neoEdge(trx, false,
1322                                                                 templateId,
1323                                                                 "Type",
1324                                                                 new JSONObject()
1325                                                                                 .put("name", nodeType),
1326                                                                 new JSONObject(),
1327                                                                 "SUBSTITUTES");
1328                                         } else {
1329                                                 errLogger.log(LogLevel.WARN, this.getClass().getName(), "neoProc, Template {} substitution_mapping is missing a node_type in spec: {}", templateName, substitutionSpec);
1330                                         }
1331
1332                                         //process the rest of the mapping definition
1333                                 } else {
1334                                         errLogger.log(LogLevel.WARN, this.getClass().getName(), "neoProc, Template {} does not have a substitution mapping", templateName);
1335                                 }
1336
1337                                 //try to connect template to catalog item if information was provided
1338                                 //
1339                                 String catalogItemSelector = (String) metadata.get("asc_catalog");
1340                                 if (catalogItemSelector != null) {
1341                                         if (null == neoEdge(trx, false,
1342                                                         templateId,
1343                                                         "CatalogItem",
1344                                                         new JSONObject(catalogItemSelector),
1345                                                         new JSONObject(),
1346                                                         "MODEL_OF")) {
1347                                                 throw new IOException("No such catalog item: " + catalogItemSelector);
1348                                         }
1349                                 }
1350                         }
1351                         trx.commit();
1352                 } catch (IOException iox) {
1353                         try {
1354                                 trx.rollback();
1355                         } catch (IOException riox) {
1356                                 errLogger.log(LogLevel.ERROR, Modeled.class.getName(), riox.getMessage());
1357                         }
1358                         throw iox;
1359                 }
1360         }
1361
1362         private void annotateItem(String thePath, String theLabels) throws IOException {
1363
1364                 if (theLabels == null) {
1365                         throw new IOException("Labels ??");
1366                 }
1367
1368                 try (FileInputStream input = new FileInputStream(thePath)){
1369                         for (Object yaml : new Yaml().loadAll(input)) {
1370                                 annotateItem((Map) yaml, theLabels);
1371                         }
1372                 }
1373         }
1374
1375         private void annotateItem(Map theSpec, String theLabels) throws IOException {
1376
1377                 Map<String, Object> metadata = (Map<String, Object>) theSpec.get("metadata");
1378                 if (metadata == null) {
1379                         throw new IOException("Missing metadata, cannot register template");
1380                 }
1381
1382                 String catalogItemSelector = (String) metadata.remove("asc_catalog");
1383                 if (catalogItemSelector == null) {
1384                         throw new IOException("Missing item selector");
1385                 }
1386
1387                 JSONObject annotation = new JSONObject();
1388                 for (Map.Entry<String, Object> e : metadata.entrySet()) {
1389                         String key = e.getKey();
1390                         if (key.startsWith("asc_")) {
1391                                 annotation.put(key.substring(4, key.length()), e.getValue());
1392                         }
1393                 }
1394
1395                 debugLogger.log(LogLevel.DEBUG, this.getClass().getName(), "annotation: " + annotation);
1396
1397                 NeoTransaction trx = new NeoTransaction(this.neoUri);
1398                 try {
1399                         String id = neoCreateNode(trx, false, annotation, ("Annotation:" + theLabels).split(":"));
1400                         if (id == null) {
1401                                 throw new IOException("No such catalog item: " + catalogItemSelector);
1402                         }
1403
1404                         id = neoEdge(trx, false,
1405                                         id,
1406                                         "CatalogItem",
1407                                         new JSONObject(catalogItemSelector),
1408                                         new JSONObject(),
1409                                         "ANNOTATION_OF");
1410                         if (id == null) {
1411                                 throw new IOException("No such catalog item: " + catalogItemSelector);
1412                         }
1413
1414                         trx.commit();
1415                 } catch (IOException iox) {
1416                         try {
1417                                 trx.rollback();
1418                         } catch (IOException riox) {
1419                                 errLogger.log(LogLevel.ERROR, this.getClass().getName(), riox.getMessage());
1420                         }
1421                         throw iox;
1422                 }
1423         }
1424
1425         private void listTemplates(String theSelector) throws IOException {
1426
1427                 JSONObject selector = null;
1428
1429                 if (theSelector != null) {
1430                         selector = new JSONObject(theSelector);
1431                 }
1432
1433                 NeoTransaction trx = new NeoTransaction(this.neoUri);
1434
1435                 JSONObject res = trx.statement(new JSONObject()
1436                                 .put("statement",
1437                                                 "MATCH (t:TOSCA:Template" +
1438                                                                 (selector != null ? neoLiteralMap(selector) : "") + ") RETURN t, id(t)")
1439                                 .put("parameters",
1440                                                 new JSONObject()
1441                                                                 .put("props", selector != null ? selector : new JSONObject())))
1442                                 .commit()
1443                                 .result();
1444
1445                 JSONArray data = res
1446                                 .getJSONArray("results")
1447                                 .getJSONObject(0)
1448                                 .getJSONArray("data");
1449                 if (data.length() == 0) {
1450                         return;
1451                 }
1452
1453                 for (int i = 0; i < data.length(); i++) {
1454                         JSONArray row = data.getJSONObject(i)
1455                                         .getJSONArray("row");
1456                         debugLogger.log(LogLevel.DEBUG, this.getClass().getName(), "{}: {}", row.getInt(1), row.getJSONObject(0));
1457                 }
1458         }
1459
1460
1461         private void removeTemplate(String theId) throws IOException {
1462
1463                 //find the nodes to delete and then use 'detach delete'
1464
1465                 NeoTransaction trx = new NeoTransaction(this.neoUri);
1466
1467                 try {
1468                         //Template elements are never more then three hops away and point towards the template
1469                         JSONObject res = trx.statement(new JSONObject()
1470                                         .put("statement",
1471                                                         "MATCH (t:TOSCA:Template)<-[*0..3]-(x) " +
1472                                                                         "WHERE id(t)=" + theId + " RETURN {labels:labels(x),id:id(x)} as tgt"))
1473                                         .execute()
1474                                         .result();
1475
1476                         JSONArray data = res
1477                                         .getJSONArray("results")
1478                                         .getJSONObject(0)
1479                                         .getJSONArray("data");
1480                         if (data.length() == 0) {
1481                                 return;
1482                         }
1483
1484                         for (int i = data.length() - 1; i >= 0; i--) {
1485                                 JSONArray row = data.getJSONObject(i)
1486                                                 .getJSONArray("row");
1487                                 debugLogger.log(LogLevel.DEBUG, this.getClass().getName(), "> {}", row.getJSONObject(0));
1488
1489                                 //double check
1490
1491
1492                                 res = trx.statement(new JSONObject()
1493                                                 .put("statement",
1494                                                                 "MATCH (n) " +
1495                                                                                 "WHERE id(n)=" + row.getJSONObject(0).getInt("id") + " " +
1496                                                                                 "DETACH DELETE n"))
1497                                                 .execute()
1498                                                 .result();
1499
1500                                 debugLogger.log(LogLevel.DEBUG, this.getClass().getName(), "> {}", res);
1501                         }
1502
1503                         trx.commit();
1504                 } catch (IOException iox) {
1505                         try {
1506                                 trx.rollback();
1507                         } catch (IOException riox) {
1508                                 debugLogger.log(LogLevel.DEBUG, this.getClass().getName(), "Rollback failed: {}", riox);
1509                         }
1510                         throw iox;
1511                 }
1512         }
1513
1514         /*
1515          */
1516         private static void ignoreMissing(String theTarget) throws IOException {
1517
1518                 for (String prefix : ignoreMissing) {
1519                         //make sure they are only one name element away
1520                         if ((theTarget.startsWith(prefix)) && (theTarget.substring(prefix.length()).lastIndexOf('.') == 0)) {
1521                                 return;
1522                         }
1523                 }
1524
1525                 throw new IOException("Not configured to ignore missing " + theTarget);
1526         }
1527
1528         private static JSONArray encodeRange(List theRange) throws IOException {
1529                 JSONArray range = new JSONArray();
1530                 for (Object value : theRange) {
1531                         if (value instanceof Number) {
1532                                 range.put(((Number) value).intValue());
1533                         } else if (value instanceof String &&
1534                                         "UNBOUNDED".equals(value)) {
1535                                 range.put(Integer.MAX_VALUE);
1536                         } else {
1537                                 throw new IOException("Unexpected value in range definition: " + value);
1538                         }
1539                 }
1540                 return range;
1541         }
1542
1543         private static String neoLiteralMap(JSONObject theProps) {
1544                 return neoLiteralMap(theProps, "props");
1545         }
1546
1547         private static String neoLiteralMap(JSONObject theProps, String theArg) {
1548                 if (theProps.length() == 0) {
1549                         return "";
1550                 }
1551                 StringBuilder sb = new StringBuilder("");
1552                 for (Iterator i = theProps.keys(); i.hasNext(); ) {
1553                         String key = (String) i.next();
1554                         sb.append("`")
1555                                         .append(key)
1556                                         .append("`: {")
1557                                         .append(theArg)
1558                                         .append("}.`")
1559                                         .append(key)
1560                                         .append("`,");
1561                 }
1562                 return "{ " + sb.substring(0, sb.length() - 1) + " }";
1563         }
1564
1565         private static String neoLabelsString(int theStartPos, String... theLabels) {
1566                 StringBuffer lbls = new StringBuffer("");
1567                 for (int i = theStartPos; i < theLabels.length; i++) {
1568                         lbls.append(":")
1569                                         .append(theLabels[i]);
1570                 }
1571                 return lbls.toString();
1572         }
1573
1574         private String neoCreateNode(
1575                         JSONObject theProperties,
1576                         String... theLabels) throws IOException {
1577                 return neoNode("CREATE", theProperties, theLabels);
1578         }
1579
1580         /* executes the (up to 2) statements required to construct a node
1581                  in a dedicated transaction */
1582         private String neoNode(
1583                         String theVerb,
1584                         JSONObject theProperties,
1585                         String... theLabels) throws IOException {
1586                 NeoTransaction trx = new NeoTransaction(this.neoUri);
1587                 try {
1588                         return neoNode(trx, true,
1589                                         theVerb, theProperties, theLabels);
1590                 } catch (IOException iox) {
1591                         try {
1592                                 trx.rollback();
1593                         } catch (IOException ioxx) {
1594                                 errLogger.log(LogLevel.ERROR, Modeled.class.getName(), ioxx.getMessage());
1595                         }
1596                         throw iox;
1597                 }
1598         }
1599
1600         private String neoCreateNode(
1601                         NeoTransaction theTransaction,
1602                         boolean doCommit,
1603                         JSONObject theProperties,
1604                         String... theLabels) throws IOException {
1605                 return neoNode(theTransaction, doCommit, "CREATE", theProperties, theLabels);
1606         }
1607
1608         private String neoMergeNode(
1609                         NeoTransaction theTransaction,
1610                         boolean doCommit,
1611                         JSONObject theProperties,
1612                         String... theLabels) throws IOException {
1613                 return neoNode(theTransaction, doCommit, "MERGE", theProperties, theLabels);
1614         }
1615
1616         /* execute the statements required to construct a node as part of the
1617         given transaction
1618
1619          */
1620         private String neoNode(
1621                         NeoTransaction theTransaction,
1622                         boolean doCommit,
1623                         String theVerb,
1624                         JSONObject theProperties,
1625                         String... theLabels) throws IOException {
1626
1627                 debugLogger.log(LogLevel.DEBUG, this.getClass().getName(), "neoNode {}", new Object[]{theProperties, theLabels});
1628
1629                 JSONObject node;
1630                 String nodeId;
1631
1632                 node = theTransaction
1633                                 .statement(
1634                                                 new JSONObject()
1635                                                                 .put("statement",
1636                                                                                 theVerb + " (n:" + theLabels[0] + neoLiteralMap(theProperties) + " ) RETURN id(n)")
1637                                                                 .put("parameters",
1638                                                                                 new JSONObject()
1639                                                                                                 .put("props", theProperties)))
1640                                 .execute()
1641                                 .result();
1642
1643
1644                 nodeId = neoId(node);
1645                 debugLogger.log(LogLevel.DEBUG, this.getClass().getName(), "neoNode, node: {}", nodeId);
1646
1647                 if (theLabels.length > 1) {
1648                         theTransaction.statement(
1649                                         new JSONObject()
1650                                                         .put("statement",
1651                                                                         "START n=node(" + nodeId + ") SET n " + neoLabelsString(1, theLabels)));
1652                 }
1653                 theTransaction.execute(doCommit);
1654
1655                 return nodeId;
1656         }
1657
1658         private void neoNodeProperties(
1659                         NeoTransaction theTransaction,
1660                         boolean doCommit,
1661                         String theId,
1662                         JSONObject theProperties) throws IOException {
1663
1664                 debugLogger.log(LogLevel.DEBUG, this.getClass().getName(), "neoNodeProperties {}", new Object[]{theId, theProperties});
1665                 theTransaction
1666                                 .statement(
1667                                                 new JSONObject()
1668                                                                 .put("statement",
1669                                                                                 "START n=node(" + theId + ") SET n+= " +
1670                                                                                                 neoLiteralMap(theProperties) + " RETURN id(n)")
1671                                                                 .put("parameters",
1672                                                                                 new JSONObject()
1673                                                                                                 .put("props", theProperties)))
1674                                 .execute(doCommit);
1675         }
1676
1677         private String neoEdge(
1678                         NeoTransaction theTransaction,
1679                         boolean doCommit,
1680                         String theFrom, String theTo,
1681                         JSONObject theProperties,
1682                         String... theLabels) throws IOException {
1683
1684                 debugLogger.log(LogLevel.DEBUG, this.getClass().getName(), "neoEdge: {}", new Object[]{theFrom, theTo, theProperties, theLabels});
1685
1686                 return neoEdge(
1687                                 theTransaction, doCommit,
1688                                 new JSONObject()
1689                                                 .put("statement",
1690                                                                 "START a=node(" + theFrom + "),b=node(" + theTo + ") " +
1691                                                                                 "MERGE (a)-[r:" + theLabels[0] + neoLiteralMap(theProperties) + "]->(b) " +
1692                                                                                 "RETURN id(r)")
1693                                                 .put("parameters",
1694                                                                 new JSONObject()
1695                                                                                 .put("props", theProperties)));
1696         }
1697
1698         private String neoEdge(
1699                         NeoTransaction theTransaction, boolean doCommit,
1700                         String theFromId,
1701                         String theToLabel, JSONObject theToProps,
1702                         JSONObject theProperties,
1703                         String... theLabels) throws IOException {
1704
1705                 return neoEdge(theTransaction, doCommit,
1706                                 new JSONObject()
1707                                                 .put("statement",
1708                                                                 //"START a=node(" + theFromId + ") " +
1709                                                                 "MATCH (a),(b:" + theToLabel + neoLiteralMap(theToProps, "toProps") + ") " +
1710                                                                                 "WHERE id(a)=" + theFromId + " " +
1711                                                                                 "MERGE (a)-[r:" + theLabels[0] + neoLiteralMap(theProperties) + "]->(b) " +
1712                                                                                 "RETURN id(r)")
1713                                                 .put("parameters",
1714                                                                 new JSONObject()
1715                                                                                 .put("toProps", theToProps)
1716                                                                                 .put("props", theProperties)));
1717         }
1718
1719         private String neoEdge(NeoTransaction theTransaction,
1720                                                    boolean doCommit,
1721                                                    JSONObject theEdgeStatement)
1722                         throws IOException {
1723
1724                 debugLogger.log(LogLevel.DEBUG, this.getClass().getName(), "neoEdge {}", new Object[]{theEdgeStatement});
1725
1726                 return neoId(
1727                                 theTransaction
1728                                                 .statement(theEdgeStatement)
1729                                                 .execute(doCommit)
1730                                                 .result()
1731                 );
1732         }
1733
1734         private static String neoId(JSONObject theResult) throws IOException {
1735                 try {
1736                         JSONArray data = theResult
1737                                         .getJSONArray("results")
1738                                         .getJSONObject(0)
1739                                         .getJSONArray("data");
1740                         if (data.length() == 0) {
1741                                 return null;
1742                         }
1743
1744                         return String.valueOf(
1745                                         data.getJSONObject(0)
1746                                                         .getJSONArray("row")
1747                                                         .getInt(0));
1748                 } catch (JSONException jsonx) {
1749                         errLogger.log(LogLevel.WARN, Modeled.class.getName(), "neoId, No 'id' in result: {} {}", theResult, jsonx);
1750                         throw new IOException("no 'id' in result", jsonx);
1751                 }
1752         }
1753
1754         private static String[] neoIds(JSONObject theResult) throws IOException {
1755                 try {
1756                         JSONArray data = theResult
1757                                         .getJSONArray("results")
1758                                         .getJSONObject(0)
1759                                         .getJSONArray("data");
1760                         if (data.length() == 0) {
1761                                 return new String[]{};
1762                         }
1763
1764                         JSONArray array = data.getJSONObject(0)
1765                                         .getJSONArray("row");
1766
1767                         String[] res = new String[array.length()];
1768                         for (int i = 0; i < array.length(); i++) {
1769                                 res[i] = String.valueOf(array.getInt(i));
1770                         }
1771                         return res;
1772                 } catch (JSONException jsonx) {
1773                         errLogger.log(LogLevel.WARN, Modeled.class.getName(), "neoId, No 'id' in result: {} {}", theResult, jsonx);
1774                         throw new IOException("no 'id' in result", jsonx);
1775                 }
1776         }
1777
1778         private static class NeoTransaction {
1779
1780                 private HttpClient client = null;
1781                 private String uri = null;
1782                 private String auth = null;
1783                 private JSONObject result = null;
1784                 private JSONArray stmts = new JSONArray();
1785
1786                 NeoTransaction(URI theTarget) {
1787
1788                         client = httpClientBuilder.build();
1789                         this.uri = theTarget.getScheme() + "://" + theTarget.getHost() + ":" + theTarget.getPort() + "/db/data/transaction";
1790
1791                         String userInfo = theTarget.getUserInfo();
1792                         if (userInfo != null) {
1793                                 this.auth = "Basic " + new String(
1794                                                 Base64.encodeBase64(
1795                                                                 userInfo.getBytes(Charset.forName("ISO-8859-1"))));
1796                         }
1797                 }
1798
1799                 /* adds a statement to the next execution cycle */
1800                 NeoTransaction statement(JSONObject theStatement) {
1801                         if (this.client == null) {
1802                                 throw new IllegalStateException("Transaction was completed");
1803                         }
1804                         this.stmts.put(theStatement);
1805                         return this;
1806                 }
1807
1808                 /* executes all pending statements but does not commit the transaction */
1809                 /* executing a transaction with no statements refreshes the transaction timer in order to keep the transaction alive */
1810                 NeoTransaction execute() throws IOException {
1811                         if (this.client == null) {
1812                                 throw new IllegalStateException("Transaction was completed");
1813                         }
1814                         post(this.uri);
1815                         return this;
1816                 }
1817
1818                 /* executes all pending statements and commits the transaction */
1819                 NeoTransaction commit() throws IOException {
1820                         if (this.client == null) {
1821                                 throw new IllegalStateException("Transaction was completed");
1822                         }
1823                         post(this.uri + "/commit");
1824                         //mark the transaction as terminated
1825                         this.client = null;
1826                         return this;
1827                 }
1828
1829                 /* just to simplify some code written on top of NeoTransaction */
1830                 NeoTransaction execute(boolean doCommit) throws IOException {
1831                         return doCommit ? commit() : execute();
1832                 }
1833
1834                 private void post(String theUri) throws IOException {
1835                         HttpPost post = new HttpPost(theUri);
1836                         JSONObject payload = new JSONObject()
1837                                         .put("statements", this.stmts);
1838                         debugLogger.log(LogLevel.DEBUG, this.getClass().getName(), "post> " + payload);
1839                         post.setEntity(new StringEntity(payload.toString(),
1840                                         ContentType.APPLICATION_JSON));
1841                         run(post);
1842                 }
1843
1844                 /* rollbacks the transaction changes */
1845                 NeoTransaction rollback() throws IOException {
1846                         if (this.client == null) {
1847                                 throw new IllegalStateException("Transaction was completed");
1848                         }
1849                         if (this.uri == null) {
1850                                 throw new IllegalStateException("Transaction not started");
1851                         }
1852                         run(new HttpDelete(this.uri));
1853                         return this;
1854                 }
1855
1856                 /* retrieve the (raw) results of the last execute/commit cycle */
1857                 JSONObject result() {
1858                         return this.result;
1859                 }
1860
1861                 private void run(HttpUriRequest theRequest) throws IOException {
1862                         theRequest.setHeader(HttpHeaders.ACCEPT, "application/json; charset=UTF-8");
1863                         if (this.auth != null) {
1864                                 theRequest.setHeader(HttpHeaders.AUTHORIZATION, this.auth);
1865                         }
1866
1867                         HttpResponse response = this.client.execute(theRequest);
1868                         int statusCode = response.getStatusLine().getStatusCode();
1869                         if (statusCode >= 300) {
1870                                 try {
1871                                         this.result = new JSONObject(IOUtils.toString(response.getEntity().getContent(), "UTF-8"));
1872                                 } catch (Exception x) {
1873                                         errLogger.log(LogLevel.ERROR, Modeled.class.getName(), x.getMessage());
1874                                 }
1875                                 throw new IOException("Neo statement(s) '" + this.stmts + "' failed: " + response.getStatusLine());
1876                         }
1877
1878                         try {
1879                                 this.result = new JSONObject(
1880                                                 IOUtils.toString(response.getEntity().getContent(), "UTF-8"));
1881                         } catch (Exception x) {
1882                                 throw new IOException("no json in response", x);
1883                         }
1884
1885                         JSONArray errors = this.result.getJSONArray("errors");
1886                         if (errors.length() > 0) {
1887                                 throw new IOException("Neo statement(s) '" + this.stmts + "' have errors: " + errors);
1888                         }
1889                         //we only get a header if this was not a one statement transaction
1890                         Header hdr = response.getFirstHeader("Location");
1891                         if (hdr != null) {
1892                                 if (!hdr.getValue().startsWith(this.uri)) {
1893                                         debugLogger.log(LogLevel.DEBUG, this.getClass().getName(), "new transaction location?? : {} vs. {}", this.uri, hdr.getValue());
1894                                 }
1895                                 this.uri = hdr.getValue();
1896                         }
1897                         this.stmts = new JSONArray();
1898                 }
1899         }
1900
1901         private static JSONObject pack(Map theRule, Map theDef) {
1902                 JSONObject pack = new JSONObject();
1903
1904                 if (theRule == null) {
1905                         return pack;
1906                 }
1907
1908                 //these are the facets of the construct definition
1909                 Map facets = (Map) theRule.get("mapping");
1910                 if (facets == null) {
1911                         return pack;
1912                 }
1913
1914                 facets.entrySet().stream()
1915                                 .forEach(
1916                                                 theEntry ->
1917                                                 {
1918                                                         Map.Entry entry = (Map.Entry) theEntry;
1919                                                         Map facetDef = (Map) entry.getValue();
1920
1921                                                         String storage = (String) facetDef.getOrDefault("storage", "");
1922                                                         String type = (String) facetDef.get("type");
1923
1924                                                         if ("none".equals(storage)) {
1925                                                                 return;
1926                                                         }
1927                                                         if ("map".equals(type)) {
1928                                                                 //maps are used for cross-references between constructs or for
1929                                                                 //constructs facets
1930                                                                 return;
1931                                                         }
1932                                                         Object val = theDef.get(entry.getKey());
1933                                                         if ("seq".equals(type)) {
1934                                                                 //sequences can be stored inlined, if so instructed ..
1935                                                                 if ("inline".equals(storage)) {
1936                                                                         val = JSONObject.valueToString(val);
1937                                                                 } else {
1938                                                                         return;
1939                                                                 }
1940                                                         }
1941                                                         if ("no".equals(facetDef.getOrDefault("required", "no"))) {
1942                                                                 pack.putOpt((String) entry.getKey(), theDef.get(entry.getKey()));
1943                                                         } else {
1944                                                                 pack.putOnce((String) entry.getKey(), theDef.get(entry.getKey()));
1945                                                         }
1946                                                 });
1947                 return pack;
1948         }
1949
1950         /* a sort of catalog of neo identifiers generated for the different
1951          * constructs (or their types) we store
1952          */
1953         private static class Tracker<T> {
1954
1955                 private Table<String, String, T>
1956                                 typeTracker = HashBasedTable.create(),
1957                                 templateTracker = HashBasedTable.create();
1958
1959                 void trackType(String theConstruct, String theName, T theInfo) {
1960                         typeTracker.put(theConstruct, theName, theInfo);
1961                 }
1962
1963                 T lookupType(String theConstruct, String theName) {
1964                         return typeTracker.get(theConstruct, theName);
1965                 }
1966
1967                 boolean tracksType(String theConstruct, String theName) {
1968                         return typeTracker.contains(theConstruct, theName);
1969                 }
1970
1971                 void trackTemplate(String theConstruct, String theName, T theInfo) {
1972                         templateTracker.put(theConstruct, theName, theInfo);
1973                 }
1974
1975                 T lookupTemplate(String theConstruct, String theName) {
1976                         return templateTracker.get(theConstruct, theName);
1977                 }
1978
1979         }
1980 }