dcaedt_catalog/asdc/src/main/java/org/onap/sdc/dcae/catalog/asdc/ASDCUtils.java

   1 package org.onap.sdc.dcae.catalog.asdc;
   2
   3 import org.apache.commons.jxpath.JXPathContext;
   4 import org.apache.commons.lang3.StringUtils;
   5 import org.json.JSONArray;
   6 import org.json.JSONObject;
   7 import org.onap.sdc.common.onaplog.OnapLoggerDebug;
   8 import org.onap.sdc.common.onaplog.OnapLoggerError;
   9 import org.onap.sdc.common.onaplog.Enums.LogLevel;
  10 import org.onap.sdc.dcae.catalog.commons.Actions;
  11 import org.onap.sdc.dcae.catalog.commons.Future;
  12 import org.onap.sdc.dcae.catalog.commons.Futures;
  13 import org.onap.sdc.dcae.catalog.commons.Recycler;
  14 import org.onap.sdc.dcae.checker.*;
  15 import org.springframework.beans.factory.annotation.Autowired;
  16 import org.springframework.boot.context.properties.ConfigurationProperties;
  17 import org.springframework.context.annotation.Scope;
  18 import org.springframework.stereotype.Component;
  19 import org.springframework.util.Base64Utils;
  20
  21 import java.io.*;
  22 import java.net.URI;
  23 import java.util.*;
  24 import java.util.function.BiFunction;
  25 import java.util.function.Function;
  26 import java.util.stream.Collectors;
  27 import java.util.stream.Stream;
  28 import java.util.stream.StreamSupport;
  29 import java.util.zip.ZipEntry;
  30 import java.util.zip.ZipInputStream;
  31
  32
  33 @Component("asdcutils")
  34 @Scope("singleton")
  35 @ConfigurationProperties(prefix="asdcutils")
  36 public class ASDCUtils {
  37
  38         private static OnapLoggerError errLogger = OnapLoggerError.getInstance();
  39         private static OnapLoggerDebug debugLogger = OnapLoggerDebug.getInstance();
  40
  41         @Autowired
  42         private ASDC asdc;
  43
  44         @Autowired
  45         private Blueprinter blueprint;
  46
  47         public ASDCUtils() {
  48                 // Making sonar happy
  49         }
  50
  51         public ASDCUtils(URI theASDCURI) {
  52                 this(theASDCURI, null);
  53         }
  54
  55         public ASDCUtils(URI theASDCURI, URI theBlueprinterURI) {
  56                 this.asdc = new ASDC();
  57                 this.asdc.setUri(theASDCURI);
  58                 if (theBlueprinterURI != null) {
  59                         this.blueprint = new Blueprinter();
  60                         this.blueprint.setUri(theBlueprinterURI);
  61                 }
  62         }
  63
  64         public ASDCUtils(ASDC theASDC) {
  65                 this(theASDC, null);
  66         }
  67
  68         public ASDCUtils(ASDC theASDC, Blueprinter theBlueprinter) {
  69                 this.asdc = theASDC;
  70                 this.blueprint = theBlueprinter;
  71         }
  72
  73         public CloneAssetArtifactsAction cloneAssetArtifacts(ASDC.AssetType theAssetType, UUID theSourceId, UUID theTargetId) {
  74                 return new CloneAssetArtifactsAction(this.asdc, theAssetType, theSourceId, theTargetId);
  75         }
  76
  77         public static class CloneAssetArtifactsAction extends ASDC.ASDCAction<CloneAssetArtifactsAction, List<JSONObject>> {
  78
  79                 private ASDC.AssetType assetType;
  80                 private UUID sourceId, targetId;
  81
  82                 protected CloneAssetArtifactsAction(ASDC theASDC, ASDC.AssetType theAssetType, UUID theSourceId, UUID theTargetId) {
  83                         theASDC.super(new JSONObject());
  84                         this.assetType = theAssetType;
  85                         this.sourceId = theSourceId;
  86                         this.targetId = theTargetId;
  87                 }
  88
  89                 protected CloneAssetArtifactsAction self() {
  90                         return this;
  91                 }
  92
  93                 public CloneAssetArtifactsAction withLabel(String theLabel) {
  94                         return with("artifactLabel", theLabel);
  95                 }
  96
  97                 protected String[] mandatoryInfoEntries() {
  98                         return new String[] {};
  99                 }
 100
 101                 public Future<List<JSONObject>> execute() {
 102                         checkMandatory();
 103
 104                         final Actions.Sequence<JSONObject> sequencer = new Actions.Sequence<JSONObject>();
 105
 106                         new Actions.Sequence().add(super.asdc().getAssetArchiveAction(this.assetType, this.sourceId)).add(super.asdc().getAssetAction(this.assetType, this.sourceId, JSONObject.class)).execute().setHandler(assetFuture -> {
 107                                 debugLogger.log(LogLevel.DEBUG, this.getClass().getName(), "*** {}", assetFuture.result());
 108                                 processArtifacts((List) assetFuture.result(), (JSONObject theInfo, byte[] theData) -> {
 109                                         theInfo.remove("artifactChecksum");
 110                                         theInfo.remove("artifactUUID");
 111                                         theInfo.remove("artifactVersion");
 112                                         theInfo.remove("artifactURL");
 113                                         theInfo.put("description", theInfo.remove("artifactDescription"));
 114                                         theInfo.put("payloadData", Base64Utils.encodeToString(theData));
 115                                         return theInfo;
 116                                 }, null).forEach(artifactInfo -> sequencer.add(super.asdc().createAssetArtifact(this.assetType, this.targetId).withInfo(ASDC.merge(artifactInfo, this.info)).withOperator(this.operatorId)));
 117                                 sequencer.execute();
 118                         });
 119
 120                         return sequencer.future();
 121                 }
 122         } //the Action class
 123
 124         /* */
 125         private static JSONObject lookupArtifactInfo(JSONArray theArtifacts, String theName) {
 126
 127                 for (int i = 0; theArtifacts != null && i < theArtifacts.length(); i++) {
 128                         JSONObject artifactInfo = theArtifacts.getJSONObject(i);
 129                         if (theName.equals(artifactInfo.getString("artifactName"))) {
 130                                 debugLogger.log(LogLevel.DEBUG, ASDCUtils.class.getName(), "Found artifact info {}", artifactInfo);
 131                                 return artifactInfo;
 132                         }
 133                 }
 134
 135                 return null;
 136         }
 137
 138         private static byte[] extractArtifactData(InputStream theEntryStream) throws IOException {
 139                 ByteArrayOutputStream baos = new ByteArrayOutputStream();
 140                 try {
 141                         byte[] buff = new byte[4096];
 142                         int cnt = 0;
 143                         while ((cnt = theEntryStream.read(buff)) != -1) {
 144                                 baos.write(buff, 0, cnt);
 145                         }
 146                 } finally {
 147                         baos.close();
 148                 }
 149                 return baos.toByteArray();
 150         }
 151
 152         /**
 153          * Recycle a cdump, fetch all relevant ASDC artifacts, interact with Shu's toscalib service in order to generate
 154          * a blueprint. No 'Action' object here as there is nothig to set up.
 155          */
 156         public Future<Future<String>> buildBlueprint(Reader theCdump) {
 157
 158                 final Recycler recycler = new Recycler();
 159                 Object template = null;
 160
 161                 try {
 162                         template = recycler.recycle(theCdump);
 163
 164                 } catch (Exception x) {
 165                         return Futures.failedFuture(x);
 166                 }
 167
 168                 JXPathContext jxroot = JXPathContext.newContext(template);
 169                 jxroot.setLenient(true);
 170
 171                 //based on the output of ASDCCatalog the node description will contain the UUID of the resource declaring it
 172                 List uuids = (List) StreamSupport.stream(Spliterators.spliteratorUnknownSize(jxroot.iterate("topology_template/node_templates/*/description"), 16), false).distinct().filter(desc -> desc != null)
 173                                 //the desc contains the full URI and the resource uuid is the 5th path element
 174                                 .map(desc -> desc.toString().split("/")[5]).collect(Collectors.toList());
 175
 176                 //prepare fetching all archives/resource details
 177                 final Futures.Accumulator accumulator = new Futures.Accumulator();
 178                 uuids.stream().forEach(uuid -> {
 179                         UUID rid = UUID.fromString((String) uuid);
 180                         accumulator.add(this.asdc.getAssetArchive(ASDC.AssetType.resource, rid));
 181                         accumulator.add(this.asdc.getAsset(ASDC.AssetType.resource, rid, JSONObject.class));
 182                 });
 183
 184                 final byte[] templateData = recycler.toString(template).getBytes(/*"UTF-8"*/);
 185                 //retrieve all resource archive+details, prepare blueprint service request and send its request
 186                 return Futures.advance(accumulator.accumulate(), (List theArchives) -> {
 187                         Blueprinter.BlueprintAction action = blueprint.generateBlueprint();
 188                         processArtifacts(theArchives, (JSONObject theInfo, byte[] theData) -> new JSONObject().put(theInfo.getString("artifactName").split("\\.")[0], Base64Utils.encodeToString(theData)),
 189                                         (Stream<JSONObject> theAssetArtifacts) -> theAssetArtifacts.reduce(new JSONObject(), ASDC::merge)).forEach(artifactInfo -> action.withModelInfo(artifactInfo));
 190
 191                         return action.withTemplateData(templateData).execute();
 192                 });
 193         }
 194
 195         public Future<Future<String>> buildBlueprintViaToscaLab(Reader theCdump) {
 196                 return processCdump(theCdump, (theTemplate, theArchives) -> {
 197                         Blueprinter.BlueprintAction action = blueprint.generateBlueprint();
 198                         processArtifacts(theArchives, (JSONObject theInfo, byte[] theData) -> new JSONObject().put(theInfo.getString("artifactName").split("\\.")[0], Base64Utils.encodeToString(theData)),
 199                                         (Stream<JSONObject> theAssetArtifacts) -> theAssetArtifacts.reduce(new JSONObject(), ASDC::merge)).forEach(artifactInfo -> action.withModelInfo(artifactInfo));
 200
 201                         return action.withTemplateData(Recycler.toString(theTemplate).getBytes()).execute();
 202
 203                 });
 204         }
 205
 206         private static class Tracker implements TargetLocator {
 207
 208                 private static enum Position {
 209                         SCHEMA, TEMPLATE, TRANSLATE;
 210                 }
 211
 212                 private static final int Positions = Position.values().length;
 213
 214                 private List<Target> tgts = new ArrayList<Target>(3);
 215
 216                 public Tracker() {
 217                         clear();
 218                 }
 219
 220                 public boolean addSearchPath(URI theURI) {
 221                         return false;
 222                 }
 223
 224                 public boolean addSearchPath(String thePath) {
 225                         return false;
 226                 }
 227
 228                 public Iterable<URI> searchPaths() {
 229                         return Collections.emptyList();
 230                 }
 231
 232                 protected int position(String... theKeys) {
 233                         for (String key : theKeys) {
 234                                 if ("schema".equals(key)) {
 235                                         return Position.SCHEMA.ordinal();
 236                                 }
 237                                 if ("template".equals(key)) {
 238                                         return Position.TEMPLATE.ordinal();
 239                                 }
 240                                 if ("translate".equals(key)) {
 241                                         return Position.TRANSLATE.ordinal();
 242                                 }
 243                         }
 244                         return -1;
 245                 }
 246
 247                 public Target resolve(String theName) {
 248                         for (Target tgt : tgts) {
 249                                 if (tgt != null && tgt.getName().equals(theName)) {
 250                                         return tgt;
 251                                 }
 252                         }
 253                         return null;
 254                 }
 255
 256                 public void track(JSONObject theInfo, final byte[] theData) {
 257                         String uri = theInfo.getString("artifactURL").split("/")[5];
 258                         String name = theInfo.getString("artifactName"), desc = theInfo.getString("artifactDescription"), label = theInfo.getString("artifactLabel");
 259                         int pos = position(desc, label);
 260
 261                         debugLogger.log(LogLevel.DEBUG, ASDCUtils.class.getName(), "Tracking {} at {}, {}", name, pos, theInfo.optString("artifactURL"));
 262
 263                         if (pos > -1) {
 264                                 tgts.set(pos, new Target(name, URI.create("asdc:" + uri + "/" + name)) {
 265                                         @Override
 266                                         public Reader open(){
 267                                                 return new BufferedReader(new InputStreamReader(new ByteArrayInputStream(theData)));
 268                                         }
 269                                 });
 270                         }
 271                 }
 272
 273                 public boolean hasSchema() {
 274                         return tgts.get(Position.SCHEMA.ordinal()) != null;
 275                 }
 276
 277                 public Target schema() {
 278                         return tgts.get(Position.SCHEMA.ordinal());
 279                 }
 280
 281                 public boolean hasTemplate() {
 282                         return tgts.get(Position.TEMPLATE.ordinal()) != null;
 283                 }
 284
 285                 public Target template() {
 286                         return tgts.get(Position.TEMPLATE.ordinal());
 287                 }
 288
 289                 public boolean hasTranslation() {
 290                         return tgts.get(Position.TRANSLATE.ordinal()) != null;
 291                 }
 292
 293                 public Target translation() {
 294                         return tgts.get(Position.TRANSLATE.ordinal());
 295                 }
 296
 297                 public void clear() {
 298                         if (tgts.isEmpty()) {
 299                                 for (int i = 0; i < Positions; i++) {
 300                                         tgts.add(null);
 301                                 }
 302                         } else {
 303                                 Collections.fill(tgts, null);
 304                         }
 305                 }
 306         }
 307
 308         private Checker buildChecker() {
 309                 try {
 310                         return new Checker();
 311                 } catch (CheckerException cx) {
 312                         errLogger.log(LogLevel.ERROR, this.getClass().getName(), "CheckerException while creating Checker {}", cx);
 313                         return null;
 314                 }
 315         }
 316
 317         public Future<Catalog> buildCatalog(Reader theCdump) {
 318
 319                 //
 320                 //the purpose of the tracking is to be able to resolve import references within the 'space' of an
 321                 //asset's artifacts
 322                 //processing order is important too so we 'order the targets: schema, template, translation
 323                 //
 324                 final Tracker tracker = new Tracker();
 325                 final Catalog catalog = Checker.buildCatalog();
 326
 327                 return processCdump(theCdump, (theTemplate, theArchives) -> {
 328
 329                         final Checker checker = buildChecker();
 330                         if (checker == null) {
 331                                 return null;
 332                         }
 333                         checker.setTargetLocator(tracker);
 334
 335                         processArtifacts(theArchives, (JSONObject theInfo, byte[] theData) -> {
 336                                                 tracker.track(theInfo, theData);
 337                                                 return (Catalog) null;
 338                                         },
 339                                         // aggregation: this is where the actual processing takes place now that
 340                                         // we have all the targets
 341                                         (Stream<Catalog> theAssetArtifacts) -> {
 342                                                 //the stream is full of nulls, ignore it, work with the tracker
 343
 344                                                 try {
 345                                                         if (tracker.hasSchema()) {
 346                                                                 checker.check(tracker.schema(), catalog);
 347                                                         }
 348                                                         if (tracker.hasTemplate()) {
 349                                                                 checker.check(tracker.template(), catalog);
 350                                                         }
 351                                                         if (tracker.hasTranslation()) {
 352                                                                 checker.check(tracker.translation(), catalog);
 353                                                         }
 354                                                 } catch (CheckerException cx) {
 355                                                         //got to do better than this
 356                                                         errLogger.log(LogLevel.ERROR, ASDC.class.getName(),"CheckerException while checking catalog:{}", cx);
 357                                                 } finally {
 358                                                         tracker.clear();
 359                                                 }
 360                                                 return checker.catalog();
 361                                         });
 362
 363                         Target cdump = new Target("cdump", URI.create("asdc:cdump"));
 364                         cdump.setTarget(theTemplate);
 365
 366                         validateCatalog(catalog, checker, cdump);
 367
 368                         return catalog;
 369                 });
 370         }
 371
 372         private void validateCatalog(Catalog catalog, Checker checker, Target cdump) {
 373                 try {
 374             checker.validate(cdump, catalog);
 375         } catch (CheckerException cx) {
 376             errLogger.log(LogLevel.ERROR, ASDC.class.getName(),"CheckerException while building catalog:{}", cx);
 377         }
 378         }
 379
 380         /* The common process of recycling, retrieving all related artifacts and then doing 'something' */
 381         private <T> Future<T> processCdump(Reader theCdump, BiFunction<Object, List, T> theProcessor) {
 382
 383                 final Recycler recycler = new Recycler();
 384                 Object template = null;
 385                 try {
 386                         template = recycler.recycle(theCdump);
 387
 388                 } catch (Exception x) {
 389                         return Futures.failedFuture(x);
 390                 }
 391
 392                 JXPathContext jxroot = JXPathContext.newContext(template);
 393                 jxroot.setLenient(true);
 394
 395                 //based on the output of ASDCCatalog the node description will contain the UUID of the resource declaring it
 396                 //the desc contains the full URI and the resource uuid is the 5th path element
 397                 List uuids = (List) StreamSupport.stream(Spliterators.spliteratorUnknownSize(jxroot.iterate("topology_template/node_templates/*/description"), 16), false).distinct().filter(desc -> desc != null)
 398                                 .map(desc -> desc.toString().split("/")[5]).collect(Collectors.toList());
 399
 400                 //serialized fetch version
 401                 final Actions.Sequence sequencer = new Actions.Sequence();
 402                 uuids.stream().forEach(uuid -> {
 403                         UUID rid = UUID.fromString((String) uuid);
 404                         sequencer.add(this.asdc.getAssetArchiveAction(ASDC.AssetType.resource, rid));
 405                         sequencer.add(this.asdc.getAssetAction(ASDC.AssetType.resource, rid, JSONObject.class));
 406                 });
 407
 408                 final Object tmpl = template;
 409                 return Futures.advance(sequencer.execute(), (List theArchives) -> theProcessor.apply(tmpl, theArchives));
 410         }
 411
 412         private static <T> Stream<T> processArtifacts(List theArtifactData, BiFunction<JSONObject, byte[], T> theProcessor, Function<Stream<T>, T> theAggregator) {
 413
 414                 Stream.Builder<T> assetBuilder = Stream.builder();
 415
 416                 for (int i = 0; i < theArtifactData.size(); i = i + 2) { //cute old style loop
 417
 418                         JSONObject assetInfo = (JSONObject) theArtifactData.get(i + 1);
 419                         byte[] assetData = (byte[]) theArtifactData.get(i + 0);
 420
 421                         JSONArray artifacts = assetInfo.optJSONArray("artifacts");
 422
 423                         Stream.Builder<T> artifactBuilder = Stream.builder();
 424
 425                         try (ZipInputStream zipper = new ZipInputStream(new ByteArrayInputStream(assetData))){
 426                                 //we process the artifacts in the order they are stored in the archive .. fugly
 427                                 for (ZipEntry zipped = zipper.getNextEntry(); zipped != null; zipped = zipper.getNextEntry()) {
 428                                         JSONObject artifactInfo = lookupArtifactInfo(artifacts, StringUtils.substringAfterLast(zipped.getName(), "/"));
 429                                         if (artifactInfo != null) {
 430                                                 artifactBuilder.add(theProcessor.apply(artifactInfo, extractArtifactData(zipper)));
 431                                         }
 432                                         zipper.closeEntry();
 433                                 }
 434                         } catch (IOException iox) {
 435                                 errLogger.log(LogLevel.ERROR, ASDC.class.getName(), "IOException: {}", iox);
 436                                 return null;
 437                         }
 438
 439                         if (theAggregator != null) {
 440                                 assetBuilder.add(theAggregator.apply(artifactBuilder.build()));
 441                         } else {
 442                                 artifactBuilder.build().forEach(entry -> assetBuilder.add(entry));
 443                         }
 444                 }
 445
 446                 return assetBuilder.build();
 447         }
 448 }