ueb-listener/src/main/java/org/onap/ccsdk/sli/northbound/uebclient/SdncUebCallback.java

   1 /*-
   2  * ============LICENSE_START=======================================================
   3  * openECOMP : SDN-C
   4  * ================================================================================
   5  * Copyright (C) 2017 AT&T Intellectual Property. All rights
   6  *                      reserved.
   7  * ================================================================================
   8  * Licensed under the Apache License, Version 2.0 (the "License");
   9  * you may not use this file except in compliance with the License.
  10  * You may obtain a copy of the License at
  11  *
  12  *      http://www.apache.org/licenses/LICENSE-2.0
  13  *
  14  * Unless required by applicable law or agreed to in writing, software
  15  * distributed under the License is distributed on an "AS IS" BASIS,
  16  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  17  * See the License for the specific language governing permissions and
  18  * limitations under the License.
  19  * ============LICENSE_END=========================================================
  20  */
  21
  22 package org.onap.ccsdk.sli.northbound.uebclient;
  23
  24 import java.io.BufferedReader;
  25 import java.io.ByteArrayInputStream;
  26 import java.io.File;
  27 import java.io.FileInputStream;
  28 import java.io.FileNotFoundException;
  29 import java.io.FileReader;
  30 import java.io.FileWriter;
  31 import java.io.IOException;
  32 import java.net.Authenticator;
  33 import java.net.HttpURLConnection;
  34 import java.net.PasswordAuthentication;
  35 import java.net.URL;
  36 import java.nio.file.DirectoryStream;
  37 import java.nio.file.Files;
  38 import java.nio.file.Path;
  39 import java.nio.file.StandardCopyOption;
  40 import java.sql.SQLException;
  41 import java.text.SimpleDateFormat;
  42 import java.util.Date;
  43 import java.util.HashMap;
  44 import java.util.LinkedList;
  45 import java.util.List;
  46 import java.util.Map;
  47 import java.util.Properties;
  48
  49 import javax.sql.rowset.CachedRowSet;
  50 import javax.xml.parsers.DocumentBuilder;
  51 import javax.xml.parsers.DocumentBuilderFactory;
  52 import javax.xml.transform.Source;
  53 import javax.xml.transform.Transformer;
  54 import javax.xml.transform.TransformerFactory;
  55 import javax.xml.transform.stream.StreamResult;
  56 import javax.xml.transform.stream.StreamSource;
  57 import javax.xml.xpath.XPath;
  58 import javax.xml.xpath.XPathFactory;
  59
  60 import org.apache.commons.codec.binary.Base64;
  61 import org.apache.commons.lang3.tuple.Pair;
  62 import org.onap.ccsdk.sli.core.dblib.DBResourceManager;
  63 import org.onap.ccsdk.sli.northbound.uebclient.SdncArtifactMap.SdncArtifactType;
  64 import org.openecomp.sdc.api.IDistributionClient;
  65 import org.openecomp.sdc.api.consumer.IDistributionStatusMessage;
  66 import org.openecomp.sdc.api.consumer.INotificationCallback;
  67 import org.openecomp.sdc.api.notification.IArtifactInfo;
  68 import org.openecomp.sdc.api.notification.INotificationData;
  69 import org.openecomp.sdc.api.notification.IResourceInstance;
  70 import org.openecomp.sdc.api.results.IDistributionClientDownloadResult;
  71 import org.openecomp.sdc.api.results.IDistributionClientResult;
  72 import org.openecomp.sdc.tosca.parser.api.ISdcCsarHelper;
  73 import org.openecomp.sdc.tosca.parser.exceptions.SdcToscaParserException;
  74 import org.openecomp.sdc.tosca.parser.impl.SdcPropertyNames;
  75 import org.openecomp.sdc.tosca.parser.impl.SdcToscaParserFactory;
  76 import org.openecomp.sdc.toscaparser.api.Group;
  77 import org.openecomp.sdc.toscaparser.api.Metadata;
  78 import org.openecomp.sdc.toscaparser.api.NodeTemplate;
  79 import org.openecomp.sdc.utils.ArtifactTypeEnum;
  80 import org.openecomp.sdc.utils.DistributionActionResultEnum;
  81 import org.openecomp.sdc.utils.DistributionStatusEnum;
  82 import org.slf4j.Logger;
  83 import org.slf4j.LoggerFactory;
  84 import org.w3c.dom.Document;
  85 import org.w3c.dom.Element;
  86
  87 public class SdncUebCallback implements INotificationCallback {
  88
  89     private static final Logger LOG = LoggerFactory
  90             .getLogger(SdncUebCallback.class);
  91
  92         private static DBResourceManager jdbcDataSource = null;
  93         private static final String SDNC_CONFIG_DIR = "SDNC_CONFIG_DIR";
  94
  95
  96     private class SdncAuthenticator extends Authenticator {
  97
  98         private final String user;
  99         private final String passwd;
 100
 101         SdncAuthenticator(String user, String passwd) {
 102             this.user = user;
 103             this.passwd = passwd;
 104         }
 105         @Override
 106         protected PasswordAuthentication getPasswordAuthentication() {
 107             return new PasswordAuthentication(user, passwd.toCharArray());
 108         }
 109
 110     }
 111
 112     private class DeployableArtifact {
 113         SdncArtifactType type;
 114         IArtifactInfo artifactInfo;
 115         String svcName;
 116         String resourceName;
 117         String artifactName;
 118         String artifactVersion;
 119         File file;
 120
 121         public String getArtifactName() {
 122             return artifactName;
 123         }
 124
 125
 126
 127         public String getArtifactVersion() {
 128             return artifactVersion;
 129         }
 130
 131
 132         public SdncArtifactType getType() {
 133             return type;
 134         }
 135
 136
 137
 138         public IArtifactInfo getArtifactInfo() {
 139             return artifactInfo;
 140         }
 141
 142
 143         public File getFile() {
 144             return file;
 145         }
 146
 147
 148
 149
 150         public DeployableArtifact(SdncArtifactType type, String svcName, String resourceName, IArtifactInfo artifactInfo, File file) {
 151             this.type = type;
 152             this.artifactInfo = artifactInfo;
 153                         this.svcName = svcName;
 154                         this.resourceName = resourceName;
 155             this.artifactName = artifactInfo.getArtifactName();
 156             this.artifactVersion = artifactInfo.getArtifactVersion();
 157             this.file = file;
 158         }
 159
 160
 161         public DeployableArtifact(SdncArtifactType type, String svcName, String resourceName, String artifactName, String artifactVersion, File file) {
 162             this.type = type;
 163             this.artifactInfo = null;
 164                         this.svcName = svcName;
 165                         this.resourceName = resourceName;
 166             this.artifactName = artifactName;
 167             this.artifactVersion = artifactVersion;
 168             this.file = file;
 169         }
 170
 171
 172
 173         public String getSvcName() {
 174             return svcName;
 175         }
 176
 177
 178
 179         public String getResourceName() {
 180             return resourceName;
 181         }
 182
 183     }
 184
 185     private final IDistributionClient client;
 186     private final SdncUebConfiguration config;
 187
 188     private LinkedList<DeployableArtifact> deployList[];
 189
 190         private static void setJdbcDataSource() throws IOException {
 191
 192                 String propPath;
 193                 String propDir = System.getenv(SDNC_CONFIG_DIR);
 194                 if (propDir == null) {
 195                         propDir = "/opt/sdnc/data/properties";
 196                 }
 197                 propPath = propDir + "/dblib.properties";
 198                 File propFile = new File(propPath);
 199
 200                 if (!propFile.exists()) {
 201
 202                         throw new FileNotFoundException(
 203                                         "Missing configuration properties file : "
 204                                                         + propFile);
 205                 }
 206
 207                 Properties props = new Properties();
 208                 props.load(new FileInputStream(propFile));
 209
 210                 jdbcDataSource = new DBResourceManager(props);
 211
 212                 if(jdbcDataSource.isActive()){
 213                         LOG.warn( "DBLIB: JDBC DataSource has been initialized.");
 214                 } else {
 215                         LOG.warn( "DBLIB: JDBC DataSource did not initialize successfully.");
 216                 }
 217         }
 218
 219         private static void loadArtifactMap() {
 220
 221         }
 222
 223     public SdncUebCallback(IDistributionClient client, SdncUebConfiguration config) {
 224         this.client = client;
 225         this.config = config;
 226
 227     }
 228
 229     @Override
 230         public void activateCallback(INotificationData data) {
 231
 232         LOG.info("Received notification : ("+data.getDistributionID()+","+data.getServiceName()+","+data.getServiceVersion()+
 233                                 ","+data.getServiceDescription() +  ")");
 234
 235         String incomingDirName = config.getIncomingDir();
 236         String archiveDirName = config.getArchiveDir();
 237
 238         File incomingDir = null;
 239         File archiveDir = null;
 240
 241         // Process service level artifacts
 242         List<IArtifactInfo> artifactList = data.getServiceArtifacts();
 243
 244         if (artifactList != null) {
 245
 246             incomingDir = new File(incomingDirName + "/" + escapeFilename(data.getServiceName()));
 247             if (!incomingDir.exists()) {
 248                 incomingDir.mkdirs();
 249             }
 250
 251             archiveDir = new File(archiveDirName + "/" + escapeFilename(data.getServiceName()));
 252             if (!archiveDir.exists()) {
 253                 archiveDir.mkdirs();
 254             }
 255             for (IArtifactInfo curArtifact : artifactList)
 256             {
 257
 258                 LOG.info("Received artifact " + curArtifact.getArtifactName());
 259
 260                                 handleArtifact(data, data.getServiceName(), null, null, curArtifact, incomingDir, archiveDir);
 261             }
 262         }
 263
 264
 265         // Process resource level artifacts
 266         for (IResourceInstance curResource : data.getResources()) {
 267
 268             LOG.info("Received resource : "+curResource.getResourceName());
 269             artifactList = curResource.getArtifacts();
 270
 271             if (artifactList != null) {
 272
 273                 incomingDir = new File(incomingDirName + "/" + escapeFilename(data.getServiceName()) + "/"
 274                     + escapeFilename(curResource.getResourceName()));
 275                 if (!incomingDir.exists()) {
 276                     incomingDir.mkdirs();
 277                 }
 278
 279                 archiveDir = new File(archiveDirName + "/" + escapeFilename(data.getServiceName()) + "/"
 280                     + escapeFilename(curResource.getResourceName()));
 281                 if (!archiveDir.exists()) {
 282                     archiveDir.mkdirs();
 283                 }
 284                 for (IArtifactInfo curArtifact : artifactList)
 285                 {
 286
 287                     LOG.info("Received artifact " + curArtifact.getArtifactName());
 288
 289                                         handleArtifact(data, data.getServiceName(), curResource.getResourceName(), curResource.getResourceType(), curArtifact, incomingDir, archiveDir);
 290                 }
 291             }
 292         }
 293
 294         deployDownloadedFiles(incomingDir, archiveDir, data);
 295     }
 296
 297
 298     public void deployDownloadedFiles(File incomingDir, File archiveDir, INotificationData data) {
 299
 300         if (incomingDir == null) {
 301             incomingDir = new File(config.getIncomingDir());
 302
 303             if (!incomingDir.exists()) {
 304                 incomingDir.mkdirs();
 305             }
 306
 307         }
 308
 309         if (archiveDir == null) {
 310             archiveDir = new File(config.getArchiveDir());
 311
 312             if (!archiveDir.exists()) {
 313                 archiveDir.mkdirs();
 314             }
 315         }
 316
 317         String curFileName = "";
 318         try (DirectoryStream<Path> stream = Files.newDirectoryStream(incomingDir.toPath())) {
 319             for (Path file: stream) {
 320                 curFileName = file.toString();
 321                 handleSuccessfulDownload(null,null, null, null, file.toFile(), archiveDir);
 322             }
 323         } catch (Exception x) {
 324             // IOException can never be thrown by the iteration.
 325             // In this snippet, it can only be thrown by newDirectoryStream.
 326             LOG.warn("Cannot process spool file {}", curFileName, x);
 327         }
 328
 329         // Deploy scheduled deployments
 330         int numPasses = config.getMaxPasses();
 331
 332         deployList = new LinkedList[numPasses];
 333
 334         for (int i = 0 ; i < numPasses ; i++) {
 335                         deployList[i] = new LinkedList<>();
 336         }
 337         for (int pass = 0 ; pass < config.getMaxPasses() ; pass++) {
 338
 339             if (deployList[pass] != null) {
 340                 while (! deployList[pass].isEmpty()) {
 341                     DeployableArtifact artifact = deployList[pass].pop();
 342
 343                     DistributionStatusEnum deployResult = DistributionStatusEnum.DEPLOY_ERROR;
 344
 345
 346                     try {
 347
 348                         deployResult = deploySpoolFile(artifact);
 349                     } catch (Exception e) {
 350                         LOG.error("Caught exception trying to deploy file", e);
 351                     }
 352
 353
 354                     IArtifactInfo artifactInfo = artifact.getArtifactInfo();
 355
 356                                         if ((artifactInfo != null) && (data != null)) {
 357                         IDistributionClientResult deploymentStatus;
 358                             deploymentStatus = client.sendDeploymentStatus(buildStatusMessage(
 359                                     client, data, artifactInfo,
 360                                     deployResult));
 361                     }
 362
 363                 }
 364             }
 365         }
 366     }
 367
 368         private void handleArtifact(INotificationData data, String svcName, String resourceName, String resourceType,
 369         IArtifactInfo artifact, File incomingDir, File archiveDir) {
 370
 371         // Download Artifact
 372         IDistributionClientDownloadResult downloadResult = client.download(artifact);
 373
 374                 if (downloadResult == null) {
 375
 376                         handleFailedDownload(data, artifact);
 377                         return;
 378                 }
 379
 380                 byte[] payloadBytes = downloadResult.getArtifactPayload();
 381
 382                 if (payloadBytes == null) {
 383                         handleFailedDownload(data, artifact);
 384                         return;
 385                 }
 386
 387                 String payload = new String(payloadBytes);
 388
 389
 390         File spoolFile = new File(incomingDir.getAbsolutePath() + "/" + artifact.getArtifactName());
 391
 392         boolean writeSucceeded = false;
 393
 394         try (FileWriter spoolFileWriter = new FileWriter(spoolFile)) {
 395             spoolFileWriter.write(payload);
 396             spoolFileWriter.close();
 397             writeSucceeded = true;
 398         } catch (Exception e) {
 399             LOG.error("Unable to save downloaded file to spool directory ("+ incomingDir.getAbsolutePath() +")", e);
 400         }
 401
 402
 403                 if (writeSucceeded && (downloadResult.getDistributionActionResult() == DistributionActionResultEnum.SUCCESS)) {
 404             handleSuccessfulDownload(data, svcName, resourceName, artifact, spoolFile, archiveDir);
 405         } else {
 406             handleFailedDownload(data, artifact);
 407         }
 408
 409     }
 410
 411     private void handleFailedDownload(INotificationData data,
 412             IArtifactInfo relevantArtifact) {
 413         // Send Download Status
 414         IDistributionClientResult sendDownloadStatus = client
 415                 .sendDownloadStatus(buildStatusMessage(client, data,
 416                         relevantArtifact, DistributionStatusEnum.DOWNLOAD_ERROR));
 417     }
 418
 419     private void handleSuccessfulDownload(INotificationData data, String svcName, String resourceName,
 420             IArtifactInfo artifact, File spoolFile, File archiveDir) {
 421
 422                 if ((data != null) && (artifact != null)) {
 423             // Send Download Status
 424             IDistributionClientResult sendDownloadStatus = client
 425                     .sendDownloadStatus(buildStatusMessage(client, data, artifact, DistributionStatusEnum.DOWNLOAD_OK));
 426         }
 427
 428         // If an override file exists, read that instead of the file we just downloaded
 429         ArtifactTypeEnum artifactEnum = ArtifactTypeEnum.YANG_XML;
 430
 431                 boolean toscaYamlType = false;
 432         if (artifact != null) {
 433                         String artifactTypeString = artifact.getArtifactType();
 434                         if (artifactTypeString.contains("TOSCA_TEMPLATE")) {
 435                                 toscaYamlType = true;
 436                         }
 437                 } else {
 438                         if (spoolFile.toString().contains(".yml") || spoolFile.toString().contains(".csar")) {
 439                                 toscaYamlType = true;
 440                         }
 441         }
 442         String overrideFileName = config.getOverrideFile();
 443                 if ((overrideFileName != null) && (overrideFileName.length() > 0)) {
 444             File overrideFile = new File(overrideFileName);
 445
 446             if (overrideFile.exists()) {
 447                 artifactEnum = ArtifactTypeEnum.YANG_XML;
 448                 spoolFile = overrideFile;
 449             }
 450
 451         }
 452
 453                 if (toscaYamlType) {
 454                         processToscaYaml (data, svcName, resourceName, artifact, spoolFile, archiveDir);
 455
 456                         try {
 457                                 Path source = spoolFile.toPath();
 458                                 Path targetDir = archiveDir.toPath();
 459
 460                                 Files.move(source, targetDir.resolve(source.getFileName()), StandardCopyOption.REPLACE_EXISTING);
 461                         } catch (IOException e) {
 462                                 LOG.warn("Could not move "+spoolFile.getAbsolutePath()+" to "+archiveDir.getAbsolutePath(), e);
 463                         }
 464
 465                         return;
 466                 }
 467
 468         // Process spool file
 469         Document spoolDoc = null;
 470         File transformedFile = null;
 471
 472         // Apply XSLTs and get Doc object
 473         try {
 474                         if (!spoolFile.isDirectory()) {
 475             transformedFile = applyXslts(spoolFile);
 476                         }
 477         } catch (Exception e) {
 478             LOG.error("Caught exception trying to parse XML file", e);
 479         }
 480
 481         if (transformedFile != null) {
 482             try {
 483                 DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
 484                 DocumentBuilder db = dbf.newDocumentBuilder();
 485
 486                 spoolDoc = db.parse(transformedFile);
 487             } catch (Exception e) {
 488                 LOG.error("Caught exception trying to parse transformed XML file {}",
 489                           transformedFile.getAbsolutePath(), e);
 490             }
 491         }
 492
 493
 494         if (spoolDoc != null) {
 495             // Analyze file type
 496             SdncArtifactType artifactType = analyzeFileType(artifactEnum, spoolFile, spoolDoc);
 497
 498             if (artifactType != null) {
 499
 500                 scheduleDeployment(artifactType, svcName, resourceName, artifact, spoolFile.getName(), transformedFile);
 501
 502             }
 503
 504             // SDNGC-2660 : Move file to archive directory even if it is an unrecognized type so that
 505             // we do not keep trying and failing to process it.
 506             try {
 507                 Path source = spoolFile.toPath();
 508                 Path targetDir = archiveDir.toPath();
 509
 510                 Files.move(source, targetDir.resolve(source.getFileName()), StandardCopyOption.REPLACE_EXISTING);
 511             } catch (IOException e) {
 512                 LOG.warn("Could not move "+spoolFile.getAbsolutePath()+" to "+archiveDir.getAbsolutePath(), e);
 513             }
 514         }
 515
 516
 517     }
 518
 519
 520         private void processToscaYaml(INotificationData data, String svcName, String resourceName,
 521                         IArtifactInfo artifact, File spoolFile, File archiveDir) {
 522
 523                 // Use ASDC Dist Client 1.1.5 with TOSCA parsing APIs to extract relevant TOSCA model data
 524
 525                 // TOSCA data extraction flow 1707:
 526                 // Use ASDC dist-client to get yaml string - not yet available
 527                 String model_yaml = null;
 528                 LOG.info("Process TOSCA YAML file: "+spoolFile.toString());
 529
 530                 SdcToscaParserFactory factory = SdcToscaParserFactory.getInstance();
 531                 ISdcCsarHelper sdcCsarHelper = null;
 532                 try {
 533                         sdcCsarHelper = factory.getSdcCsarHelper(spoolFile.getAbsolutePath());
 534                 } catch (SdcToscaParserException e) {
 535                         LOG.error("Could not create SDC TOSCA Parser ", e);
 536                         factory.close();
 537                         return;
 538                 }
 539
 540                 // Ingest Service Data - 1707
 541                 Metadata serviceMetadata = sdcCsarHelper.getServiceMetadata();
 542                 SdncServiceModel serviceModel = new SdncServiceModel(sdcCsarHelper, serviceMetadata);
 543                 serviceModel.setFilename(spoolFile.toString().substring(spoolFile
 544                                                                         .toString().lastIndexOf('/')+1));  // will be csar file name
 545                 serviceModel.setServiceInstanceNamePrefix(SdncBaseModel.extractSubstitutionMappingTypeName(sdcCsarHelper).substring(SdncBaseModel.extractSubstitutionMappingTypeName(sdcCsarHelper).lastIndexOf(".")+1));
 546
 547                 try {
 548                         cleanUpExistingToscaServiceData(serviceModel.getServiceUUID());
 549                         LOG.info("Call insertToscaData for SERVICE_MODEL serviceUUID = " + serviceModel.getServiceUUID());
 550                         insertToscaData(serviceModel.getSql(model_yaml));
 551                 } catch (IOException e) {
 552                         LOG.error("Could not insert Tosca YAML data into the SERVICE_MODEL table ", e);
 553                         factory.close();
 554                         return;
 555                 }
 556
 557                 // Ingest Network (VL) Data - 1707
 558                 //List<NodeTemplate> vlNodeTemplatesList = sdcCsarHelper.getServiceNodeTemplatesByType("VL");
 559                 List<NodeTemplate> vlNodeTemplatesList = sdcCsarHelper.getServiceVlList();
 560
 561                 for (NodeTemplate nodeTemplate :  vlNodeTemplatesList) {
 562                         SdncNodeModel nodeModel = new SdncNodeModel (sdcCsarHelper, nodeTemplate);
 563                         nodeModel.setServiceUUID(serviceModel.getServiceUUID());
 564                         nodeModel.setEcompGeneratedNaming(SdncBaseModel.extractBooleanInputDefaultValue(sdcCsarHelper, SdcPropertyNames.PROPERTY_NAME_SERVICENAMING_DEFAULT_ECOMPGENERATEDNAMING));//service_naming#default#ecomp_generated_naming
 565
 566                         try {
 567                                 cleanUpExistingToscaData("NETWORK_MODEL", "customization_uuid", nodeModel.getCustomizationUUID());
 568                                 cleanUpExistingToscaData("VPN_BINDINGS", "network_customization_uuid", nodeModel.getCustomizationUUID());
 569                                 LOG.info("Call insertToscaData for NETWORK_MODEL customizationUUID = " + nodeModel.getCustomizationUUID());
 570                                 // using ASDC dist-client use method for get yaml string
 571                                 insertToscaData(nodeModel.getSql(model_yaml));
 572                                 insertToscaData(nodeModel.getVpnBindingsSql());
 573                         } catch (IOException e) {
 574                                 LOG.error("Could not insert Tosca YAML data into the NETWORK_MODEL table ", e);
 575                         }
 576                 }
 577
 578                 // Ingest Allotted Resource Data - 1707
 579                 List<NodeTemplate> arNodeTemplatesList = sdcCsarHelper.getAllottedResources();
 580
 581                 for (NodeTemplate nodeTemplate :  arNodeTemplatesList) {
 582                         SdncARModel nodeModel = new SdncARModel (sdcCsarHelper, nodeTemplate);
 583
 584                         try {
 585                                 cleanUpExistingToscaData("ALLOTTED_RESOURCE_MODEL", "customization_uuid", nodeModel.getCustomizationUUID());
 586                                 LOG.info("Call insertToscaData for ALLOTTED_RESOURCE_MODEL customizationUUID = " + nodeModel.getCustomizationUUID());
 587                                 // using ASDC dist-client use method for get yaml string
 588                                 insertToscaData(nodeModel.getSql("ALLOTTED_RESOURCE_MODEL", model_yaml));
 589                         } catch (IOException e) {
 590                                 LOG.error("Could not insert Tosca YAML data into the NETWORK_MODEL table ", e);
 591                         }
 592                 }
 593
 594                 // Ingest Network (VF) Data - 1707
 595                 //List<NodeTemplate> nodeTemplatesList = sdcCsarHelper.getServiceNodeTemplatesByType("VF");
 596                 List<NodeTemplate> vfNodeTemplatesList = sdcCsarHelper.getServiceVfList();
 597
 598                 for (NodeTemplate nodeTemplate :  vfNodeTemplatesList) {
 599                         SdncVFModel vfNodeModel = new SdncVFModel (sdcCsarHelper, nodeTemplate);
 600
 601                         try {
 602                                 cleanUpExistingToscaData("VF_MODEL", "customization_uuid", vfNodeModel.getCustomizationUUID()) ;
 603                                 LOG.info("Call insertToscaData for VF_MODEL customizationUUID = " + vfNodeModel.getCustomizationUUID());
 604                                 insertToscaData(vfNodeModel.getSql("VF_MODEL", model_yaml));
 605                         } catch (IOException e) {
 606                                 LOG.error("Could not insert Tosca YAML data into the VF_MODEL table ", e);
 607                         }
 608
 609                         // For each VF, insert VF_MODULE_MODEL data
 610                         List<Group> vfModules = sdcCsarHelper.getVfModulesByVf(vfNodeModel.getCustomizationUUIDNoQuotes());
 611                         for (Group group : vfModules){
 612                                 SdncVFModuleModel vfModuleModel = new SdncVFModuleModel(sdcCsarHelper, group);
 613
 614                                 try {
 615                                         cleanUpExistingToscaData("VF_MODULE_MODEL", "customization_uuid", vfModuleModel.getCustomizationUUID());
 616                                         LOG.info("Call insertToscaData for VF_MODULE_MODEL customizationUUID = " + vfModuleModel.getCustomizationUUID());
 617                                         insertToscaData(vfModuleModel.getSql("VF_MODULE_MODEL", model_yaml));
 618                                 } catch (IOException e) {
 619                                         LOG.error("Could not insert Tosca YAML data into the VF_MODULE_MODEL table ", e);
 620                                 }
 621
 622                                 // For each VF Module, get the VFC list, insert VF_MODULE_TO_VFC_MAPPING data
 623                                 // List<NodeTemplate> groupMembers = sdcCsarHelper.getMembersOfGroup(group); - old version
 624                                 // For each vfcNode (group member) in the groupMembers list, extract vm_type and vm_count.
 625                                 // Insert vf_module.customizationUUID, vfcNode.customizationUUID and vm_type and vm_count into VF_MODULE_TO_VFC_MAPPING
 626                                 List<NodeTemplate> groupMembers = sdcCsarHelper.getMembersOfVfModule(nodeTemplate, group); // not yet available
 627                                 for (NodeTemplate vfcNode : groupMembers){
 628                                         SdncVFCModel vfcModel = new SdncVFCModel(sdcCsarHelper, vfcNode);
 629
 630                                         try {
 631                                                 cleanUpExistingToscaData("VF_MODULE_TO_VFC_MAPPING", "vf_module_customization_uuid", vfModuleModel.getCustomizationUUID());
 632                                                 LOG.info("Call insertToscaData for VF_MODULE_TO_VFC_MAPPING customizationUUID = " + vfModuleModel.getCustomizationUUID());
 633                                                 insertToscaData("insert into VF_MODULE_TO_VFC_MAPPING (vf_module_customization_uuid, vfc_customization_uuid, vm_type, vm_count) values (" +
 634                                                                 vfModuleModel.getCustomizationUUID() + ", " + vfcModel.getCustomizationUUID() + ", \"" + vfcModel.getVmType() + "\", \"" + vfcModel.getVmCount() + "\")");
 635                                         } catch (IOException e) {
 636                                                 LOG.error("Could not insert Tosca YAML data into the VF_MODULE_TO_VFC_MAPPING table ", e);
 637                                         }
 638
 639                                 }
 640
 641                         }
 642
 643                         // For each VF, insert VFC_MODEL data
 644                         List<NodeTemplate> vfcNodes = sdcCsarHelper.getVfcListByVf(vfNodeModel.getCustomizationUUIDNoQuotes());
 645                         for (NodeTemplate vfcNode : vfcNodes){
 646                                 SdncVFCModel vfcModel = new SdncVFCModel(sdcCsarHelper, vfcNode);
 647
 648                                 try {
 649                                         cleanUpExistingToscaData("VFC_MODEL", "customization_uuid", vfcModel.getCustomizationUUID());
 650                                         LOG.info("Call insertToscaData for VFC_MODEL customizationUUID = " + vfcModel.getCustomizationUUID());
 651                                         insertToscaData(vfcModel.getSql("VFC_MODEL", model_yaml));
 652                                 } catch (IOException e) {
 653                                         LOG.error("Could not insert Tosca YAML data into the VFC_MODEL table ", e);
 654                                 }
 655
 656                         }
 657
 658                         // For each VF, insert VF_TO_NETWORK_ROLE_MAPPING data
 659                         List<NodeTemplate> cpNodes = sdcCsarHelper.getCpListByVf(vfNodeModel.getCustomizationUUIDNoQuotes());
 660                         for (NodeTemplate cpNode : cpNodes){
 661
 662                                 // Insert into VF_TO_NETWORK_ROLE_MAPPING vf_customization_uuid and network_role
 663                                 String cpNetworkRole = sdcCsarHelper.getNodeTemplatePropertyLeafValue(cpNode, "network_role_tag");
 664
 665                                 try {
 666                                         cleanUpExistingToscaData("VF_TO_NETWORK_ROLE_MAPPING", "vf_customization_uuid", vfNodeModel.getCustomizationUUID());
 667                                         LOG.info("Call insertToscaData for VF_TO_NETWORK_ROLE_MAPPING vfCustomizationUUID = " + vfNodeModel.getCustomizationUUID());
 668                                         insertToscaData("insert into VF_TO_NETWORK_ROLE_MAPPING (vf_customization_uuid, network_role) values (" +
 669                                         vfNodeModel.getCustomizationUUID() + ", \"" + cpNetworkRole + "\")");
 670                                 } catch (IOException e) {
 671                                         LOG.error("Could not insert Tosca YAML data into the VF_TO_NETWORK_ROLE_MAPPING table ", e);
 672                                 }
 673
 674                                 // Insert VFC_TO_NETWORK_ROLE_MAPPING data
 675                                 Map<String, String> mappingParams = new HashMap<>();
 676                                 //String cpNetworkRoleTag = "\"" + sdcCsarHelper.getNodeTemplatePropertyLeafValue(cpNode, SdcPropertyNames.PROPERTY_NAME_NETWORKROLETAG) + "\"";
 677                                 // extract network_role, network_role_tag and virtual_binding from this cpNode
 678                                 SdncBaseModel.addParameter("network_role", SdncBaseModel.extractValue(sdcCsarHelper, cpNode, "network_role"), mappingParams);
 679                                 SdncBaseModel.addParameter("network_role_tag", SdncBaseModel.extractValue(sdcCsarHelper, cpNode, "network_role_tag"), mappingParams);
 680                                 String virtualBinding = "\"" + SdncBaseModel.extractValue(sdcCsarHelper, cpNode, "requirements#virtualBinding") + "\"";
 681
 682                                 // get list of cpNodes and vfcNodes with matching virtualBinding
 683                                 List<Pair<NodeTemplate, NodeTemplate>> matchList = sdcCsarHelper.getNodeTemplatePairsByReqName(sdcCsarHelper.getCpListByVf(vfNodeModel.getCustomizationUUIDNoQuotes()), sdcCsarHelper.getVfcListByVf(vfNodeModel.getCustomizationUUIDNoQuotes()), virtualBinding);
 684                                 for (Pair<NodeTemplate, NodeTemplate> match : matchList) {  // should be 1 match?
 685
 686                                         // extract values from the left "CP" Node
 687                                         SdncBaseModel.addParameter("ipv4_use_dhcp", SdncBaseModel.extractBooleanValue(sdcCsarHelper, match.getLeft(), SdcPropertyNames.PROPERTY_NAME_NETWORKASSIGNMENTS_IPV4SUBNETDEFAULTASSIGNMENTS_DHCPENABLED), mappingParams);
 688                                         //SdncBaseModel.addParameter("ipv4_ip_version", SdncBaseModel.extractValue(sdcCsarHelper, match.getLeft(), SdcPropertyNames.PROPERTY_NAME_NETWORKASSIGNMENTS_IPV4SUBNETDEFAULTASSIGNMENTS_IPVERSION), mappingParams);
 689                                         SdncBaseModel.addParameter("ipv4_ip_version", "dummy_ipv4_vers", mappingParams);
 690                                         SdncBaseModel.addParameter("ipv6_use_dhcp", SdncBaseModel.extractBooleanValue(sdcCsarHelper, match.getLeft(), SdcPropertyNames.PROPERTY_NAME_NETWORKASSIGNMENTS_IPV6SUBNETDEFAULTASSIGNMENTS_DHCPENABLED), mappingParams);
 691                                         //SdncBaseModel.addParameter("ipv6_ip_version", SdncBaseModel.extractValue(sdcCsarHelper, match.getLeft(), SdcPropertyNames.PROPERTY_NAME_NETWORKASSIGNMENTS_IPV6SUBNETDEFAULTASSIGNMENTS_IPVERSION), mappingParams);
 692                                         SdncBaseModel.addParameter("ipv6_ip_version", "dummy_ipv6_vers", mappingParams);
 693                                         //String extcp_subnetpool_id = "\"" + SdncBaseModel.extractValue(sdcCsarHelper, match.getLeft(), SdcPropertyNames.PROPERTY_NAME_SUBNETPOOLID) + "\""; // need path to subnetpoolid
 694
 695                                         // extract values from the right "VFC" Node
 696                                         String vfcCustomizationUuid = "\"" + SdncBaseModel.extractValue(sdcCsarHelper, match.getRight().getMetadata(), "customization_uuid") + "\"";
 697                                         SdncBaseModel.addParameter("vm_type", SdncBaseModel.extractValue(sdcCsarHelper, match.getRight(), SdcPropertyNames.PROPERTY_NAME_VMTYPE), mappingParams);
 698                                         SdncBaseModel.addIntParameter("ipv4_count", SdncBaseModel.extractValue(sdcCsarHelper, match.getRight(), SdcPropertyNames.PROPERTY_NAME_NETWORKASSIGNMENTS_IPV4SUBNETDEFAULTASSIGNMENTS_MINSUBNETSCOUNT), mappingParams);
 699                                         SdncBaseModel.addIntParameter("ipv6_count", SdncBaseModel.extractValue(sdcCsarHelper, match.getRight(), SdcPropertyNames.PROPERTY_NAME_NETWORKASSIGNMENTS_IPV6SUBNETDEFAULTASSIGNMENTS_MINSUBNETSCOUNT), mappingParams);
 700
 701                                         try {
 702                                                 cleanUpExistingToscaData("VFC_TO_NETWORK_ROLE_MAPPING", "vfc_customization_uuid", vfcCustomizationUuid);
 703                                                 LOG.info("Call insertToscaData for VFC_TO_NETWORK_ROLE_MAPPING vfcCustomizationUUID = " + vfcCustomizationUuid);
 704                                                 insertToscaData(SdncBaseModel.getSql("VFC_TO_NETWORK_ROLE_MAPPING", "vfc_customization_uuid", vfcCustomizationUuid, "", mappingParams));
 705                                         } catch (IOException e) {
 706                                                 LOG.error("Could not insert Tosca YAML data into the VFC_TO_NETWORK_ROLE_MAPPING table ", e);
 707                                         }
 708
 709                                 }
 710
 711                         } // CP loop
 712
 713                 } // VF loop
 714
 715                 // Close ASDC TOSCA Parser factory - we are done processing this distribution
 716                 factory.close();
 717
 718                 if ((artifact != null) && (data != null)) {
 719                         LOG.info("Update to SDN-C succeeded");
 720                         IDistributionClientResult deploymentStatus;
 721                                 deploymentStatus = client.sendDeploymentStatus(buildStatusMessage(
 722                                                 client, data, artifact,
 723                                                 DistributionStatusEnum.DEPLOY_OK));
 724                 }
 725
 726         }
 727
 728          private void cleanUpExistingToscaData(String tableName, String keyName, String keyValue) throws IOException
 729      {
 730
 731             if (jdbcDataSource == null) {
 732                  setJdbcDataSource();
 733             }
 734              try {
 735                 int rowCount = 0;
 736                 CachedRowSet data = jdbcDataSource.getData("SELECT * from " + tableName + " where " + keyName + " = " + keyValue + ";", null, "");
 737                 while(data.next()) {
 738                                 rowCount ++;
 739                 }
 740                 if (rowCount != 0) {
 741                     LOG.info("cleanUpExistingToscaData: " + keyValue);
 742                         jdbcDataSource.writeData("DELETE from " + tableName + " where " + keyName + " = " + keyValue + ";", null, null);
 743                 }
 744
 745                         } catch (SQLException e) {
 746                                 LOG.error("Could not clean up existing " + tableName  + " for " + keyValue, e);
 747                         }
 748
 749      }
 750
 751
 752          private void cleanUpExistingToscaServiceData(String serviceUUID) throws IOException
 753      {
 754
 755             if (jdbcDataSource == null) {
 756                  setJdbcDataSource();
 757             }
 758              try {
 759                 int rowCount = 0;
 760                 CachedRowSet data = jdbcDataSource.getData("SELECT * from SERVICE_MODEL where service_uuid = " + serviceUUID + ";", null, "");
 761                 while(data.next()) {
 762                                 rowCount ++;
 763                 }
 764                 if (rowCount != 0) {
 765                     LOG.info("cleanUpExistingToscaData: " + serviceUUID);
 766                         jdbcDataSource.writeData("DELETE from NETWORK_MODEL where service_uuid = " + serviceUUID + ";", null, null);
 767                         jdbcDataSource.writeData("DELETE from SERVICE_MODEL where service_uuid = " + serviceUUID + ";", null, null);
 768                 }
 769
 770                         } catch (SQLException e) {
 771                                 LOG.error("Could not clean up existing NETWORK_MODEL and SERVICE_MODEL for service_UUID " + serviceUUID, e);
 772                         }
 773
 774      }
 775
 776
 777          private void insertToscaData(String toscaDataString) throws IOException
 778      {
 779             LOG.debug("insertToscaData: " + toscaDataString);
 780
 781             if (jdbcDataSource == null) {
 782                  setJdbcDataSource();
 783             }
 784              try {
 785
 786                                 jdbcDataSource.writeData(toscaDataString, null, null);
 787
 788                         } catch (SQLException e) {
 789                                 LOG.error("Could not insert Tosca YAML data into the database ", e);
 790                         }
 791
 792      }
 793
 794
 795     private SdncArtifactType analyzeFileType(ArtifactTypeEnum artifactType, File spoolFile, Document spoolDoc) {
 796
 797         if (artifactType != ArtifactTypeEnum.YANG_XML) {
 798             LOG.error("Unexpected artifact type - expecting YANG_XML, got "+artifactType);
 799                         return (null);
 800         }
 801
 802         // Examine outer tag
 803
 804         try {
 805
 806
 807             Element root = spoolDoc.getDocumentElement();
 808
 809             String rootName = root.getTagName();
 810
 811             if (rootName.contains(":")) {
 812                 String[] rootNameElems = rootName.split(":");
 813                 rootName = rootNameElems[rootNameElems.length - 1];
 814             }
 815
 816             if (rootName != null) {
 817                 SdncArtifactType mapEntry = config.getMapping(rootName);
 818
 819
 820                 if (mapEntry == null) {
 821
 822                     LOG.error("Unexpected file contents - root tag is "+rootName);
 823                 }
 824                                 return(mapEntry);
 825             } else {
 826                 LOG.error("Cannot get root tag from file");
 827                                 return(null);
 828             }
 829
 830         } catch (Exception e) {
 831             LOG.error("Could not parse YANG_XML file "+spoolFile.getName(), e);
 832                         return(null);
 833         }
 834     }
 835
 836     private void scheduleDeployment(SdncArtifactType type, String svcName, String resourceName, IArtifactInfo artifactInfo, String spoolFileName, File spoolFile) {
 837
 838         if (type.getPass() < deployList.length) {
 839
 840             if (artifactInfo != null) {
 841                 LOG.debug("Scheduling "+artifactInfo.getArtifactName()+" version "+artifactInfo.getArtifactVersion()+" for deployment");
 842
 843                 deployList[type.getPass()].add(new DeployableArtifact(type, svcName, resourceName, artifactInfo, spoolFile));
 844             } else {
 845                 SimpleDateFormat sdfDate = new SimpleDateFormat("yyyy-MM-dd_HH:mm:ss.SSS");//dd/MM/yyyy
 846                 Date now = new Date();
 847                 String artifactVersion = sdfDate.format(now);
 848                 LOG.debug("Scheduling "+spoolFileName+" version "+artifactVersion+" for deployment");
 849                 deployList[type.getPass()].add(new DeployableArtifact(type, svcName, resourceName, spoolFileName,
 850                     artifactVersion, spoolFile));
 851             }
 852         } else {
 853             LOG.info("Pass for type "+type.getTag()+" is "+type.getPass()+" which is not <= "+deployList.length);
 854         }
 855     }
 856
 857
 858     private DistributionStatusEnum deploySpoolFile(DeployableArtifact artifact) {
 859
 860         DistributionStatusEnum deployResult = DistributionStatusEnum.DEPLOY_OK;
 861
 862         StringBuffer msgBuffer = new StringBuffer();
 863
 864
 865         String namespace = config.getAsdcApiNamespace();
 866                 if ((namespace == null) || (namespace.length() == 0)) {
 867             namespace="com:att:sdnctl:asdcapi";
 868         }
 869
 870         msgBuffer.append("<input xmlns='");
 871         msgBuffer.append(namespace);
 872         msgBuffer.append("'>\n");
 873
 874         String svcName = artifact.getSvcName();
 875         String resourceName = artifact.getResourceName();
 876         String artifactName = artifact.getArtifactName();
 877
 878         if (svcName != null) {
 879             if (resourceName != null) {
 880                 artifactName = svcName + "/" + resourceName + "/" + artifactName;
 881             } else {
 882                 artifactName = svcName + "/" + artifactName;
 883             }
 884         }
 885
 886         msgBuffer.append("<artifact-name>"+artifactName+"</artifact-name>\n");
 887         msgBuffer.append("<artifact-version>"+artifact.getArtifactVersion()+"</artifact-version>\n");
 888
 889
 890         try (BufferedReader rdr = new BufferedReader(new FileReader(artifact.getFile()))){
 891             String curLine = rdr.readLine();
 892             while (curLine != null) {
 893
 894                 if (!curLine.startsWith("<?")) {
 895                     msgBuffer.append(curLine+"\n");
 896                 }
 897                 curLine = rdr.readLine();
 898             }
 899         } catch (Exception e) {
 900             LOG.error("Could not process spool file "+artifact.getFile().getName(), e);
 901                         return(DistributionStatusEnum.DEPLOY_ERROR);
 902         }
 903
 904         msgBuffer.append("</input>\n");
 905
 906
 907         byte[] msgBytes = msgBuffer.toString().getBytes();
 908
 909         Document results = postRestXml(artifact.getType().getRpcUrl(config.getAsdcApiBaseUrl()), msgBytes);
 910
 911         if (results == null) {
 912
 913             deployResult = DistributionStatusEnum.DEPLOY_ERROR;
 914         } else {
 915
 916             XPathFactory xpf = XPathFactory.newInstance();
 917             XPath xp = xpf.newXPath();
 918
 919             String asdcApiResponseCode = "500";
 920
 921             try {
 922
 923                 asdcApiResponseCode = xp.evaluate("//asdc-api-response-code[position()=1]/text()", results.getDocumentElement());
 924             } catch (Exception e) {
 925                 LOG.error("Caught exception retrying to evaluate xpath", e);
 926             }
 927
 928             if (asdcApiResponseCode.contains("200")) {
 929                 LOG.info("Update to SDN-C succeeded");
 930                 deployResult = DistributionStatusEnum.DEPLOY_OK;
 931             } else {
 932                 LOG.info("Update to SDN-C failed (response code "+asdcApiResponseCode+")");
 933
 934                 if (asdcApiResponseCode.contains("409")) {
 935                     deployResult = DistributionStatusEnum.ALREADY_DEPLOYED;
 936                 } else {
 937
 938                     deployResult = DistributionStatusEnum.DEPLOY_ERROR;
 939                 }
 940             }
 941         }
 942
 943
 944
 945                 return(deployResult);
 946     }
 947
 948
 949
 950
 951
 952     public static IDistributionStatusMessage buildStatusMessage(
 953             final IDistributionClient client, final INotificationData data,
 954             final IArtifactInfo relevantArtifact,
 955             final DistributionStatusEnum status) {
 956         IDistributionStatusMessage statusMessage = new IDistributionStatusMessage() {
 957
 958             @Override
 959                         public long getTimestamp() {
 960                 return System.currentTimeMillis();
 961             }
 962
 963             @Override
 964                         public DistributionStatusEnum getStatus() {
 965                 return status;
 966             }
 967
 968             @Override
 969                         public String getDistributionID() {
 970                 return data.getDistributionID();
 971             }
 972
 973             @Override
 974                         public String getConsumerID() {
 975                 return client.getConfiguration().getConsumerID();
 976             }
 977
 978             @Override
 979                         public String getArtifactURL() {
 980                 return relevantArtifact.getArtifactURL();
 981             }
 982         };
 983         return statusMessage;
 984
 985     }
 986
 987     private HttpURLConnection getRestXmlConnection(String urlString, String method) throws IOException
 988     {
 989         URL sdncUrl = new URL(urlString);
 990         Authenticator.setDefault(new SdncAuthenticator(config.getSdncUser(), config.getSdncPasswd()));
 991
 992         HttpURLConnection conn = (HttpURLConnection) sdncUrl.openConnection();
 993
 994         String authStr = config.getSdncUser()+":"+config.getSdncPasswd();
 995         String encodedAuthStr = new String(Base64.encodeBase64(authStr.getBytes()));
 996
 997         conn.addRequestProperty("Authentication", "Basic "+encodedAuthStr);
 998
 999         conn.setRequestMethod(method);
1000         conn.setRequestProperty("Content-Type", "application/xml");
1001         conn.setRequestProperty("Accept", "application/xml");
1002
1003         conn.setDoInput(true);
1004         conn.setDoOutput(true);
1005         conn.setUseCaches(false);
1006
1007                 return(conn);
1008
1009     }
1010
1011     private Document postRestXml(String urlString, byte[] msgBytes) {
1012         Document response = null;
1013
1014         try {
1015                         SdncOdlConnection odlConn = SdncOdlConnection.newInstance(urlString, config.getSdncUser(), config.getSdncPasswd());
1016
1017                         String sdncResp = odlConn.send("POST", "application/xml", new String(msgBytes));
1018
1019             DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
1020             DocumentBuilder db = dbf.newDocumentBuilder();
1021
1022
1023                         response = db.parse(new ByteArrayInputStream(sdncResp.getBytes()));
1024         } catch (Exception e) {
1025                         LOG.error("Caught exception posting to ODL tier", e);
1026         }
1027
1028                 return(response);
1029
1030     }
1031
1032     private File applyXslts(File srcFile) {
1033
1034         Document doc;
1035
1036         File inFile = srcFile;
1037         File outFile = null;
1038
1039         String xsltPathList = config.getXsltPathList();
1040
1041                 if ((xsltPathList == null) || (xsltPathList.length() == 0)) {
1042             outFile = inFile;
1043         } else {
1044
1045             String[] xsltPaths = xsltPathList.split(",");
1046
1047             for (String xsltPath : xsltPaths) {
1048                 try{
1049
1050                     outFile = File.createTempFile("tmp", "xml");
1051                     TransformerFactory factory = TransformerFactory.newInstance();
1052                     Source xslt = new StreamSource(new File(xsltPath));
1053                     Transformer transformer = factory.newTransformer(xslt);
1054                     Source text = new StreamSource(inFile);
1055
1056
1057                     transformer.transform(text, new StreamResult(outFile));
1058
1059                     inFile = outFile;
1060
1061                 } catch (Exception e) {
1062                     LOG.error("Caught exception trying to apply XSLT template "+xsltPath, e);
1063
1064                 }
1065
1066             }
1067         }
1068
1069         // After transformations, parse transformed XML
1070
1071
1072                 return(outFile);
1073     }
1074
1075     private String escapeFilename(String str) {
1076         StringBuffer retval = new StringBuffer();
1077
1078         for (int i = 0 ; i < str.length() ; i++) {
1079             char curchar = str.charAt(i);
1080             if (Character.isJavaIdentifierPart(curchar)) {
1081                 retval.append(curchar);
1082             }
1083         }
1084
1085                 return(retval.toString());
1086
1087     }
1088
1089 }