d04704aaba6216760fb5580f5f9f6760680b072e
[sdnc/northbound.git] / ueb-listener / src / main / java / org / openecomp / sdnc / uebclient / SdncUebCallback.java
1 /*-
2  * ============LICENSE_START=======================================================
3  * openECOMP : SDN-C
4  * ================================================================================
5  * Copyright (C) 2017 AT&T Intellectual Property. All rights
6  *                                                      reserved.
7  * ================================================================================
8  * Licensed under the Apache License, Version 2.0 (the "License");
9  * you may not use this file except in compliance with the License.
10  * You may obtain a copy of the License at
11  * 
12  *      http://www.apache.org/licenses/LICENSE-2.0
13  * 
14  * Unless required by applicable law or agreed to in writing, software
15  * distributed under the License is distributed on an "AS IS" BASIS,
16  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17  * See the License for the specific language governing permissions and
18  * limitations under the License.
19  * ============LICENSE_END=========================================================
20  */
21
22 package org.openecomp.sdnc.uebclient;
23
24 import java.io.BufferedReader;
25 import java.io.ByteArrayInputStream;
26 import java.io.DataOutputStream;
27 import java.io.File;
28 import java.io.FileInputStream;
29 import java.io.FileNotFoundException;
30 import java.io.FileReader;
31 import java.io.FileWriter;
32 import java.io.IOException;
33 import java.io.InputStream;
34 import java.io.InputStreamReader;
35 import java.net.Authenticator;
36 import java.net.HttpURLConnection;
37 import java.net.PasswordAuthentication;
38 import java.net.URL;
39 import java.nio.file.DirectoryStream;
40 import java.nio.file.Files;
41 import java.nio.file.Path;
42 import java.nio.file.StandardCopyOption;
43 import java.sql.ResultSet;
44 import java.sql.SQLException;
45 import java.text.SimpleDateFormat;
46 import java.util.Date;
47 import java.util.HashMap;
48 import java.util.LinkedList;
49 import java.util.List;
50 import java.util.Map;
51 import java.util.Properties;
52
53 import javax.net.ssl.HostnameVerifier;
54 import javax.net.ssl.HttpsURLConnection;
55 import javax.net.ssl.SSLSession;
56 import javax.sql.rowset.CachedRowSet;
57 import javax.xml.parsers.DocumentBuilder;
58 import javax.xml.parsers.DocumentBuilderFactory;
59 import javax.xml.transform.Source;
60 import javax.xml.transform.Transformer;
61 import javax.xml.transform.TransformerFactory;
62 import javax.xml.transform.stream.StreamResult;
63 import javax.xml.transform.stream.StreamSource;
64 import javax.xml.xpath.XPath;
65 import javax.xml.xpath.XPathFactory;
66
67 import org.apache.commons.codec.binary.Base64;
68 import org.apache.commons.lang3.tuple.Pair;
69 import org.openecomp.sdc.api.IDistributionClient;
70 import org.openecomp.sdc.api.consumer.IDistributionStatusMessage;
71 import org.openecomp.sdc.api.consumer.INotificationCallback;
72 import org.openecomp.sdc.api.notification.IArtifactInfo;
73 import org.openecomp.sdc.api.notification.INotificationData;
74 import org.openecomp.sdc.api.notification.IResourceInstance;
75 import org.openecomp.sdc.api.results.IDistributionClientDownloadResult;
76 import org.openecomp.sdc.api.results.IDistributionClientResult;
77 import org.openecomp.sdc.tosca.parser.api.ISdcCsarHelper;
78 import org.openecomp.sdc.tosca.parser.exceptions.SdcToscaParserException;
79 import org.openecomp.sdc.tosca.parser.impl.SdcPropertyNames;
80 import org.openecomp.sdc.tosca.parser.impl.SdcToscaParserFactory;
81 import org.openecomp.sdc.toscaparser.api.Group;
82 import org.openecomp.sdc.toscaparser.api.Metadata;
83 import org.openecomp.sdc.toscaparser.api.NodeTemplate;
84 import org.openecomp.sdc.utils.ArtifactTypeEnum;
85 import org.openecomp.sdc.utils.DistributionActionResultEnum;
86 import org.openecomp.sdc.utils.DistributionStatusEnum;
87 import org.openecomp.sdnc.sli.resource.dblib.DBResourceManager;
88 import org.openecomp.sdnc.uebclient.SdncArtifactMap.SdncArtifactType;
89 import org.slf4j.Logger;
90 import org.slf4j.LoggerFactory;
91 import org.w3c.dom.Document;
92 import org.w3c.dom.Element;
93 import org.yaml.snakeyaml.Yaml;
94 import org.yaml.snakeyaml.constructor.Constructor;
95
96 public class SdncUebCallback implements INotificationCallback {
97
98     private static final Logger LOG = LoggerFactory
99             .getLogger(SdncUebCallback.class);
100
101         private static DBResourceManager jdbcDataSource = null;
102         private static final String SDNC_CONFIG_DIR = "SDNC_CONFIG_DIR";
103
104
105     private class SdncAuthenticator extends Authenticator {
106
107         private final String user;
108         private final String passwd;
109
110         SdncAuthenticator(String user, String passwd) {
111             this.user = user;
112             this.passwd = passwd;
113         }
114         @Override
115         protected PasswordAuthentication getPasswordAuthentication() {
116             return new PasswordAuthentication(user, passwd.toCharArray());
117         }
118
119     }
120
121     private class DeployableArtifact {
122         SdncArtifactType type;
123         IArtifactInfo artifactInfo;
124         String svcName;
125         String resourceName;
126         String artifactName;
127         String artifactVersion;
128         File file;
129
130         public String getArtifactName() {
131             return artifactName;
132         }
133
134
135
136         public String getArtifactVersion() {
137             return artifactVersion;
138         }
139
140
141         public SdncArtifactType getType() {
142             return type;
143         }
144
145
146
147         public IArtifactInfo getArtifactInfo() {
148             return artifactInfo;
149         }
150
151
152         public File getFile() {
153             return file;
154         }
155
156
157
158
159         public DeployableArtifact(SdncArtifactType type, String svcName, String resourceName, IArtifactInfo artifactInfo, File file) {
160             this.type = type;
161             this.artifactInfo = artifactInfo;
162                         this.svcName = svcName;
163                         this.resourceName = resourceName;
164             this.artifactName = artifactInfo.getArtifactName();
165             this.artifactVersion = artifactInfo.getArtifactVersion();
166             this.file = file;
167         }
168
169
170         public DeployableArtifact(SdncArtifactType type, String svcName, String resourceName, String artifactName, String artifactVersion, File file) {
171             this.type = type;
172             this.artifactInfo = null;
173                         this.svcName = svcName;
174                         this.resourceName = resourceName;
175             this.artifactName = artifactName;
176             this.artifactVersion = artifactVersion;
177             this.file = file;
178         }
179
180
181
182         public String getSvcName() {
183             return svcName;
184         }
185
186
187
188         public String getResourceName() {
189             return resourceName;
190         }
191
192     }
193
194     private final IDistributionClient client;
195     private final SdncUebConfiguration config;
196
197     private LinkedList<DeployableArtifact> deployList[];
198
199         private static void setJdbcDataSource() throws IOException {
200
201                 String propPath = null;
202                 String propDir = System.getenv(SDNC_CONFIG_DIR);
203                 if (propDir == null) {
204
205                         propDir = "/opt/sdnc/data/properties";
206                 }
207                 propPath = propDir + "/dblib.properties";
208                 File propFile = new File(propPath);
209
210                 if (!propFile.exists()) {
211
212                         throw new FileNotFoundException(
213                                         "Missing configuration properties file : "
214                                                         + propFile);
215                 }
216
217                 Properties props = new Properties();
218                 props.load(new FileInputStream(propFile));
219
220                 try {
221                         jdbcDataSource = DBResourceManager.create(props);
222                 } catch(Throwable exc) {
223                         LOG.error("", exc);
224                 }
225
226                 if(((DBResourceManager)jdbcDataSource).isActive()){
227                         LOG.warn( "DBLIB: JDBC DataSource has been initialized.");
228                 } else {
229                         LOG.warn( "DBLIB: JDBC DataSource did not initialize successfully.");
230                 }
231         }
232
233         private static void loadArtifactMap() {
234
235         }
236
237     public SdncUebCallback(IDistributionClient client, SdncUebConfiguration config) {
238         this.client = client;
239         this.config = config;
240
241     }
242
243     @Override
244         public void activateCallback(INotificationData data) {
245
246         LOG.info("Received notification : ("+data.getDistributionID()+","+data.getServiceName()+","+data.getServiceVersion()+
247                                 ","+data.getServiceDescription() +  ")");
248
249         String incomingDirName = config.getIncomingDir();
250         String archiveDirName = config.getArchiveDir();
251
252         File incomingDir = null;
253         File archiveDir = null;
254
255         if (!incomingDir.exists()) {
256             incomingDir.mkdirs();
257         }
258
259
260         if (!archiveDir.exists()) {
261             archiveDir.mkdirs();
262         }
263
264         // Process service level artifacts
265         List<IArtifactInfo> artifactList = data.getServiceArtifacts();
266
267         if (artifactList != null) {
268
269             incomingDir = new File(incomingDirName + "/" + escapeFilename(data.getServiceName()));
270             if (!incomingDir.exists()) {
271                 incomingDir.mkdirs();
272             }
273
274             archiveDir = new File(archiveDirName + "/" + escapeFilename(data.getServiceName()));
275             if (!archiveDir.exists()) {
276                 archiveDir.mkdirs();
277             }
278             for (IArtifactInfo curArtifact : artifactList)
279             {
280
281                 LOG.info("Received artifact " + curArtifact.getArtifactName());
282
283                                 handleArtifact(data, data.getServiceName(), null, null, curArtifact, incomingDir, archiveDir);
284             }
285         }
286
287
288         // Process resource level artifacts
289         for (IResourceInstance curResource : data.getResources()) {
290
291             LOG.info("Received resource : "+curResource.getResourceName());
292             artifactList = curResource.getArtifacts();
293
294             if (artifactList != null) {
295
296                 incomingDir = new File(incomingDirName + "/" + escapeFilename(data.getServiceName()) + "/" + escapeFilename(curResource.getResourceName()));
297                 if (!incomingDir.exists()) {
298                     incomingDir.mkdirs();
299                 }
300
301                 archiveDir = new File(archiveDirName + "/" + escapeFilename(data.getServiceName()) + "/" + escapeFilename(curResource.getResourceName()));
302                 if (!archiveDir.exists()) {
303                     archiveDir.mkdirs();
304                 }
305                 for (IArtifactInfo curArtifact : artifactList)
306                 {
307
308                     LOG.info("Received artifact " + curArtifact.getArtifactName());
309
310                                         handleArtifact(data, data.getServiceName(), curResource.getResourceName(), curResource.getResourceType(), curArtifact, incomingDir, archiveDir);
311                 }
312             }
313         }
314
315         deployDownloadedFiles(incomingDir, archiveDir, data);
316
317
318     }
319
320
321     public void deployDownloadedFiles(File incomingDir, File archiveDir, INotificationData data) {
322
323         if (incomingDir == null) {
324             incomingDir = new File(config.getIncomingDir());
325
326             if (!incomingDir.exists()) {
327                 incomingDir.mkdirs();
328             }
329
330         }
331
332         if (archiveDir == null) {
333             archiveDir = new File(config.getArchiveDir());
334
335             if (!archiveDir.exists()) {
336                 archiveDir.mkdirs();
337             }
338         }
339
340         String curFileName = "";
341         try (DirectoryStream<Path> stream = Files.newDirectoryStream(incomingDir.toPath())) {
342             for (Path file: stream) {
343                 curFileName = file.toString();
344                 handleSuccessfulDownload(null,null, null, null, file.toFile(), archiveDir);
345             }
346         } catch (Exception x) {
347             // IOException can never be thrown by the iteration.
348             // In this snippet, it can only be thrown by newDirectoryStream.
349             LOG.warn("Cannot process spool file "+ curFileName, x);
350         }
351
352         // Deploy scheduled deployments
353         int numPasses = config.getMaxPasses();
354
355         deployList = new LinkedList[numPasses];
356
357         for (int i = 0 ; i < numPasses ; i++) {
358                         deployList[i] = new LinkedList<DeployableArtifact>();
359         }
360         for (int pass = 0 ; pass < config.getMaxPasses() ; pass++) {
361
362             if (deployList[pass] != null) {
363                 while (! deployList[pass].isEmpty()) {
364                     DeployableArtifact artifact = deployList[pass].pop();
365
366                     DistributionStatusEnum deployResult = DistributionStatusEnum.DEPLOY_ERROR;
367
368
369                     try {
370
371                         deployResult = deploySpoolFile(artifact);
372                     } catch (Exception e) {
373                         LOG.error("Caught exception trying to deploy file", e);
374                     }
375
376
377                     IArtifactInfo artifactInfo = artifact.getArtifactInfo();
378
379                                         if ((artifactInfo != null) && (data != null)) {
380                         IDistributionClientResult deploymentStatus;
381                             deploymentStatus = client.sendDeploymentStatus(buildStatusMessage(
382                                     client, data, artifactInfo,
383                                     deployResult));
384                     }
385
386                 }
387             }
388         }
389     }
390
391         private void handleArtifact(INotificationData data, String svcName, String resourceName, String resourceType, IArtifactInfo artifact, File incomingDir, File archiveDir) {
392
393         // Download Artifact
394         IDistributionClientDownloadResult downloadResult = client
395                 .download(artifact);
396
397                 if (downloadResult == null) {
398
399                         handleFailedDownload(data, artifact);
400                         return;
401                 }
402
403                 byte[] payloadBytes = downloadResult.getArtifactPayload();
404
405                 if (payloadBytes == null) {
406                         handleFailedDownload(data, artifact);
407                         return;
408                 }
409
410                 String payload = new String(payloadBytes);
411
412
413         File spoolFile = new File(incomingDir.getAbsolutePath() + "/" + artifact.getArtifactName());
414
415         boolean writeSucceeded = false;
416
417         try {
418             FileWriter spoolFileWriter = new FileWriter(spoolFile);
419             spoolFileWriter.write(payload);
420             spoolFileWriter.close();
421             writeSucceeded = true;
422         } catch (Exception e) {
423             LOG.error("Unable to save downloaded file to spool directory ("+ incomingDir.getAbsolutePath() +")", e);
424         }
425
426
427                 if (writeSucceeded && (downloadResult.getDistributionActionResult() == DistributionActionResultEnum.SUCCESS)) {
428             handleSuccessfulDownload(data, svcName, resourceName, artifact, spoolFile, archiveDir);
429
430
431         } else {
432             handleFailedDownload(data, artifact);
433         }
434
435     }
436
437     private void handleFailedDownload(INotificationData data,
438             IArtifactInfo relevantArtifact) {
439         // Send Download Status
440         IDistributionClientResult sendDownloadStatus = client
441                 .sendDownloadStatus(buildStatusMessage(client, data,
442                         relevantArtifact, DistributionStatusEnum.DOWNLOAD_ERROR));
443     }
444
445     private void handleSuccessfulDownload(INotificationData data, String svcName, String resourceName,
446             IArtifactInfo artifact, File spoolFile, File archiveDir) {
447
448                 if ((data != null) && (artifact != null)) {
449             // Send Download Status
450             IDistributionClientResult sendDownloadStatus = client
451                     .sendDownloadStatus(buildStatusMessage(client, data, artifact, DistributionStatusEnum.DOWNLOAD_OK));
452         }
453
454         // If an override file exists, read that instead of the file we just downloaded
455         ArtifactTypeEnum artifactEnum = ArtifactTypeEnum.YANG_XML;
456
457                 boolean toscaYamlType = false;
458         if (artifact != null) {
459                         String artifactTypeString = artifact.getArtifactType();
460                         if (artifactTypeString.contains("TOSCA_TEMPLATE")) {
461                                 toscaYamlType = true;
462                         }
463                 } else {
464                         if (spoolFile.toString().contains(".yml") || spoolFile.toString().contains(".csar")) {
465                                 toscaYamlType = true;
466                         }
467         }
468         String overrideFileName = config.getOverrideFile();
469                 if ((overrideFileName != null) && (overrideFileName.length() > 0)) {
470             File overrideFile = new File(overrideFileName);
471
472             if (overrideFile.exists()) {
473                 artifactEnum = ArtifactTypeEnum.YANG_XML;
474                 spoolFile = overrideFile;
475             }
476
477         }
478
479                 if (toscaYamlType == true) {
480                         processToscaYaml (data, svcName, resourceName, artifact, spoolFile, archiveDir);
481
482                         try {
483                                 Path source = spoolFile.toPath();
484                                 Path targetDir = archiveDir.toPath();
485
486                                 Files.move(source, targetDir.resolve(source.getFileName()), StandardCopyOption.REPLACE_EXISTING);
487                         } catch (IOException e) {
488                                 LOG.warn("Could not move "+spoolFile.getAbsolutePath()+" to "+archiveDir.getAbsolutePath(), e);
489                         }
490
491                         return;
492                 }
493
494         // Process spool file
495         Document spoolDoc = null;
496         File transformedFile = null;
497
498         // Apply XSLTs and get Doc object
499         try {
500                         if (!spoolFile.isDirectory()) {
501             transformedFile = applyXslts(spoolFile);
502                         }
503         } catch (Exception e) {
504             LOG.error("Caught exception trying to parse XML file", e);
505         }
506
507         if (transformedFile != null) {
508             try {
509
510                 try {
511
512                     DocumentBuilderFactory dbf = DocumentBuilderFactory
513                             .newInstance();
514                     DocumentBuilder db = dbf.newDocumentBuilder();
515
516                     spoolDoc = db.parse(transformedFile);
517                 } catch (Exception e) {
518                     LOG.error(
519                             "Caught exception trying to parse transformed XML file "
520                                     + transformedFile.getAbsolutePath(), e);
521                 }
522
523             } catch (Exception e) {
524                 LOG.error("Caught exception trying to deploy file", e);
525             }
526         }
527
528
529         if (spoolDoc != null) {
530             // Analyze file type
531             SdncArtifactType artifactType = analyzeFileType(artifactEnum,
532                     spoolFile, spoolDoc);
533
534             if (artifactType != null) {
535
536                 scheduleDeployment(artifactType, svcName, resourceName, artifact, spoolFile.getName(), transformedFile);
537
538             }
539
540             // SDNGC-2660 : Move file to archive directory even if it is an unrecognized type so that
541             // we do not keep trying and failing to process it.
542             try {
543                 Path source = spoolFile.toPath();
544                 Path targetDir = archiveDir.toPath();
545
546                 Files.move(source, targetDir.resolve(source.getFileName()), StandardCopyOption.REPLACE_EXISTING);
547             } catch (IOException e) {
548                 LOG.warn("Could not move "+spoolFile.getAbsolutePath()+" to "+archiveDir.getAbsolutePath(), e);
549             }
550         }
551
552
553     }
554
555
556         private void processToscaYaml(INotificationData data, String svcName, String resourceName,
557                         IArtifactInfo artifact, File spoolFile, File archiveDir) {
558                 
559                 // Use ASDC Dist Client 1.1.5 with TOSCA parsing APIs to extract relevant TOSCA model data
560                 
561                 // TOSCA data extraction flow 1707:
562                 // Use ASDC dist-client to get yaml string - not yet available
563                 String model_yaml = null;
564                 LOG.info("Process TOSCA YAML file: "+spoolFile.toString());
565                 
566                 SdcToscaParserFactory factory = SdcToscaParserFactory.getInstance();
567                 ISdcCsarHelper sdcCsarHelper = null;
568                 try {
569                         sdcCsarHelper = factory.getSdcCsarHelper(spoolFile.getAbsolutePath());
570                 } catch (SdcToscaParserException e) {
571                         LOG.error("Could not create SDC TOSCA Parser ", e);
572                         factory.close();
573                         return;
574                 }
575
576                 // Ingest Service Data - 1707
577                 Metadata serviceMetadata = sdcCsarHelper.getServiceMetadata();
578                 SdncServiceModel serviceModel = new SdncServiceModel(sdcCsarHelper, serviceMetadata);
579                 serviceModel.setFilename(spoolFile.toString().substring(spoolFile.toString().lastIndexOf("/")+1));  // will be csar file name
580                 serviceModel.setServiceInstanceNamePrefix(SdncBaseModel.extractSubstitutionMappingTypeName(sdcCsarHelper).substring(SdncBaseModel.extractSubstitutionMappingTypeName(sdcCsarHelper).lastIndexOf(".")+1));
581
582                 try {
583                         cleanUpExistingToscaServiceData(serviceModel.getServiceUUID());
584                         LOG.info("Call insertToscaData for SERVICE_MODEL serviceUUID = " + serviceModel.getServiceUUID());
585                         insertToscaData(serviceModel.getSql(model_yaml));
586                 } catch (IOException e) {
587                         LOG.error("Could not insert Tosca YAML data into the SERVICE_MODEL table ", e);
588                         factory.close();
589                         return;
590                 }       
591
592                 // Ingest Network (VL) Data - 1707
593                 //List<NodeTemplate> vlNodeTemplatesList = sdcCsarHelper.getServiceNodeTemplatesByType("VL");
594                 List<NodeTemplate> vlNodeTemplatesList = sdcCsarHelper.getServiceVlList();
595
596                 for (NodeTemplate nodeTemplate :  vlNodeTemplatesList) {
597                         SdncNodeModel nodeModel = new SdncNodeModel (sdcCsarHelper, nodeTemplate);
598                         nodeModel.setServiceUUID(serviceModel.getServiceUUID());
599                         nodeModel.setEcompGeneratedNaming(SdncBaseModel.extractBooleanInputDefaultValue(sdcCsarHelper, SdcPropertyNames.PROPERTY_NAME_SERVICENAMING_DEFAULT_ECOMPGENERATEDNAMING));//service_naming#default#ecomp_generated_naming
600                         
601                         try {
602                                 cleanUpExistingToscaData("NETWORK_MODEL", "customization_uuid", nodeModel.getCustomizationUUID());
603                                 cleanUpExistingToscaData("VPN_BINDINGS", "network_customization_uuid", nodeModel.getCustomizationUUID());
604                                 LOG.info("Call insertToscaData for NETWORK_MODEL customizationUUID = " + nodeModel.getCustomizationUUID());
605                                 // using ASDC dist-client use method for get yaml string
606                                 insertToscaData(nodeModel.getSql(model_yaml));
607                                 insertToscaData(nodeModel.getVpnBindingsSql());
608                         } catch (IOException e) {
609                                 LOG.error("Could not insert Tosca YAML data into the NETWORK_MODEL table ", e);
610                         }
611                 }
612                 
613                 // Ingest Allotted Resource Data - 1707
614                 List<NodeTemplate> arNodeTemplatesList = sdcCsarHelper.getAllottedResources();
615
616                 for (NodeTemplate nodeTemplate :  arNodeTemplatesList) {
617                         SdncARModel nodeModel = new SdncARModel (sdcCsarHelper, nodeTemplate);
618                         
619                         try {
620                                 cleanUpExistingToscaData("ALLOTTED_RESOURCE_MODEL", "customization_uuid", nodeModel.getCustomizationUUID());
621                                 LOG.info("Call insertToscaData for ALLOTTED_RESOURCE_MODEL customizationUUID = " + nodeModel.getCustomizationUUID());
622                                 // using ASDC dist-client use method for get yaml string
623                                 insertToscaData(nodeModel.getSql("ALLOTTED_RESOURCE_MODEL", model_yaml));
624                         } catch (IOException e) {
625                                 LOG.error("Could not insert Tosca YAML data into the NETWORK_MODEL table ", e);
626                         }
627                 }
628                 
629                 // Ingest Network (VF) Data - 1707
630                 //List<NodeTemplate> nodeTemplatesList = sdcCsarHelper.getServiceNodeTemplatesByType("VF");
631                 List<NodeTemplate> vfNodeTemplatesList = sdcCsarHelper.getServiceVfList();
632
633                 for (NodeTemplate nodeTemplate :  vfNodeTemplatesList) {
634                         SdncVFModel vfNodeModel = new SdncVFModel (sdcCsarHelper, nodeTemplate);
635                         
636                         try {
637                                 cleanUpExistingToscaData("VF_MODEL", "customization_uuid", vfNodeModel.getCustomizationUUID()) ;
638                                 LOG.info("Call insertToscaData for VF_MODEL customizationUUID = " + vfNodeModel.getCustomizationUUID());
639                                 insertToscaData(vfNodeModel.getSql("VF_MODEL", model_yaml));
640                         } catch (IOException e) {
641                                 LOG.error("Could not insert Tosca YAML data into the VF_MODEL table ", e);
642                         }
643                         
644                         // For each VF, insert VF_MODULE_MODEL data
645                         List<Group> vfModules = sdcCsarHelper.getVfModulesByVf(vfNodeModel.getCustomizationUUIDNoQuotes());
646                         for (Group group : vfModules){
647                                 SdncVFModuleModel vfModuleModel = new SdncVFModuleModel(sdcCsarHelper, group);
648                         
649                                 try {
650                                         cleanUpExistingToscaData("VF_MODULE_MODEL", "customization_uuid", vfModuleModel.getCustomizationUUID());
651                                         LOG.info("Call insertToscaData for VF_MODULE_MODEL customizationUUID = " + vfModuleModel.getCustomizationUUID());
652                                         insertToscaData(vfModuleModel.getSql("VF_MODULE_MODEL", model_yaml));
653                                 } catch (IOException e) {
654                                         LOG.error("Could not insert Tosca YAML data into the VF_MODULE_MODEL table ", e);
655                                 }
656                         
657                                 // For each VF Module, get the VFC list, insert VF_MODULE_TO_VFC_MAPPING data
658                                 // List<NodeTemplate> groupMembers = sdcCsarHelper.getMembersOfGroup(group); - old version
659                                 // For each vfcNode (group member) in the groupMembers list, extract vm_type and vm_count.  
660                                 // Insert vf_module.customizationUUID, vfcNode.customizationUUID and vm_type and vm_count into VF_MODULE_TO_VFC_MAPPING
661                                 List<NodeTemplate> groupMembers = sdcCsarHelper.getMembersOfVfModule(nodeTemplate, group); // not yet available
662                                 for (NodeTemplate vfcNode : groupMembers){
663                                         SdncVFCModel vfcModel = new SdncVFCModel(sdcCsarHelper, vfcNode);
664                                 
665                                         try {
666                                                 cleanUpExistingToscaData("VF_MODULE_TO_VFC_MAPPING", "vf_module_customization_uuid", vfModuleModel.getCustomizationUUID());
667                                                 LOG.info("Call insertToscaData for VF_MODULE_TO_VFC_MAPPING customizationUUID = " + vfModuleModel.getCustomizationUUID());
668                                                 insertToscaData("insert into VF_MODULE_TO_VFC_MAPPING (vf_module_customization_uuid, vfc_customization_uuid, vm_type, vm_count) values (" + 
669                                                                 vfModuleModel.getCustomizationUUID() + ", " + vfcModel.getCustomizationUUID() + ", \"" + vfcModel.getVmType() + "\", \"" + vfcModel.getVmCount() + "\")");
670                                         } catch (IOException e) {
671                                                 LOG.error("Could not insert Tosca YAML data into the VF_MODULE_TO_VFC_MAPPING table ", e);
672                                         }
673
674                                 }
675
676                         }
677                         
678                         // For each VF, insert VFC_MODEL data
679                         List<NodeTemplate> vfcNodes = sdcCsarHelper.getVfcListByVf(vfNodeModel.getCustomizationUUIDNoQuotes());
680                         for (NodeTemplate vfcNode : vfcNodes){
681                                 SdncVFCModel vfcModel = new SdncVFCModel(sdcCsarHelper, vfcNode);
682                         
683                                 try {
684                                         cleanUpExistingToscaData("VFC_MODEL", "customization_uuid", vfcModel.getCustomizationUUID());
685                                         LOG.info("Call insertToscaData for VFC_MODEL customizationUUID = " + vfcModel.getCustomizationUUID());
686                                         insertToscaData(vfcModel.getSql("VFC_MODEL", model_yaml));
687                                 } catch (IOException e) {
688                                         LOG.error("Could not insert Tosca YAML data into the VFC_MODEL table ", e);
689                                 }
690
691                         }
692                         
693                         // For each VF, insert VF_TO_NETWORK_ROLE_MAPPING data
694                         List<NodeTemplate> cpNodes = sdcCsarHelper.getCpListByVf(vfNodeModel.getCustomizationUUIDNoQuotes());
695                         for (NodeTemplate cpNode : cpNodes){
696                 
697                                 // Insert into VF_TO_NETWORK_ROLE_MAPPING vf_customization_uuid and network_role
698                                 String cpNetworkRole = sdcCsarHelper.getNodeTemplatePropertyLeafValue(cpNode, "network_role_tag");
699                         
700                                 try {
701                                         cleanUpExistingToscaData("VF_TO_NETWORK_ROLE_MAPPING", "vf_customization_uuid", vfNodeModel.getCustomizationUUID());
702                                         LOG.info("Call insertToscaData for VF_TO_NETWORK_ROLE_MAPPING vfCustomizationUUID = " + vfNodeModel.getCustomizationUUID());
703                                         insertToscaData("insert into VF_TO_NETWORK_ROLE_MAPPING (vf_customization_uuid, network_role) values (" + 
704                                         vfNodeModel.getCustomizationUUID() + ", \"" + cpNetworkRole + "\")");
705                                 } catch (IOException e) {
706                                         LOG.error("Could not insert Tosca YAML data into the VF_TO_NETWORK_ROLE_MAPPING table ", e);
707                                 }
708                                 
709                                 // Insert VFC_TO_NETWORK_ROLE_MAPPING data
710                                 Map<String, String> mappingParams = new HashMap<String, String>();
711                                 //String cpNetworkRoleTag = "\"" + sdcCsarHelper.getNodeTemplatePropertyLeafValue(cpNode, SdcPropertyNames.PROPERTY_NAME_NETWORKROLETAG) + "\"";
712                                 // extract network_role, network_role_tag and virtual_binding from this cpNode
713                                 SdncBaseModel.addParameter("network_role", SdncBaseModel.extractValue(sdcCsarHelper, cpNode, "network_role"), mappingParams);
714                                 SdncBaseModel.addParameter("network_role_tag", SdncBaseModel.extractValue(sdcCsarHelper, cpNode, "network_role_tag"), mappingParams);
715                                 String virtualBinding = "\"" + SdncBaseModel.extractValue(sdcCsarHelper, cpNode, "requirements#virtualBinding") + "\""; 
716
717                                 // get list of cpNodes and vfcNodes with matching virtualBinding
718                                 List<Pair<NodeTemplate, NodeTemplate>> matchList = sdcCsarHelper.getNodeTemplatePairsByReqName(sdcCsarHelper.getCpListByVf(vfNodeModel.getCustomizationUUIDNoQuotes()), sdcCsarHelper.getVfcListByVf(vfNodeModel.getCustomizationUUIDNoQuotes()), virtualBinding);                              
719                                 for (Pair<NodeTemplate, NodeTemplate> match : matchList) {  // should be 1 match?
720                                         
721                                         // extract values from the left "CP" Node
722                                         SdncBaseModel.addParameter("ipv4_use_dhcp", SdncBaseModel.extractBooleanValue(sdcCsarHelper, match.getLeft(), SdcPropertyNames.PROPERTY_NAME_NETWORKASSIGNMENTS_IPV4SUBNETDEFAULTASSIGNMENTS_DHCPENABLED), mappingParams);
723                                         //SdncBaseModel.addParameter("ipv4_ip_version", SdncBaseModel.extractValue(sdcCsarHelper, match.getLeft(), SdcPropertyNames.PROPERTY_NAME_NETWORKASSIGNMENTS_IPV4SUBNETDEFAULTASSIGNMENTS_IPVERSION), mappingParams);
724                                         SdncBaseModel.addParameter("ipv4_ip_version", "dummy_ipv4_vers", mappingParams);
725                                         SdncBaseModel.addParameter("ipv6_use_dhcp", SdncBaseModel.extractBooleanValue(sdcCsarHelper, match.getLeft(), SdcPropertyNames.PROPERTY_NAME_NETWORKASSIGNMENTS_IPV6SUBNETDEFAULTASSIGNMENTS_DHCPENABLED), mappingParams);
726                                         //SdncBaseModel.addParameter("ipv6_ip_version", SdncBaseModel.extractValue(sdcCsarHelper, match.getLeft(), SdcPropertyNames.PROPERTY_NAME_NETWORKASSIGNMENTS_IPV6SUBNETDEFAULTASSIGNMENTS_IPVERSION), mappingParams);
727                                         SdncBaseModel.addParameter("ipv6_ip_version", "dummy_ipv6_vers", mappingParams);
728                                         //String extcp_subnetpool_id = "\"" + SdncBaseModel.extractValue(sdcCsarHelper, match.getLeft(), SdcPropertyNames.PROPERTY_NAME_SUBNETPOOLID) + "\""; // need path to subnetpoolid 
729                                         
730                                         // extract values from the right "VFC" Node
731                                         String vfcCustomizationUuid = "\"" + SdncBaseModel.extractValue(sdcCsarHelper, match.getRight().getMetadata(), "customization_uuid") + "\"";
732                                         SdncBaseModel.addParameter("vm_type", SdncBaseModel.extractValue(sdcCsarHelper, match.getRight(), SdcPropertyNames.PROPERTY_NAME_VMTYPE), mappingParams);
733                                         SdncBaseModel.addIntParameter("ipv4_count", SdncBaseModel.extractValue(sdcCsarHelper, match.getRight(), SdcPropertyNames.PROPERTY_NAME_NETWORKASSIGNMENTS_IPV4SUBNETDEFAULTASSIGNMENTS_MINSUBNETSCOUNT), mappingParams);
734                                         SdncBaseModel.addIntParameter("ipv6_count", SdncBaseModel.extractValue(sdcCsarHelper, match.getRight(), SdcPropertyNames.PROPERTY_NAME_NETWORKASSIGNMENTS_IPV6SUBNETDEFAULTASSIGNMENTS_MINSUBNETSCOUNT), mappingParams);
735                                 
736                                         try {
737                                                 cleanUpExistingToscaData("VFC_TO_NETWORK_ROLE_MAPPING", "vfc_customization_uuid", vfcCustomizationUuid);
738                                                 LOG.info("Call insertToscaData for VFC_TO_NETWORK_ROLE_MAPPING vfcCustomizationUUID = " + vfcCustomizationUuid);
739                                                 insertToscaData(SdncBaseModel.getSql("VFC_TO_NETWORK_ROLE_MAPPING", "vfc_customization_uuid", vfcCustomizationUuid, "", mappingParams));
740                                         } catch (IOException e) {
741                                                 LOG.error("Could not insert Tosca YAML data into the VFC_TO_NETWORK_ROLE_MAPPING table ", e);
742                                         }
743                                 
744                                 }       
745                                 
746                         } // CP loop    
747                         
748                 } // VF loop
749                 
750                 // Close ASDC TOSCA Parser factory - we are done processing this distribution
751                 factory.close();
752                 
753                 if ((artifact != null) && (data != null)) {
754                         LOG.info("Update to SDN-C succeeded");
755                         IDistributionClientResult deploymentStatus;
756                                 deploymentStatus = client.sendDeploymentStatus(buildStatusMessage(
757                                                 client, data, artifact,
758                                                 DistributionStatusEnum.DEPLOY_OK));
759                 }
760         
761         }
762         
763          private void cleanUpExistingToscaData(String tableName, String keyName, String keyValue) throws IOException
764      {
765          
766             if (jdbcDataSource == null) {
767                  setJdbcDataSource();
768             }
769              try {
770                 int rowCount = 0;
771                 CachedRowSet data = jdbcDataSource.getData("SELECT * from " + tableName + " where " + keyName + " = " + keyValue + ";", null, "");
772                 while(data.next()) { 
773                                 rowCount ++; 
774                 }
775                 if (rowCount != 0) {
776                     LOG.info("cleanUpExistingToscaData: " + keyValue);
777                         jdbcDataSource.writeData("DELETE from " + tableName + " where " + keyName + " = " + keyValue + ";", null, null);
778                 }
779                  
780                         } catch (SQLException e) {                              
781                                 LOG.error("Could not clean up existing " + tableName  + " for " + keyValue, e);
782                         }    
783                         
784      }
785
786         
787          private void cleanUpExistingToscaServiceData(String serviceUUID) throws IOException
788      {
789          
790             if (jdbcDataSource == null) {
791                  setJdbcDataSource();
792             }
793              try {
794                 int rowCount = 0;
795                 CachedRowSet data = jdbcDataSource.getData("SELECT * from SERVICE_MODEL where service_uuid = " + serviceUUID + ";", null, "");
796                 while(data.next()) { 
797                                 rowCount ++; 
798                 }
799                 if (rowCount != 0) {
800                     LOG.info("cleanUpExistingToscaData: " + serviceUUID);
801                         jdbcDataSource.writeData("DELETE from NETWORK_MODEL where service_uuid = " + serviceUUID + ";", null, null);
802                         jdbcDataSource.writeData("DELETE from SERVICE_MODEL where service_uuid = " + serviceUUID + ";", null, null);
803                 }
804                  
805                         } catch (SQLException e) {                              
806                                 LOG.error("Could not clean up existing NETWORK_MODEL and SERVICE_MODEL for service_UUID " + serviceUUID, e);
807                         }    
808                         
809      }
810
811         
812          private void insertToscaData(String toscaDataString) throws IOException
813      {
814             LOG.debug("insertToscaData: " + toscaDataString);
815
816             if (jdbcDataSource == null) {
817                  setJdbcDataSource();
818             }
819              try {
820
821                                 jdbcDataSource.writeData(toscaDataString, null, null);
822
823                         } catch (SQLException e) {
824                                 LOG.error("Could not insert Tosca YAML data into the database ", e);
825                         }
826
827      }
828
829
830     private SdncArtifactType analyzeFileType(ArtifactTypeEnum artifactType, File spoolFile, Document spoolDoc) {
831
832         if (artifactType != ArtifactTypeEnum.YANG_XML) {
833             LOG.error("Unexpected artifact type - expecting YANG_XML, got "+artifactType);
834                         return (null);
835         }
836
837         // Examine outer tag
838
839         try {
840
841
842             Element root = spoolDoc.getDocumentElement();
843
844             String rootName = root.getTagName();
845
846             if (rootName.contains(":")) {
847                 String[] rootNameElems = rootName.split(":");
848                 rootName = rootNameElems[rootNameElems.length - 1];
849             }
850
851             if (rootName != null) {
852                 SdncArtifactType mapEntry = config.getMapping(rootName);
853
854
855                 if (mapEntry == null) {
856
857                     LOG.error("Unexpected file contents - root tag is "+rootName);
858                 }
859                                 return(mapEntry);
860             } else {
861                 LOG.error("Cannot get root tag from file");
862                                 return(null);
863             }
864
865         } catch (Exception e) {
866             LOG.error("Could not parse YANG_XML file "+spoolFile.getName(), e);
867                         return(null);
868         }
869     }
870
871     private void scheduleDeployment(SdncArtifactType type, String svcName, String resourceName, IArtifactInfo artifactInfo, String spoolFileName, File spoolFile) {
872
873         if (type.getPass() < deployList.length) {
874
875             if (artifactInfo != null) {
876                 LOG.debug("Scheduling "+artifactInfo.getArtifactName()+" version "+artifactInfo.getArtifactVersion()+" for deployment");
877
878                 deployList[type.getPass()].add(new DeployableArtifact(type, svcName, resourceName, artifactInfo, spoolFile));
879             } else {
880                 SimpleDateFormat sdfDate = new SimpleDateFormat("yyyy-MM-dd_HH:mm:ss.SSS");//dd/MM/yyyy
881                 Date now = new Date();
882                 String artifactVersion = sdfDate.format(now);
883                 LOG.debug("Scheduling "+spoolFileName+" version "+artifactVersion+" for deployment");
884                 String artifactName = spoolFileName;
885                 if (artifactInfo != null) {
886                     artifactName = artifactInfo.getArtifactName();
887                 }
888                 deployList[type.getPass()].add(new DeployableArtifact(type, svcName, resourceName, artifactName, artifactVersion, spoolFile));
889             }
890         } else {
891             LOG.info("Pass for type "+type.getTag()+" is "+type.getPass()+" which is not <= "+deployList.length);
892         }
893     }
894
895
896     private DistributionStatusEnum deploySpoolFile(DeployableArtifact artifact) {
897
898         DistributionStatusEnum deployResult = DistributionStatusEnum.DEPLOY_OK;
899
900         StringBuffer msgBuffer = new StringBuffer();
901
902
903         String namespace = config.getAsdcApiNamespace();
904                 if ((namespace == null) || (namespace.length() == 0)) {
905             namespace="com:att:sdnctl:asdcapi";
906         }
907
908         msgBuffer.append("<input xmlns='");
909         msgBuffer.append(namespace);
910         msgBuffer.append("'>\n");
911
912         String svcName = artifact.getSvcName();
913         String resourceName = artifact.getResourceName();
914         String artifactName = artifact.getArtifactName();
915
916         if (svcName != null) {
917             if (resourceName != null) {
918                 artifactName = svcName + "/" + resourceName + "/" + artifactName;
919             } else {
920                 artifactName = svcName + "/" + artifactName;
921             }
922         }
923
924         msgBuffer.append("<artifact-name>"+artifactName+"</artifact-name>\n");
925         msgBuffer.append("<artifact-version>"+artifact.getArtifactVersion()+"</artifact-version>\n");
926
927
928         try {
929             BufferedReader rdr = new BufferedReader(new FileReader(artifact.getFile()));
930
931             String curLine = rdr.readLine();
932
933             while (curLine != null) {
934
935                 if (!curLine.startsWith("<?")) {
936                     msgBuffer.append(curLine+"\n");
937                 }
938                 curLine = rdr.readLine();
939             }
940             rdr.close();
941
942         } catch (Exception e) {
943             LOG.error("Could not process spool file "+artifact.getFile().getName(), e);
944                         return(DistributionStatusEnum.DEPLOY_ERROR);
945         }
946
947         msgBuffer.append("</input>\n");
948
949
950         byte[] msgBytes = msgBuffer.toString().getBytes();
951
952         Document results = postRestXml(artifact.getType().getRpcUrl(config.getAsdcApiBaseUrl()), msgBytes);
953
954         if (results == null) {
955
956             deployResult = DistributionStatusEnum.DEPLOY_ERROR;
957         } else {
958
959             XPathFactory xpf = XPathFactory.newInstance();
960             XPath xp = xpf.newXPath();
961
962             String asdcApiResponseCode = "500";
963
964             try {
965
966                 asdcApiResponseCode = xp.evaluate("//asdc-api-response-code[position()=1]/text()", results.getDocumentElement());
967             } catch (Exception e) {
968                 LOG.error("Caught exception retrying to evaluate xpath", e);
969             }
970
971             if (asdcApiResponseCode.contains("200")) {
972                 LOG.info("Update to SDN-C succeeded");
973                 deployResult = DistributionStatusEnum.DEPLOY_OK;
974             } else {
975                 LOG.info("Update to SDN-C failed (response code "+asdcApiResponseCode+")");
976
977                 if (asdcApiResponseCode.contains("409")) {
978                     deployResult = DistributionStatusEnum.ALREADY_DEPLOYED;
979                 } else {
980
981                     deployResult = DistributionStatusEnum.DEPLOY_ERROR;
982                 }
983             }
984         }
985
986
987
988                 return(deployResult);
989     }
990
991
992
993
994
995     public static IDistributionStatusMessage buildStatusMessage(
996             final IDistributionClient client, final INotificationData data,
997             final IArtifactInfo relevantArtifact,
998             final DistributionStatusEnum status) {
999         IDistributionStatusMessage statusMessage = new IDistributionStatusMessage() {
1000
1001             @Override
1002                         public long getTimestamp() {
1003                 long currentTimeMillis = System.currentTimeMillis();
1004                 return currentTimeMillis;
1005             }
1006
1007             @Override
1008                         public DistributionStatusEnum getStatus() {
1009                 return status;
1010             }
1011
1012             @Override
1013                         public String getDistributionID() {
1014                 return data.getDistributionID();
1015             }
1016
1017             @Override
1018                         public String getConsumerID() {
1019                 return client.getConfiguration().getConsumerID();
1020             }
1021
1022             @Override
1023                         public String getArtifactURL() {
1024                 return relevantArtifact.getArtifactURL();
1025             }
1026         };
1027         return statusMessage;
1028
1029     }
1030
1031     private HttpURLConnection getRestXmlConnection(String urlString, String method) throws IOException
1032     {
1033         URL sdncUrl = new URL(urlString);
1034         Authenticator.setDefault(new SdncAuthenticator(config.getSdncUser(), config.getSdncPasswd()));
1035
1036         HttpURLConnection conn = (HttpURLConnection) sdncUrl.openConnection();
1037
1038         String authStr = config.getSdncUser()+":"+config.getSdncPasswd();
1039         String encodedAuthStr = new String(Base64.encodeBase64(authStr.getBytes()));
1040
1041         conn.addRequestProperty("Authentication", "Basic "+encodedAuthStr);
1042
1043         conn.setRequestMethod(method);
1044         conn.setRequestProperty("Content-Type", "application/xml");
1045         conn.setRequestProperty("Accept", "application/xml");
1046
1047         conn.setDoInput(true);
1048         conn.setDoOutput(true);
1049         conn.setUseCaches(false);
1050
1051                 return(conn);
1052
1053     }
1054
1055     private Document postRestXml(String urlString, byte[] msgBytes) {
1056         Document response = null;
1057
1058         try {
1059                         SdncOdlConnection odlConn = SdncOdlConnection.newInstance(urlString, config.getSdncUser(), config.getSdncPasswd());
1060
1061                         String sdncResp = odlConn.send("POST", "application/xml", new String(msgBytes));
1062
1063             DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
1064             DocumentBuilder db = dbf.newDocumentBuilder();
1065
1066
1067                         response = db.parse(new ByteArrayInputStream(sdncResp.getBytes()));
1068         } catch (Exception e) {
1069                         LOG.error("Caught exception posting to ODL tier", e);
1070         }
1071
1072                 return(response);
1073
1074     }
1075
1076     private File applyXslts(File srcFile) {
1077
1078         Document doc = null;
1079
1080
1081         File inFile = srcFile;
1082         File outFile = null;
1083
1084         String xsltPathList = config.getXsltPathList();
1085
1086                 if ((xsltPathList == null) || (xsltPathList.length() == 0)) {
1087             outFile = inFile;
1088         } else {
1089
1090             String[] xsltPaths = xsltPathList.split(",");
1091
1092             for (String xsltPath : xsltPaths) {
1093                 try{
1094
1095                     outFile = File.createTempFile("tmp", "xml");
1096                     TransformerFactory factory = TransformerFactory.newInstance();
1097                     Source xslt = new StreamSource(new File(xsltPath));
1098                     Transformer transformer = factory.newTransformer(xslt);
1099                     Source text = new StreamSource(inFile);
1100
1101
1102                     transformer.transform(text, new StreamResult(outFile));
1103
1104                     inFile = outFile;
1105
1106                 } catch (Exception e) {
1107                     LOG.error("Caught exception trying to apply XSLT template "+xsltPath, e);
1108
1109                 }
1110
1111             }
1112         }
1113
1114         // After transformations, parse transformed XML
1115
1116
1117                 return(outFile);
1118     }
1119
1120     private String escapeFilename(String str) {
1121         StringBuffer retval = new StringBuffer();
1122
1123         for (int i = 0 ; i < str.length() ; i++) {
1124             char curchar = str.charAt(i);
1125             if (Character.isJavaIdentifierPart(curchar)) {
1126                 retval.append(curchar);
1127             }
1128         }
1129
1130                 return(retval.toString());
1131
1132     }
1133
1134 }