1 /*******************************************************************************
2 * Copyright (c) 2012-2013,2015 University of Stuttgart.
3 * All rights reserved. This program and the accompanying materials
4 * are made available under the terms of the Eclipse Public License v1.0
5 * and the Apache License 2.0 which both accompany this distribution,
6 * and are available at http://www.eclipse.org/legal/epl-v10.html
7 * and http://www.apache.org/licenses/LICENSE-2.0
10 * Kálmán Képes - initial API and implementation and/or initial documentation
11 * Oliver Kopp - adapted to new storage model and to TOSCA v1.0
12 *******************************************************************************/
13 package org.eclipse.winery.repository.importing;
15 import static java.nio.file.FileVisitResult.CONTINUE;
16 import static java.nio.file.FileVisitResult.SKIP_SUBTREE;
18 import java.io.BufferedInputStream;
19 import java.io.IOException;
20 import java.io.InputStream;
21 import java.lang.reflect.Method;
23 import java.net.URISyntaxException;
24 import java.nio.file.FileVisitResult;
25 import java.nio.file.Files;
26 import java.nio.file.Path;
27 import java.nio.file.PathMatcher;
28 import java.nio.file.SimpleFileVisitor;
29 import java.nio.file.attribute.BasicFileAttributes;
30 import java.util.ArrayList;
31 import java.util.Collection;
32 import java.util.HashSet;
33 import java.util.Iterator;
34 import java.util.List;
36 import java.util.concurrent.ExecutorService;
37 import java.util.concurrent.Executors;
38 import java.util.regex.Matcher;
39 import java.util.regex.Pattern;
40 import java.util.zip.ZipEntry;
41 import java.util.zip.ZipInputStream;
43 import javax.ws.rs.core.MediaType;
44 import javax.xml.XMLConstants;
45 import javax.xml.bind.JAXBException;
46 import javax.xml.bind.Unmarshaller;
48 import org.apache.commons.configuration.ConfigurationException;
49 import org.apache.commons.configuration.PropertiesConfiguration;
50 import org.apache.commons.io.FilenameUtils;
51 import org.eclipse.winery.common.ModelUtilities;
52 import org.eclipse.winery.common.RepositoryFileReference;
53 import org.eclipse.winery.common.Util;
54 import org.eclipse.winery.common.constants.MimeTypes;
55 import org.eclipse.winery.common.constants.Namespaces;
56 import org.eclipse.winery.common.ids.XMLId;
57 import org.eclipse.winery.common.ids.definitions.ArtifactTemplateId;
58 import org.eclipse.winery.common.ids.definitions.EntityTypeId;
59 import org.eclipse.winery.common.ids.definitions.NodeTypeId;
60 import org.eclipse.winery.common.ids.definitions.RelationshipTypeId;
61 import org.eclipse.winery.common.ids.definitions.ServiceTemplateId;
62 import org.eclipse.winery.common.ids.definitions.TOSCAComponentId;
63 import org.eclipse.winery.common.ids.definitions.imports.GenericImportId;
64 import org.eclipse.winery.common.ids.definitions.imports.XSDImportId;
65 import org.eclipse.winery.common.ids.elements.PlanId;
66 import org.eclipse.winery.common.ids.elements.PlansId;
67 import org.eclipse.winery.common.propertydefinitionkv.WinerysPropertiesDefinition;
68 import org.eclipse.winery.model.csar.toscametafile.TOSCAMetaFile;
69 import org.eclipse.winery.model.csar.toscametafile.TOSCAMetaFileParser;
70 import org.eclipse.winery.model.tosca.Definitions;
71 import org.eclipse.winery.model.tosca.TArtifactReference;
72 import org.eclipse.winery.model.tosca.TArtifactReference.Exclude;
73 import org.eclipse.winery.model.tosca.TArtifactReference.Include;
74 import org.eclipse.winery.model.tosca.TArtifactTemplate;
75 import org.eclipse.winery.model.tosca.TArtifactTemplate.ArtifactReferences;
76 import org.eclipse.winery.model.tosca.TDefinitions;
77 import org.eclipse.winery.model.tosca.TDefinitions.Types;
78 import org.eclipse.winery.model.tosca.TEntityType;
79 import org.eclipse.winery.model.tosca.TEntityType.PropertiesDefinition;
80 import org.eclipse.winery.model.tosca.TExtensibleElements;
81 import org.eclipse.winery.model.tosca.TImport;
82 import org.eclipse.winery.model.tosca.TNodeType;
83 import org.eclipse.winery.model.tosca.TPlan;
84 import org.eclipse.winery.model.tosca.TPlan.PlanModelReference;
85 import org.eclipse.winery.model.tosca.TPlans;
86 import org.eclipse.winery.model.tosca.TRelationshipType;
87 import org.eclipse.winery.model.tosca.TServiceTemplate;
88 import org.eclipse.winery.repository.Constants;
89 import org.eclipse.winery.repository.JAXBSupport;
90 import org.eclipse.winery.repository.Utils;
91 import org.eclipse.winery.repository.backend.BackendUtils;
92 import org.eclipse.winery.repository.backend.Repository;
93 import org.eclipse.winery.repository.backend.constants.Filename;
94 import org.eclipse.winery.repository.backend.filebased.FileUtils;
95 import org.eclipse.winery.repository.datatypes.ids.elements.ArtifactTemplateDirectoryId;
96 import org.eclipse.winery.repository.datatypes.ids.elements.SelfServiceMetaDataId;
97 import org.eclipse.winery.repository.datatypes.ids.elements.VisualAppearanceId;
98 import org.eclipse.winery.repository.export.CSARExporter;
99 import org.eclipse.winery.repository.resources.admin.NamespacesResource;
100 import org.slf4j.Logger;
101 import org.slf4j.LoggerFactory;
102 import org.w3c.dom.Element;
105 * Imports a CSAR into the storage. As the internal storage format does not have
106 * CSARs as the topmost artifacts, but one TDefinition, the CSAR has to be split
107 * up into several components.
109 * Existing components are <em>not</em> replaced, but silently skipped
111 * Minor errors are logged and not further propagated / notified. That means, a
112 * user cannot see minor errors. Major errors are immediately thrown.
114 * One instance for each import
116 public class CSARImporter {
118 private static final Logger logger = LoggerFactory.getLogger(CSARImporter.class);
120 // ExecutorService for XSD schema initialization
121 // Threads set to 1 to avoid testing for parallel processing of the same XSD file
122 private static final ExecutorService xsdParsingService = Executors.newFixedThreadPool(1);
124 private static final ExecutorService entityTypeAdjestmentService = Executors.newFixedThreadPool(10);
128 * Reads the CSAR from the given inputstream
130 * @param in the inputstream to read from
131 * @param errorList the list of errors during the import. Has to be non-null
132 * @param overwrite if true: contents of the repo are overwritten
134 * @throws InvalidCSARException if the CSAR is invalid
136 public void readCSAR(InputStream in, List<String> errors, boolean overwrite, final boolean asyncWPDParsing) throws IOException {
137 // we have to extract the file to a temporary directory as
138 // the .definitions file does not necessarily have to be the first entry in the archive
139 Path csarDir = Files.createTempDirectory("winery");
141 try (ZipInputStream zis = new ZipInputStream(in)) {
143 while ((entry = zis.getNextEntry()) != null) {
144 if (!entry.isDirectory()) {
145 Path targetPath = csarDir.resolve(entry.getName());
146 Files.createDirectories(targetPath.getParent());
147 Files.copy(zis, targetPath);
150 this.importFromDir(csarDir, errors, overwrite, asyncWPDParsing);
151 } catch (Exception e) {
152 CSARImporter.logger.debug("Could not import CSAR", e);
155 // cleanup: delete all contents of the temporary directory
156 FileUtils.forceDelete(csarDir);
161 * Import an extracted CSAR from a directory
163 * @param path the root path of an extracted CSAR file
164 * @param overwrite if true: contents of the repo are overwritten
165 * @param asyncWPDParsing true if WPD should be parsed asynchronously to
166 * speed up the import. Required, because JUnit terminates the
167 * used ExecutorService
168 * @throws InvalidCSARException
169 * @throws IOException
171 void importFromDir(final Path path, final List<String> errors, final boolean overwrite, final boolean asyncWPDParsing) throws IOException {
172 Path toscaMetaPath = path.resolve("TOSCA-Metadata/TOSCA.meta");
173 if (!Files.exists(toscaMetaPath)) {
174 errors.add("TOSCA.meta does not exist");
177 final TOSCAMetaFileParser tmfp = new TOSCAMetaFileParser();
178 final TOSCAMetaFile tmf = tmfp.parse(toscaMetaPath);
180 // we do NOT do any sanity checks, of TOSAC.meta
181 // and just start parsing
183 if (tmf.getEntryDefinitions() != null) {
184 // we obey the entry definitions and "just" import that
185 // imported definitions are added recursively
186 Path defsPath = path.resolve(tmf.getEntryDefinitions());
187 this.importDefinitions(tmf, defsPath, errors, overwrite, asyncWPDParsing);
189 this.importSelfServiceMetaData(tmf, path, defsPath, errors);
191 // no explicit entry definitions found
192 // we import all available definitions
193 // The specification says (cos01, Section 16.1, line 2935) that all definitions are contained in the "Definitions" directory
194 // The alternative is to go through all entries in the TOSCA Meta File, but there is no guarantee that this list is complete
195 Path definitionsDir = path.resolve("Definitions");
196 if (!Files.exists(definitionsDir)) {
197 errors.add("No entry definitions defined and Definitions directory does not exist.");
200 final List<IOException> exceptions = new ArrayList<IOException>();
201 Files.walkFileTree(definitionsDir, new SimpleFileVisitor<Path>() {
204 public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) {
205 if (dir.endsWith("Definitions")) {
206 return FileVisitResult.CONTINUE;
208 return FileVisitResult.SKIP_SUBTREE;
213 public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) {
215 CSARImporter.this.importDefinitions(tmf, file, errors, overwrite, asyncWPDParsing);
216 } catch (IOException e) {
218 return FileVisitResult.TERMINATE;
220 return FileVisitResult.CONTINUE;
224 if (!exceptions.isEmpty()) {
225 // something went wrong during parsing
226 // we rethrow the exception
227 throw exceptions.get(0);
231 this.importNamespacePrefixes(path);
235 private static final Pattern GENERATED_PREFIX_PATTERN = Pattern.compile("^ns\\d+$");
239 * Import namespace prefixes. This is kind of a quick hack. TODO: during the
240 * import, the prefixes should be extracted using JAXB and stored in the
243 * @param rootPath the root path of the extracted CSAR
245 private void importNamespacePrefixes(Path rootPath) {
246 Path properties = rootPath.resolve(CSARExporter.PATH_TO_NAMESPACES_PROPERTIES);
247 if (Files.exists(properties)) {
248 PropertiesConfiguration pconf;
250 pconf = new PropertiesConfiguration(properties.toFile());
251 } catch (ConfigurationException e) {
252 CSARImporter.logger.debug(e.getMessage(), e);
255 Iterator<String> namespaces = pconf.getKeys();
256 while (namespaces.hasNext()) {
257 boolean addToStorage = false;
258 String namespace = namespaces.next();
259 if (NamespacesResource.INSTANCE.getIsPrefixKnownForNamespace(namespace)) {
260 String storedPrefix = NamespacesResource.getPrefix(namespace);
261 // QUICK HACK to check whether the prefix is a generated one
262 // We assume we know the internal generation routine
263 Matcher m = CSARImporter.GENERATED_PREFIX_PATTERN.matcher(storedPrefix);
265 // the stored prefix is a generated one
266 // replace it by the one stored in the exported properties
273 String prefix = pconf.getString(namespace);
274 NamespacesResource.INSTANCE.addNamespace(namespace, prefix);
281 * Imports a self-service meta data description (if available)
283 * The first service template in the provided entry definitions is taken
289 private void importSelfServiceMetaData(final TOSCAMetaFile tmf, final Path rootPath, Path entryDefinitions, final List<String> errors) {
290 final Path selfServiceDir = rootPath.resolve(Constants.DIRNAME_SELF_SERVICE_METADATA);
291 if (!Files.exists(selfServiceDir)) {
292 CSARImporter.logger.debug("Self-service Portal directory does not exist in CSAR");
295 if (!Files.exists(entryDefinitions)) {
296 CSARImporter.logger.debug("Entry definitions does not exist.");
300 Unmarshaller um = JAXBSupport.createUnmarshaller();
303 defs = (TDefinitions) um.unmarshal(entryDefinitions.toFile());
304 } catch (JAXBException e) {
305 errors.add("Could not unmarshal definitions " + entryDefinitions.getFileName() + " " + e.getMessage());
307 } catch (ClassCastException e) {
308 errors.add("Definitions " + entryDefinitions.getFileName() + " is not a TDefinitions " + e.getMessage());
312 final int cutLength = selfServiceDir.toString().length() + 1;
313 Iterator<TExtensibleElements> iterator = defs.getServiceTemplateOrNodeTypeOrNodeTypeImplementation().iterator();
314 boolean found = false;
315 TExtensibleElements next = null;
316 while (iterator.hasNext() && !found) {
317 next = iterator.next();
318 if (next instanceof TServiceTemplate) {
324 TServiceTemplate serviceTemplate = (TServiceTemplate) next;
325 String namespace = serviceTemplate.getTargetNamespace();
326 if (namespace == null) {
327 namespace = defs.getTargetNamespace();
329 ServiceTemplateId stId = new ServiceTemplateId(namespace, serviceTemplate.getId(), false);
330 final SelfServiceMetaDataId id = new SelfServiceMetaDataId(stId);
332 // QUICK HACK: We just import all data without any validation
333 // Reason: the metadata resource can deal with nearly arbitrary formats of the data, therefore we do not do any checking here
336 Files.walkFileTree(selfServiceDir, new SimpleFileVisitor<Path>() {
339 public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) {
340 String name = file.toString().substring(cutLength);
341 // check: if name contains "/", this could lead to exceptions
342 RepositoryFileReference ref = new RepositoryFileReference(id, name);
344 if (name.equals("data.xml")) {
345 // we have to check whether the data.xml contains
346 // (uri:"http://opentosca.org/self-service", local:"application")
348 // (uri:"http://www.eclipse.org/winery/model/selfservice", local:"Application"
349 // We quickly replace it via String replacement instead of XSLT
351 String oldContent = org.apache.commons.io.FileUtils.readFileToString(file.toFile(), "UTF-8");
352 String newContent = oldContent.replace("http://opentosca.org/self-service", "http://www.eclipse.org/winery/model/selfservice");
353 newContent = newContent.replace(":application", ":Application");
354 if (!oldContent.equals(newContent)) {
355 // we replaced something -> write new content to old file
356 org.apache.commons.io.FileUtils.writeStringToFile(file.toFile(), newContent, "UTF-8");
358 } catch (IOException e) {
359 CSARImporter.logger.debug("Could not replace content in data.xml", e);
362 CSARImporter.this.importFile(file, ref, tmf, rootPath, errors);
363 return FileVisitResult.CONTINUE;
366 } catch (IOException e) {
367 CSARImporter.logger.debug(e.getMessage(), e);
368 errors.add("Self-service Meta Data: " + e.getMessage());
375 * Recursively imports the given definitions
377 * @param tmf the TOSCAMetaFile object holding the parsed content of a TOSCA
378 * meta file. If null, no files must be referenced from the given
380 * @param overwrite true: existing contents are overwritten
381 * @param asyncWPDParsing
382 * @param definitions the path to the definitions to import
384 * @throws IOException
386 public void importDefinitions(TOSCAMetaFile tmf, Path defsPath, final List<String> errors, boolean overwrite, boolean asyncWPDParsing) throws IOException {
387 if (defsPath == null) {
388 throw new IllegalStateException("path to definitions must not be null");
390 if (!Files.exists(defsPath)) {
391 errors.add(String.format("Definitions %1$s does not exist", defsPath.getFileName()));
395 Unmarshaller um = JAXBSupport.createUnmarshaller();
398 defs = (TDefinitions) um.unmarshal(defsPath.toFile());
399 } catch (JAXBException e) {
403 String msg = cause.getMessage();
405 eMsg = eMsg + msg + "; ";
407 cause = cause.getCause();
408 } while (cause != null);
409 errors.add("Could not unmarshal definitions " + defsPath.getFileName() + " " + eMsg);
410 CSARImporter.logger.debug("Unmarshalling error", e);
412 } catch (ClassCastException e) {
413 errors.add("Definitions " + defsPath.getFileName() + " is not a TDefinitions " + e.getMessage());
417 List<TImport> imports = defs.getImport();
418 this.importImports(defsPath.getParent(), tmf, imports, errors, overwrite, asyncWPDParsing);
419 // imports has been modified to contain necessary imports only
421 // this method adds new imports to defs which may not be imported using "importImports".
422 // Therefore, "importTypes" has to be called *after* importImports
423 this.importTypes(defs, errors);
425 String defaultNamespace = defs.getTargetNamespace();
426 List<TExtensibleElements> componentInstanceList = defs.getServiceTemplateOrNodeTypeOrNodeTypeImplementation();
427 for (final TExtensibleElements ci : componentInstanceList) {
428 // Determine namespace
429 String namespace = this.getNamespace(ci, defaultNamespace);
430 // Ensure that element has the namespace
431 this.setNamespace(ci, namespace);
434 String id = ModelUtilities.getId(ci);
436 // Determine WineryId
437 Class<? extends TOSCAComponentId> widClass = org.eclipse.winery.repository.Utils.getComponentIdClassForTExtensibleElements(ci.getClass());
438 final TOSCAComponentId wid = BackendUtils.getTOSCAcomponentId(widClass, namespace, id, false);
440 if (Repository.INSTANCE.exists(wid)) {
442 Repository.INSTANCE.forceDelete(wid);
443 String msg = String.format("Deleted %1$s %2$s to enable replacement", ci.getClass().getName(), wid.getQName().toString());
444 CSARImporter.logger.debug(msg);
446 String msg = String.format("Skipped %1$s %2$s, because it already exists", ci.getClass().getName(), wid.getQName().toString());
447 CSARImporter.logger.debug(msg);
448 // this is not displayed in the UI as we currently do not distinguish between pre-existing types and types created during the import.
453 // Create a fresh definitions object without the other data.
454 final Definitions newDefs = BackendUtils.createWrapperDefinitions(wid);
456 // copy over the inputs determined by this.importImports
457 newDefs.getImport().addAll(imports);
459 // add the current TExtensibleElements as the only content to it
460 newDefs.getServiceTemplateOrNodeTypeOrNodeTypeImplementation().add(ci);
462 if (ci instanceof TArtifactTemplate) {
463 // convention: Definitions are stored in the "Definitions" directory, therefore going to levels up (Definitions dir -> root dir) resolves to the root dir
464 // COS01, line 2663 states that the path has to be resolved from the *root* of the CSAR
465 this.adjustArtifactTemplate(defsPath.getParent().getParent(), tmf, (ArtifactTemplateId) wid, (TArtifactTemplate) ci, errors);
466 } else if (ci instanceof TNodeType) {
467 this.adjustNodeType(defsPath.getParent().getParent(), (TNodeType) ci, (NodeTypeId) wid, tmf, errors);
468 } else if (ci instanceof TRelationshipType) {
469 this.adjustRelationshipType(defsPath.getParent().getParent(), (TRelationshipType) ci, (RelationshipTypeId) wid, tmf, errors);
470 } else if (ci instanceof TServiceTemplate) {
471 this.adjustServiceTemplate(defsPath.getParent().getParent(), tmf, (ServiceTemplateId) wid, (TServiceTemplate) ci, errors);
474 // node types and relationship types are subclasses of TEntityType
475 // Therefore, we check the entity type separately here
476 if (ci instanceof TEntityType) {
477 if (asyncWPDParsing) {
478 // Adjusting takes a long time
479 // Therefore, we first save the type as is and convert to Winery-Property-Definitions in the background
480 CSARImporter.storeDefinitions(wid, newDefs);
481 CSARImporter.entityTypeAdjestmentService.submit(new Runnable() {
485 CSARImporter.adjustEntityType((TEntityType) ci, (EntityTypeId) wid, newDefs, errors);
486 CSARImporter.storeDefinitions(wid, newDefs);
490 CSARImporter.adjustEntityType((TEntityType) ci, (EntityTypeId) wid, newDefs, errors);
491 CSARImporter.storeDefinitions(wid, newDefs);
494 CSARImporter.storeDefinitions(wid, newDefs);
500 * Imports the specified types into the repository. The types are converted
501 * to an import statement
503 * @param errors Container for error messages
505 private void importTypes(TDefinitions defs, final List<String> errors) {
506 Types typesContainer = defs.getTypes();
507 if (typesContainer != null) {
508 List<Object> types = typesContainer.getAny();
509 for (Object type : types) {
510 if (type instanceof Element) {
511 Element element = (Element) type;
513 // generate id part of ImportId out of definitions' id
514 // we do not use the name as the name has to be URLencoded again and we have issues with the interplay with org.eclipse.winery.common.ids.definitions.imports.GenericImportId.getId(TImport) then.
515 String id = defs.getId();
516 // try to make the id unique by hashing the "content" of the definition
517 id = id + "-" + Integer.toHexString(element.hashCode());
520 TOSCAComponentId importId;
522 if (element.getNamespaceURI().equals(XMLConstants.W3C_XML_SCHEMA_NS_URI)) {
523 ns = element.getAttribute("targetNamespace");
524 importId = new XSDImportId(ns, id, false);
526 // Quick hack for non-XML-Schema-definitions
528 importId = new GenericImportId(ns, id, false, element.getNamespaceURI());
531 // Following code is adapted from importOtherImports
533 TDefinitions wrapperDefs = BackendUtils.createWrapperDefinitions(importId);
534 TImport imp = new TImport();
535 String fileName = id + ".xsd";
536 imp.setLocation(fileName);
537 imp.setImportType(XMLConstants.W3C_XML_SCHEMA_NS_URI);
538 imp.setNamespace(ns);
539 wrapperDefs.getImport().add(imp);
540 CSARImporter.storeDefinitions(importId, wrapperDefs);
542 // put the file itself to the repo
543 // ref is required to generate fileRef
544 RepositoryFileReference ref = BackendUtils.getRefOfDefinitions(importId);
545 RepositoryFileReference fileRef = new RepositoryFileReference(ref.getParent(), fileName);
546 // convert element to document
547 // QUICK HACK. Alternative: Add new method Repository.INSTANCE.getOutputStream and transform DOM node to OuptputStream
548 String content = Util.getXMLAsString(element);
550 Repository.INSTANCE.putContentToFile(fileRef, content, MediaType.APPLICATION_XML_TYPE);
551 } catch (IOException e) {
552 CSARImporter.logger.debug("Could not put XML Schema definition to file " + fileRef.toString(), e);
553 errors.add("Could not put XML Schema definition to file " + fileRef.toString());
556 // add import to definitions
558 // adapt path - similar to importOtherImport
559 String newLoc = "../" + Utils.getURLforPathInsideRepo(BackendUtils.getPathInsideRepo(fileRef));
560 imp.setLocation(newLoc);
561 defs.getImport().add(imp);
563 // This is a known type. Otherwise JAX-B would render it as Element
564 errors.add("There is a Type of class " + type.getClass().toString() + " which is unknown to Winery. The type element is imported as is");
571 * All EntityTypes may contain properties definition. In case a winery
572 * properties definition is found, the TOSCA conforming properties
573 * definition is removed
575 * @param ci the entity type
576 * @param wid the Winery id of the entitytype
577 * @param newDefs the definitions, the entiy type is contained in. The
578 * imports might be adjusted here
581 private static void adjustEntityType(TEntityType ci, EntityTypeId wid, Definitions newDefs, final List<String> errors) {
582 PropertiesDefinition propertiesDefinition = ci.getPropertiesDefinition();
583 if (propertiesDefinition != null) {
584 WinerysPropertiesDefinition winerysPropertiesDefinition = ModelUtilities.getWinerysPropertiesDefinition(ci);
586 if (winerysPropertiesDefinition == null) {
589 if (winerysPropertiesDefinition.getIsDerivedFromXSD() == null) {
590 // if the winery's properties are defined by Winery itself,
591 // remove the TOSCA conforming properties definition as a Winery properties definition exists (and which takes precedence)
592 ci.setPropertiesDefinition(null);
594 // no derivation from properties required as the properties are generated by Winery
597 // we have to remove the import, too
598 // Determine the location
599 String elementName = winerysPropertiesDefinition.getElementName();
600 String loc = BackendUtils.getImportLocationForWinerysPropertiesDefinitionXSD(wid, null, elementName);
601 // remove the import matching that location
602 List<TImport> imports = newDefs.getImport();
603 boolean found = false;
604 if (imports != null) {
605 Iterator<TImport> iterator = imports.iterator();
607 while (iterator.hasNext()) {
608 imp = iterator.next();
609 // TODO: add check for QNames.QNAME_WINERYS_PROPERTIES_DEFINITION_ATTRIBUTE instead of import location. The current routine, however, works, too.
610 if (imp.getLocation().equals(loc)) {
616 // imp with Winery's k/v location found
618 // the XSD has been imported in importOtherImport
619 // it was too difficult to do the location check there, therefore we just remove the XSD from the repository here
620 XSDImportId importId = new XSDImportId(winerysPropertiesDefinition.getNamespace(), elementName, false);
622 Repository.INSTANCE.forceDelete(importId);
623 } catch (IOException e) {
624 CSARImporter.logger.debug("Could not delete Winery's generated XSD definition", e);
625 errors.add("Could not delete Winery's generated XSD definition");
628 // K/V properties definition was incomplete
632 // winery's properties are derived from an XSD
633 // The export does NOT add an imports statement: only the wpd exists
635 ModelUtilities.removeWinerysPropertiesDefinition(ci);
636 // derive the WPDs again from the properties definition
641 BackendUtils.deriveWPD(ci, errors);
647 * In case plans are provided, the plans are imported into Winery's storage
649 * @param rootPath the root path of the extracted csar
650 * @param tmf the TOSCAMetaFile object used to determine the mime type of
652 * @param wid Winery's internal id of the service template
653 * @param st the the service template to be imported {@inheritDoc}
655 * @throws InvalidCSARException
657 private void adjustServiceTemplate(Path rootPath, TOSCAMetaFile tmf, ServiceTemplateId wid, TServiceTemplate st, final List<String> errors) {
658 TPlans plans = st.getPlans();
660 for (TPlan plan : plans.getPlan()) {
661 PlanModelReference refContainer = plan.getPlanModelReference();
662 if (refContainer != null) {
663 String ref = refContainer.getReference();
665 // URLs are stored encoded -> undo the encoding
666 ref = Util.URLdecode(ref);
669 refURI = new URI(ref);
670 } catch (URISyntaxException e) {
671 errors.add(String.format("Invalid URI %1$s", ref));
674 if (refURI.isAbsolute()) {
675 // Points to somewhere external
676 // This is a linked plan
677 // We have to do nothing
680 Path path = rootPath.resolve(ref);
681 if (!Files.exists(path)) {
682 // possibly, the reference is relative to the Definitions subfolder
683 // COS01 does not make any explicit statement how to resolve the reference here
684 path = rootPath.resolve("Definitions").resolve(ref);
685 if (!Files.exists(path)) {
686 errors.add(String.format("Plan reference %1$s not found", ref));
687 // we quickly remove the reference to reflect the not-found in the data
688 refContainer.setReference(null);
692 PlansId plansId = new PlansId(wid);
693 PlanId pid = new PlanId(plansId, new XMLId(plan.getId(), false));
694 if (Files.isDirectory(path)) {
695 errors.add(String.format("Reference %1$s is a directory and Winery currently does not support importing directories", ref));
698 RepositoryFileReference fref = new RepositoryFileReference(pid, path.getFileName().toString());
699 this.importFile(path, fref, tmf, rootPath, errors);
702 // Adjust the reference
703 refContainer.setReference("../" + Utils.getURLforPathInsideRepo(BackendUtils.getPathInsideRepo(fref)));
711 * Adds a color to the given relationship type
713 private void adjustRelationshipType(Path rootPath, TRelationshipType ci, RelationshipTypeId wid, TOSCAMetaFile tmf, final List<String> errors) {
714 VisualAppearanceId visId = new VisualAppearanceId(wid);
715 this.importIcons(rootPath, visId, tmf, errors);
718 private void adjustNodeType(Path rootPath, TNodeType ci, NodeTypeId wid, TOSCAMetaFile tmf, final List<String> errors) {
719 VisualAppearanceId visId = new VisualAppearanceId(wid);
720 this.importIcons(rootPath, visId, tmf, errors);
723 private void importIcons(Path rootPath, VisualAppearanceId visId, TOSCAMetaFile tmf, final List<String> errors) {
724 String pathInsideRepo = BackendUtils.getPathInsideRepo(visId);
725 Path visPath = rootPath.resolve(pathInsideRepo);
726 this.importIcon(visId, visPath, Filename.FILENAME_BIG_ICON, tmf, rootPath, errors);
729 private void importIcon(VisualAppearanceId visId, Path visPath, String fileName, TOSCAMetaFile tmf, Path rootPath, final List<String> errors) {
730 Path file = visPath.resolve(fileName);
731 if (Files.exists(file)) {
732 RepositoryFileReference ref = new RepositoryFileReference(visId, fileName);
733 this.importFile(file, ref, tmf, rootPath, errors);
738 * Adjusts the given artifact template to conform with the repository format
740 * We import the files given at the artifact references
742 * @throws InvalidCSARException
743 * @throws IOException
745 private void adjustArtifactTemplate(Path rootPath, TOSCAMetaFile tmf, ArtifactTemplateId atid, TArtifactTemplate ci, final List<String> errors) throws IOException {
746 ArtifactReferences refs = ci.getArtifactReferences();
748 // no references stored - break
751 List<TArtifactReference> refList = refs.getArtifactReference();
752 Iterator<TArtifactReference> iterator = refList.iterator();
753 while (iterator.hasNext()) {
754 TArtifactReference ref = iterator.next();
755 String reference = ref.getReference();
756 // URLs are stored encoded -> undo the encoding
757 reference = Util.URLdecode(reference);
761 refURI = new URI(reference);
762 } catch (URISyntaxException e) {
763 errors.add(String.format("Invalid URI %1$s", ref));
766 if (refURI.isAbsolute()) {
767 // Points to somewhere external
768 // We have to do nothing
772 // we remove the current element as it will be handled during the export
775 Path path = rootPath.resolve(reference);
776 if (!Files.exists(path)) {
777 errors.add(String.format("Reference %1$s not found", reference));
781 if (Files.isRegularFile(path)) {
782 allFiles = new HashSet<Path>();
785 assert (Files.isDirectory(path));
786 Path localRoot = rootPath.resolve(path);
787 List<Object> includeOrExclude = ref.getIncludeOrExclude();
789 if (includeOrExclude.get(0) instanceof TArtifactReference.Exclude) {
790 // Implicit semantics of an exclude listed first:
791 // include all files and then exclude the files matched by the pattern
792 allFiles = this.getAllFiles(localRoot);
794 // semantics if include lited as first:
795 // same as listed at other places
796 allFiles = new HashSet<>();
799 for (Object object : includeOrExclude) {
800 if (object instanceof TArtifactReference.Include) {
801 this.handleInclude((TArtifactReference.Include) object, localRoot, allFiles);
803 assert (object instanceof TArtifactReference.Exclude);
804 this.handleExclude((TArtifactReference.Exclude) object, localRoot, allFiles);
808 this.importAllFiles(allFiles, atid, tmf, rootPath, errors);
811 if (refList.isEmpty()) {
812 // everything is imported and is a file stored locally
813 // we don't need the references stored locally: they are generated on the fly when exporting
814 ci.setArtifactReferences(null);
819 * Imports a file from the filesystem to the repository
821 * @param p the file to read from
822 * @param fref the "file" to put the content to
823 * @param tmf the TOSCAMetaFile object used to determine the mimetype. Must
825 * @param rootPath used to relativize p to determine the mime type
826 * @throws InvalidCSARException
828 private void importFile(Path p, RepositoryFileReference fref, TOSCAMetaFile tmf, Path rootPath, final List<String> errors) {
830 throw new IllegalStateException("tmf must not be null");
832 try (InputStream is = Files.newInputStream(p);
833 BufferedInputStream bis = new BufferedInputStream(is)) {
834 String mediaType = tmf.getMimeType(p.relativize(rootPath).toString());
835 if (mediaType == null) {
836 // Manually find out mime type
838 mediaType = Utils.getMimeType(bis, p.getFileName().toString());
839 } catch (IOException e) {
840 errors.add(String.format("No MimeType given for %1$s (%2$s)", p.getFileName(), e.getMessage()));
843 if (mediaType == null) {
844 errors.add(String.format("No MimeType given for %1$s", p.getFileName()));
849 Repository.INSTANCE.putContentToFile(fref, bis, MediaType.valueOf(mediaType));
850 } catch (IllegalArgumentException | IOException e) {
851 throw new IllegalStateException(e);
853 } catch (IOException e1) {
854 throw new IllegalStateException("Could not work on generated temporary files", e1);
858 private void importAllFiles(Collection<Path> allFiles, ArtifactTemplateId atid, TOSCAMetaFile tmf, Path rootPath, final List<String> errors) {
859 // import all files to repository
860 ArtifactTemplateDirectoryId fileDir = new ArtifactTemplateDirectoryId(atid);
861 for (Path p : allFiles) {
862 if (!Files.exists(p)) {
863 errors.add(String.format("File %1$s does not exist", p.toString()));
866 RepositoryFileReference fref = new RepositoryFileReference(fileDir, p.getFileName().toString());
867 this.importFile(p, fref, tmf, rootPath, errors);
873 * Modifies given allFiles object to exclude all files given by the excl
876 * Semantics: Remove all files from the set, which match the given pattern
878 private void handleExclude(Exclude excl, Path localRoot, Set<Path> allFiles) {
879 PathMatcher pathMatcher = localRoot.getFileSystem().getPathMatcher("glob:" + excl.getPattern());
880 Iterator<Path> it = allFiles.iterator();
881 while (it.hasNext()) {
882 Path curPath = it.next();
883 if (pathMatcher.matches(curPath)) {
890 * Modifies given allFiles object to include all files given by the incl
893 * Semantics: Add all files from localRoot to allFiles matching the pattern
895 private void handleInclude(final Include incl, final Path localRoot, final Set<Path> allFiles) {
896 final PathMatcher pathMatcher = localRoot.getFileSystem().getPathMatcher("glob:" + incl.getPattern());
898 Files.walkFileTree(localRoot, new SimpleFileVisitor<Path>() {
901 public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
902 Path relFile = localRoot.relativize(file);
903 if (pathMatcher.matches(relFile)) {
910 public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
911 if (pathMatcher.matches(dir)) {
912 Set<Path> filesToAdd = CSARImporter.this.getAllFiles(dir);
913 allFiles.addAll(filesToAdd);
920 } catch (IOException e) {
921 throw new IllegalStateException(e);
926 * Lists all files contained in the given path
928 private Set<Path> getAllFiles(Path startPath) {
929 final Set<Path> res = new HashSet<>();
931 Files.walkFileTree(startPath, new SimpleFileVisitor<Path>() {
934 public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
939 } catch (IOException e) {
940 throw new IllegalStateException(e);
946 * Sets the namespace on the CI if CI offers the method "setTargetNamespace"
948 * @param ci the component instance to set the namespace
949 * @param namespace the namespace to set
951 private void setNamespace(TExtensibleElements ci, String namespace) {
954 method = ci.getClass().getMethod("setTargetNamespace", String.class);
955 method.invoke(ci, namespace);
956 } catch (NoSuchMethodException ne) {
957 // this is OK, because we do not check, whether the method really exists
958 // Special case for TArtifactTemplate not offering setTargetNamespace
960 } catch (Exception e) {
961 throw new IllegalStateException("Could not set target namespace", e);
966 * @param ci the component instance to get the namespace from
967 * @param defaultNamespace the namespace to use if the TExtensibleElements
968 * has no targetNamespace
970 private String getNamespace(TExtensibleElements ci, String defaultNamespace) {
974 method = ci.getClass().getMethod("getTargetNamespace");
975 res = method.invoke(ci);
976 } catch (Exception e) {
977 // we are at TArtifactTemplate, which does not offer getTargetNamespace
980 String ns = (String) res;
982 ns = defaultNamespace;
988 * @param basePath the base path where to resolve files from. This is the
989 * directory of the Definitions
990 * @param imports the list of imports to import. SIDE EFFECT: this list is
991 * modified. After this method has run, the list contains the
992 * imports to be put into the wrapper element
994 private void importImports(Path basePath, TOSCAMetaFile tmf, List<TImport> imports, final List<String> errors, boolean overwrite, final boolean asyncWPDParsing) throws IOException {
995 for (Iterator<TImport> iterator = imports.iterator(); iterator.hasNext();) {
996 TImport imp = iterator.next();
997 String importType = imp.getImportType();
998 String namespace = imp.getNamespace();
999 String loc = imp.getLocation();
1001 if (namespace == null) {
1002 errors.add("not namespace-qualified imports are not supported.");
1007 errors.add("Empty location imports are not supported.");
1009 if (importType.equals(Namespaces.TOSCA_NAMESPACE)) {
1010 if (!Util.isRelativeURI(loc)) {
1011 errors.add("Absolute URIs for definitions import not supported.");
1016 loc = Util.URLdecode(loc);
1018 Path defsPath = basePath.resolve(loc);
1019 // fallback for older CSARs, where the location is given from the root
1020 if (!Files.exists(defsPath)) {
1021 defsPath = basePath.getParent().resolve(loc);
1022 // the real existence check is done in importDefinitions
1024 this.importDefinitions(tmf, defsPath, errors, overwrite, asyncWPDParsing);
1025 // imports of definitions don't have to be kept as these are managed by Winery
1028 this.importOtherImport(basePath, imp, errors, importType, overwrite);
1035 * SIDE EFFECT: modifies the location of imp to point to the correct
1036 * relative location (when read from the exported CSAR)
1038 * @param rootPath the absolute path where to resolve files from
1040 private void importOtherImport(Path rootPath, TImport imp, final List<String> errors, String type, boolean overwrite) {
1041 assert (!type.equals(Namespaces.TOSCA_NAMESPACE));
1042 String loc = imp.getLocation();
1044 if (!Util.isRelativeURI(loc)) {
1045 // This is just an information message
1046 errors.add("Absolute URIs are not resolved by Winery (" + loc + ")");
1050 // location URLs are encoded: http://www.w3.org/TR/2001/WD-charmod-20010126/#sec-URIs, RFC http://www.ietf.org/rfc/rfc2396.txt
1051 loc = Util.URLdecode(loc);
1054 path = rootPath.resolve(loc);
1055 } catch (Exception e) {
1056 // java.nio.file.InvalidPathException could be thrown which is a RuntimeException
1057 errors.add(e.getMessage());
1060 if (!Files.exists(path)) {
1061 // fallback for older CSARs, where the location is given from the root
1062 path = rootPath.getParent().resolve(loc);
1063 if (!Files.exists(path)) {
1064 errors.add(String.format("File %1$s does not exist", loc));
1068 String namespace = imp.getNamespace();
1069 String fileName = path.getFileName().toString();
1070 String id = fileName;
1071 id = FilenameUtils.removeExtension(id);
1072 // Convention: id of import is filename without extension
1074 GenericImportId rid;
1075 if (type.equals(XMLConstants.W3C_XML_SCHEMA_NS_URI)) {
1076 rid = new XSDImportId(namespace, id, false);
1078 rid = new GenericImportId(namespace, id, false, type);
1081 boolean importDataExistsInRepo = Repository.INSTANCE.exists(rid);
1083 if (!importDataExistsInRepo) {
1085 // a) create a .definitions file
1086 // b) put the file itself in the repo
1088 // Create the definitions file
1089 TDefinitions defs = BackendUtils.createWrapperDefinitions(rid);
1090 defs.getImport().add(imp);
1091 // QUICK HACK: We change the imp object's location here and below again
1092 // This is "OK" as "storeDefinitions" serializes the current state and not the future state of the imp object
1093 // change the location to point to the file in the folder of the .definitions file
1094 imp.setLocation(fileName);
1096 // put the definitions file to the repository
1097 CSARImporter.storeDefinitions(rid, defs);
1100 // put the file itself to the repo
1101 // ref is required to generate fileRef
1102 RepositoryFileReference ref = BackendUtils.getRefOfDefinitions(rid);
1103 RepositoryFileReference fileRef = new RepositoryFileReference(ref.getParent(), fileName);
1105 // location is relative to Definitions/
1106 // even if the import already exists, we have to adapt the path
1108 String newLoc = "../" + Utils.getURLforPathInsideRepo(BackendUtils.getPathInsideRepo(fileRef));
1109 imp.setLocation(newLoc);
1111 if (!importDataExistsInRepo || overwrite) {
1112 // finally write the file to the storage
1113 try (InputStream is = Files.newInputStream(path);
1114 BufferedInputStream bis = new BufferedInputStream(is)) {
1115 MediaType mediaType;
1116 if (type.equals(XMLConstants.W3C_XML_SCHEMA_NS_URI)) {
1117 mediaType = MediaType.valueOf(MimeTypes.MIMETYPE_XSD);
1119 String mimeType = Utils.getMimeType(bis, path.getFileName().toString());
1120 mediaType = MediaType.valueOf(mimeType);
1122 Repository.INSTANCE.putContentToFile(fileRef, bis, mediaType);
1123 } catch (IllegalArgumentException | IOException e) {
1124 throw new IllegalStateException(e);
1127 // we have to update the cache in case of a new XSD to speedup usage of winery
1128 if (rid instanceof XSDImportId) {
1129 // We do the initialization asynchronously
1130 // We do not check whether the XSD has already been checked
1131 // We cannot just checck whether an XSD already has been handled since the XSD could change over time
1132 // Synchronization at org.eclipse.winery.repository.resources.imports.xsdimports.XSDImportResource.getAllDefinedLocalNames(short) also isn't feasible as the backend doesn't support locks
1133 CSARImporter.xsdParsingService.submit(new Runnable() {
1137 CSARImporter.logger.debug("Updating XSD import cache data");
1138 // We call the queries without storing the result:
1139 // We use the SIDEEFFECT that a cache is created
1140 Utils.getAllXSDElementDefinitionsForTypeAheadSelection();
1141 Utils.getAllXSDTypeDefinitionsForTypeAheadSelection();
1142 CSARImporter.logger.debug("Updated XSD import cache data");
1149 private static void storeDefinitions(TOSCAComponentId id, TDefinitions defs) {
1150 RepositoryFileReference ref = BackendUtils.getRefOfDefinitions(id);
1151 String s = Utils.getXMLAsString(defs, true);
1153 Repository.INSTANCE.putContentToFile(ref, s, MediaType.valueOf(MimeTypes.MIMETYPE_TOSCA_DEFINITIONS));
1154 } catch (IllegalArgumentException | IOException e) {
1155 throw new IllegalStateException(e);