1 /*******************************************************************************
2 * Copyright (c) 2012-2014 University of Stuttgart.
3 * All rights reserved. This program and the accompanying materials
4 * are made available under the terms of the Eclipse Public License v1.0
5 * and the Apache License 2.0 which both accompany this distribution,
6 * and are available at http://www.eclipse.org/legal/epl-v10.html
7 * and http://www.apache.org/licenses/LICENSE-2.0
10 * Oliver Kopp - initial API and implementation
11 *******************************************************************************/
12 package org.eclipse.winery.repository.backend.filebased;
15 import java.io.IOException;
16 import java.io.InputStream;
17 import java.io.OutputStream;
18 import java.io.Reader;
19 import java.lang.reflect.Constructor;
20 import java.lang.reflect.InvocationTargetException;
21 import java.nio.charset.Charset;
22 import java.nio.file.DirectoryStream;
23 import java.nio.file.FileSystem;
24 import java.nio.file.FileVisitResult;
25 import java.nio.file.Files;
26 import java.nio.file.NoSuchFileException;
27 import java.nio.file.Path;
28 import java.nio.file.SimpleFileVisitor;
29 import java.nio.file.StandardCopyOption;
30 import java.nio.file.attribute.BasicFileAttributes;
31 import java.nio.file.attribute.FileTime;
32 import java.nio.file.spi.FileSystemProvider;
33 import java.util.Collection;
34 import java.util.Date;
35 import java.util.HashSet;
36 import java.util.SortedSet;
37 import java.util.TreeSet;
38 import java.util.zip.ZipEntry;
39 import java.util.zip.ZipInputStream;
40 import java.util.zip.ZipOutputStream;
42 import javax.ws.rs.core.MediaType;
44 import org.apache.commons.configuration.Configuration;
45 import org.apache.commons.configuration.ConfigurationException;
46 import org.apache.commons.configuration.PropertiesConfiguration;
47 import org.apache.commons.lang3.SystemUtils;
48 import org.eclipse.winery.common.RepositoryFileReference;
49 import org.eclipse.winery.common.Util;
50 import org.eclipse.winery.common.ids.GenericId;
51 import org.eclipse.winery.common.ids.Namespace;
52 import org.eclipse.winery.common.ids.XMLId;
53 import org.eclipse.winery.common.ids.definitions.ArtifactTemplateId;
54 import org.eclipse.winery.common.ids.definitions.ArtifactTypeId;
55 import org.eclipse.winery.common.ids.definitions.CapabilityTypeId;
56 import org.eclipse.winery.common.ids.definitions.NodeTypeId;
57 import org.eclipse.winery.common.ids.definitions.NodeTypeImplementationId;
58 import org.eclipse.winery.common.ids.definitions.PolicyTemplateId;
59 import org.eclipse.winery.common.ids.definitions.PolicyTypeId;
60 import org.eclipse.winery.common.ids.definitions.RelationshipTypeId;
61 import org.eclipse.winery.common.ids.definitions.RelationshipTypeImplementationId;
62 import org.eclipse.winery.common.ids.definitions.RequirementTypeId;
63 import org.eclipse.winery.common.ids.definitions.ServiceTemplateId;
64 import org.eclipse.winery.common.ids.definitions.TOSCAComponentId;
65 import org.eclipse.winery.common.ids.elements.TOSCAElementId;
66 import org.eclipse.winery.repository.Constants;
67 import org.eclipse.winery.repository.backend.AbstractRepository;
68 import org.eclipse.winery.repository.backend.BackendUtils;
69 import org.eclipse.winery.repository.backend.IRepository;
70 import org.eclipse.winery.repository.backend.IRepositoryAdministration;
71 import org.slf4j.Logger;
72 import org.slf4j.LoggerFactory;
75 * When it comes to a storage of plain files, we use Java 7's nio internally.
76 * Therefore, we intend to expose the stream types offered by java.nio.Files:
77 * BufferedReader/BufferedWriter
79 public class FilebasedRepository extends AbstractRepository implements IRepository, IRepositoryAdministration {
81 private static final Logger logger = LoggerFactory.getLogger(FilebasedRepository.class);
83 protected final Path repositoryRoot;
85 // convenience variables to have a clean code
86 private final FileSystem fileSystem;
87 private final FileSystemProvider provider;
90 private Path makeAbsolute(Path relativePath) {
91 return this.repositoryRoot.resolve(relativePath);
95 public boolean flagAsExisting(GenericId id) {
96 Path path = this.id2AbsolutePath(id);
98 FileUtils.createDirectory(path);
99 } catch (IOException e) {
100 FilebasedRepository.logger.debug(e.toString());
106 private Path id2AbsolutePath(GenericId id) {
107 Path relativePath = this.fileSystem.getPath(BackendUtils.getPathInsideRepo(id));
108 return this.makeAbsolute(relativePath);
112 * Converts the given reference to an absolute path of the underlying
115 public Path ref2AbsolutePath(RepositoryFileReference ref) {
116 return this.id2AbsolutePath(ref.getParent()).resolve(ref.getFileName());
121 * @param repositoryLocation a string pointing to a location on the file
122 * system. May be null.
124 public FilebasedRepository(String repositoryLocation) {
125 this.repositoryRoot = this.determineRepositoryPath(repositoryLocation);
126 this.fileSystem = this.repositoryRoot.getFileSystem();
127 this.provider = this.fileSystem.provider();
130 private Path determineRepositoryPath(String repositoryLocation) {
132 if (repositoryLocation == null) {
133 if (SystemUtils.IS_OS_WINDOWS) {
134 if (new File(Constants.GLOBAL_REPO_PATH_WINDOWS).exists()) {
135 repositoryLocation = Constants.GLOBAL_REPO_PATH_WINDOWS;
136 File repo = new File(repositoryLocation);
138 org.apache.commons.io.FileUtils.forceMkdir(repo);
139 } catch (IOException e) {
140 FilebasedRepository.logger.error("Could not create repository directory", e);
142 repositoryPath = repo.toPath();
144 repositoryPath = this.createDefaultRepositoryPath();
147 repositoryPath = this.createDefaultRepositoryPath();
150 File repo = new File(repositoryLocation);
152 org.apache.commons.io.FileUtils.forceMkdir(repo);
153 } catch (IOException e) {
154 FilebasedRepository.logger.error("Could not create repository directory", e);
156 repositoryPath = repo.toPath();
158 return repositoryPath;
161 public static File getDefaultRepositoryFilePath() {
162 return new File(org.apache.commons.io.FileUtils.getUserDirectory(), Constants.DEFAULT_REPO_NAME);
165 private Path createDefaultRepositoryPath() {
167 boolean operationalFileSystemAccess;
169 repo = FilebasedRepository.getDefaultRepositoryFilePath();
170 operationalFileSystemAccess = true;
171 } catch (NullPointerException e) {
172 // it seems, we run at a system, where we do not have any filesystem
174 operationalFileSystemAccess = false;
177 // operationalFileSystemAccess = false;
180 if (operationalFileSystemAccess) {
182 org.apache.commons.io.FileUtils.forceMkdir(repo);
183 } catch (IOException e) {
184 FilebasedRepository.logger.error("Could not create directory", e);
186 repositoryPath = repo.toPath();
188 assert (!operationalFileSystemAccess);
189 // we do not have access to the file system
190 throw new IllegalStateException("No write access to file system");
193 return repositoryPath;
197 public void forceDelete(RepositoryFileReference ref) throws IOException {
198 Path relativePath = this.fileSystem.getPath(BackendUtils.getPathInsideRepo(ref));
199 Path fileToDelete = this.makeAbsolute(relativePath);
201 this.provider.delete(fileToDelete);
202 // Quick hack for deletion of the mime type information
203 // Alternative: superclass: protected void deleteMimeTypeInformation(RepositoryFileReference ref) throws IOException
204 // However, this would again call this method, where we would have to check for the extension, too.
205 // Therefore, we directly delete the information file
206 Path mimeTypeFile = fileToDelete.getParent().resolve(ref.getFileName() + Constants.SUFFIX_MIMETYPE);
207 this.provider.delete(mimeTypeFile);
208 } catch (IOException e) {
209 if (!(e instanceof NoSuchFileException)) {
210 // only if file did exist and something else went wrong: complain :)
211 // (otherwise, silently ignore the error)
212 FilebasedRepository.logger.debug("Could not delete file", e);
219 public void forceDelete(GenericId id) throws IOException {
221 FileUtils.forceDelete(this.id2AbsolutePath(id));
222 } catch (IOException e) {
223 FilebasedRepository.logger.debug("Could not delete id", id);
229 public boolean exists(GenericId id) {
230 Path absolutePath = this.id2AbsolutePath(id);
231 boolean result = Files.exists(absolutePath);
239 public void putContentToFile(RepositoryFileReference ref, String content, MediaType mediaType) throws IOException {
240 if (mediaType == null) {
241 // quick hack for storing mime type called this method
242 assert (ref.getFileName().endsWith(Constants.SUFFIX_MIMETYPE));
243 // we do not need to store the mime type of the file containing the mime type information
245 this.setMimeType(ref, mediaType);
247 Path path = this.ref2AbsolutePath(ref);
248 FileUtils.createDirectory(path.getParent());
249 Files.write(path, content.getBytes());
256 public void putContentToFile(RepositoryFileReference ref, InputStream inputStream, MediaType mediaType) throws IOException {
257 if (mediaType == null) {
258 // quick hack for storing mime type called this method
259 assert (ref.getFileName().endsWith(Constants.SUFFIX_MIMETYPE));
260 // we do not need to store the mime type of the file containing the mime type information
262 this.setMimeType(ref, mediaType);
264 Path targetPath = this.ref2AbsolutePath(ref);
265 // ensure that parent directory exists
266 FileUtils.createDirectory(targetPath.getParent());
269 Files.copy(inputStream, targetPath, StandardCopyOption.REPLACE_EXISTING);
270 } catch (IllegalStateException e) {
271 FilebasedRepository.logger.debug("Guessing that stream with length 0 is to be written to a file", e);
272 // copy throws an "java.lang.IllegalStateException: Stream already closed" if the InputStream contains 0 bytes
273 // For instance, this case happens if SugarCE-6.4.2.zip.removed is tried to be uploaded
274 // We work around the Java7 issue and create an empty file
275 if (Files.exists(targetPath)) {
276 // semantics of putContentToFile: existing content is replaced without notification
277 Files.delete(targetPath);
279 Files.createFile(targetPath);
284 public boolean exists(RepositoryFileReference ref) {
285 return Files.exists(this.ref2AbsolutePath(ref));
289 public <T extends TOSCAComponentId> SortedSet<T> getAllTOSCAComponentIds(Class<T> idClass) {
290 SortedSet<T> res = new TreeSet<T>();
291 String rootPathFragment = Util.getRootPathFragment(idClass);
292 Path dir = this.repositoryRoot.resolve(rootPathFragment);
293 if (!Files.exists(dir)) {
294 // return empty list if no ids are available
297 assert (Files.isDirectory(dir));
299 final OnlyNonHiddenDirectories onhdf = new OnlyNonHiddenDirectories();
301 // list all directories contained in this directory
302 try (DirectoryStream<Path> ds = Files.newDirectoryStream(dir, onhdf)) {
303 for (Path nsP : ds) {
304 // the current path is the namespace
305 Namespace ns = new Namespace(nsP.getFileName().toString(), true);
306 try (DirectoryStream<Path> idDS = Files.newDirectoryStream(nsP, onhdf)) {
307 for (Path idP : idDS) {
308 XMLId xmlId = new XMLId(idP.getFileName().toString(), true);
309 Constructor<T> constructor;
311 constructor = idClass.getConstructor(Namespace.class, XMLId.class);
312 } catch (Exception e) {
313 FilebasedRepository.logger.debug("Internal error at determining id constructor", e);
314 // abort everything, return invalid result
319 id = constructor.newInstance(ns, xmlId);
320 } catch (InstantiationException
321 | IllegalAccessException
322 | IllegalArgumentException
323 | InvocationTargetException e) {
324 FilebasedRepository.logger.debug("Internal error at invocation of id constructor", e);
325 // abort everything, return invalid result
332 } catch (IOException e) {
333 FilebasedRepository.logger.debug("Cannot close ds", e);
340 public SortedSet<RepositoryFileReference> getContainedFiles(GenericId id) {
341 Path dir = this.id2AbsolutePath(id);
342 SortedSet<RepositoryFileReference> res = new TreeSet<RepositoryFileReference>();
343 if (!Files.exists(dir)) {
346 assert (Files.isDirectory(dir));
347 // list all directories contained in this directory
348 try (DirectoryStream<Path> ds = Files.newDirectoryStream(dir, new OnlyNonHiddenFiles())) {
350 RepositoryFileReference ref = new RepositoryFileReference(id, p.getFileName().toString());
353 } catch (IOException e) {
354 FilebasedRepository.logger.debug("Cannot close ds", e);
360 public Configuration getConfiguration(RepositoryFileReference ref) {
361 Path path = this.ref2AbsolutePath(ref);
363 PropertiesConfiguration configuration = new PropertiesConfiguration();
364 if (Files.exists(path)) {
365 try (Reader r = Files.newBufferedReader(path, Charset.defaultCharset())) {
366 configuration.load(r);
367 } catch (ConfigurationException | IOException e) {
368 FilebasedRepository.logger.error("Could not read config file", e);
369 throw new IllegalStateException("Could not read config file", e);
373 configuration.addConfigurationListener(new AutoSaveListener(path, configuration));
375 // We do NOT implement reloading as the configuration is only accessed
376 // in JAX-RS resources, which are created on a per-request basis
378 return configuration;
382 * @return null if an error occurred
385 public Date getLastUpdate(RepositoryFileReference ref) {
386 Path path = this.ref2AbsolutePath(ref);
388 if (Files.exists(path)) {
389 FileTime lastModifiedTime;
391 lastModifiedTime = Files.getLastModifiedTime(path);
392 res = new Date(lastModifiedTime.toMillis());
393 } catch (IOException e) {
394 FilebasedRepository.logger.debug(e.getMessage(), e);
398 // this branch is taken if the resource directory exists, but the
399 // configuration itself does not exist.
400 // For instance, this happens if icons are manually put for a node
401 // type, but no color configuration is made.
402 res = Constants.LASTMODIFIEDDATE_FOR_404;
408 public <T extends TOSCAElementId> SortedSet<T> getNestedIds(GenericId ref, Class<T> idClass) {
409 Path dir = this.id2AbsolutePath(ref);
410 SortedSet<T> res = new TreeSet<T>();
411 if (!Files.exists(dir)) {
412 // the id has been generated by the exporter without existance test.
413 // This test is done here.
416 assert (Files.isDirectory(dir));
417 // list all directories contained in this directory
418 try (DirectoryStream<Path> ds = Files.newDirectoryStream(dir, new OnlyNonHiddenDirectories())) {
420 XMLId xmlId = new XMLId(p.getFileName().toString(), true);
421 @SuppressWarnings("unchecked")
422 Constructor<T>[] constructors = (Constructor<T>[]) idClass.getConstructors();
423 assert (constructors.length == 1);
424 Constructor<T> constructor = constructors[0];
425 assert (constructor.getParameterTypes().length == 2);
428 id = constructor.newInstance(ref, xmlId);
429 } catch (InstantiationException | IllegalAccessException
430 | IllegalArgumentException | InvocationTargetException e) {
431 FilebasedRepository.logger.debug("Internal error at invocation of id constructor", e);
432 // abort everything, return invalid result
437 } catch (IOException e) {
438 FilebasedRepository.logger.debug("Cannot close ds", e);
444 // below, toscaComponents is an array, which is used in an iterator
445 // As Java does not allow generic arrays, we have to suppress the warning when fetching an element out of the list
446 @SuppressWarnings("unchecked")
447 public Collection<Namespace> getUsedNamespaces() {
449 @SuppressWarnings("rawtypes")
450 Class[] toscaComponentIds = {
451 ArtifactTemplateId.class,
452 ArtifactTypeId.class,
453 CapabilityTypeId.class,
455 NodeTypeImplementationId.class,
456 PolicyTemplateId.class,
458 RelationshipTypeId.class,
459 RelationshipTypeImplementationId.class,
460 RequirementTypeId.class,
461 ServiceTemplateId.class
465 // we use a HashSet to avoid reporting duplicate namespaces
466 Collection<Namespace> res = new HashSet<Namespace>();
468 for (Class<? extends TOSCAComponentId> id : toscaComponentIds) {
469 String rootPathFragment = Util.getRootPathFragment(id);
470 Path dir = this.repositoryRoot.resolve(rootPathFragment);
471 if (!Files.exists(dir)) {
474 assert (Files.isDirectory(dir));
476 final OnlyNonHiddenDirectories onhdf = new OnlyNonHiddenDirectories();
478 // list all directories contained in this directory
479 try (DirectoryStream<Path> ds = Files.newDirectoryStream(dir, onhdf)) {
480 for (Path nsP : ds) {
481 // the current path is the namespace
482 Namespace ns = new Namespace(nsP.getFileName().toString(), true);
485 } catch (IOException e) {
486 FilebasedRepository.logger.debug("Cannot close ds", e);
493 public void doDump(OutputStream out) throws IOException {
494 final ZipOutputStream zout = new ZipOutputStream(out);
495 final int cutLength = this.repositoryRoot.toString().length() + 1;
497 Files.walkFileTree(this.repositoryRoot, new SimpleFileVisitor<Path>() {
500 public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) {
501 if (dir.endsWith(".git")) {
502 return FileVisitResult.SKIP_SUBTREE;
504 return FileVisitResult.CONTINUE;
509 public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) {
510 String name = file.toString().substring(cutLength);
511 ZipEntry ze = new ZipEntry(name);
513 ze.setTime(Files.getLastModifiedTime(file).toMillis());
514 ze.setSize(Files.size(file));
515 zout.putNextEntry(ze);
516 Files.copy(file, zout);
518 } catch (IOException e) {
519 FilebasedRepository.logger.debug(e.getMessage());
521 return FileVisitResult.CONTINUE;
528 * Removes all files and dirs except the .git directory
531 public void doClear() {
533 DirectoryStream.Filter<Path> noGitDirFilter = new DirectoryStream.Filter<Path>() {
536 public boolean accept(Path entry) throws IOException {
537 return !(entry.getFileName().toString().equals(".git"));
541 DirectoryStream<Path> ds = Files.newDirectoryStream(this.repositoryRoot, noGitDirFilter);
543 FileUtils.forceDelete(p);
545 } catch (IOException e) {
546 FilebasedRepository.logger.error(e.getMessage());
552 public void doImport(InputStream in) {
553 ZipInputStream zis = new ZipInputStream(in);
556 while ((entry = zis.getNextEntry()) != null) {
557 if (!entry.isDirectory()) {
558 Path path = this.repositoryRoot.resolve(entry.getName());
559 FileUtils.createDirectory(path.getParent());
560 Files.copy(zis, path);
563 } catch (IOException e) {
564 FilebasedRepository.logger.error(e.getMessage());
572 public long getSize(RepositoryFileReference ref) throws IOException {
573 return Files.size(this.ref2AbsolutePath(ref));
580 public FileTime getLastModifiedTime(RepositoryFileReference ref) throws IOException {
581 Path path = this.ref2AbsolutePath(ref);
582 FileTime res = Files.getLastModifiedTime(path);
590 public InputStream newInputStream(RepositoryFileReference ref) throws IOException {
591 Path path = this.ref2AbsolutePath(ref);
592 InputStream res = Files.newInputStream(path);