696468144fb3913294b07345cf1afb8d83ab7454
[vfc/nfvo/wfengine.git] /
1 /*******************************************************************************
2  * Copyright (c) 2012-2014 University of Stuttgart.
3  * All rights reserved. This program and the accompanying materials
4  * are made available under the terms of the Eclipse Public License v1.0
5  * and the Apache License 2.0 which both accompany this distribution,
6  * and are available at http://www.eclipse.org/legal/epl-v10.html
7  * and http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Contributors:
10  *     Oliver Kopp - initial API and implementation
11  *******************************************************************************/
12 package org.eclipse.winery.repository.backend.filebased;
13
14 import java.io.File;
15 import java.io.IOException;
16 import java.io.InputStream;
17 import java.io.OutputStream;
18 import java.io.Reader;
19 import java.lang.reflect.Constructor;
20 import java.lang.reflect.InvocationTargetException;
21 import java.nio.charset.Charset;
22 import java.nio.file.DirectoryStream;
23 import java.nio.file.FileSystem;
24 import java.nio.file.FileVisitResult;
25 import java.nio.file.Files;
26 import java.nio.file.NoSuchFileException;
27 import java.nio.file.Path;
28 import java.nio.file.SimpleFileVisitor;
29 import java.nio.file.StandardCopyOption;
30 import java.nio.file.attribute.BasicFileAttributes;
31 import java.nio.file.attribute.FileTime;
32 import java.nio.file.spi.FileSystemProvider;
33 import java.util.Collection;
34 import java.util.Date;
35 import java.util.HashSet;
36 import java.util.SortedSet;
37 import java.util.TreeSet;
38 import java.util.zip.ZipEntry;
39 import java.util.zip.ZipInputStream;
40 import java.util.zip.ZipOutputStream;
41
42 import javax.ws.rs.core.MediaType;
43
44 import org.apache.commons.configuration.Configuration;
45 import org.apache.commons.configuration.ConfigurationException;
46 import org.apache.commons.configuration.PropertiesConfiguration;
47 import org.apache.commons.lang3.SystemUtils;
48 import org.eclipse.winery.common.RepositoryFileReference;
49 import org.eclipse.winery.common.Util;
50 import org.eclipse.winery.common.ids.GenericId;
51 import org.eclipse.winery.common.ids.Namespace;
52 import org.eclipse.winery.common.ids.XMLId;
53 import org.eclipse.winery.common.ids.definitions.ArtifactTemplateId;
54 import org.eclipse.winery.common.ids.definitions.ArtifactTypeId;
55 import org.eclipse.winery.common.ids.definitions.CapabilityTypeId;
56 import org.eclipse.winery.common.ids.definitions.NodeTypeId;
57 import org.eclipse.winery.common.ids.definitions.NodeTypeImplementationId;
58 import org.eclipse.winery.common.ids.definitions.PolicyTemplateId;
59 import org.eclipse.winery.common.ids.definitions.PolicyTypeId;
60 import org.eclipse.winery.common.ids.definitions.RelationshipTypeId;
61 import org.eclipse.winery.common.ids.definitions.RelationshipTypeImplementationId;
62 import org.eclipse.winery.common.ids.definitions.RequirementTypeId;
63 import org.eclipse.winery.common.ids.definitions.ServiceTemplateId;
64 import org.eclipse.winery.common.ids.definitions.TOSCAComponentId;
65 import org.eclipse.winery.common.ids.elements.TOSCAElementId;
66 import org.eclipse.winery.repository.Constants;
67 import org.eclipse.winery.repository.backend.AbstractRepository;
68 import org.eclipse.winery.repository.backend.BackendUtils;
69 import org.eclipse.winery.repository.backend.IRepository;
70 import org.eclipse.winery.repository.backend.IRepositoryAdministration;
71 import org.slf4j.Logger;
72 import org.slf4j.LoggerFactory;
73
74 /**
75  * When it comes to a storage of plain files, we use Java 7's nio internally.
76  * Therefore, we intend to expose the stream types offered by java.nio.Files:
77  * BufferedReader/BufferedWriter
78  */
79 public class FilebasedRepository extends AbstractRepository implements IRepository, IRepositoryAdministration {
80         
81         private static final Logger logger = LoggerFactory.getLogger(FilebasedRepository.class);
82         
83         protected final Path repositoryRoot;
84         
85         // convenience variables to have a clean code
86         private final FileSystem fileSystem;
87         private final FileSystemProvider provider;
88         
89         
90         private Path makeAbsolute(Path relativePath) {
91                 return this.repositoryRoot.resolve(relativePath);
92         }
93         
94         @Override
95         public boolean flagAsExisting(GenericId id) {
96                 Path path = this.id2AbsolutePath(id);
97                 try {
98                         FileUtils.createDirectory(path);
99                 } catch (IOException e) {
100                         FilebasedRepository.logger.debug(e.toString());
101                         return false;
102                 }
103                 return true;
104         }
105         
106         private Path id2AbsolutePath(GenericId id) {
107                 Path relativePath = this.fileSystem.getPath(BackendUtils.getPathInsideRepo(id));
108                 return this.makeAbsolute(relativePath);
109         }
110         
111         /**
112          * Converts the given reference to an absolute path of the underlying
113          * FileSystem
114          */
115         public Path ref2AbsolutePath(RepositoryFileReference ref) {
116                 return this.id2AbsolutePath(ref.getParent()).resolve(ref.getFileName());
117         }
118         
119         /**
120          * 
121          * @param repositoryLocation a string pointing to a location on the file
122          *            system. May be null.
123          */
124         public FilebasedRepository(String repositoryLocation) {
125                 this.repositoryRoot = this.determineRepositoryPath(repositoryLocation);
126                 this.fileSystem = this.repositoryRoot.getFileSystem();
127                 this.provider = this.fileSystem.provider();
128         }
129         
130         private Path determineRepositoryPath(String repositoryLocation) {
131                 Path repositoryPath;
132                 if (repositoryLocation == null) {
133                         if (SystemUtils.IS_OS_WINDOWS) {
134                                 if (new File(Constants.GLOBAL_REPO_PATH_WINDOWS).exists()) {
135                                         repositoryLocation = Constants.GLOBAL_REPO_PATH_WINDOWS;
136                                         File repo = new File(repositoryLocation);
137                                         try {
138                                                 org.apache.commons.io.FileUtils.forceMkdir(repo);
139                                         } catch (IOException e) {
140                                                 FilebasedRepository.logger.error("Could not create repository directory", e);
141                                         }
142                                         repositoryPath = repo.toPath();
143                                 } else {
144                                         repositoryPath = this.createDefaultRepositoryPath();
145                                 }
146                         } else {
147                                 repositoryPath = this.createDefaultRepositoryPath();
148                         }
149                 } else {
150                         File repo = new File(repositoryLocation);
151                         try {
152                                 org.apache.commons.io.FileUtils.forceMkdir(repo);
153                         } catch (IOException e) {
154                                 FilebasedRepository.logger.error("Could not create repository directory", e);
155                         }
156                         repositoryPath = repo.toPath();
157                 }
158                 return repositoryPath;
159         }
160         
161         public static File getDefaultRepositoryFilePath() {
162                 return new File(org.apache.commons.io.FileUtils.getUserDirectory(), Constants.DEFAULT_REPO_NAME);
163         }
164         
165         private Path createDefaultRepositoryPath() {
166                 File repo = null;
167                 boolean operationalFileSystemAccess;
168                 try {
169                         repo = FilebasedRepository.getDefaultRepositoryFilePath();
170                         operationalFileSystemAccess = true;
171                 } catch (NullPointerException e) {
172                         // it seems, we run at a system, where we do not have any filesystem
173                         // access
174                         operationalFileSystemAccess = false;
175                 }
176                 
177                 // operationalFileSystemAccess = false;
178                 
179                 Path repositoryPath;
180                 if (operationalFileSystemAccess) {
181                         try {
182                                 org.apache.commons.io.FileUtils.forceMkdir(repo);
183                         } catch (IOException e) {
184                                 FilebasedRepository.logger.error("Could not create directory", e);
185                         }
186                         repositoryPath = repo.toPath();
187                 } else {
188                         assert (!operationalFileSystemAccess);
189                         // we do not have access to the file system
190                         throw new IllegalStateException("No write access to file system");
191                 }
192                 
193                 return repositoryPath;
194         }
195         
196         @Override
197         public void forceDelete(RepositoryFileReference ref) throws IOException {
198                 Path relativePath = this.fileSystem.getPath(BackendUtils.getPathInsideRepo(ref));
199                 Path fileToDelete = this.makeAbsolute(relativePath);
200                 try {
201                         this.provider.delete(fileToDelete);
202                         // Quick hack for deletion of the mime type information
203                         // Alternative: superclass: protected void deleteMimeTypeInformation(RepositoryFileReference ref) throws IOException
204                         // However, this would again call this method, where we would have to check for the extension, too.
205                         // Therefore, we directly delete the information file
206                         Path mimeTypeFile = fileToDelete.getParent().resolve(ref.getFileName() + Constants.SUFFIX_MIMETYPE);
207                         this.provider.delete(mimeTypeFile);
208                 } catch (IOException e) {
209                         if (!(e instanceof NoSuchFileException)) {
210                                 // only if file did exist and something else went wrong: complain :)
211                                 // (otherwise, silently ignore the error)
212                                 FilebasedRepository.logger.debug("Could not delete file", e);
213                                 throw e;
214                         }
215                 }
216         }
217         
218         @Override
219         public void forceDelete(GenericId id) throws IOException {
220                 try {
221                         FileUtils.forceDelete(this.id2AbsolutePath(id));
222                 } catch (IOException e) {
223                         FilebasedRepository.logger.debug("Could not delete id", id);
224                         throw e;
225                 }
226         }
227         
228         @Override
229         public boolean exists(GenericId id) {
230                 Path absolutePath = this.id2AbsolutePath(id);
231                 boolean result = Files.exists(absolutePath);
232                 return result;
233         }
234         
235         /**
236          * {@inheritDoc}
237          */
238         @Override
239         public void putContentToFile(RepositoryFileReference ref, String content, MediaType mediaType) throws IOException {
240                 if (mediaType == null) {
241                         // quick hack for storing mime type called this method
242                         assert (ref.getFileName().endsWith(Constants.SUFFIX_MIMETYPE));
243                         // we do not need to store the mime type of the file containing the mime type information
244                 } else {
245                         this.setMimeType(ref, mediaType);
246                 }
247                 Path path = this.ref2AbsolutePath(ref);
248                 FileUtils.createDirectory(path.getParent());
249                 Files.write(path, content.getBytes());
250         }
251         
252         /**
253          * {@inheritDoc}
254          */
255         @Override
256         public void putContentToFile(RepositoryFileReference ref, InputStream inputStream, MediaType mediaType) throws IOException {
257                 if (mediaType == null) {
258                         // quick hack for storing mime type called this method
259                         assert (ref.getFileName().endsWith(Constants.SUFFIX_MIMETYPE));
260                         // we do not need to store the mime type of the file containing the mime type information
261                 } else {
262                         this.setMimeType(ref, mediaType);
263                 }
264                 Path targetPath = this.ref2AbsolutePath(ref);
265                 // ensure that parent directory exists
266                 FileUtils.createDirectory(targetPath.getParent());
267                 
268                 try {
269                         Files.copy(inputStream, targetPath, StandardCopyOption.REPLACE_EXISTING);
270                 } catch (IllegalStateException e) {
271                         FilebasedRepository.logger.debug("Guessing that stream with length 0 is to be written to a file", e);
272                         // copy throws an "java.lang.IllegalStateException: Stream already closed" if the InputStream contains 0 bytes
273                         // For instance, this case happens if SugarCE-6.4.2.zip.removed is tried to be uploaded
274                         // We work around the Java7 issue and create an empty file
275                         if (Files.exists(targetPath)) {
276                                 // semantics of putContentToFile: existing content is replaced without notification
277                                 Files.delete(targetPath);
278                         }
279                         Files.createFile(targetPath);
280                 }
281         }
282         
283         @Override
284         public boolean exists(RepositoryFileReference ref) {
285                 return Files.exists(this.ref2AbsolutePath(ref));
286         }
287         
288         @Override
289         public <T extends TOSCAComponentId> SortedSet<T> getAllTOSCAComponentIds(Class<T> idClass) {
290                 SortedSet<T> res = new TreeSet<T>();
291                 String rootPathFragment = Util.getRootPathFragment(idClass);
292                 Path dir = this.repositoryRoot.resolve(rootPathFragment);
293                 if (!Files.exists(dir)) {
294                         // return empty list if no ids are available
295                         return res;
296                 }
297                 assert (Files.isDirectory(dir));
298                 
299                 final OnlyNonHiddenDirectories onhdf = new OnlyNonHiddenDirectories();
300                 
301                 // list all directories contained in this directory
302                 try (DirectoryStream<Path> ds = Files.newDirectoryStream(dir, onhdf)) {
303                         for (Path nsP : ds) {
304                                 // the current path is the namespace
305                                 Namespace ns = new Namespace(nsP.getFileName().toString(), true);
306                                 try (DirectoryStream<Path> idDS = Files.newDirectoryStream(nsP, onhdf)) {
307                                         for (Path idP : idDS) {
308                                                 XMLId xmlId = new XMLId(idP.getFileName().toString(), true);
309                                                 Constructor<T> constructor;
310                                                 try {
311                                                         constructor = idClass.getConstructor(Namespace.class, XMLId.class);
312                                                 } catch (Exception e) {
313                                                         FilebasedRepository.logger.debug("Internal error at determining id constructor", e);
314                                                         // abort everything, return invalid result
315                                                         return res;
316                                                 }
317                                                 T id;
318                                                 try {
319                                                         id = constructor.newInstance(ns, xmlId);
320                                                 } catch (InstantiationException
321                                                                 | IllegalAccessException
322                                                                 | IllegalArgumentException
323                                                                 | InvocationTargetException e) {
324                                                         FilebasedRepository.logger.debug("Internal error at invocation of id constructor", e);
325                                                         // abort everything, return invalid result
326                                                         return res;
327                                                 }
328                                                 res.add(id);
329                                         }
330                                 }
331                         }
332                 } catch (IOException e) {
333                         FilebasedRepository.logger.debug("Cannot close ds", e);
334                 }
335                 
336                 return res;
337         }
338         
339         @Override
340         public SortedSet<RepositoryFileReference> getContainedFiles(GenericId id) {
341                 Path dir = this.id2AbsolutePath(id);
342                 SortedSet<RepositoryFileReference> res = new TreeSet<RepositoryFileReference>();
343                 if (!Files.exists(dir)) {
344                         return res;
345                 }
346                 assert (Files.isDirectory(dir));
347                 // list all directories contained in this directory
348                 try (DirectoryStream<Path> ds = Files.newDirectoryStream(dir, new OnlyNonHiddenFiles())) {
349                         for (Path p : ds) {
350                                 RepositoryFileReference ref = new RepositoryFileReference(id, p.getFileName().toString());
351                                 res.add(ref);
352                         }
353                 } catch (IOException e) {
354                         FilebasedRepository.logger.debug("Cannot close ds", e);
355                 }
356                 return res;
357         }
358         
359         @Override
360         public Configuration getConfiguration(RepositoryFileReference ref) {
361                 Path path = this.ref2AbsolutePath(ref);
362                 
363                 PropertiesConfiguration configuration = new PropertiesConfiguration();
364                 if (Files.exists(path)) {
365                         try (Reader r = Files.newBufferedReader(path, Charset.defaultCharset())) {
366                                 configuration.load(r);
367                         } catch (ConfigurationException | IOException e) {
368                                 FilebasedRepository.logger.error("Could not read config file", e);
369                                 throw new IllegalStateException("Could not read config file", e);
370                         }
371                 }
372                 
373                 configuration.addConfigurationListener(new AutoSaveListener(path, configuration));
374                 
375                 // We do NOT implement reloading as the configuration is only accessed
376                 // in JAX-RS resources, which are created on a per-request basis
377                 
378                 return configuration;
379         }
380         
381         /**
382          * @return null if an error occurred
383          */
384         @Override
385         public Date getLastUpdate(RepositoryFileReference ref) {
386                 Path path = this.ref2AbsolutePath(ref);
387                 Date res;
388                 if (Files.exists(path)) {
389                         FileTime lastModifiedTime;
390                         try {
391                                 lastModifiedTime = Files.getLastModifiedTime(path);
392                                 res = new Date(lastModifiedTime.toMillis());
393                         } catch (IOException e) {
394                                 FilebasedRepository.logger.debug(e.getMessage(), e);
395                                 res = null;
396                         }
397                 } else {
398                         // this branch is taken if the resource directory exists, but the
399                         // configuration itself does not exist.
400                         // For instance, this happens if icons are manually put for a node
401                         // type, but no color configuration is made.
402                         res = Constants.LASTMODIFIEDDATE_FOR_404;
403                 }
404                 return res;
405         }
406         
407         @Override
408         public <T extends TOSCAElementId> SortedSet<T> getNestedIds(GenericId ref, Class<T> idClass) {
409                 Path dir = this.id2AbsolutePath(ref);
410                 SortedSet<T> res = new TreeSet<T>();
411                 if (!Files.exists(dir)) {
412                         // the id has been generated by the exporter without existance test.
413                         // This test is done here.
414                         return res;
415                 }
416                 assert (Files.isDirectory(dir));
417                 // list all directories contained in this directory
418                 try (DirectoryStream<Path> ds = Files.newDirectoryStream(dir, new OnlyNonHiddenDirectories())) {
419                         for (Path p : ds) {
420                                 XMLId xmlId = new XMLId(p.getFileName().toString(), true);
421                                 @SuppressWarnings("unchecked")
422                                 Constructor<T>[] constructors = (Constructor<T>[]) idClass.getConstructors();
423                                 assert (constructors.length == 1);
424                                 Constructor<T> constructor = constructors[0];
425                                 assert (constructor.getParameterTypes().length == 2);
426                                 T id;
427                                 try {
428                                         id = constructor.newInstance(ref, xmlId);
429                                 } catch (InstantiationException | IllegalAccessException
430                                                 | IllegalArgumentException | InvocationTargetException e) {
431                                         FilebasedRepository.logger.debug("Internal error at invocation of id constructor", e);
432                                         // abort everything, return invalid result
433                                         return res;
434                                 }
435                                 res.add(id);
436                         }
437                 } catch (IOException e) {
438                         FilebasedRepository.logger.debug("Cannot close ds", e);
439                 }
440                 return res;
441         }
442         
443         @Override
444         // below, toscaComponents is an array, which is used in an iterator
445         // As Java does not allow generic arrays, we have to suppress the warning when fetching an element out of the list
446         @SuppressWarnings("unchecked")
447         public Collection<Namespace> getUsedNamespaces() {
448                 // @formatter:off
449                 @SuppressWarnings("rawtypes")
450                 Class[] toscaComponentIds = {
451                         ArtifactTemplateId.class,
452                         ArtifactTypeId.class,
453                         CapabilityTypeId.class,
454                         NodeTypeId.class,
455                         NodeTypeImplementationId.class,
456                         PolicyTemplateId.class,
457                         PolicyTypeId.class,
458                         RelationshipTypeId.class,
459                         RelationshipTypeImplementationId.class,
460                         RequirementTypeId.class,
461                         ServiceTemplateId.class
462                 };
463                 // @formatter:on
464                 
465                 // we use a HashSet to avoid reporting duplicate namespaces
466                 Collection<Namespace> res = new HashSet<Namespace>();
467                 
468                 for (Class<? extends TOSCAComponentId> id : toscaComponentIds) {
469                         String rootPathFragment = Util.getRootPathFragment(id);
470                         Path dir = this.repositoryRoot.resolve(rootPathFragment);
471                         if (!Files.exists(dir)) {
472                                 continue;
473                         }
474                         assert (Files.isDirectory(dir));
475                         
476                         final OnlyNonHiddenDirectories onhdf = new OnlyNonHiddenDirectories();
477                         
478                         // list all directories contained in this directory
479                         try (DirectoryStream<Path> ds = Files.newDirectoryStream(dir, onhdf)) {
480                                 for (Path nsP : ds) {
481                                         // the current path is the namespace
482                                         Namespace ns = new Namespace(nsP.getFileName().toString(), true);
483                                         res.add(ns);
484                                 }
485                         } catch (IOException e) {
486                                 FilebasedRepository.logger.debug("Cannot close ds", e);
487                         }
488                 }
489                 return res;
490         }
491         
492         @Override
493         public void doDump(OutputStream out) throws IOException {
494                 final ZipOutputStream zout = new ZipOutputStream(out);
495                 final int cutLength = this.repositoryRoot.toString().length() + 1;
496                 
497                 Files.walkFileTree(this.repositoryRoot, new SimpleFileVisitor<Path>() {
498                         
499                         @Override
500                         public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) {
501                                 if (dir.endsWith(".git")) {
502                                         return FileVisitResult.SKIP_SUBTREE;
503                                 } else {
504                                         return FileVisitResult.CONTINUE;
505                                 }
506                         }
507                         
508                         @Override
509                         public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) {
510                                 String name = file.toString().substring(cutLength);
511                                 ZipEntry ze = new ZipEntry(name);
512                                 try {
513                                         ze.setTime(Files.getLastModifiedTime(file).toMillis());
514                                         ze.setSize(Files.size(file));
515                                         zout.putNextEntry(ze);
516                                         Files.copy(file, zout);
517                                         zout.closeEntry();
518                                 } catch (IOException e) {
519                                         FilebasedRepository.logger.debug(e.getMessage());
520                                 }
521                                 return FileVisitResult.CONTINUE;
522                         }
523                 });
524                 zout.close();
525         }
526         
527         /**
528          * Removes all files and dirs except the .git directory
529          */
530         @Override
531         public void doClear() {
532                 try {
533                         DirectoryStream.Filter<Path> noGitDirFilter = new DirectoryStream.Filter<Path>() {
534                                 
535                                 @Override
536                                 public boolean accept(Path entry) throws IOException {
537                                         return !(entry.getFileName().toString().equals(".git"));
538                                 }
539                         };
540                         
541                         DirectoryStream<Path> ds = Files.newDirectoryStream(this.repositoryRoot, noGitDirFilter);
542                         for (Path p : ds) {
543                                 FileUtils.forceDelete(p);
544                         }
545                 } catch (IOException e) {
546                         FilebasedRepository.logger.error(e.getMessage());
547                         e.printStackTrace();
548                 }
549         }
550         
551         @Override
552         public void doImport(InputStream in) {
553                 ZipInputStream zis = new ZipInputStream(in);
554                 ZipEntry entry;
555                 try {
556                         while ((entry = zis.getNextEntry()) != null) {
557                                 if (!entry.isDirectory()) {
558                                         Path path = this.repositoryRoot.resolve(entry.getName());
559                                         FileUtils.createDirectory(path.getParent());
560                                         Files.copy(zis, path);
561                                 }
562                         }
563                 } catch (IOException e) {
564                         FilebasedRepository.logger.error(e.getMessage());
565                 }
566         }
567         
568         /**
569          * {@inheritDoc}
570          */
571         @Override
572         public long getSize(RepositoryFileReference ref) throws IOException {
573                 return Files.size(this.ref2AbsolutePath(ref));
574         }
575         
576         /**
577          * {@inheritDoc}
578          */
579         @Override
580         public FileTime getLastModifiedTime(RepositoryFileReference ref) throws IOException {
581                 Path path = this.ref2AbsolutePath(ref);
582                 FileTime res = Files.getLastModifiedTime(path);
583                 return res;
584         }
585         
586         /**
587          * {@inheritDoc}
588          */
589         @Override
590         public InputStream newInputStream(RepositoryFileReference ref) throws IOException {
591                 Path path = this.ref2AbsolutePath(ref);
592                 InputStream res = Files.newInputStream(path);
593                 return res;
594         }
595         
596 }