Add tar.gz compression capability

The change modifies ArchiveUtils by change of
the library responsible for compression and
decompression of zip files. After change of
library for appache compression tar.gz was
added as another format which archive utils
can support. For ZIP files there is kept
backward compatibility because new lib
keeps implementation of ZipFile similar
to the one used before in the utils.

Issue-ID: INT-1458
Signed-off-by: Lukasz Rajewski <lukasz.rajewski@orange.com>
Change-Id: I78388ef8c5e7a23ac6664ae49c00638af38d8c8a

diff --git a/ms/blueprintsprocessor/modules/blueprints/blueprint-core/pom.xml b/ms/blueprintsprocessor/modules/blueprints/blueprint-core/pom.xml
index 28060ef..fb2daab 100644 (file)
--- a/ms/blueprintsprocessor/modules/blueprints/blueprint-core/pom.xml
+++ b/ms/blueprintsprocessor/modules/blueprints/blueprint-core/pom.xml
@@ -91,6 +91,10 @@
             <groupId>org.springframework.boot</groupId>
             <artifactId>spring-boot-starter-logging</artifactId>
         </dependency>
+        <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-compress</artifactId>
+        </dependency>
         <!--Testing dependencies-->
         <dependency>
             <groupId>org.jetbrains.kotlin</groupId>

diff --git a/ms/blueprintsprocessor/modules/blueprints/blueprint-core/src/main/kotlin/org/onap/ccsdk/cds/controllerblueprints/core/utils/BluePrintArchiveUtils.kt b/ms/blueprintsprocessor/modules/blueprints/blueprint-core/src/main/kotlin/org/onap/ccsdk/cds/controllerblueprints/core/utils/BluePrintArchiveUtils.kt
index 9ccf856..595dbce 100755 (executable)
--- a/ms/blueprintsprocessor/modules/blueprints/blueprint-core/src/main/kotlin/org/onap/ccsdk/cds/controllerblueprints/core/utils/BluePrintArchiveUtils.kt
+++ b/ms/blueprintsprocessor/modules/blueprints/blueprint-core/src/main/kotlin/org/onap/ccsdk/cds/controllerblueprints/core/utils/BluePrintArchiveUtils.kt
@@ -22,22 +22,39 @@ import com.google.common.base.Predicates
 import org.onap.ccsdk.cds.controllerblueprints.core.BluePrintProcessorException
 import org.slf4j.LoggerFactory
 import java.io.BufferedInputStream
+import java.io.ByteArrayInputStream
 import java.io.ByteArrayOutputStream
+import java.io.Closeable
 import java.io.File
 import java.io.FileOutputStream
+import java.io.InputStream
+import java.io.InputStreamReader
 import java.io.IOException
 import java.io.OutputStream
-import java.nio.charset.Charset
 import java.nio.file.FileVisitResult
 import java.nio.file.Files
 import java.nio.file.Path
 import java.nio.file.SimpleFileVisitor
 import java.nio.file.attribute.BasicFileAttributes
 import java.util.function.Predicate
+import org.apache.commons.compress.archivers.ArchiveEntry
+import org.apache.commons.compress.archivers.ArchiveInputStream
+import org.apache.commons.compress.archivers.ArchiveOutputStream
+import org.apache.commons.compress.archivers.tar.TarArchiveEntry
+import org.apache.commons.compress.archivers.tar.TarArchiveInputStream
+import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream
+import org.apache.commons.compress.archivers.zip.ZipArchiveEntry
+import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream
+import org.apache.commons.compress.archivers.zip.ZipFile
+import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream
+import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream
+import java.util.Enumeration
 import java.util.zip.Deflater
-import java.util.zip.ZipEntry
-import java.util.zip.ZipFile
-import java.util.zip.ZipOutputStream
+
+enum class ArchiveType {
+    TarGz,
+    Zip
+}
 
 class BluePrintArchiveUtils {
 
@@ -51,7 +68,7 @@ class BluePrintArchiveUtils {
          * @param destination the output filename
          * @return True if OK
          */
-        fun compress(source: File, destination: File): Boolean {
+        fun compress(source: File, destination: File, archiveType: ArchiveType = ArchiveType.Zip): Boolean {
             try {
                 if (!destination.parentFile.exists()) {
                     destination.parentFile.mkdirs()
@@ -59,7 +76,7 @@ class BluePrintArchiveUtils {
                 destination.createNewFile()
                 val ignoreZipFiles = Predicate<Path> { path -> !path.endsWith(".zip") && !path.endsWith(".ZIP") }
                 FileOutputStream(destination).use { out ->
-                    compressFolder(source.toPath(), out, pathFilter = ignoreZipFiles)
+                    compressFolder(source.toPath(), out, archiveType, pathFilter = ignoreZipFiles)
                 }
             } catch (e: Exception) {
                 log.error("Fail to compress folder($source) to path(${destination.path})", e)
@@ -71,8 +88,12 @@ class BluePrintArchiveUtils {
         /**
          * In-memory compress an entire folder.
          */
-        fun compressToBytes(baseDir: Path, compressionLevel: Int = Deflater.NO_COMPRESSION): ByteArray {
-            return compressFolder(baseDir, ByteArrayOutputStream(), compressionLevel = compressionLevel)
+        fun compressToBytes(
+            baseDir: Path,
+            archiveType: ArchiveType = ArchiveType.Zip,
+            compressionLevel: Int = Deflater.NO_COMPRESSION
+        ): ByteArray {
+            return compressFolder(baseDir, ByteArrayOutputStream(), archiveType, compressionLevel = compressionLevel)
                 .toByteArray()
         }
 
@@ -89,38 +110,51 @@ class BluePrintArchiveUtils {
         private fun <T> compressFolder(
             baseDir: Path,
             output: T,
+            archiveType: ArchiveType,
             pathFilter: Predicate<Path> = Predicates.alwaysTrue(),
             compressionLevel: Int = Deflater.DEFAULT_COMPRESSION,
             fixedModificationTime: Long? = null
         ): T
                 where T : OutputStream {
-            ZipOutputStream(output)
-                .apply { setLevel(compressionLevel) }
-                .use { zos ->
+            val stream: ArchiveOutputStream = if (archiveType == ArchiveType.Zip)
+                ZipArchiveOutputStream(output).apply { setLevel(compressionLevel) }
+            else
+                TarArchiveOutputStream(GzipCompressorOutputStream(output))
+            stream
+                .use { aos ->
                     Files.walkFileTree(baseDir, object : SimpleFileVisitor<Path>() {
                         @Throws(IOException::class)
                         override fun visitFile(file: Path, attrs: BasicFileAttributes): FileVisitResult {
                             if (pathFilter.test(file)) {
-                                val zipEntry = ZipEntry(baseDir.relativize(file).toString())
-                                fixedModificationTime?.let {
-                                    zipEntry.time = it
+                                var archiveEntry: ArchiveEntry = aos.createArchiveEntry(file.toFile(),
+                                        baseDir.relativize(file).toString())
+                                if (archiveType == ArchiveType.Zip) {
+                                    val entry = archiveEntry as ZipArchiveEntry
+                                    fixedModificationTime?.let {
+                                        entry.time = it
+                                    }
+                                    entry.time = 0
                                 }
-                                zipEntry.time = 0
-                                zos.putNextEntry(zipEntry)
-                                Files.copy(file, zos)
-                                zos.closeEntry()
+                                aos.putArchiveEntry(archiveEntry)
+                                Files.copy(file, aos)
+                                aos.closeArchiveEntry()
                             }
                             return FileVisitResult.CONTINUE
                         }
 
                         @Throws(IOException::class)
                         override fun preVisitDirectory(dir: Path, attrs: BasicFileAttributes): FileVisitResult {
-                            val zipEntry = ZipEntry(baseDir.relativize(dir).toString() + "/")
-                            fixedModificationTime?.let {
-                                zipEntry.time = it
-                            }
-                            zos.putNextEntry(zipEntry)
-                            zos.closeEntry()
+                            var archiveEntry: ArchiveEntry?
+                            if (archiveType == ArchiveType.Zip) {
+                                val entry = ZipArchiveEntry(baseDir.relativize(dir).toString() + "/")
+                                fixedModificationTime?.let {
+                                    entry.time = it
+                                }
+                                archiveEntry = entry
+                            } else
+                                archiveEntry = TarArchiveEntry(baseDir.relativize(dir).toString() + "/")
+                            aos.putArchiveEntry(archiveEntry)
+                            aos.closeArchiveEntry()
                             return FileVisitResult.CONTINUE
                         }
                     })
@@ -128,31 +162,111 @@ class BluePrintArchiveUtils {
             return output
         }
 
-        fun deCompress(zipFile: File, targetPath: String): File {
-            val zip = ZipFile(zipFile, Charset.defaultCharset())
-            val enumeration = zip.entries()
-            while (enumeration.hasMoreElements()) {
-                val entry = enumeration.nextElement()
-                val destFilePath = File(targetPath, entry.name)
-                destFilePath.parentFile.mkdirs()
+        private fun getDefaultEncoding(): String? {
+            val bytes = byteArrayOf('D'.toByte())
+            val inputStream: InputStream = ByteArrayInputStream(bytes)
+            val reader = InputStreamReader(inputStream)
+            return reader.encoding
+        }
 
-                if (entry.isDirectory)
-                    continue
+        fun deCompress(archiveFile: File, targetPath: String, archiveType: ArchiveType = ArchiveType.Zip): File {
+            var enumeration: ArchiveEnumerator? = null
+            if (archiveType == ArchiveType.Zip) {
+                val zipArchive = ZipFile(archiveFile, getDefaultEncoding())
+                enumeration = ArchiveEnumerator(zipArchive)
+            } else { // Tar Gz
+                var tarGzArchiveIs: InputStream = BufferedInputStream(archiveFile.inputStream())
+                tarGzArchiveIs = GzipCompressorInputStream(tarGzArchiveIs)
+                val tarGzArchive: ArchiveInputStream = TarArchiveInputStream(tarGzArchiveIs)
+                enumeration = ArchiveEnumerator(tarGzArchive)
+            }
+
+            enumeration.use {
+                while (enumeration!!.hasMoreElements()) {
+                    val entry: ArchiveEntry? = enumeration.nextElement()
+                    val destFilePath = File(targetPath, entry!!.name)
+                    destFilePath.parentFile.mkdirs()
 
-                val bufferedIs = BufferedInputStream(zip.getInputStream(entry))
-                bufferedIs.use {
+                    if (entry!!.isDirectory)
+                        continue
+
+                    val bufferedIs = BufferedInputStream(enumeration.getInputStream(entry))
                     destFilePath.outputStream().buffered(1024).use { bos ->
                         bufferedIs.copyTo(bos)
                     }
+
+                    if (!enumeration.getHasSharedEntryInputStream())
+                        bufferedIs.close()
                 }
             }
 
             val destinationDir = File(targetPath)
             check(destinationDir.isDirectory && destinationDir.exists()) {
-                throw BluePrintProcessorException("failed to decompress blueprint(${zipFile.absolutePath}) to ($targetPath) ")
+                throw BluePrintProcessorException("failed to decompress blueprint(${archiveFile.absolutePath}) to ($targetPath) ")
             }
 
             return destinationDir
         }
     }
+
+    class ArchiveEnumerator : Enumeration<ArchiveEntry>, Closeable {
+        private val zipArchive: ZipFile?
+        private val zipEnumeration: Enumeration<ZipArchiveEntry>?
+        private val archiveStream: ArchiveInputStream?
+        private var nextEntry: ArchiveEntry? = null
+        private val hasSharedEntryInputStream: Boolean
+
+        constructor(zipFile: ZipFile) {
+            zipArchive = zipFile
+            zipEnumeration = zipFile.entries
+            archiveStream = null
+            hasSharedEntryInputStream = false
+        }
+
+        constructor(archiveStream: ArchiveInputStream) {
+            this.archiveStream = archiveStream
+            zipArchive = null
+            zipEnumeration = null
+            hasSharedEntryInputStream = true
+        }
+
+        fun getHasSharedEntryInputStream(): Boolean {
+            return hasSharedEntryInputStream
+        }
+
+        fun getInputStream(entry: ArchiveEntry): InputStream? {
+            return if (zipArchive != null)
+                zipArchive?.getInputStream(entry as ZipArchiveEntry?)
+            else
+                archiveStream
+        }
+
+        override fun hasMoreElements(): Boolean {
+            if (zipEnumeration != null)
+                return zipEnumeration?.hasMoreElements()
+            else if (archiveStream != null) {
+                nextEntry = archiveStream.nextEntry
+                if (nextEntry != null && !archiveStream.canReadEntryData(nextEntry))
+                    return hasMoreElements()
+                return nextEntry != null
+            }
+            return false
+        }
+
+        override fun nextElement(): ArchiveEntry? {
+            if (zipEnumeration != null)
+                nextEntry = zipEnumeration.nextElement()
+            else if (archiveStream != null) {
+                if (nextEntry == null)
+                    nextEntry = archiveStream.nextEntry
+            }
+            return nextEntry
+        }
+
+        override fun close() {
+            if (zipArchive != null)
+                zipArchive.close()
+            else archiveStream?.close()
+        }
+    }
 }

diff --git a/ms/blueprintsprocessor/parent/pom.xml b/ms/blueprintsprocessor/parent/pom.xml
index 8301fbc..d47889a 100755 (executable)
--- a/ms/blueprintsprocessor/parent/pom.xml
+++ b/ms/blueprintsprocessor/parent/pom.xml
@@ -60,6 +60,7 @@
         <json-smart.version>2.3</json-smart.version>
 
         <commons-io-version>2.6</commons-io-version>
+        <commons-compress-version>1.20</commons-compress-version>
         <commons-collections-version>3.2.2</commons-collections-version>
     </properties>
 
@@ -129,6 +130,11 @@
                 <artifactId>commons-io</artifactId>
                 <version>${commons-io-version}</version>
             </dependency>
+            <dependency>
+                <groupId>org.apache.commons</groupId>
+                <artifactId>commons-compress</artifactId>
+                <version>${commons-compress-version}</version>
+            </dependency>
             <dependency>
                 <groupId>com.hubspot.jinjava</groupId>
                 <artifactId>jinjava</artifactId>