ms/blueprintsprocessor/modules/blueprints/blueprint-core/src/main/kotlin/org/onap/ccsdk/cds/controllerblueprints/core/utils/BlueprintArchiveUtils.kt

   1 /*
   2  * Copyright © 2017-2018 AT&T Intellectual Property.
   3  * Modifications Copyright © 2019 Bell Canada.
   4  * Modifications Copyright © 2019 Nordix Foundation.
   5  *
   6  * Licensed under the Apache License, Version 2.0 (the "License");
   7  * you may not use this file except in compliance with the License.
   8  * You may obtain a copy of the License at
   9  *
  10  *     http://www.apache.org/licenses/LICENSE-2.0
  11  *
  12  * Unless required by applicable law or agreed to in writing, software
  13  * distributed under the License is distributed on an "AS IS" BASIS,
  14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15  * See the License for the specific language governing permissions and
  16  * limitations under the License.
  17  */
  18
  19 package org.onap.ccsdk.cds.controllerblueprints.core.utils
  20
  21 import com.google.common.base.Predicates
  22 import org.apache.commons.compress.archivers.ArchiveEntry
  23 import org.apache.commons.compress.archivers.ArchiveInputStream
  24 import org.apache.commons.compress.archivers.ArchiveOutputStream
  25 import org.apache.commons.compress.archivers.tar.TarArchiveEntry
  26 import org.apache.commons.compress.archivers.tar.TarArchiveInputStream
  27 import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream
  28 import org.apache.commons.compress.archivers.zip.ZipArchiveEntry
  29 import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream
  30 import org.apache.commons.compress.archivers.zip.ZipFile
  31 import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream
  32 import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream
  33 import org.onap.ccsdk.cds.controllerblueprints.core.BlueprintProcessorException
  34 import org.slf4j.LoggerFactory
  35 import java.io.BufferedInputStream
  36 import java.io.ByteArrayInputStream
  37 import java.io.ByteArrayOutputStream
  38 import java.io.Closeable
  39 import java.io.File
  40 import java.io.FileOutputStream
  41 import java.io.IOException
  42 import java.io.InputStream
  43 import java.io.InputStreamReader
  44 import java.io.OutputStream
  45 import java.nio.file.FileVisitResult
  46 import java.nio.file.Files
  47 import java.nio.file.Path
  48 import java.nio.file.SimpleFileVisitor
  49 import java.nio.file.attribute.BasicFileAttributes
  50 import java.util.Enumeration
  51 import java.util.function.Predicate
  52 import java.util.zip.Deflater
  53
  54 enum class ArchiveType {
  55     TarGz,
  56     Zip
  57 }
  58
  59 class BlueprintArchiveUtils {
  60
  61     companion object {
  62
  63         private val log = LoggerFactory.getLogger(BlueprintArchiveUtils::class.java)
  64
  65         /**
  66          * Create a new Zip from a root directory
  67          *
  68          * @param source the base directory
  69          * @param destination the output filename
  70          * @return True if OK
  71          */
  72         fun compress(source: File, destination: File, archiveType: ArchiveType = ArchiveType.Zip): Boolean {
  73             try {
  74                 if (!destination.parentFile.exists()) {
  75                     destination.parentFile.mkdirs()
  76                 }
  77                 destination.createNewFile()
  78                 val ignoreZipFiles = Predicate<Path> { path -> !path.endsWith(".zip") && !path.endsWith(".ZIP") }
  79                 FileOutputStream(destination).use { out ->
  80                     compressFolder(source.toPath(), out, archiveType, pathFilter = ignoreZipFiles)
  81                 }
  82             } catch (e: Exception) {
  83                 log.error("Fail to compress folder($source) to path(${destination.path})", e)
  84                 return false
  85             }
  86             return true
  87         }
  88
  89         /**
  90          * In-memory compress an entire folder.
  91          */
  92         fun compressToBytes(
  93             baseDir: Path,
  94             archiveType: ArchiveType = ArchiveType.Zip,
  95             compressionLevel: Int = Deflater.NO_COMPRESSION
  96         ): ByteArray {
  97             return compressFolder(baseDir, ByteArrayOutputStream(), archiveType, compressionLevel = compressionLevel)
  98                 .toByteArray()
  99         }
 100
 101         /**
 102          * Compress an entire folder.
 103          *
 104          * @param baseDir path of base folder to be packaged.
 105          * @param output the output stream
 106          * @param pathFilter filter to ignore files based on its path.
 107          * @param compressionLevel the wanted compression level.
 108          * @param fixedModificationTime to force every entry to have this modification time.
 109          * Useful for reproducible operations, like tests, for example.
 110          */
 111         private fun <T> compressFolder(
 112             baseDir: Path,
 113             output: T,
 114             archiveType: ArchiveType,
 115             pathFilter: Predicate<Path> = Predicates.alwaysTrue(),
 116             compressionLevel: Int = Deflater.DEFAULT_COMPRESSION,
 117             fixedModificationTime: Long? = null
 118         ): T
 119             where T : OutputStream {
 120             val stream: ArchiveOutputStream = if (archiveType == ArchiveType.Zip)
 121                 ZipArchiveOutputStream(output).apply { setLevel(compressionLevel) }
 122             else
 123                 TarArchiveOutputStream(GzipCompressorOutputStream(output))
 124             stream
 125                 .use { aos ->
 126                     Files.walkFileTree(
 127                         baseDir,
 128                         object : SimpleFileVisitor<Path>() {
 129                             @Throws(IOException::class)
 130                             override fun visitFile(file: Path, attrs: BasicFileAttributes): FileVisitResult {
 131                                 if (pathFilter.test(file)) {
 132                                     var archiveEntry: ArchiveEntry = aos.createArchiveEntry(
 133                                         file.toFile(),
 134                                         baseDir.relativize(file).toString()
 135                                     )
 136                                     if (archiveType == ArchiveType.Zip) {
 137                                         val entry = archiveEntry as ZipArchiveEntry
 138                                         fixedModificationTime?.let {
 139                                             entry.time = it
 140                                         }
 141                                         entry.time = 0
 142                                     }
 143                                     aos.putArchiveEntry(archiveEntry)
 144                                     Files.copy(file, aos)
 145                                     aos.closeArchiveEntry()
 146                                 }
 147                                 return FileVisitResult.CONTINUE
 148                             }
 149
 150                             @Throws(IOException::class)
 151                             override fun preVisitDirectory(dir: Path, attrs: BasicFileAttributes): FileVisitResult {
 152                                 var archiveEntry: ArchiveEntry?
 153                                 if (archiveType == ArchiveType.Zip) {
 154                                     val entry = ZipArchiveEntry(baseDir.relativize(dir).toString() + "/")
 155                                     fixedModificationTime?.let {
 156                                         entry.time = it
 157                                     }
 158                                     archiveEntry = entry
 159                                 } else
 160                                     archiveEntry = TarArchiveEntry(baseDir.relativize(dir).toString() + "/")
 161                                 aos.putArchiveEntry(archiveEntry)
 162                                 aos.closeArchiveEntry()
 163                                 return FileVisitResult.CONTINUE
 164                             }
 165                         }
 166                     )
 167                 }
 168             return output
 169         }
 170
 171         private fun getDefaultEncoding(): String? {
 172             val bytes = byteArrayOf('D'.toByte())
 173             val inputStream: InputStream = ByteArrayInputStream(bytes)
 174             val reader = InputStreamReader(inputStream)
 175             return reader.encoding
 176         }
 177
 178         fun deCompress(archiveFile: File, targetPath: String, archiveType: ArchiveType = ArchiveType.Zip): File {
 179             var enumeration: ArchiveEnumerator? = null
 180             if (archiveType == ArchiveType.Zip) {
 181                 val zipArchive = ZipFile(archiveFile, getDefaultEncoding())
 182                 enumeration = ArchiveEnumerator(zipArchive)
 183             } else { // Tar Gz
 184                 var tarGzArchiveIs: InputStream = BufferedInputStream(archiveFile.inputStream())
 185                 tarGzArchiveIs = GzipCompressorInputStream(tarGzArchiveIs)
 186                 val tarGzArchive: ArchiveInputStream = TarArchiveInputStream(tarGzArchiveIs)
 187                 enumeration = ArchiveEnumerator(tarGzArchive)
 188             }
 189
 190             enumeration.use {
 191                 while (enumeration!!.hasMoreElements()) {
 192                     val entry: ArchiveEntry? = enumeration.nextElement()
 193                     val destFilePath = File(targetPath, entry!!.name)
 194                     destFilePath.parentFile.mkdirs()
 195
 196                     if (entry!!.isDirectory)
 197                         continue
 198
 199                     val bufferedIs = BufferedInputStream(enumeration.getInputStream(entry))
 200                     destFilePath.outputStream().buffered(1024).use { bos ->
 201                         bufferedIs.copyTo(bos)
 202                     }
 203
 204                     if (!enumeration.getHasSharedEntryInputStream())
 205                         bufferedIs.close()
 206                 }
 207             }
 208
 209             val destinationDir = File(targetPath)
 210             check(destinationDir.isDirectory && destinationDir.exists()) {
 211                 throw BlueprintProcessorException("failed to decompress blueprint(${archiveFile.absolutePath}) to ($targetPath) ")
 212             }
 213
 214             return destinationDir
 215         }
 216     }
 217
 218     class ArchiveEnumerator : Enumeration<ArchiveEntry>, Closeable {
 219
 220         private val zipArchive: ZipFile?
 221         private val zipEnumeration: Enumeration<ZipArchiveEntry>?
 222         private val archiveStream: ArchiveInputStream?
 223         private var nextEntry: ArchiveEntry? = null
 224         private val hasSharedEntryInputStream: Boolean
 225
 226         constructor(zipFile: ZipFile) {
 227             zipArchive = zipFile
 228             zipEnumeration = zipFile.entries
 229             archiveStream = null
 230             hasSharedEntryInputStream = false
 231         }
 232
 233         constructor(archiveStream: ArchiveInputStream) {
 234             this.archiveStream = archiveStream
 235             zipArchive = null
 236             zipEnumeration = null
 237             hasSharedEntryInputStream = true
 238         }
 239
 240         fun getHasSharedEntryInputStream(): Boolean {
 241             return hasSharedEntryInputStream
 242         }
 243
 244         fun getInputStream(entry: ArchiveEntry): InputStream? {
 245             return if (zipArchive != null)
 246                 zipArchive?.getInputStream(entry as ZipArchiveEntry?)
 247             else
 248                 archiveStream
 249         }
 250
 251         override fun hasMoreElements(): Boolean {
 252             if (zipEnumeration != null)
 253                 return zipEnumeration?.hasMoreElements()
 254             else if (archiveStream != null) {
 255                 nextEntry = archiveStream.nextEntry
 256                 if (nextEntry != null && !archiveStream.canReadEntryData(nextEntry))
 257                     return hasMoreElements()
 258                 return nextEntry != null
 259             }
 260             return false
 261         }
 262
 263         override fun nextElement(): ArchiveEntry? {
 264             if (zipEnumeration != null)
 265                 nextEntry = zipEnumeration.nextElement()
 266             else if (archiveStream != null) {
 267                 if (nextEntry == null)
 268                     nextEntry = archiveStream.nextEntry
 269             }
 270             return nextEntry
 271         }
 272
 273         override fun close() {
 274             if (zipArchive != null)
 275                 zipArchive.close()
 276             else archiveStream?.close()
 277         }
 278     }
 279 }