Added option to disable kotlin script cache 38/99238/3
authorSebastien Premont-Tendland <sebastien.premont@bell.ca>
Thu, 5 Dec 2019 21:27:15 +0000 (16:27 -0500)
committerSebastien Premont-Tendland <sebastien.premont@bell.ca>
Mon, 9 Dec 2019 15:22:56 +0000 (10:22 -0500)
using environment variable USE_SCRIPT_COMPILE_CACHE.

Disabling the cache allow to scale out the BP pod
without having lock file issue when using k8s
NFS provisioner for PV.

It also forces each pod to read the jar from disk
at every request. This way they always create an instance
with the latest jar file.

Issue-ID: CCSDK-1969

Signed-off-by: Sebastien Premont-Tendland <sebastien.premont@bell.ca>
Change-Id: I05afbdb4d49847761142542d1beca78947f032f3

ms/blueprintsprocessor/modules/blueprints/blueprint-core/src/main/kotlin/org/onap/ccsdk/cds/controllerblueprints/core/BluePrintConstants.kt
ms/blueprintsprocessor/modules/blueprints/blueprint-core/src/main/kotlin/org/onap/ccsdk/cds/controllerblueprints/core/scripts/BluePrintCompileService.kt
ms/blueprintsprocessor/modules/blueprints/blueprint-core/src/main/kotlin/org/onap/ccsdk/cds/controllerblueprints/core/scripts/BluePrintCompilerCache.kt
ms/blueprintsprocessor/modules/blueprints/blueprint-core/src/main/kotlin/org/onap/ccsdk/cds/controllerblueprints/core/utils/BluePrintFileUtils.kt

index 9520f67..3016228 100644 (file)
@@ -210,4 +210,6 @@ object BluePrintConstants {
     const val MODEL_TYPE_ARTIFACT_SCRIPT_KOTLIN = "artifact-script-kotlin"
     const val MODEL_TYPE_ARTIFACT_DIRECTED_GRAPH = "artifact-directed-graph"
     const val MODEL_TYPE_ARTIFACT_COMPONENT_JAR = "artifact-component-jar"
+
+    val USE_SCRIPT_COMPILE_CACHE: Boolean = (System.getenv("USE_SCRIPT_COMPILE_CACHE") ?: "true").toBoolean()
 }
index b2857c6..a8c6303 100644 (file)
@@ -25,9 +25,11 @@ import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity
 import org.jetbrains.kotlin.cli.common.messages.MessageCollector
 import org.jetbrains.kotlin.cli.jvm.K2JVMCompiler
 import org.jetbrains.kotlin.config.Services
+import org.onap.ccsdk.cds.controllerblueprints.core.BluePrintConstants
 import org.onap.ccsdk.cds.controllerblueprints.core.BluePrintException
 import org.onap.ccsdk.cds.controllerblueprints.core.checkFileExists
 import org.onap.ccsdk.cds.controllerblueprints.core.logger
+import org.onap.ccsdk.cds.controllerblueprints.core.utils.BluePrintFileUtils
 import java.io.File
 import java.net.URLClassLoader
 import java.util.ArrayList
@@ -50,11 +52,21 @@ open class BluePrintCompileService {
         kClassName: String,
         args: ArrayList<Any?>?
     ): T {
-        /** Compile the source code */
-        compile(bluePrintSourceCode)
-        /** Get the class loader with compiled jar */
-        val classLoaderWithDependencies = BluePrintCompileCache.classLoader(bluePrintSourceCode.cacheKey)
-        /* Create the instance from the class loader */
+        /** Compile the source code if needed */
+        log.debug("Jar Exists : ${bluePrintSourceCode.targetJarFile.exists()}, Regenerate : ${bluePrintSourceCode.regenerate}")
+        if (!bluePrintSourceCode.targetJarFile.exists() || bluePrintSourceCode.regenerate) {
+            compile(bluePrintSourceCode)
+        }
+
+        val classLoaderWithDependencies = if (BluePrintConstants.USE_SCRIPT_COMPILE_CACHE) {
+            /** Get the class loader with compiled jar from cache */
+            BluePrintCompileCache.classLoader(bluePrintSourceCode.cacheKey)
+        } else {
+            /** Get the class loader with compiled jar from disk */
+            BluePrintFileUtils.getURLClassLoaderFromDirectory(bluePrintSourceCode.cacheKey)
+        }
+
+        /** Create the instance from the class loader */
         return instance(classLoaderWithDependencies, kClassName, args)
     }
 
@@ -64,51 +76,41 @@ open class BluePrintCompileService {
         val sourcePath = bluePrintSourceCode.blueprintKotlinSources.first()
         val compiledJarFile = bluePrintSourceCode.targetJarFile
 
-        /** Check cache is present for the blueprint scripts */
-        val hasCompiledCache = BluePrintCompileCache.hasClassLoader(bluePrintSourceCode.cacheKey)
-
-        log.debug(
-            "Jar Exists : ${compiledJarFile.exists()}, Regenerate : ${bluePrintSourceCode.regenerate}," +
-                    " Compiled hash(${bluePrintSourceCode.cacheKey}) : $hasCompiledCache"
-        )
-
-        if (!compiledJarFile.exists() || bluePrintSourceCode.regenerate || !hasCompiledCache) {
-            log.info("compiling for cache key(${bluePrintSourceCode.cacheKey})")
-            coroutineScope {
-                val timeTaken = measureTimeMillis {
-                    /** Create compile arguments */
-                    val args = mutableListOf<String>().apply {
-                        add("-no-stdlib")
-                        add("-no-reflect")
-                        add("-module-name")
-                        add(bluePrintSourceCode.moduleName)
-                        add("-cp")
-                        add(classPaths!!)
-                        add(sourcePath)
-                        add("-d")
-                        add(compiledJarFile.absolutePath)
-                    }
-                    val deferredCompile = async {
-                        val k2jvmCompiler = K2JVMCompiler()
-                        /** Construct Arguments */
-                        val arguments = k2jvmCompiler.createArguments()
-                        parseCommandLineArguments(args, arguments)
-                        val messageCollector = CompilationMessageCollector()
-                        /** Compile with arguments */
-                        val exitCode: ExitCode = k2jvmCompiler.exec(messageCollector, Services.EMPTY, arguments)
-                        when (exitCode) {
-                            ExitCode.OK -> {
-                                checkFileExists(compiledJarFile) { "couldn't generate compiled jar(${compiledJarFile.absolutePath})" }
-                            }
-                            else -> {
-                                throw BluePrintException("$exitCode :${messageCollector.errors().joinToString("\n")}")
-                            }
+        log.info("compiling for cache key(${bluePrintSourceCode.cacheKey})")
+        coroutineScope {
+            val timeTaken = measureTimeMillis {
+                /** Create compile arguments */
+                val args = mutableListOf<String>().apply {
+                    add("-no-stdlib")
+                    add("-no-reflect")
+                    add("-module-name")
+                    add(bluePrintSourceCode.moduleName)
+                    add("-cp")
+                    add(classPaths!!)
+                    add(sourcePath)
+                    add("-d")
+                    add(compiledJarFile.absolutePath)
+                }
+                val deferredCompile = async {
+                    val k2jvmCompiler = K2JVMCompiler()
+                    /** Construct Arguments */
+                    val arguments = k2jvmCompiler.createArguments()
+                    parseCommandLineArguments(args, arguments)
+                    val messageCollector = CompilationMessageCollector()
+                    /** Compile with arguments */
+                    val exitCode: ExitCode = k2jvmCompiler.exec(messageCollector, Services.EMPTY, arguments)
+                    when (exitCode) {
+                        ExitCode.OK -> {
+                            checkFileExists(compiledJarFile) { "couldn't generate compiled jar(${compiledJarFile.absolutePath})" }
+                        }
+                        else -> {
+                            throw BluePrintException("$exitCode :${messageCollector.errors().joinToString("\n")}")
                         }
                     }
-                    deferredCompile.await()
                 }
-                log.info("compiled in ($timeTaken)mSec for cache key(${bluePrintSourceCode.cacheKey})")
+                deferredCompile.await()
             }
+            log.info("compiled in ($timeTaken)mSec for cache key(${bluePrintSourceCode.cacheKey})")
         }
     }
 
@@ -123,6 +125,10 @@ open class BluePrintCompileService {
             kClazz.constructors
                 .single().newInstance(*args.toArray())
         } ?: throw BluePrintException("failed to create class($kClassName) instance for constructor argument($args).")
+
+        if (!BluePrintConstants.USE_SCRIPT_COMPILE_CACHE) {
+            classLoader.close()
+        }
         return instance as T
     }
 }
index fbfcfb9..cb6616f 100644 (file)
@@ -21,8 +21,7 @@ import com.google.common.cache.CacheLoader
 import com.google.common.cache.LoadingCache
 import org.onap.ccsdk.cds.controllerblueprints.core.BluePrintException
 import org.onap.ccsdk.cds.controllerblueprints.core.logger
-import org.onap.ccsdk.cds.controllerblueprints.core.normalizedFile
-import java.net.URL
+import org.onap.ccsdk.cds.controllerblueprints.core.utils.BluePrintFileUtils
 import java.net.URLClassLoader
 
 object BluePrintCompileCache {
@@ -58,19 +57,10 @@ object BluePrintClassLoader : CacheLoader<String, URLClassLoader>() {
 
     val log = logger(BluePrintClassLoader::class)
 
-    override fun load(key: String): URLClassLoader {
+    override fun load(key: String) = try {
         log.info("loading compiled cache($key)")
-        val keyPath = normalizedFile(key)
-        if (!keyPath.exists()) {
-            throw BluePrintException("failed to load cache($key), missing files.")
-        }
-        val urls = arrayListOf<URL>()
-        keyPath.walkTopDown()
-            .filter { it.name.endsWith("cba-kts.jar") }
-            .forEach {
-                log.debug("Adding (${it.absolutePath}) to cache($key)")
-                urls.add(it.toURI().toURL())
-            }
-        return URLClassLoader(urls.toTypedArray(), this.javaClass.classLoader)
+        BluePrintFileUtils.getURLClassLoaderFromDirectory(key)
+    } catch (e: Exception) {
+        throw BluePrintException("failed to load cache($key) with Exception($e)")
     }
 }
index 6605e8e..9e10473 100755 (executable)
@@ -32,7 +32,11 @@ import org.onap.ccsdk.cds.controllerblueprints.core.service.BluePrintContext
 import org.slf4j.LoggerFactory
 import java.io.File
 import java.io.FileFilter
+import java.io.FileNotFoundException
+import java.net.URL
+import java.net.URLClassLoader
 import java.nio.file.Files
+import java.nio.file.NotDirectoryException
 import java.nio.file.Path
 import java.nio.file.Paths
 import java.nio.file.StandardOpenOption
@@ -40,6 +44,8 @@ import java.nio.file.StandardOpenOption
 class BluePrintFileUtils {
     companion object {
 
+        const val COMPILED_JAR_SUFFIX = "cba-kts.jar"
+
         private val log = LoggerFactory.getLogger(this::class.toString())
 
         fun createEmptyBluePrint(basePath: String) {
@@ -263,7 +269,7 @@ class BluePrintFileUtils {
         }
 
         private fun compileJarFileName(artifactName: String, artifactVersion: String): String {
-            return "$artifactName-$artifactVersion-cba-kts.jar"
+            return "$artifactName-$artifactVersion-$COMPILED_JAR_SUFFIX"
         }
 
         fun compileJarFilePathName(basePath: String, artifactName: String, artifactVersion: String): String {
@@ -288,5 +294,28 @@ class BluePrintFileUtils {
             // In case dot is in first position, we are dealing with a hidden file rather than an extension
             return if (dotIndexe > 0) fileName.substring(0, dotIndexe) else fileName
         }
+
+        fun getURLClassLoaderFromDirectory(directory: File): URLClassLoader {
+            if (!directory.exists()) {
+                throw FileNotFoundException(directory.absolutePath)
+            } else if (!directory.isDirectory) {
+                throw NotDirectoryException(directory.absolutePath)
+            }
+
+            val urls = arrayListOf<URL>()
+            directory.walkTopDown()
+                    .filter { it.name.endsWith(COMPILED_JAR_SUFFIX) }
+                    .forEach {
+                        log.debug("Adding (${it.absolutePath}) to classLoader (${directory.absolutePath})")
+
+                        urls.add(it.toURI().toURL())
+                    }
+            return URLClassLoader(urls.toTypedArray(), this.javaClass.classLoader)
+        }
+
+        fun getURLClassLoaderFromDirectory(path: String): URLClassLoader {
+            val directory = normalizedFile(path)
+            return getURLClassLoaderFromDirectory(directory)
+        }
     }
 }