2 * Copyright © 2019 Bell Canada.
3 * Modifications Copyright © 2018-2019 IBM.
5 * Licensed under the Apache License, Version 2.0 (the "License");
6 * you may not use this file except in compliance with the License.
7 * You may obtain a copy of the License at
9 * http://www.apache.org/licenses/LICENSE-2.0
11 * Unless required by applicable law or agreed to in writing, software
12 * distributed under the License is distributed on an "AS IS" BASIS,
13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 * See the License for the specific language governing permissions and
15 * limitations under the License.
18 package org.onap.ccsdk.cds.blueprintsprocessor.functions.ansible.executor
20 import com.fasterxml.jackson.databind.JsonNode
21 import com.fasterxml.jackson.databind.ObjectMapper
22 import com.fasterxml.jackson.databind.node.TextNode
23 import org.onap.ccsdk.cds.blueprintsprocessor.core.api.data.ExecutionServiceInput
24 import org.onap.ccsdk.cds.blueprintsprocessor.rest.service.BluePrintRestLibPropertyService
25 import org.onap.ccsdk.cds.blueprintsprocessor.rest.service.BlueprintWebClientService
26 import org.onap.ccsdk.cds.blueprintsprocessor.services.execution.AbstractComponentFunction
27 import org.onap.ccsdk.cds.controllerblueprints.core.*
28 import org.onap.ccsdk.cds.controllerblueprints.core.utils.JacksonUtils
29 import org.slf4j.LoggerFactory
30 import org.springframework.beans.factory.config.ConfigurableBeanFactory
31 import org.springframework.context.annotation.Scope
32 import org.springframework.http.HttpMethod
33 import org.springframework.stereotype.Component
35 import java.net.URLEncoder
39 * ComponentRemoteAnsibleExecutor
41 * Component that launches a run of a job template (INPUT_JOB_TEMPLATE_NAME) representing an Ansible playbook,
42 * and its parameters, via the AWX server identified by the INPUT_ENDPOINT_SELECTOR parameter.
44 * It supports extra_vars, limit, tags, skip-tags, inventory (by name or Id) Ansible parameters.
45 * It reports the results of the execution via properties, named execute-command-status and execute-command-logs
47 * @author Serge Simard
49 @Component("component-remote-ansible-executor")
50 @Scope(value = ConfigurableBeanFactory.SCOPE_PROTOTYPE)
51 open class ComponentRemoteAnsibleExecutor(private val blueprintRestLibPropertyService: BluePrintRestLibPropertyService,
52 private val mapper: ObjectMapper)
53 : AbstractComponentFunction() {
55 // HTTP related constants
56 private val HTTP_SUCCESS = 200..202
57 private val GET = HttpMethod.GET.name
58 private val POST = HttpMethod.POST.name
60 var checkDelay: Long = 1_000
63 private val log = LoggerFactory.getLogger(ComponentRemoteAnsibleExecutor::class.java)
65 // input fields names accepted by this executor
66 const val INPUT_ENDPOINT_SELECTOR = "endpoint-selector"
67 const val INPUT_JOB_TEMPLATE_NAME = "job-template-name"
68 const val INPUT_WORKFLOW_JOB_TEMPLATE_NAME = "workflow-job-template-id"
69 const val INPUT_LIMIT_TO_HOST = "limit"
70 const val INPUT_INVENTORY = "inventory"
71 const val INPUT_EXTRA_VARS = "extra-vars"
72 const val INPUT_TAGS = "tags"
73 const val INPUT_SKIP_TAGS = "skip-tags"
75 // output fields names (and values) populated by this executor; aligned with job details status field values.
76 const val ATTRIBUTE_EXEC_CMD_STATUS = "ansible-command-status"
77 const val ATTRIBUTE_EXEC_CMD_LOG = "ansible-command-logs"
78 const val ATTRIBUTE_EXEC_CMD_STATUS_ERROR = "error"
81 override suspend fun processNB(executionRequest: ExecutionServiceInput) {
84 val restClientService = getAWXRestClient()
86 // Get either a job template name or a workflow template name property
87 var workflowURIPrefix = ""
88 var jobTemplateName = getOperationInput(INPUT_JOB_TEMPLATE_NAME).returnNullIfMissing()?.textValue() ?: ""
89 val isWorkflowJT = jobTemplateName.isBlank()
91 jobTemplateName = getOperationInput(INPUT_WORKFLOW_JOB_TEMPLATE_NAME).asText()
92 workflowURIPrefix = "workflow_"
95 val jtId = lookupJobTemplateIDByName(restClientService, jobTemplateName, workflowURIPrefix)
96 if (jtId.isNotEmpty()) {
97 runJobTemplateOnAWX(restClientService, jobTemplateName, jtId, workflowURIPrefix)
99 val message = "Workflow/Job template ${jobTemplateName} does not exists"
101 setNodeOutputErrors(ATTRIBUTE_EXEC_CMD_STATUS_ERROR, message)
103 } catch (e: Exception) {
104 log.error("Failed to process on remote executor (${e.message})", e)
105 setNodeOutputErrors(ATTRIBUTE_EXEC_CMD_STATUS_ERROR, "Failed to process on remote executor (${e.message})")
110 override suspend fun recoverNB(runtimeException: RuntimeException, executionRequest: ExecutionServiceInput) {
111 val message = "Error in ComponentRemoteAnsibleExecutor : ${runtimeException.message}"
112 log.error(message, runtimeException)
113 setNodeOutputErrors(ATTRIBUTE_EXEC_CMD_STATUS_ERROR, message)
116 /** Creates a TokenAuthRestClientService, since this executor expect type property to be "token-auth" and the
117 * token to be an OAuth token (access_token response field) generated via the AWX /api/o/token rest endpoint
118 * The token field is of the form "Bearer access_token_from_response", for example :
119 * "blueprintsprocessor.restclient.awx.type=token-auth"
120 * "blueprintsprocessor.restclient.awx.url=http://awx-endpoint"
121 * "blueprintsprocessor.restclient.awx.token=Bearer J9gEtMDzxcqw25574fioY9VAhLDIs1"
123 * Also supports json endpoint definition via DSL entry, e.g.:
124 * "ansible-remote-endpoint": {
125 * "type": "token-auth",
126 * "url": "http://awx-endpoint",
127 * "token": "Bearer J9gEtMDzxcqw25574fioY9VAhLDIs1"
130 private fun getAWXRestClient(): BlueprintWebClientService {
132 val endpointSelector = getOperationInput(INPUT_ENDPOINT_SELECTOR)
135 return blueprintRestLibPropertyService.blueprintWebClientService(endpointSelector)
136 } catch (e: NoSuchElementException) {
137 throw IllegalArgumentException("No value provided for input selector $endpointSelector", e)
142 * Finds the job template ID based on the job template name provided in the request
144 private fun lookupJobTemplateIDByName(awxClient: BlueprintWebClientService, job_template_name: String?,
145 workflowPrefix : String) : String {
146 val encodedJTName = URI(null, null,
147 "/api/v2/${workflowPrefix}job_templates/${job_template_name}/",
150 // Get Job Template details by name
151 var response = awxClient.exchangeResource(GET, encodedJTName, "")
152 val jtDetails: JsonNode = mapper.readTree(response.body)
153 return jtDetails.at("/id").asText()
157 * Performs the job template execution on AWX, ie. prepare arguments as per job template
158 * requirements (ask fields) and provided overriding values. Then it launches the run, and monitors
159 * its execution. Finally, it retrieves the job results via the stdout api.
160 * The status and output attributes are populated in the process.
162 private fun runJobTemplateOnAWX(awxClient: BlueprintWebClientService, job_template_name: String?, jtId: String,
163 workflowPrefix : String) {
164 setNodeOutputProperties("preparing".asJsonPrimitive(), "".asJsonPrimitive())
166 // Get Job Template requirements
167 var response = awxClient.exchangeResource(GET, "/api/v2/${workflowPrefix}job_templates/${jtId}/launch/", "")
168 // FIXME: handle non-successful SC
169 val jtLaunchReqs: JsonNode = mapper.readTree(response.body)
170 val payload = prepareLaunchPayload(awxClient, jtLaunchReqs, workflowPrefix.isBlank())
171 log.info("Running job with $payload, for requestId $processId.")
173 // Launch the job for the targeted template
174 var jtLaunched: JsonNode = JacksonUtils.objectMapper.createObjectNode()
175 response = awxClient.exchangeResource(POST, "/api/v2/${workflowPrefix}job_templates/${jtId}/launch/", payload)
176 if (response.status in HTTP_SUCCESS) {
177 jtLaunched = mapper.readTree(response.body)
178 val fieldsIgnored: JsonNode = jtLaunched.at("/ignored_fields")
179 if (fieldsIgnored.rootFieldsToMap().isNotEmpty()) {
180 log.warn("Ignored fields : $fieldsIgnored, for requestId $processId.")
184 if (response.status in HTTP_SUCCESS) {
185 val jobId: String = jtLaunched.at("/id").asText()
187 // Poll current job status while job is not executed
188 var jobStatus = "unknown"
189 var jobEndTime = "null"
190 while (jobEndTime == "null") {
191 response = awxClient.exchangeResource(GET, "/api/v2/${workflowPrefix}jobs/${jobId}/", "")
192 val jobLaunched: JsonNode = mapper.readTree(response.body)
193 jobStatus = jobLaunched.at("/status").asText()
194 jobEndTime = jobLaunched.at("/finished").asText()
195 Thread.sleep(checkDelay)
198 log.info("Execution of job template $job_template_name in job #$jobId finished with status ($jobStatus) for requestId $processId")
200 // Get workflow/job execution results
201 val collectedOutput = extractJobRunResponse(awxClient, jobId, workflowPrefix)
203 setNodeOutputProperties(jobStatus.asJsonPrimitive(), collectedOutput.asJsonPrimitive())
205 // The job template requirements were not fulfilled with the values passed in. The message below will
206 // provide more information via the response, like the ignored_fields, or variables_needed_to_start,
207 // or resources_needed_to_start, in order to help user pinpoint the problems with the request.
208 val message = "Execution of job template $job_template_name could not be started for requestId $processId." +
209 " (Response: ${response.body}) "
211 setNodeOutputErrors(ATTRIBUTE_EXEC_CMD_STATUS_ERROR, message)
216 * Extracts the response from either a job stdout call OR collects the workflow run output
218 private fun extractJobRunResponse(awxClient: BlueprintWebClientService, jobId: String, workflowPrefix: String): String {
220 // First, collect all job ID from either the job template run or the workflow nodes that ran
221 var jobIds : Array<String>
222 var collectedResponses = StringBuilder()
223 if (workflowPrefix.isNotEmpty()) {
224 var response = awxClient.exchangeResource(GET, "/api/v2/${workflowPrefix}jobs/${jobId}/workflow_nodes/", "")
225 val jobDetails = mapper.readTree(response.body).at("/results")
226 jobIds = emptyArray()
227 for (jobDetail in jobDetails.elements()) {
228 jobIds = jobIds.plus( jobDetail.at("/summary_fields/job/id").asText() )
231 jobIds = arrayOf(jobId)
234 // Then collect the response text from the corresponding jobIds
235 val plainTextHeaders = mutableMapOf<String, String>()
236 plainTextHeaders["Content-Type"] = "text/plain ;utf-8"
237 for (aJobId in jobIds) {
238 var response = awxClient.exchangeResource(GET, "/api/v2/jobs/${aJobId}/stdout/?format=txt", "", plainTextHeaders)
239 collectedResponses.append("Output for job ${aJobId}:")
240 collectedResponses.append(response.body)
242 return collectedResponses.toString()
246 * Prepares the JSON payload expected by the job template api,
247 * by applying the overrides that were provided
248 * and allowed by the template definition flags in jtLaunchReqs
250 private fun prepareLaunchPayload(awxClient: BlueprintWebClientService, jtLaunchReqs: JsonNode,
251 isWorkflow : Boolean): String {
252 val payload = JacksonUtils.objectMapper.createObjectNode()
254 // Parameter defaults
255 val inventoryProp = getOptionalOperationInput(INPUT_INVENTORY)
256 val extraArgs = getOperationInput(INPUT_EXTRA_VARS)
259 val limitProp = getOptionalOperationInput(INPUT_LIMIT_TO_HOST)
260 val tagsProp = getOptionalOperationInput(INPUT_TAGS)
261 val skipTagsProp = getOptionalOperationInput(INPUT_SKIP_TAGS)
263 val askLimitOnLaunch = jtLaunchReqs.at("/ask_limit_on_launch").asBoolean()
264 if (askLimitOnLaunch && limitProp.isNotNull()) {
265 payload.set(INPUT_LIMIT_TO_HOST, limitProp)
267 val askTagsOnLaunch = jtLaunchReqs.at("/ask_tags_on_launch").asBoolean()
268 if (askTagsOnLaunch && tagsProp.isNotNull()) {
269 payload.set(INPUT_TAGS, tagsProp)
271 if (askTagsOnLaunch && skipTagsProp.isNotNull()) {
272 payload.set("skip_tags", skipTagsProp)
276 val askInventoryOnLaunch = jtLaunchReqs.at("/ask_inventory_on_launch").asBoolean()
277 if (askInventoryOnLaunch && inventoryProp.isNotNull()) {
278 var inventoryKeyId = if (inventoryProp is TextNode) {
279 resolveInventoryIdByName(awxClient, inventoryProp.textValue())?.asJsonPrimitive()
283 payload.set(INPUT_INVENTORY, inventoryKeyId)
285 val askVariablesOnLaunch = jtLaunchReqs.at("/ask_variables_on_launch").asBoolean()
286 if (askVariablesOnLaunch) {
287 payload.set("extra_vars", extraArgs)
289 return payload.asJsonString(false)
292 private fun resolveInventoryIdByName(awxClient: BlueprintWebClientService, inventoryProp: String): Int? {
293 var invId: Int? = null
295 // Get Inventory by name
296 val encoded = URLEncoder.encode(inventoryProp)
297 val response = awxClient.exchangeResource(GET, "/api/v2/inventories/?name=$encoded", "")
298 if (response.status in HTTP_SUCCESS) {
299 // Extract the inventory ID from response
300 val invDetails = mapper.readTree(response.body)
301 val nbInvFound = invDetails.at("/count").asInt()
302 if (nbInvFound == 1) {
303 invId = invDetails["results"][0]["id"].asInt()
304 log.info("Resolved inventory $inventoryProp to ID #: $invId")
309 val message = "Could not resolve inventory $inventoryProp by name..."
311 throw IllegalArgumentException(message)
318 * Utility function to set the output properties of the executor node
320 private fun setNodeOutputProperties(status: JsonNode, message: JsonNode) {
321 setAttribute(ATTRIBUTE_EXEC_CMD_STATUS, status)
322 log.info("Executor status: $status")
323 setAttribute(ATTRIBUTE_EXEC_CMD_LOG, message)
324 log.info("Executor message: $message")
328 * Utility function to set the output properties and errors of the executor node, in cas of errors
330 private fun setNodeOutputErrors(status: String, message: String) {
331 setAttribute(ATTRIBUTE_EXEC_CMD_STATUS, status.asJsonPrimitive())
332 setAttribute(ATTRIBUTE_EXEC_CMD_LOG, message.asJsonPrimitive())
334 addError(status, ATTRIBUTE_EXEC_CMD_LOG, message)