2 * ============LICENSE_START=======================================================
4 * ================================================================================
5 * Copyright (C) 2017 AT&T Intellectual Property. All rights reserved.
6 * ================================================================================
7 * Licensed under the Apache License, Version 2.0 (the "License");
8 * you may not use this file except in compliance with the License.
9 * You may obtain a copy of the License at
11 * http://www.apache.org/licenses/LICENSE-2.0
13 * Unless required by applicable law or agreed to in writing, software
14 * distributed under the License is distributed on an "AS IS" BASIS,
15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 * See the License for the specific language governing permissions and
17 * limitations under the License.
18 * ============LICENSE_END=========================================================
21 package org.openecomp.mso.openstack.utils;
23 import java.io.Serializable;
24 import java.util.ArrayList;
25 import java.util.Calendar;
26 import java.util.HashMap;
27 import java.util.List;
30 import org.codehaus.jackson.map.ObjectMapper;
31 import org.codehaus.jackson.JsonNode;
32 import org.codehaus.jackson.JsonParseException;
34 import org.openecomp.mso.cloud.CloudConfig;
35 import org.openecomp.mso.cloud.CloudConfigFactory;
36 import org.openecomp.mso.cloud.CloudIdentity;
37 import org.openecomp.mso.cloud.CloudSite;
38 import org.openecomp.mso.db.catalog.beans.HeatTemplate;
39 import org.openecomp.mso.db.catalog.beans.HeatTemplateParam;
40 import org.openecomp.mso.logger.MessageEnum;
41 import org.openecomp.mso.logger.MsoAlarmLogger;
42 import org.openecomp.mso.logger.MsoLogger;
43 import org.openecomp.mso.openstack.beans.HeatStatus;
44 import org.openecomp.mso.openstack.beans.StackInfo;
45 import org.openecomp.mso.openstack.exceptions.MsoAdapterException;
46 import org.openecomp.mso.openstack.exceptions.MsoCloudSiteNotFound;
47 import org.openecomp.mso.openstack.exceptions.MsoException;
48 import org.openecomp.mso.openstack.exceptions.MsoIOException;
49 import org.openecomp.mso.openstack.exceptions.MsoOpenstackException;
50 import org.openecomp.mso.openstack.exceptions.MsoStackAlreadyExists;
51 import org.openecomp.mso.openstack.exceptions.MsoTenantNotFound;
52 import org.openecomp.mso.properties.MsoJavaProperties;
53 import org.openecomp.mso.properties.MsoPropertiesException;
54 import org.openecomp.mso.properties.MsoPropertiesFactory;
55 import com.woorea.openstack.base.client.OpenStackConnectException;
56 import com.woorea.openstack.base.client.OpenStackRequest;
57 import com.woorea.openstack.base.client.OpenStackResponseException;
58 import com.woorea.openstack.heat.Heat;
59 import com.woorea.openstack.heat.model.CreateStackParam;
60 import com.woorea.openstack.heat.model.Stack;
61 import com.woorea.openstack.heat.model.Stack.Output;
62 import com.woorea.openstack.heat.model.Stacks;
63 import com.woorea.openstack.keystone.Keystone;
64 import com.woorea.openstack.keystone.model.Access;
65 import com.woorea.openstack.keystone.model.Authentication;
66 import com.woorea.openstack.keystone.utils.KeystoneUtils;
68 public class MsoHeatUtils extends MsoCommonUtils {
70 private MsoPropertiesFactory msoPropertiesFactory;
72 private CloudConfigFactory cloudConfigFactory;
74 private static final String TOKEN_AUTH = "TokenAuth";
76 private static final String QUERY_ALL_STACKS = "QueryAllStacks";
78 private static final String DELETE_STACK = "DeleteStack";
80 private static final String HEAT_ERROR = "HeatError";
82 private static final String CREATE_STACK = "CreateStack";
84 // Cache Heat Clients statically. Since there is just one MSO user, there is no
85 // benefit to re-authentication on every request (or across different flows). The
86 // token will be used until it expires.
88 // The cache key is "tenantId:cloudId"
89 private static Map <String, HeatCacheEntry> heatClientCache = new HashMap <String, HeatCacheEntry> ();
91 // Fetch cloud configuration each time (may be cached in CloudConfig class)
92 protected CloudConfig cloudConfig;
94 private static MsoLogger LOGGER = MsoLogger.getMsoLogger (MsoLogger.Catalog.RA);
96 protected MsoJavaProperties msoProps = null;
98 // Properties names and variables (with default values)
99 protected String createPollIntervalProp = "ecomp.mso.adapters.heat.create.pollInterval";
100 private String deletePollIntervalProp = "ecomp.mso.adapters.heat.delete.pollInterval";
101 private String deletePollTimeoutProp = "ecomp.mso.adapters.heat.delete.pollTimeout";
103 protected int createPollIntervalDefault = 15;
104 private int deletePollIntervalDefault = 15;
105 private int deletePollTimeoutDefault = 300;
106 private String msoPropID;
108 private static final ObjectMapper JSON_MAPPER = new ObjectMapper();
111 * This constructor MUST be used ONLY in the JUNIT tests, not for real code.
112 * The MsoPropertiesFactory will be added by EJB injection.
114 * @param msoPropID ID of the mso pro config as defined in web.xml
115 * @param msoPropFactory The mso properties factory instanciated by EJB injection
116 * @param cloudConfFactory the Cloud Config instantiated by EJB injection
118 public MsoHeatUtils (String msoPropID, MsoPropertiesFactory msoPropFactory, CloudConfigFactory cloudConfFactory) {
119 msoPropertiesFactory = msoPropFactory;
120 cloudConfigFactory = cloudConfFactory;
121 this.msoPropID = msoPropID;
122 // Dynamically get properties each time (in case reloaded).
125 msoProps = msoPropertiesFactory.getMsoJavaProperties (msoPropID);
126 } catch (MsoPropertiesException e) {
127 LOGGER.error (MessageEnum.LOAD_PROPERTIES_FAIL, "Unknown. Mso Properties ID not found in cache: " + msoPropID, "", "", MsoLogger.ErrorCode.DataError, "Exception - Mso Properties ID not found in cache", e);
129 cloudConfig = cloudConfigFactory.getCloudConfig ();
130 LOGGER.debug("MsoHeatUtils:" + msoPropID);
136 * keep this old method signature here to maintain backwards compatibility. keep others as well.
137 * this method does not include environment, files, or heatFiles
139 public StackInfo createStack (String cloudSiteId,
143 Map <String, ? extends Object> stackInputs,
144 boolean pollForCompletion,
145 int timeoutMinutes) throws MsoException {
146 // Just call the new method with the environment & files variable set to null
147 return this.createStack (cloudSiteId,
160 // This method has environment, but not files or heatFiles
161 public StackInfo createStack (String cloudSiteId,
165 Map <String, ? extends Object> stackInputs,
166 boolean pollForCompletion,
168 String environment) throws MsoException {
169 // Just call the new method with the files/heatFiles variables set to null
170 return this.createStack (cloudSiteId,
183 // This method has environment and files, but not heatFiles.
184 public StackInfo createStack (String cloudSiteId,
188 Map <String, ? extends Object> stackInputs,
189 boolean pollForCompletion,
192 Map <String, Object> files) throws MsoException {
193 return this.createStack (cloudSiteId,
206 // This method has environment, files, heatfiles
207 public StackInfo createStack (String cloudSiteId,
211 Map <String, ? extends Object> stackInputs,
212 boolean pollForCompletion,
215 Map <String, Object> files,
216 Map <String, Object> heatFiles) throws MsoException {
217 return this.createStack (cloudSiteId,
231 * Create a new Stack in the specified cloud location and tenant. The Heat template
232 * and parameter map are passed in as arguments, along with the cloud access credentials.
233 * It is expected that parameters have been validated and contain at minimum the required
234 * parameters for the given template with no extra (undefined) parameters..
236 * The Stack name supplied by the caller must be unique in the scope of this tenant.
237 * However, it should also be globally unique, as it will be the identifier for the
238 * resource going forward in Inventory. This latter is managed by the higher levels
239 * invoking this function.
241 * The caller may choose to let this function poll Openstack for completion of the
242 * stack creation, or may handle polling itself via separate calls to query the status.
243 * In either case, a StackInfo object will be returned containing the current status.
244 * When polling is enabled, a status of CREATED is expected. When not polling, a
245 * status of BUILDING is expected.
247 * An error will be thrown if the requested Stack already exists in the specified
250 * For 1510 - add "environment", "files" (nested templates), and "heatFiles" (get_files) as
251 * parameters for createStack. If environment is non-null, it will be added to the stack.
252 * The nested templates and get_file entries both end up being added to the "files" on the
253 * stack. We must combine them before we add them to the stack if they're both non-null.
255 * @param cloudSiteId The cloud (may be a region) in which to create the stack.
256 * @param tenantId The Openstack ID of the tenant in which to create the Stack
257 * @param stackName The name of the stack to create
258 * @param stackTemplate The Heat template
259 * @param stackInputs A map of key/value inputs
260 * @param pollForCompletion Indicator that polling should be handled in Java vs. in the client
261 * @param environment An optional yaml-format string to specify environmental parameters
262 * @param files a Map<String, Object> that lists the child template IDs (file is the string, object is an int of
264 * @param heatFiles a Map<String, Object> that lists the get_file entries (fileName, fileBody)
265 * @param backout Donot delete stack on create Failure - defaulted to True
266 * @return A StackInfo object
267 * @throws MsoOpenstackException Thrown if the Openstack API call returns an exception.
270 @SuppressWarnings("unchecked")
271 public StackInfo createStack (String cloudSiteId,
275 Map <String, ? extends Object> stackInputs,
276 boolean pollForCompletion,
279 Map <String, Object> files,
280 Map <String, Object> heatFiles,
281 boolean backout) throws MsoException {
282 // Create local variables checking to see if we have an environment, nested, get_files
283 // Could later add some checks to see if it's valid.
284 boolean haveEnvtVariable = true;
285 if (environment == null || "".equalsIgnoreCase (environment.trim ())) {
286 haveEnvtVariable = false;
287 LOGGER.debug ("createStack called with no environment variable");
289 LOGGER.debug ("createStack called with an environment variable: " + environment);
292 boolean haveFiles = true;
293 if (files == null || files.isEmpty ()) {
295 LOGGER.debug ("createStack called with no files / child template ids");
297 LOGGER.debug ("createStack called with " + files.size () + " files / child template ids");
300 boolean haveHeatFiles = true;
301 if (heatFiles == null || heatFiles.isEmpty ()) {
302 haveHeatFiles = false;
303 LOGGER.debug ("createStack called with no heatFiles");
305 LOGGER.debug ("createStack called with " + heatFiles.size () + " heatFiles");
308 // Obtain the cloud site information where we will create the stack
309 CloudSite cloudSite = cloudConfig.getCloudSite (cloudSiteId);
310 if (cloudSite == null) {
311 throw new MsoCloudSiteNotFound (cloudSiteId);
313 // Get a Heat client. They are cached between calls (keyed by tenantId:cloudId)
314 // This could throw MsoTenantNotFound or MsoOpenstackException (both propagated)
315 Heat heatClient = getHeatClient (cloudSite, tenantId);
317 LOGGER.debug ("Ready to Create Stack (" + heatTemplate + ") with input params: " + stackInputs);
319 // Build up the stack to create
320 // Disable auto-rollback, because error reason is lost. Always rollback in the code.
321 CreateStackParam stack = new CreateStackParam ();
322 stack.setStackName (stackName);
323 stack.setTimeoutMinutes (timeoutMinutes);
324 stack.setParameters ((Map <String, Object>) stackInputs);
325 stack.setTemplate (heatTemplate);
326 stack.setDisableRollback (true);
327 // TJM New for PO Adapter - add envt variable
328 if (haveEnvtVariable) {
329 LOGGER.debug ("Found an environment variable - value: " + environment);
330 stack.setEnvironment (environment);
332 // Now handle nested templates or get_files - have to combine if we have both
333 // as they're both treated as "files:" on the stack.
334 if (haveFiles && haveHeatFiles) {
335 // Let's do this here - not in the bean
336 LOGGER.debug ("Found files AND heatFiles - combine and add!");
337 Map <String, Object> combinedFiles = new HashMap <String, Object> ();
338 for (String keyString : files.keySet ()) {
339 combinedFiles.put (keyString, files.get (keyString));
341 for (String keyString : heatFiles.keySet ()) {
342 combinedFiles.put (keyString, heatFiles.get (keyString));
344 stack.setFiles (combinedFiles);
346 // Handle if we only have one or neither:
348 LOGGER.debug ("Found files - adding to stack");
349 stack.setFiles (files);
352 LOGGER.debug ("Found heatFiles - adding to stack");
353 // the setFiles was modified to handle adding the entries
354 stack.setFiles (heatFiles);
358 Stack heatStack = null;
360 // Execute the actual Openstack command to create the Heat stack
361 OpenStackRequest <Stack> request = heatClient.getStacks ().create (stack);
363 // Obtain an MSO token for the tenant
364 CloudIdentity cloudIdentity = cloudSite.getIdentityService ();
365 // cloudIdentity.getMsoId(), cloudIdentity.getMsoPass()
367 request.header ("X-Auth-User", cloudIdentity.getMsoId ());
368 request.header ("X-Auth-Key", cloudIdentity.getMsoPass ());
369 LOGGER.debug ("headers added, about to executeAndRecordOpenstackRequest");
370 LOGGER.debug(this.requestToStringBuilder(stack).toString());
371 // END - try to fix X-Auth-User
372 heatStack = executeAndRecordOpenstackRequest (request, msoProps);
373 } catch (OpenStackResponseException e) {
374 // Since this came on the 'Create Stack' command, nothing was changed
375 // in the cloud. Return the error as an exception.
376 if (e.getStatus () == 409) {
377 // Stack already exists. Return a specific error for this case
378 MsoStackAlreadyExists me = new MsoStackAlreadyExists (stackName, tenantId, cloudSiteId);
379 me.addContext (CREATE_STACK);
382 // Convert the OpenStackResponseException to an MsoOpenstackException
383 LOGGER.debug("ERROR STATUS = " + e.getStatus() + ",\n" + e.getMessage() + "\n" + e.getLocalizedMessage());
384 throw heatExceptionToMsoException (e, CREATE_STACK);
386 } catch (OpenStackConnectException e) {
387 // Error connecting to Openstack instance. Convert to an MsoException
388 throw heatExceptionToMsoException (e, CREATE_STACK);
389 } catch (RuntimeException e) {
391 throw runtimeExceptionToMsoException (e, CREATE_STACK);
394 // Subsequent access by the canonical name "<stack name>/<stack-id>".
395 // Otherwise, simple query by name returns a 302 redirect.
396 // NOTE: This is specific to the v1 Orchestration API.
397 String canonicalName = stackName + "/" + heatStack.getId ();
399 // If client has requested a final response, poll for stack completion
400 if (pollForCompletion) {
401 // Set a time limit on overall polling.
402 // Use the resource (template) timeout for Openstack (expressed in minutes)
403 // and add one poll interval to give Openstack a chance to fail on its own.
404 int createPollInterval = msoProps.getIntProperty (createPollIntervalProp, createPollIntervalDefault);
405 int pollTimeout = (timeoutMinutes * 60) + createPollInterval;
406 // New 1610 - poll on delete if we rollback - use same values for now
407 int deletePollInterval = createPollInterval;
408 int deletePollTimeout = pollTimeout;
409 boolean createTimedOut = false;
410 StringBuilder stackErrorStatusReason = new StringBuilder("");
411 LOGGER.debug("createPollInterval=" + createPollInterval + ", pollTimeout=" + pollTimeout);
415 heatStack = queryHeatStack (heatClient, canonicalName);
416 LOGGER.debug (heatStack.getStackStatus () + " (" + canonicalName + ")");
418 LOGGER.debug("Current stack " + this.getOutputsAsStringBuilder(heatStack).toString());
419 } catch (Exception e) {
420 LOGGER.debug("an error occurred trying to print out the current outputs of the stack");
423 if ("CREATE_IN_PROGRESS".equals (heatStack.getStackStatus ())) {
424 // Stack creation is still running.
425 // Sleep and try again unless timeout has been reached
426 if (pollTimeout <= 0) {
427 // Note that this should not occur, since there is a timeout specified
428 // in the Openstack call.
429 LOGGER.error (MessageEnum.RA_CREATE_STACK_TIMEOUT, cloudSiteId, tenantId, stackName, heatStack.getStackStatus (), "", "", MsoLogger.ErrorCode.AvailabilityError, "Create stack timeout");
430 createTimedOut = true;
434 Thread.sleep (createPollInterval * 1000L);
435 } catch (InterruptedException e) {
436 LOGGER.debug ("Thread interrupted while sleeping", e);
439 pollTimeout -= createPollInterval;
440 LOGGER.debug("pollTimeout remaining: " + pollTimeout);
442 //save off the status & reason msg before we attempt delete
443 stackErrorStatusReason.append("Stack error (" + heatStack.getStackStatus() + "): " + heatStack.getStackStatusReason());
446 } catch (MsoException me) {
447 // Cannot query the stack status. Something is wrong.
448 // Try to roll back the stack
451 LOGGER.warn(MessageEnum.RA_CREATE_STACK_ERR, "Create Stack errored, stack deletion suppressed", "", "", MsoLogger.ErrorCode.BusinessProcesssError, "Exception in Create Stack, stack deletion suppressed");
456 LOGGER.debug("Create Stack error - unable to query for stack status - attempting to delete stack: " + canonicalName + " - This will likely fail and/or we won't be able to query to see if delete worked");
457 OpenStackRequest <Void> request = heatClient.getStacks ().deleteByName (canonicalName);
458 executeAndRecordOpenstackRequest (request, msoProps);
459 // this may be a waste of time - if we just got an exception trying to query the stack - we'll just
460 // get another one, n'est-ce pas?
461 boolean deleted = false;
464 heatStack = queryHeatStack(heatClient, canonicalName);
465 if (heatStack != null) {
466 LOGGER.debug(heatStack.getStackStatus());
467 if ("DELETE_IN_PROGRESS".equals(heatStack.getStackStatus())) {
468 if (deletePollTimeout <= 0) {
469 LOGGER.error (MessageEnum.RA_CREATE_STACK_TIMEOUT, cloudSiteId, tenantId, stackName,
470 heatStack.getStackStatus (), "", "", MsoLogger.ErrorCode.AvailabilityError,
471 "Rollback: DELETE stack timeout");
475 Thread.sleep(deletePollInterval * 1000L);
476 } catch (InterruptedException ie) {
477 LOGGER.debug("Thread interrupted while sleeping", ie);
479 deletePollTimeout -= deletePollInterval;
481 } else if ("DELETE_COMPLETE".equals(heatStack.getStackStatus())){
482 LOGGER.debug("DELETE_COMPLETE for " + canonicalName);
486 //got a status other than DELETE_IN_PROGRESS or DELETE_COMPLETE - so break and evaluate
490 // assume if we can't find it - it's deleted
491 LOGGER.debug("heatStack returned null - assume the stack " + canonicalName + " has been deleted");
496 } catch (Exception e3) {
497 // Just log this one. We will report the original exception.
498 LOGGER.error (MessageEnum.RA_CREATE_STACK_ERR, "Create Stack: Nested exception rolling back stack: " + e3, "", "", MsoLogger.ErrorCode.BusinessProcesssError, "Create Stack: Nested exception rolling back stack on error on query");
502 } catch (Exception e2) {
503 // Just log this one. We will report the original exception.
504 LOGGER.error (MessageEnum.RA_CREATE_STACK_ERR, "Create Stack: Nested exception rolling back stack: " + e2, "", "", MsoLogger.ErrorCode.BusinessProcesssError, "Create Stack: Nested exception rolling back stack");
508 // Propagate the original exception from Stack Query.
509 me.addContext (CREATE_STACK);
514 if (!"CREATE_COMPLETE".equals (heatStack.getStackStatus ())) {
515 LOGGER.error (MessageEnum.RA_CREATE_STACK_ERR, "Create Stack error: Polling complete with non-success status: "
516 + heatStack.getStackStatus () + ", " + heatStack.getStackStatusReason (), "", "", MsoLogger.ErrorCode.BusinessProcesssError, "Create Stack error");
518 // Rollback the stack creation, since it is in an indeterminate state.
521 LOGGER.warn(MessageEnum.RA_CREATE_STACK_ERR, "Create Stack errored, stack deletion suppressed", "", "", MsoLogger.ErrorCode.BusinessProcesssError, "Create Stack error, stack deletion suppressed");
526 LOGGER.debug("Create Stack errored - attempting to DELETE stack: " + canonicalName);
527 LOGGER.debug("deletePollInterval=" + deletePollInterval + ", deletePollTimeout=" + deletePollTimeout);
528 OpenStackRequest <Void> request = heatClient.getStacks ().deleteByName (canonicalName);
529 executeAndRecordOpenstackRequest (request, msoProps);
530 boolean deleted = false;
533 heatStack = queryHeatStack(heatClient, canonicalName);
534 if (heatStack != null) {
535 LOGGER.debug(heatStack.getStackStatus() + " (" + canonicalName + ")");
536 if ("DELETE_IN_PROGRESS".equals(heatStack.getStackStatus())) {
537 if (deletePollTimeout <= 0) {
538 LOGGER.error (MessageEnum.RA_CREATE_STACK_TIMEOUT, cloudSiteId, tenantId, stackName,
539 heatStack.getStackStatus (), "", "", MsoLogger.ErrorCode.AvailabilityError,
540 "Rollback: DELETE stack timeout");
544 Thread.sleep(deletePollInterval * 1000L);
545 } catch (InterruptedException ie) {
546 LOGGER.debug("Thread interrupted while sleeping", ie);
548 deletePollTimeout -= deletePollInterval;
549 LOGGER.debug("deletePollTimeout remaining: " + deletePollTimeout);
551 } else if ("DELETE_COMPLETE".equals(heatStack.getStackStatus())){
552 LOGGER.debug("DELETE_COMPLETE for " + canonicalName);
555 } else if ("DELETE_FAILED".equals(heatStack.getStackStatus())) {
556 // Warn about this (?) - but still throw the original exception
557 LOGGER.warn(MessageEnum.RA_CREATE_STACK_ERR, "Create Stack errored, stack deletion FAILED", "", "", MsoLogger.ErrorCode.BusinessProcesssError, "Create Stack error, stack deletion FAILED");
558 LOGGER.debug("Stack deletion FAILED on a rollback of a create - " + canonicalName + ", status=" + heatStack.getStackStatus() + ", reason=" + heatStack.getStackStatusReason());
561 //got a status other than DELETE_IN_PROGRESS or DELETE_COMPLETE - so break and evaluate
565 // assume if we can't find it - it's deleted
566 LOGGER.debug("heatStack returned null - assume the stack " + canonicalName + " has been deleted");
571 } catch (MsoException me2) {
572 // We got an exception on the delete - don't throw this exception - throw the original - just log.
573 LOGGER.debug("Exception thrown trying to delete " + canonicalName + " on a create->rollback: " + me2.getContextMessage());
574 LOGGER.warn(MessageEnum.RA_CREATE_STACK_ERR, "Create Stack errored, then stack deletion FAILED - exception thrown", "", "", MsoLogger.ErrorCode.BusinessProcesssError, me2.getContextMessage());
577 } // end while !deleted
578 StringBuilder errorContextMessage = null;
579 if (createTimedOut) {
580 errorContextMessage = new StringBuilder("Stack Creation Timeout");
582 errorContextMessage = stackErrorStatusReason;
585 errorContextMessage.append(" - stack successfully deleted");
587 errorContextMessage.append(" - encountered an error trying to delete the stack");
589 // MsoOpenstackException me = new MsoOpenstackException(0, "", stackErrorStatusReason.toString());
590 // me.addContext(CREATE_STACK);
591 // alarmLogger.sendAlarm(HEAT_ERROR, MsoAlarmLogger.CRITICAL, me.getContextMessage());
593 } catch (Exception e2) {
594 // shouldn't happen - but handle
595 LOGGER.error (MessageEnum.RA_CREATE_STACK_ERR, "Create Stack: Nested exception rolling back stack: " + e2, "", "", MsoLogger.ErrorCode.BusinessProcesssError, "Exception in Create Stack: rolling back stack");
598 MsoOpenstackException me = new MsoOpenstackException(0, "", stackErrorStatusReason.toString());
599 me.addContext(CREATE_STACK);
600 alarmLogger.sendAlarm(HEAT_ERROR, MsoAlarmLogger.CRITICAL, me.getContextMessage());
605 // Get initial status, since it will have been null after the create.
606 heatStack = queryHeatStack (heatClient, canonicalName);
607 LOGGER.debug (heatStack.getStackStatus ());
610 return new StackInfo (heatStack);
614 * Query for a single stack (by Name) in a tenant. This call will always return a
615 * StackInfo object. If the stack does not exist, an "empty" StackInfo will be
616 * returned - containing only the stack name and a status of NOTFOUND.
618 * @param tenantId The Openstack ID of the tenant in which to query
619 * @param cloudSiteId The cloud identifier (may be a region) in which to query
620 * @param stackName The name of the stack to query (may be simple or canonical)
621 * @return A StackInfo object
622 * @throws MsoOpenstackException Thrown if the Openstack API call returns an exception.
624 public StackInfo queryStack (String cloudSiteId, String tenantId, String stackName) throws MsoException {
625 LOGGER.debug ("Query HEAT stack: " + stackName + " in tenant " + tenantId);
627 // Obtain the cloud site information where we will create the stack
628 CloudSite cloudSite = cloudConfig.getCloudSite (cloudSiteId);
629 if (cloudSite == null) {
630 throw new MsoCloudSiteNotFound (cloudSiteId);
633 // Get a Heat client. They are cached between calls (keyed by tenantId:cloudId)
634 Heat heatClient = null;
636 heatClient = getHeatClient (cloudSite, tenantId);
637 } catch (MsoTenantNotFound e) {
638 // Tenant doesn't exist, so stack doesn't either
639 LOGGER.debug ("Tenant with id " + tenantId + "not found.", e);
640 return new StackInfo (stackName, HeatStatus.NOTFOUND, null, null);
641 } catch (MsoException me) {
642 // Got an Openstack error. Propagate it
643 LOGGER.error (MessageEnum.RA_CONNECTION_EXCEPTION, "OpenStack", "Openstack Exception on Token request: " + me, "Openstack", "", MsoLogger.ErrorCode.AvailabilityError, "Connection Exception");
644 me.addContext ("QueryStack");
649 // An MsoException will propagate transparently to the caller.
650 Stack heatStack = queryHeatStack (heatClient, stackName);
652 if (heatStack == null) {
653 // Stack does not exist. Return a StackInfo with status NOTFOUND
654 StackInfo stackInfo = new StackInfo (stackName, HeatStatus.NOTFOUND, null, null);
658 return new StackInfo (heatStack);
662 * Delete a stack (by Name/ID) in a tenant. If the stack is not found, it will be
663 * considered a successful deletion. The return value is a StackInfo object which
664 * contains the current stack status.
666 * The client may choose to let the adapter poll Openstack for completion of the
667 * stack deletion, or may handle polling itself via separate query calls. In either
668 * case, a StackInfo object will be returned. When polling is enabled, a final
669 * status of NOTFOUND is expected. When not polling, a status of DELETING is expected.
671 * There is no rollback from a successful stack deletion. A deletion failure will
672 * also result in an undefined stack state - the components may or may not have been
673 * all or partially deleted, so the resulting stack must be considered invalid.
675 * @param tenantId The Openstack ID of the tenant in which to perform the delete
676 * @param cloudSiteId The cloud identifier (may be a region) from which to delete the stack.
677 * @param stackName The name/id of the stack to delete. May be simple or canonical
678 * @param pollForCompletion Indicator that polling should be handled in Java vs. in the client
679 * @return A StackInfo object
680 * @throws MsoOpenstackException Thrown if the Openstack API call returns an exception.
681 * @throws MsoCloudSiteNotFound
683 public StackInfo deleteStack (String tenantId,
686 boolean pollForCompletion) throws MsoException {
687 // Obtain the cloud site information where we will create the stack
688 CloudSite cloudSite = cloudConfig.getCloudSite (cloudSiteId);
689 if (cloudSite == null) {
690 throw new MsoCloudSiteNotFound (cloudSiteId);
693 // Get a Heat client. They are cached between calls (keyed by tenantId:cloudId)
694 Heat heatClient = null;
696 heatClient = getHeatClient (cloudSite, tenantId);
697 } catch (MsoTenantNotFound e) {
698 // Tenant doesn't exist, so stack doesn't either
699 LOGGER.debug ("Tenant with id " + tenantId + "not found.", e);
700 return new StackInfo (stackName, HeatStatus.NOTFOUND, null, null);
701 } catch (MsoException me) {
702 // Got an Openstack error. Propagate it
703 LOGGER.error (MessageEnum.RA_CONNECTION_EXCEPTION, "Openstack", "Openstack Exception on Token request: " + me, "Openstack", "", MsoLogger.ErrorCode.AvailabilityError, "Connection Exception");
704 me.addContext (DELETE_STACK);
708 // OK if stack not found, perform a query first
709 Stack heatStack = queryHeatStack (heatClient, stackName);
710 if (heatStack == null || "DELETE_COMPLETE".equals (heatStack.getStackStatus ())) {
711 // Not found. Return a StackInfo with status NOTFOUND
712 return new StackInfo (stackName, HeatStatus.NOTFOUND, null, null);
717 // Use canonical name "<stack name>/<stack-id>" to delete.
718 // Otherwise, deletion by name returns a 302 redirect.
719 // NOTE: This is specific to the v1 Orchestration API.
720 String canonicalName = heatStack.getStackName () + "/" + heatStack.getId ();
723 OpenStackRequest <Void> request = heatClient.getStacks ().deleteByName (canonicalName);
724 executeAndRecordOpenstackRequest (request, msoProps);
725 } catch (OpenStackResponseException e) {
726 if (e.getStatus () == 404) {
727 // Not found. We are OK with this. Return a StackInfo with status NOTFOUND
728 return new StackInfo (stackName, HeatStatus.NOTFOUND, null, null);
730 // Convert the OpenStackResponseException to an MsoOpenstackException
731 throw heatExceptionToMsoException (e, DELETE_STACK);
733 } catch (OpenStackConnectException e) {
734 // Error connecting to Openstack instance. Convert to an MsoException
735 throw heatExceptionToMsoException (e, DELETE_STACK);
736 } catch (RuntimeException e) {
738 throw runtimeExceptionToMsoException (e, DELETE_STACK);
741 // Requery the stack for current status.
742 // It will probably still exist with "DELETE_IN_PROGRESS" status.
743 heatStack = queryHeatStack (heatClient, canonicalName);
745 if (pollForCompletion) {
746 // Set a timeout on polling
747 int pollInterval = msoProps.getIntProperty (deletePollIntervalProp, deletePollIntervalDefault);
748 int pollTimeout = msoProps.getIntProperty (deletePollTimeoutProp, deletePollTimeoutDefault);
750 // When querying by canonical name, Openstack returns DELETE_COMPLETE status
751 // instead of "404" (which would result from query by stack name).
752 while (heatStack != null && !"DELETE_COMPLETE".equals (heatStack.getStackStatus ())) {
753 LOGGER.debug ("Stack status: " + heatStack.getStackStatus ());
755 if ("DELETE_FAILED".equals (heatStack.getStackStatus ())) {
756 // Throw a 'special case' of MsoOpenstackException to report the Heat status
757 String error = "Stack delete error (" + heatStack.getStackStatus ()
759 + heatStack.getStackStatusReason ();
760 MsoOpenstackException me = new MsoOpenstackException (0, "", error);
761 me.addContext (DELETE_STACK);
763 // Alarm this condition, stack deletion failed
764 alarmLogger.sendAlarm (HEAT_ERROR, MsoAlarmLogger.CRITICAL, me.getContextMessage ());
769 if (pollTimeout <= 0) {
770 LOGGER.error (MessageEnum.RA_DELETE_STACK_TIMEOUT, cloudSiteId, tenantId, stackName, heatStack.getStackStatus (), "", "", MsoLogger.ErrorCode.AvailabilityError, "Delete Stack Timeout");
772 // Throw a 'special case' of MsoOpenstackException to report the Heat status
773 MsoOpenstackException me = new MsoOpenstackException (0, "", "Stack Deletion Timeout");
774 me.addContext (DELETE_STACK);
776 // Alarm this condition, stack deletion failed
777 alarmLogger.sendAlarm (HEAT_ERROR, MsoAlarmLogger.CRITICAL, me.getContextMessage ());
783 Thread.sleep (pollInterval * 1000L);
784 } catch (InterruptedException e) {
785 LOGGER.debug ("Thread interrupted while sleeping", e);
788 pollTimeout -= pollInterval;
790 heatStack = queryHeatStack (heatClient, canonicalName);
793 // The stack is gone when this point is reached
794 return new StackInfo (stackName, HeatStatus.NOTFOUND, null, null);
797 // Return the current status (if not polling, the delete may still be in progress)
798 StackInfo stackInfo = new StackInfo (heatStack);
799 stackInfo.setName (stackName);
805 * Query for all stacks in a tenant site. This call will return a List of StackInfo
806 * objects, one for each deployed stack.
808 * Note that this is limited to a single site. To ensure that a tenant is truly
809 * empty would require looping across all tenant endpoints.
811 * @param tenantId The Openstack ID of the tenant to query
812 * @param cloudSiteId The cloud identifier (may be a region) in which to query.
813 * @return A List of StackInfo objects
814 * @throws MsoOpenstackException Thrown if the Openstack API call returns an exception.
815 * @throws MsoCloudSiteNotFound
817 public List <StackInfo> queryAllStacks (String tenantId, String cloudSiteId) throws MsoException {
818 // Obtain the cloud site information where we will create the stack
819 CloudSite cloudSite = cloudConfig.getCloudSite (cloudSiteId);
820 if (cloudSite == null) {
821 throw new MsoCloudSiteNotFound (cloudSiteId);
824 // Get a Heat client. They are cached between calls (keyed by tenantId:cloudId)
825 Heat heatClient = getHeatClient (cloudSite, tenantId);
828 OpenStackRequest <Stacks> request = heatClient.getStacks ().list ();
829 Stacks stacks = executeAndRecordOpenstackRequest (request, msoProps);
831 List <StackInfo> stackList = new ArrayList <StackInfo> ();
833 // Not sure if returns an empty list or null if no stacks exist
834 if (stacks != null) {
835 for (Stack stack : stacks) {
836 stackList.add (new StackInfo (stack));
841 } catch (OpenStackResponseException e) {
842 if (e.getStatus () == 404) {
843 // Not sure if this can happen, but return an empty list
844 LOGGER.debug ("queryAllStacks - stack not found: ");
845 return new ArrayList <StackInfo> ();
847 // Convert the OpenStackResponseException to an MsoOpenstackException
848 throw heatExceptionToMsoException (e, QUERY_ALL_STACKS);
850 } catch (OpenStackConnectException e) {
851 // Error connecting to Openstack instance. Convert to an MsoException
852 throw heatExceptionToMsoException (e, QUERY_ALL_STACKS);
853 } catch (RuntimeException e) {
855 throw runtimeExceptionToMsoException (e, QUERY_ALL_STACKS);
860 * Validate parameters to be passed to Heat template. This method performs
862 * 1. Apply default values to parameters which have them defined
863 * 2. Report any required parameters that are missing. This will generate an
864 * exception in the caller, since stack create/update operations would fail.
865 * 3. Report and remove any extraneous parameters. This will allow clients to
866 * pass supersets of parameters and not get errors.
868 * These functions depend on the HeatTemplate definition from the MSO Catalog DB,
869 * along with the input parameter Map. The output is an updated parameter map.
870 * If the parameters are invalid for the template, an IllegalArgumentException
873 public Map <String, Object> validateStackParams (Map <String, Object> inputParams,
874 HeatTemplate heatTemplate) throws IllegalArgumentException {
875 // Check that required parameters have been supplied for this template type
876 String missingParams = null;
877 List <String> paramList = new ArrayList <String> ();
879 // TODO: Enhance DB to support defaults for Heat Template parameters
881 for (HeatTemplateParam parm : heatTemplate.getParameters ()) {
882 if (parm.isRequired () && !inputParams.containsKey (parm.getParamName ())) {
883 if (missingParams == null) {
884 missingParams = parm.getParamName ();
886 missingParams += "," + parm.getParamName ();
889 paramList.add (parm.getParamName ());
891 if (missingParams != null) {
892 // Problem - missing one or more required parameters
893 String error = "Missing Required inputs for HEAT Template: " + missingParams;
894 LOGGER.error (MessageEnum.RA_MISSING_PARAM, missingParams + " for HEAT Template", "", "", MsoLogger.ErrorCode.SchemaError, "Missing Required inputs for HEAT Template: " + missingParams);
895 throw new IllegalArgumentException (error);
898 // Remove any extraneous parameters (don't throw an error)
899 Map <String, Object> updatedParams = new HashMap <String, Object> ();
900 List <String> extraParams = new ArrayList <String> ();
901 for (String key : inputParams.keySet ()) {
902 if (!paramList.contains (key)) {
903 // This is not a valid parameter for this template
904 extraParams.add (key);
906 updatedParams.put (key, inputParams.get (key));
909 if (!extraParams.isEmpty ()) {
910 LOGGER.warn (MessageEnum.RA_GENERAL_WARNING, "Heat Stack (" + heatTemplate.getTemplateName ()
911 + ") extra input params received: "
912 + extraParams, "", "", MsoLogger.ErrorCode.DataError, "Heat Stack (" + heatTemplate.getTemplateName () + ") extra input params received: "+ extraParams);
915 return updatedParams;
918 // ---------------------------------------------------------------
919 // PRIVATE FUNCTIONS FOR USE WITHIN THIS CLASS
922 * Get a Heat client for the Openstack Identity service.
923 * This requires a 'member'-level userId + password, which will be retrieved from
924 * properties based on the specified cloud Id. The tenant in which to operate
925 * must also be provided.
927 * On successful authentication, the Heat object will be cached for the
928 * tenantID + cloudId so that it can be reused without reauthenticating with
929 * Openstack every time.
933 * @return an authenticated Heat object
935 public Heat getHeatClient (CloudSite cloudSite, String tenantId) throws MsoException {
936 String cloudId = cloudSite.getId ();
938 // Check first in the cache of previously authorized clients
939 String cacheKey = cloudId + ":" + tenantId;
940 if (heatClientCache.containsKey (cacheKey)) {
941 if (!heatClientCache.get (cacheKey).isExpired ()) {
942 LOGGER.debug ("Using Cached HEAT Client for " + cacheKey);
943 return heatClientCache.get (cacheKey).getHeatClient ();
945 // Token is expired. Remove it from cache.
946 heatClientCache.remove (cacheKey);
947 LOGGER.debug ("Expired Cached HEAT Client for " + cacheKey);
951 // Obtain an MSO token for the tenant
952 CloudIdentity cloudIdentity = cloudSite.getIdentityService ();
953 String keystoneUrl = cloudIdentity.getKeystoneUrl (cloudId, msoPropID);
954 Keystone keystoneTenantClient = new Keystone (keystoneUrl);
955 Access access = null;
957 Authentication credentials = cloudIdentity.getAuthentication ();
959 OpenStackRequest <Access> request = keystoneTenantClient.tokens ()
960 .authenticate (credentials).withTenantId (tenantId);
962 access = executeAndRecordOpenstackRequest (request, msoProps);
963 } catch (OpenStackResponseException e) {
964 if (e.getStatus () == 401) {
965 // Authentication error.
966 String error = "Authentication Failure: tenant=" + tenantId + ",cloud=" + cloudIdentity.getId ();
967 alarmLogger.sendAlarm ("MsoAuthenticationError", MsoAlarmLogger.CRITICAL, error);
968 throw new MsoAdapterException (error);
970 throw keystoneErrorToMsoException (e, TOKEN_AUTH);
972 } catch (OpenStackConnectException e) {
973 // Connection to Openstack failed
974 MsoIOException me = new MsoIOException (e.getMessage (), e);
975 me.addContext (TOKEN_AUTH);
977 } catch (RuntimeException e) {
979 throw runtimeExceptionToMsoException (e, TOKEN_AUTH);
982 // For DCP/LCP, the region should be the cloudId.
983 String region = cloudSite.getRegionId ();
984 String heatUrl = null;
986 heatUrl = KeystoneUtils.findEndpointURL (access.getServiceCatalog (), "orchestration", region, "public");
987 LOGGER.debug("heatUrl=" + heatUrl + ", region=" + region);
988 } catch (RuntimeException e) {
989 // This comes back for not found (probably an incorrect region ID)
990 String error = "Orchestration service not found: region=" + region + ",cloud=" + cloudIdentity.getId ();
991 alarmLogger.sendAlarm ("MsoConfigurationError", MsoAlarmLogger.CRITICAL, error);
992 throw new MsoAdapterException (error, e);
995 Heat heatClient = new Heat (heatUrl);
996 heatClient.token (access.getToken ().getId ());
998 heatClientCache.put (cacheKey,
999 new HeatCacheEntry (heatUrl,
1000 access.getToken ().getId (),
1001 access.getToken ().getExpires ()));
1002 LOGGER.debug ("Caching HEAT Client for " + cacheKey);
1008 * Forcibly expire a HEAT client from the cache. This call is for use by
1009 * the KeystoneClient in case where a tenant is deleted. In that case,
1010 * all cached credentials must be purged so that fresh authentication is
1011 * done if a similarly named tenant is re-created.
1013 * Note: This is probably only applicable to dev/test environments where
1014 * the same Tenant Name is repeatedly used for creation/deletion.
1020 public static void expireHeatClient (String tenantId, String cloudId) {
1021 String cacheKey = cloudId + ":" + tenantId;
1022 if (heatClientCache.containsKey (cacheKey)) {
1023 heatClientCache.remove (cacheKey);
1024 LOGGER.debug ("Deleted Cached HEAT Client for " + cacheKey);
1029 * Query for a Heat Stack. This function is needed in several places, so
1030 * a common method is useful. This method takes an authenticated Heat Client
1031 * (which internally identifies the cloud & tenant to search), and returns
1032 * a Stack object if found, Null if not found, or an MsoOpenstackException
1033 * if the Openstack API call fails.
1035 * The stack name may be a simple name or a canonical name ("{name}/{id}").
1036 * When simple name is used, Openstack always returns a 302 redirect which
1037 * results in a 2nd request (to the canonical name). Note that query by
1038 * canonical name for a deleted stack returns a Stack object with status
1039 * "DELETE_COMPLETE" while query by simple name for a deleted stack returns
1042 * @param heatClient an authenticated Heat client
1044 * @param stackName the stack name to query
1046 * @return a Stack object that describes the current stack or null if the
1047 * requested stack doesn't exist.
1049 * @throws MsoOpenstackException Thrown if the Openstack API call returns an exception
1051 protected Stack queryHeatStack (Heat heatClient, String stackName) throws MsoException {
1052 if (stackName == null) {
1056 OpenStackRequest <Stack> request = heatClient.getStacks ().byName (stackName);
1057 return executeAndRecordOpenstackRequest (request, msoProps);
1058 } catch (OpenStackResponseException e) {
1059 if (e.getStatus () == 404) {
1060 LOGGER.debug ("queryHeatStack - stack not found: " + stackName);
1063 // Convert the OpenStackResponseException to an MsoOpenstackException
1064 throw heatExceptionToMsoException (e, "QueryStack");
1066 } catch (OpenStackConnectException e) {
1067 // Connection to Openstack failed
1068 throw heatExceptionToMsoException (e, "QueryAllStack");
1073 * An entry in the Heat Client Cache. It saves the Heat client object
1074 * along with the token expiration. After this interval, this cache
1075 * item will no longer be used.
1077 private static class HeatCacheEntry implements Serializable {
1079 private static final long serialVersionUID = 1L;
1081 private String heatUrl;
1082 private String token;
1083 private Calendar expires;
1085 public HeatCacheEntry (String heatUrl, String token, Calendar expires) {
1086 this.heatUrl = heatUrl;
1088 this.expires = expires;
1091 public Heat getHeatClient () {
1092 Heat heatClient = new Heat (heatUrl);
1093 heatClient.token (token);
1097 public boolean isExpired () {
1098 if (expires == null) {
1102 return System.currentTimeMillis() > expires.getTimeInMillis();
1107 * Clean up the Heat client cache to remove expired entries.
1109 public static void heatCacheCleanup () {
1110 for (String cacheKey : heatClientCache.keySet ()) {
1111 if (heatClientCache.get (cacheKey).isExpired ()) {
1112 heatClientCache.remove (cacheKey);
1113 LOGGER.debug ("Cleaned Up Cached Heat Client for " + cacheKey);
1119 * Reset the Heat client cache.
1120 * This may be useful if cached credentials get out of sync.
1122 public static void heatCacheReset () {
1123 heatClientCache = new HashMap <String, HeatCacheEntry> ();
1126 public Map<String, Object> queryStackForOutputs(String cloudSiteId,
1127 String tenantId, String stackName) throws MsoException {
1128 LOGGER.debug("MsoHeatUtils.queryStackForOutputs)");
1129 StackInfo heatStack = this.queryStack(cloudSiteId, tenantId, stackName);
1130 if (heatStack == null || heatStack.getStatus() == HeatStatus.NOTFOUND) {
1133 Map<String, Object> outputs = heatStack.getOutputs();
1137 public void queryAndCopyOutputsToInputs(String cloudSiteId,
1138 String tenantId, String stackName, Map<String, String> inputs,
1139 boolean overWrite) throws MsoException {
1140 LOGGER.debug("MsoHeatUtils.queryAndCopyOutputsToInputs");
1141 Map<String, Object> outputs = this.queryStackForOutputs(cloudSiteId,
1142 tenantId, stackName);
1143 this.copyStringOutputsToInputs(inputs, outputs, overWrite);
1147 public void copyStringOutputsToInputs(Map<String, String> inputs,
1148 Map<String, Object> otherStackOutputs, boolean overWrite) {
1149 if (inputs == null || otherStackOutputs == null)
1151 for (String key : otherStackOutputs.keySet()) {
1152 if (!inputs.containsKey(key)) {
1153 Object obj = otherStackOutputs.get(key);
1154 if (obj instanceof String) {
1155 inputs.put(key, (String) otherStackOutputs.get(key));
1156 } else if (obj instanceof JsonNode ){
1157 // This is a bit of mess - but I think it's the least impacting
1158 // let's convert it BACK to a string - then it will get converted back later
1160 String str = this.convertNode((JsonNode) obj);
1161 inputs.put(key, str);
1162 } catch (Exception e) {
1163 LOGGER.debug("DANGER WILL ROBINSON: unable to convert value for "+ key);
1164 //effect here is this value will not have been copied to the inputs - and therefore will error out downstream
1166 } else if (obj instanceof java.util.LinkedHashMap) {
1167 LOGGER.debug("LinkedHashMap - this is showing up as a LinkedHashMap instead of JsonNode");
1169 String str = JSON_MAPPER.writeValueAsString(obj);
1170 inputs.put(key, str);
1171 } catch (Exception e) {
1172 LOGGER.debug("DANGER WILL ROBINSON: unable to convert value for "+ key);
1175 // just try to cast it - could be an integer or some such
1177 String str = (String) obj;
1178 inputs.put(key, str);
1179 } catch (Exception e) {
1180 LOGGER.debug("DANGER WILL ROBINSON: unable to convert value for "+ key);
1181 //effect here is this value will not have been copied to the inputs - and therefore will error out downstream
1188 public StringBuilder requestToStringBuilder(CreateStackParam stack) {
1189 StringBuilder sb = new StringBuilder();
1190 sb.append("Stack:\n");
1191 sb.append("\tStackName: " + stack.getStackName());
1192 sb.append("\tTemplateUrl: " + stack.getTemplateUrl());
1193 sb.append("\tTemplate: " + stack.getTemplate());
1194 sb.append("\tEnvironment: " + stack.getEnvironment());
1195 sb.append("\tTimeout: " + stack.getTimeoutMinutes());
1196 sb.append("\tParameters:\n");
1197 Map<String, Object> params = stack.getParameters();
1198 if (params == null || params.size() < 1) {
1199 sb.append("\tNONE");
1201 for (String key : params.keySet()) {
1202 if (params.get(key) instanceof String) {
1203 sb.append("\t" + key + "=" + (String) params.get(key));
1204 } else if (params.get(key) instanceof JsonNode) {
1205 String jsonStringOut = this.convertNode((JsonNode)params.get(key));
1206 sb.append("\t" + key + "=" + jsonStringOut);
1208 sb.append("\t" + key + "= [some non-string/non-json]");
1215 private String convertNode(final JsonNode node) {
1217 final Object obj = JSON_MAPPER.treeToValue(node, Object.class);
1218 final String json = JSON_MAPPER.writeValueAsString(obj);
1220 } catch (JsonParseException jpe) {
1221 LOGGER.debug("Error converting json to string " + jpe.getMessage());
1222 } catch (Exception e) {
1223 LOGGER.debug("Error converting json to string " + e.getMessage());
1225 return "[Error converting json to string]";
1229 private StringBuilder getOutputsAsStringBuilder(Stack heatStack) {
1230 // This should only be used as a utility to print out the stack outputs
1232 StringBuilder sb = new StringBuilder("");
1233 if (heatStack == null) {
1234 sb.append("(heatStack is null)");
1237 List<Output> outputList = heatStack.getOutputs();
1238 if (outputList == null || outputList.isEmpty()) {
1239 sb.append("(outputs is empty)");
1242 Map<String, Object> outputs = new HashMap<String,Object>();
1243 for (Output outputItem : outputList) {
1244 outputs.put(outputItem.getOutputKey(), outputItem.getOutputValue());
1247 sb.append("OUTPUTS:");
1248 for (String key : outputs.keySet()) {
1249 sb.append("outputs[" + counter++ + "]: " + key + "=");
1250 Object obj = outputs.get(key);
1251 if (obj instanceof String) {
1252 sb.append((String)obj);
1253 } else if (obj instanceof JsonNode) {
1254 sb.append(this.convertNode((JsonNode)obj));
1255 } else if (obj instanceof java.util.LinkedHashMap) {
1257 String str = JSON_MAPPER.writeValueAsString(obj);
1259 } catch (Exception e) {
1260 sb.append("(a LinkedHashMap value that would not convert nicely)");
1266 } catch (Exception e) {
1267 str = "(a value unable to be cast as a String)";