adapters/mso-adapter-utils/src/main/java/org/openecomp/mso/openstack/utils/MsoHeatUtils.java

   1 /*-
   2  * ============LICENSE_START=======================================================
   3  * ONAP - SO
   4  * ================================================================================
   5  * Copyright (C) 2017 AT&T Intellectual Property. All rights reserved.
   6  * Copyright (C) 2017 Huawei Technologies Co., Ltd. All rights reserved.
   7  * ================================================================================
   8  * Licensed under the Apache License, Version 2.0 (the "License");
   9  * you may not use this file except in compliance with the License.
  10  * You may obtain a copy of the License at
  11  *
  12  *      http://www.apache.org/licenses/LICENSE-2.0
  13  *
  14  * Unless required by applicable law or agreed to in writing, software
  15  * distributed under the License is distributed on an "AS IS" BASIS,
  16  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  17  * See the License for the specific language governing permissions and
  18  * limitations under the License.
  19  * ============LICENSE_END=========================================================
  20  */
  21
  22 package org.openecomp.mso.openstack.utils;
  23
  24 import java.io.Serializable;
  25 import java.util.ArrayList;
  26 import java.util.Arrays;
  27 import java.util.Calendar;
  28 import java.util.HashMap;
  29 import java.util.List;
  30 import java.util.Map;
  31 import java.util.Set;
  32
  33 import org.openecomp.mso.cloud.CloudConfig;
  34 import org.openecomp.mso.cloud.CloudConfigFactory;
  35 import org.openecomp.mso.cloud.CloudIdentity;
  36 import org.openecomp.mso.cloud.CloudSite;
  37 import org.openecomp.mso.db.catalog.beans.HeatTemplate;
  38 import org.openecomp.mso.db.catalog.beans.HeatTemplateParam;
  39 import org.openecomp.mso.logger.MessageEnum;
  40 import org.openecomp.mso.logger.MsoAlarmLogger;
  41 import org.openecomp.mso.logger.MsoLogger;
  42 import org.openecomp.mso.openstack.beans.HeatStatus;
  43 import org.openecomp.mso.openstack.beans.StackInfo;
  44 import org.openecomp.mso.openstack.exceptions.MsoAdapterException;
  45 import org.openecomp.mso.openstack.exceptions.MsoCloudSiteNotFound;
  46 import org.openecomp.mso.openstack.exceptions.MsoException;
  47 import org.openecomp.mso.openstack.exceptions.MsoIOException;
  48 import org.openecomp.mso.openstack.exceptions.MsoOpenstackException;
  49 import org.openecomp.mso.openstack.exceptions.MsoStackAlreadyExists;
  50 import org.openecomp.mso.openstack.exceptions.MsoTenantNotFound;
  51 import org.openecomp.mso.properties.MsoJavaProperties;
  52 import org.openecomp.mso.properties.MsoPropertiesException;
  53 import org.openecomp.mso.properties.MsoPropertiesFactory;
  54
  55 import com.fasterxml.jackson.core.JsonParseException;
  56 import com.fasterxml.jackson.databind.JsonNode;
  57 import com.fasterxml.jackson.databind.ObjectMapper;
  58 import com.woorea.openstack.base.client.OpenStackConnectException;
  59 import com.woorea.openstack.base.client.OpenStackRequest;
  60 import com.woorea.openstack.base.client.OpenStackResponseException;
  61 import com.woorea.openstack.heat.Heat;
  62 import com.woorea.openstack.heat.model.CreateStackParam;
  63 import com.woorea.openstack.heat.model.Stack;
  64 import com.woorea.openstack.heat.model.Stack.Output;
  65 import com.woorea.openstack.heat.model.Stacks;
  66 import com.woorea.openstack.keystone.Keystone;
  67 import com.woorea.openstack.keystone.model.Access;
  68 import com.woorea.openstack.keystone.model.Authentication;
  69 import com.woorea.openstack.keystone.utils.KeystoneUtils;
  70
  71 public class MsoHeatUtils extends MsoCommonUtils {
  72
  73         private MsoPropertiesFactory msoPropertiesFactory;
  74
  75         private CloudConfigFactory cloudConfigFactory;
  76
  77     private static final String TOKEN_AUTH = "TokenAuth";
  78
  79     private static final String QUERY_ALL_STACKS = "QueryAllStacks";
  80
  81     private static final String DELETE_STACK = "DeleteStack";
  82
  83     private static final String HEAT_ERROR = "HeatError";
  84
  85     private static final String CREATE_STACK = "CreateStack";
  86
  87     // Cache Heat Clients statically. Since there is just one MSO user, there is no
  88     // benefit to re-authentication on every request (or across different flows). The
  89     // token will be used until it expires.
  90     //
  91     // The cache key is "tenantId:cloudId"
  92     private static Map <String, HeatCacheEntry> heatClientCache = new HashMap <> ();
  93
  94     // Fetch cloud configuration each time (may be cached in CloudConfig class)
  95     protected CloudConfig cloudConfig;
  96
  97     private static final MsoLogger LOGGER = MsoLogger.getMsoLogger (MsoLogger.Catalog.RA);
  98
  99     protected MsoJavaProperties msoProps = null;
 100
 101     // Properties names and variables (with default values)
 102     protected String createPollIntervalProp = "ecomp.mso.adapters.heat.create.pollInterval";
 103     private String deletePollIntervalProp = "ecomp.mso.adapters.heat.delete.pollInterval";
 104     private String deletePollTimeoutProp = "ecomp.mso.adapters.heat.delete.pollTimeout";
 105
 106     protected int createPollIntervalDefault = 15;
 107     private int deletePollIntervalDefault = 15;
 108     private int deletePollTimeoutDefault = 300;
 109     private String msoPropID;
 110
 111     private static final ObjectMapper JSON_MAPPER = new ObjectMapper();
 112
 113     /**
 114      * This constructor MUST be used ONLY in the JUNIT tests, not for real code.
 115      * The MsoPropertiesFactory will be added by EJB injection.
 116      *
 117      * @param msoPropID ID of the mso pro config as defined in web.xml
 118      * @param msoPropFactory The mso properties factory instanciated by EJB injection
 119      * @param cloudConfFactory the Cloud Config instantiated by EJB injection
 120      */
 121     public MsoHeatUtils (String msoPropID, MsoPropertiesFactory msoPropFactory, CloudConfigFactory cloudConfFactory) {
 122         msoPropertiesFactory = msoPropFactory;
 123         cloudConfigFactory = cloudConfFactory;
 124         this.msoPropID = msoPropID;
 125         // Dynamically get properties each time (in case reloaded).
 126
 127         try {
 128                         msoProps = msoPropertiesFactory.getMsoJavaProperties (msoPropID);
 129                 } catch (MsoPropertiesException e) {
 130                         LOGGER.error (MessageEnum.LOAD_PROPERTIES_FAIL, "Unknown. Mso Properties ID not found in cache: " + msoPropID, "", "", MsoLogger.ErrorCode.DataError, "Exception - Mso Properties ID not found in cache", e);
 131                 }
 132         cloudConfig = cloudConfigFactory.getCloudConfig ();
 133         LOGGER.debug("MsoHeatUtils:" + msoPropID);
 134
 135     }
 136
 137
 138     /**
 139      * keep this old method signature here to maintain backwards compatibility. keep others as well.
 140      * this method does not include environment, files, or heatFiles
 141      */
 142     public StackInfo createStack (String cloudSiteId,
 143                                   String tenantId,
 144                                   String stackName,
 145                                   String heatTemplate,
 146                                   Map <String, ?> stackInputs,
 147                                   boolean pollForCompletion,
 148                                   int timeoutMinutes) throws MsoException {
 149         // Just call the new method with the environment & files variable set to null
 150         return this.createStack (cloudSiteId,
 151                                  tenantId,
 152                                  stackName,
 153                                  heatTemplate,
 154                                  stackInputs,
 155                                  pollForCompletion,
 156                                  timeoutMinutes,
 157                                  null,
 158                                  null,
 159                                  null,
 160                                  true);
 161     }
 162
 163     // This method has environment, but not files or heatFiles
 164     public StackInfo createStack (String cloudSiteId,
 165                                   String tenantId,
 166                                   String stackName,
 167                                   String heatTemplate,
 168                                   Map <String, ?> stackInputs,
 169                                   boolean pollForCompletion,
 170                                   int timeoutMinutes,
 171                                   String environment) throws MsoException {
 172         // Just call the new method with the files/heatFiles variables set to null
 173         return this.createStack (cloudSiteId,
 174                                  tenantId,
 175                                  stackName,
 176                                  heatTemplate,
 177                                  stackInputs,
 178                                  pollForCompletion,
 179                                  timeoutMinutes,
 180                                  environment,
 181                                  null,
 182                                  null,
 183                                  true);
 184     }
 185
 186     // This method has environment and files, but not heatFiles.
 187     public StackInfo createStack (String cloudSiteId,
 188                                   String tenantId,
 189                                   String stackName,
 190                                   String heatTemplate,
 191                                   Map <String, ?> stackInputs,
 192                                   boolean pollForCompletion,
 193                                   int timeoutMinutes,
 194                                   String environment,
 195                                   Map <String, Object> files) throws MsoException {
 196         return this.createStack (cloudSiteId,
 197                                  tenantId,
 198                                  stackName,
 199                                  heatTemplate,
 200                                  stackInputs,
 201                                  pollForCompletion,
 202                                  timeoutMinutes,
 203                                  environment,
 204                                  files,
 205                                  null,
 206                                  true);
 207     }
 208
 209     // This method has environment, files, heatfiles
 210     public StackInfo createStack (String cloudSiteId,
 211                                   String tenantId,
 212                                   String stackName,
 213                                   String heatTemplate,
 214                                   Map <String, ?> stackInputs,
 215                                   boolean pollForCompletion,
 216                                   int timeoutMinutes,
 217                                   String environment,
 218                                   Map <String, Object> files,
 219                                   Map <String, Object> heatFiles) throws MsoException {
 220         return this.createStack (cloudSiteId,
 221                                  tenantId,
 222                                  stackName,
 223                                  heatTemplate,
 224                                  stackInputs,
 225                                  pollForCompletion,
 226                                  timeoutMinutes,
 227                                  environment,
 228                                  files,
 229                                  heatFiles,
 230                                  true);
 231     }
 232
 233     /**
 234      * Create a new Stack in the specified cloud location and tenant. The Heat template
 235      * and parameter map are passed in as arguments, along with the cloud access credentials.
 236      * It is expected that parameters have been validated and contain at minimum the required
 237      * parameters for the given template with no extra (undefined) parameters..
 238      *
 239      * The Stack name supplied by the caller must be unique in the scope of this tenant.
 240      * However, it should also be globally unique, as it will be the identifier for the
 241      * resource going forward in Inventory. This latter is managed by the higher levels
 242      * invoking this function.
 243      *
 244      * The caller may choose to let this function poll Openstack for completion of the
 245      * stack creation, or may handle polling itself via separate calls to query the status.
 246      * In either case, a StackInfo object will be returned containing the current status.
 247      * When polling is enabled, a status of CREATED is expected. When not polling, a
 248      * status of BUILDING is expected.
 249      *
 250      * An error will be thrown if the requested Stack already exists in the specified
 251      * Tenant and Cloud.
 252      *
 253      * For 1510 - add "environment", "files" (nested templates), and "heatFiles" (get_files) as
 254      * parameters for createStack. If environment is non-null, it will be added to the stack.
 255      * The nested templates and get_file entries both end up being added to the "files" on the
 256      * stack. We must combine them before we add them to the stack if they're both non-null.
 257      *
 258      * @param cloudSiteId The cloud (may be a region) in which to create the stack.
 259      * @param tenantId The Openstack ID of the tenant in which to create the Stack
 260      * @param stackName The name of the stack to create
 261      * @param heatTemplate The Heat template
 262      * @param stackInputs A map of key/value inputs
 263      * @param pollForCompletion Indicator that polling should be handled in Java vs. in the client
 264      * @param environment An optional yaml-format string to specify environmental parameters
 265      * @param files a Map<String, Object> that lists the child template IDs (file is the string, object is an int of
 266      *        Template id)
 267      * @param heatFiles a Map<String, Object> that lists the get_file entries (fileName, fileBody)
 268      * @param backout Donot delete stack on create Failure - defaulted to True
 269      * @return A StackInfo object
 270      * @throws MsoOpenstackException Thrown if the Openstack API call returns an exception.
 271      */
 272
 273     @SuppressWarnings("unchecked")
 274     public StackInfo createStack (String cloudSiteId,
 275                                   String tenantId,
 276                                   String stackName,
 277                                   String heatTemplate,
 278                                   Map <String, ?> stackInputs,
 279                                   boolean pollForCompletion,
 280                                   int timeoutMinutes,
 281                                   String environment,
 282                                   Map <String, Object> files,
 283                                   Map <String, Object> heatFiles,
 284                                   boolean backout) throws MsoException {
 285         // Create local variables checking to see if we have an environment, nested, get_files
 286         // Could later add some checks to see if it's valid.
 287         boolean haveEnvtVariable = true;
 288         if (environment == null || "".equalsIgnoreCase (environment.trim ())) {
 289             haveEnvtVariable = false;
 290             LOGGER.debug ("createStack called with no environment variable");
 291         } else {
 292             LOGGER.debug ("createStack called with an environment variable: " + environment);
 293         }
 294
 295         boolean haveFiles = true;
 296         if (files == null || files.isEmpty ()) {
 297             haveFiles = false;
 298             LOGGER.debug ("createStack called with no files / child template ids");
 299         } else {
 300             LOGGER.debug ("createStack called with " + files.size () + " files / child template ids");
 301         }
 302
 303         boolean haveHeatFiles = true;
 304         if (heatFiles == null || heatFiles.isEmpty ()) {
 305             haveHeatFiles = false;
 306             LOGGER.debug ("createStack called with no heatFiles");
 307         } else {
 308             LOGGER.debug ("createStack called with " + heatFiles.size () + " heatFiles");
 309         }
 310
 311         // Obtain the cloud site information where we will create the stack
 312         CloudSite cloudSite = cloudConfig.getCloudSite(cloudSiteId).orElseThrow(
 313                 () -> new MsoCloudSiteNotFound(cloudSiteId));
 314         LOGGER.debug("Found: " + cloudSite.toString());
 315         // Get a Heat client. They are cached between calls (keyed by tenantId:cloudId)
 316         // This could throw MsoTenantNotFound or MsoOpenstackException (both propagated)
 317         Heat heatClient = getHeatClient (cloudSite, tenantId);
 318         if (heatClient != null) {
 319                 LOGGER.debug("Found: " + heatClient.toString());
 320         }
 321
 322         LOGGER.debug ("Ready to Create Stack (" + heatTemplate + ") with input params: " + stackInputs);
 323
 324         // Build up the stack to create
 325         // Disable auto-rollback, because error reason is lost. Always rollback in the code.
 326         CreateStackParam stack = new CreateStackParam ();
 327         stack.setStackName (stackName);
 328         stack.setTimeoutMinutes (timeoutMinutes);
 329         stack.setParameters ((Map <String, Object>) stackInputs);
 330         stack.setTemplate (heatTemplate);
 331         stack.setDisableRollback (true);
 332         // TJM New for PO Adapter - add envt variable
 333         if (haveEnvtVariable) {
 334             LOGGER.debug ("Found an environment variable - value: " + environment);
 335             stack.setEnvironment (environment);
 336         }
 337         // Now handle nested templates or get_files - have to combine if we have both
 338         // as they're both treated as "files:" on the stack.
 339         if (haveFiles && haveHeatFiles) {
 340             // Let's do this here - not in the bean
 341             LOGGER.debug ("Found files AND heatFiles - combine and add!");
 342             Map <String, Object> combinedFiles = new HashMap <> ();
 343             for (String keyString : files.keySet ()) {
 344                 combinedFiles.put (keyString, files.get (keyString));
 345             }
 346             for (String keyString : heatFiles.keySet ()) {
 347                 combinedFiles.put (keyString, heatFiles.get (keyString));
 348             }
 349             stack.setFiles (combinedFiles);
 350         } else {
 351             // Handle if we only have one or neither:
 352             if (haveFiles) {
 353                 LOGGER.debug ("Found files - adding to stack");
 354                 stack.setFiles (files);
 355             }
 356             if (haveHeatFiles) {
 357                 LOGGER.debug ("Found heatFiles - adding to stack");
 358                 // the setFiles was modified to handle adding the entries
 359                 stack.setFiles (heatFiles);
 360             }
 361         }
 362
 363         // 1802 - attempt to add better formatted printout of request to openstack
 364         try {
 365                 Map<String, Object> inputs = new HashMap<String, Object>();
 366                 for (String key : stackInputs.keySet()) {
 367                         Object o = (Object) stackInputs.get(key);
 368                         if (o != null) {
 369                                 inputs.put(key, o);
 370                         }
 371                 }
 372                 LOGGER.debug(this.printStackRequest(tenantId, heatFiles, files, environment, inputs, stackName, heatTemplate, timeoutMinutes, backout, cloudSiteId));
 373         } catch (Exception e) {
 374                 // that's okay - this is a nice-to-have
 375                 LOGGER.debug("(had an issue printing nicely formatted request to debuglog) " + e.getMessage());
 376         }
 377
 378         Stack heatStack = null;
 379         try {
 380             // Execute the actual Openstack command to create the Heat stack
 381             OpenStackRequest <Stack> request = heatClient.getStacks ().create (stack);
 382             // Begin X-Auth-User
 383             // Obtain an MSO token for the tenant
 384             CloudIdentity cloudIdentity = cloudSite.getIdentityService ();
 385             // cloudIdentity.getMsoId(), cloudIdentity.getMsoPass()
 386             //req
 387             request.header ("X-Auth-User", cloudIdentity.getMsoId ());
 388             request.header ("X-Auth-Key", cloudIdentity.getMsoPass ());
 389             LOGGER.debug ("headers added, about to executeAndRecordOpenstackRequest");
 390             //LOGGER.debug(this.requestToStringBuilder(stack).toString());
 391             // END - try to fix X-Auth-User
 392             heatStack = executeAndRecordOpenstackRequest (request, msoProps);
 393         } catch (OpenStackResponseException e) {
 394             // Since this came on the 'Create Stack' command, nothing was changed
 395             // in the cloud. Return the error as an exception.
 396             if (e.getStatus () == 409) {
 397                 // Stack already exists. Return a specific error for this case
 398                 MsoStackAlreadyExists me = new MsoStackAlreadyExists (stackName, tenantId, cloudSiteId);
 399                 me.addContext (CREATE_STACK);
 400                 throw me;
 401             } else {
 402                 // Convert the OpenStackResponseException to an MsoOpenstackException
 403                 LOGGER.debug("ERROR STATUS = " + e.getStatus() + ",\n" + e.getMessage() + "\n" + e.getLocalizedMessage());
 404                 throw heatExceptionToMsoException (e, CREATE_STACK);
 405             }
 406         } catch (OpenStackConnectException e) {
 407             // Error connecting to Openstack instance. Convert to an MsoException
 408             throw heatExceptionToMsoException (e, CREATE_STACK);
 409         } catch (RuntimeException e) {
 410             // Catch-all
 411             throw runtimeExceptionToMsoException (e, CREATE_STACK);
 412         }
 413
 414         // Subsequent access by the canonical name "<stack name>/<stack-id>".
 415         // Otherwise, simple query by name returns a 302 redirect.
 416         // NOTE: This is specific to the v1 Orchestration API.
 417         String canonicalName = stackName + "/" + heatStack.getId ();
 418
 419         // If client has requested a final response, poll for stack completion
 420         if (pollForCompletion) {
 421             // Set a time limit on overall polling.
 422             // Use the resource (template) timeout for Openstack (expressed in minutes)
 423             // and add one poll interval to give Openstack a chance to fail on its own.
 424             int createPollInterval = msoProps.getIntProperty (createPollIntervalProp, createPollIntervalDefault);
 425             int pollTimeout = (timeoutMinutes * 60) + createPollInterval;
 426             // New 1610 - poll on delete if we rollback - use same values for now
 427             int deletePollInterval = createPollInterval;
 428             int deletePollTimeout = pollTimeout;
 429             boolean createTimedOut = false;
 430             StringBuilder stackErrorStatusReason = new StringBuilder("");
 431             LOGGER.debug("createPollInterval=" + createPollInterval + ", pollTimeout=" + pollTimeout);
 432
 433             while (true) {
 434                 try {
 435                     heatStack = queryHeatStack (heatClient, canonicalName);
 436                     LOGGER.debug (heatStack.getStackStatus () + " (" + canonicalName + ")");
 437                     try {
 438                         LOGGER.debug("Current stack " + this.getOutputsAsStringBuilder(heatStack).toString());
 439                     } catch (Exception e) {
 440                         LOGGER.debug("an error occurred trying to print out the current outputs of the stack", e);
 441                     }
 442
 443                     if ("CREATE_IN_PROGRESS".equals (heatStack.getStackStatus ())) {
 444                         // Stack creation is still running.
 445                         // Sleep and try again unless timeout has been reached
 446                         if (pollTimeout <= 0) {
 447                             // Note that this should not occur, since there is a timeout specified
 448                             // in the Openstack call.
 449                             LOGGER.error (MessageEnum.RA_CREATE_STACK_TIMEOUT, cloudSiteId, tenantId, stackName, heatStack.getStackStatus (), "", "", MsoLogger.ErrorCode.AvailabilityError, "Create stack timeout");
 450                             createTimedOut = true;
 451                             break;
 452                         }
 453                         try {
 454                             Thread.sleep (createPollInterval * 1000L);
 455                         } catch (InterruptedException e) {
 456                             LOGGER.debug ("Thread interrupted while sleeping", e);
 457                         }
 458
 459                         pollTimeout -= createPollInterval;
 460                                 LOGGER.debug("pollTimeout remaining: " + pollTimeout);
 461                     } else {
 462                         //save off the status & reason msg before we attempt delete
 463                         stackErrorStatusReason.append("Stack error (" + heatStack.getStackStatus() + "): " + heatStack.getStackStatusReason());
 464                         break;
 465                     }
 466                 } catch (MsoException me) {
 467                         // Cannot query the stack status. Something is wrong.
 468                         // Try to roll back the stack
 469                         if (!backout)
 470                         {
 471                                 LOGGER.warn(MessageEnum.RA_CREATE_STACK_ERR, "Create Stack errored, stack deletion suppressed", "", "", MsoLogger.ErrorCode.BusinessProcesssError, "Exception in Create Stack, stack deletion suppressed");
 472                         }
 473                         else
 474                         {
 475                                 try {
 476                                         LOGGER.debug("Create Stack error - unable to query for stack status - attempting to delete stack: " + canonicalName + " - This will likely fail and/or we won't be able to query to see if delete worked");
 477                                         OpenStackRequest <Void> request = heatClient.getStacks ().deleteByName (canonicalName);
 478                                         executeAndRecordOpenstackRequest (request, msoProps);
 479                                         // this may be a waste of time - if we just got an exception trying to query the stack - we'll just
 480                                         // get another one, n'est-ce pas?
 481                                         boolean deleted = false;
 482                                         while (!deleted) {
 483                                                 try {
 484                                                         heatStack = queryHeatStack(heatClient, canonicalName);
 485                                                         if (heatStack != null) {
 486                                                         LOGGER.debug(heatStack.getStackStatus());
 487                                                         if ("DELETE_IN_PROGRESS".equals(heatStack.getStackStatus())) {
 488                                                                 if (deletePollTimeout <= 0) {
 489                                                                         LOGGER.error (MessageEnum.RA_CREATE_STACK_TIMEOUT, cloudSiteId, tenantId, stackName,
 490                                                                                         heatStack.getStackStatus (), "", "", MsoLogger.ErrorCode.AvailabilityError,
 491                                                                                         "Rollback: DELETE stack timeout");
 492                                                                         break;
 493                                                                 } else {
 494                                                                         try {
 495                                                                                 Thread.sleep(deletePollInterval * 1000L);
 496                                                                         } catch (InterruptedException ie) {
 497                                                                                 LOGGER.debug("Thread interrupted while sleeping", ie);
 498                                                                         }
 499                                                                         deletePollTimeout -= deletePollInterval;
 500                                                                 }
 501                                                         } else if ("DELETE_COMPLETE".equals(heatStack.getStackStatus())){
 502                                                                 LOGGER.debug("DELETE_COMPLETE for " + canonicalName);
 503                                                                 deleted = true;
 504                                                                 continue;
 505                                                         } else {
 506                                                                 //got a status other than DELETE_IN_PROGRESS or DELETE_COMPLETE - so break and evaluate
 507                                                                 break;
 508                                                         }
 509                                                 } else {
 510                                                         // assume if we can't find it - it's deleted
 511                                                         LOGGER.debug("heatStack returned null - assume the stack " + canonicalName + " has been deleted");
 512                                                         deleted = true;
 513                                                         continue;
 514                                                         }
 515
 516                                                 } catch (Exception e3) {
 517                                                         // Just log this one. We will report the original exception.
 518                                                         LOGGER.error (MessageEnum.RA_CREATE_STACK_ERR, "Create Stack: Nested exception rolling back stack: " + e3, "", "", MsoLogger.ErrorCode.BusinessProcesssError, "Create Stack: Nested exception rolling back stack on error on query");
 519
 520                                                 }
 521                                         }
 522                                 } catch (Exception e2) {
 523                                         // Just log this one. We will report the original exception.
 524                                         LOGGER.error (MessageEnum.RA_CREATE_STACK_ERR, "Create Stack: Nested exception rolling back stack: " + e2, "", "", MsoLogger.ErrorCode.BusinessProcesssError, "Create Stack: Nested exception rolling back stack");
 525                                 }
 526                         }
 527
 528                     // Propagate the original exception from Stack Query.
 529                     me.addContext (CREATE_STACK);
 530                     throw me;
 531                 }
 532             }
 533
 534             if (!"CREATE_COMPLETE".equals (heatStack.getStackStatus ())) {
 535                 LOGGER.error (MessageEnum.RA_CREATE_STACK_ERR, "Create Stack error:  Polling complete with non-success status: "
 536                               + heatStack.getStackStatus () + ", " + heatStack.getStackStatusReason (), "", "", MsoLogger.ErrorCode.BusinessProcesssError, "Create Stack error");
 537
 538                 // Rollback the stack creation, since it is in an indeterminate state.
 539                 if (!backout)
 540                 {
 541                         LOGGER.warn(MessageEnum.RA_CREATE_STACK_ERR, "Create Stack errored, stack deletion suppressed", "", "", MsoLogger.ErrorCode.BusinessProcesssError, "Create Stack error, stack deletion suppressed");
 542                 }
 543                 else
 544                 {
 545                         try {
 546                                 LOGGER.debug("Create Stack errored - attempting to DELETE stack: " + canonicalName);
 547                                 LOGGER.debug("deletePollInterval=" + deletePollInterval + ", deletePollTimeout=" + deletePollTimeout);
 548                                 OpenStackRequest <Void> request = heatClient.getStacks ().deleteByName (canonicalName);
 549                                 executeAndRecordOpenstackRequest (request, msoProps);
 550                                 boolean deleted = false;
 551                                 while (!deleted) {
 552                                         try {
 553                                                 heatStack = queryHeatStack(heatClient, canonicalName);
 554                                                 if (heatStack != null) {
 555                                                         LOGGER.debug(heatStack.getStackStatus() + " (" + canonicalName + ")");
 556                                                         if ("DELETE_IN_PROGRESS".equals(heatStack.getStackStatus())) {
 557                                                                 if (deletePollTimeout <= 0) {
 558                                                                         LOGGER.error (MessageEnum.RA_CREATE_STACK_TIMEOUT, cloudSiteId, tenantId, stackName,
 559                                                                                         heatStack.getStackStatus (), "", "", MsoLogger.ErrorCode.AvailabilityError,
 560                                                                                         "Rollback: DELETE stack timeout");
 561                                                                         break;
 562                                                                 } else {
 563                                                                         try {
 564                                                                                 Thread.sleep(deletePollInterval * 1000L);
 565                                                                         } catch (InterruptedException ie) {
 566                                                                                 LOGGER.debug("Thread interrupted while sleeping", ie);
 567                                                                         }
 568                                                                         deletePollTimeout -= deletePollInterval;
 569                                                                         LOGGER.debug("deletePollTimeout remaining: " + deletePollTimeout);
 570                                                                 }
 571                                                         } else if ("DELETE_COMPLETE".equals(heatStack.getStackStatus())){
 572                                                                 LOGGER.debug("DELETE_COMPLETE for " + canonicalName);
 573                                                                 deleted = true;
 574                                                                 continue;
 575                                                         } else if ("DELETE_FAILED".equals(heatStack.getStackStatus())) {
 576                                                                 // Warn about this (?) - but still throw the original exception
 577                                                                 LOGGER.warn(MessageEnum.RA_CREATE_STACK_ERR, "Create Stack errored, stack deletion FAILED", "", "", MsoLogger.ErrorCode.BusinessProcesssError, "Create Stack error, stack deletion FAILED");
 578                                                                 LOGGER.debug("Stack deletion FAILED on a rollback of a create - " + canonicalName + ", status=" + heatStack.getStackStatus() + ", reason=" + heatStack.getStackStatusReason());
 579                                                                 break;
 580                                                         } else {
 581                                                                 //got a status other than DELETE_IN_PROGRESS or DELETE_COMPLETE - so break and evaluate
 582                                                                 break;
 583                                                         }
 584                                                 } else {
 585                                                         // assume if we can't find it - it's deleted
 586                                                         LOGGER.debug("heatStack returned null - assume the stack " + canonicalName + " has been deleted");
 587                                                         deleted = true;
 588                                                         continue;
 589                                                 }
 590
 591                                         } catch (MsoException me2) {
 592                                                 // We got an exception on the delete - don't throw this exception - throw the original - just log.
 593                                                 LOGGER.debug("Exception thrown trying to delete " + canonicalName + " on a create->rollback: " + me2.getContextMessage(), me2);
 594                                                 LOGGER.warn(MessageEnum.RA_CREATE_STACK_ERR, "Create Stack errored, then stack deletion FAILED - exception thrown", "", "", MsoLogger.ErrorCode.BusinessProcesssError, me2.getContextMessage());
 595                                         }
 596
 597                                 } // end while !deleted
 598                                 StringBuilder errorContextMessage;
 599                                 if (createTimedOut) {
 600                                         errorContextMessage = new StringBuilder("Stack Creation Timeout");
 601                                 } else {
 602                                         errorContextMessage  = stackErrorStatusReason;
 603                                 }
 604                                 if (deleted) {
 605                                         errorContextMessage.append(" - stack successfully deleted");
 606                                 } else {
 607                                         errorContextMessage.append(" - encountered an error trying to delete the stack");
 608                                 }
 609 //                              MsoOpenstackException me = new MsoOpenstackException(0, "", stackErrorStatusReason.toString());
 610  //                             me.addContext(CREATE_STACK);
 611   //                            alarmLogger.sendAlarm(HEAT_ERROR, MsoAlarmLogger.CRITICAL, me.getContextMessage());
 612    //                           throw me;
 613                         } catch (Exception e2) {
 614                                 // shouldn't happen - but handle
 615                                 LOGGER.error (MessageEnum.RA_CREATE_STACK_ERR, "Create Stack: Nested exception rolling back stack: " + e2, "", "", MsoLogger.ErrorCode.BusinessProcesssError, "Exception in Create Stack: rolling back stack");
 616                         }
 617                 }
 618                 MsoOpenstackException me = new MsoOpenstackException(0, "", stackErrorStatusReason.toString());
 619                 me.addContext(CREATE_STACK);
 620                 alarmLogger.sendAlarm(HEAT_ERROR, MsoAlarmLogger.CRITICAL, me.getContextMessage());
 621                 throw me;
 622             }
 623
 624         } else {
 625             // Get initial status, since it will have been null after the create.
 626             heatStack = queryHeatStack (heatClient, canonicalName);
 627             LOGGER.debug (heatStack.getStackStatus ());
 628         }
 629
 630         return new StackInfo (heatStack);
 631     }
 632
 633     /**
 634      * Query for a single stack (by Name) in a tenant. This call will always return a
 635      * StackInfo object. If the stack does not exist, an "empty" StackInfo will be
 636      * returned - containing only the stack name and a status of NOTFOUND.
 637      *
 638      * @param tenantId The Openstack ID of the tenant in which to query
 639      * @param cloudSiteId The cloud identifier (may be a region) in which to query
 640      * @param stackName The name of the stack to query (may be simple or canonical)
 641      * @return A StackInfo object
 642      * @throws MsoOpenstackException Thrown if the Openstack API call returns an exception.
 643      */
 644     public StackInfo queryStack (String cloudSiteId, String tenantId, String stackName) throws MsoException {
 645         LOGGER.debug ("Query HEAT stack: " + stackName + " in tenant " + tenantId);
 646
 647         // Obtain the cloud site information where we will create the stack
 648         CloudSite cloudSite = cloudConfig.getCloudSite(cloudSiteId).orElseThrow(
 649                 () -> new MsoCloudSiteNotFound(cloudSiteId));
 650         LOGGER.debug("Found: " + cloudSite.toString());
 651
 652         // Get a Heat client. They are cached between calls (keyed by tenantId:cloudId)
 653         Heat heatClient = null;
 654         try {
 655             heatClient = getHeatClient (cloudSite, tenantId);
 656             if (heatClient != null) {
 657                 LOGGER.debug("Found: " + heatClient.toString());
 658             }
 659         } catch (MsoTenantNotFound e) {
 660             // Tenant doesn't exist, so stack doesn't either
 661             LOGGER.debug ("Tenant with id " + tenantId + "not found.", e);
 662             return new StackInfo (stackName, HeatStatus.NOTFOUND);
 663         } catch (MsoException me) {
 664             // Got an Openstack error. Propagate it
 665             LOGGER.error (MessageEnum.RA_CONNECTION_EXCEPTION, "OpenStack", "Openstack Exception on Token request: " + me, "Openstack", "", MsoLogger.ErrorCode.AvailabilityError, "Connection Exception");
 666             me.addContext ("QueryStack");
 667             throw me;
 668         }
 669
 670         // Query the Stack.
 671         // An MsoException will propagate transparently to the caller.
 672         Stack heatStack = queryHeatStack (heatClient, stackName);
 673
 674         if (heatStack == null) {
 675             // Stack does not exist. Return a StackInfo with status NOTFOUND
 676             StackInfo stackInfo = new StackInfo (stackName, HeatStatus.NOTFOUND);
 677             return stackInfo;
 678         }
 679
 680         return new StackInfo (heatStack);
 681     }
 682
 683     /**
 684      * Delete a stack (by Name/ID) in a tenant. If the stack is not found, it will be
 685      * considered a successful deletion. The return value is a StackInfo object which
 686      * contains the current stack status.
 687      *
 688      * The client may choose to let the adapter poll Openstack for completion of the
 689      * stack deletion, or may handle polling itself via separate query calls. In either
 690      * case, a StackInfo object will be returned. When polling is enabled, a final
 691      * status of NOTFOUND is expected. When not polling, a status of DELETING is expected.
 692      *
 693      * There is no rollback from a successful stack deletion. A deletion failure will
 694      * also result in an undefined stack state - the components may or may not have been
 695      * all or partially deleted, so the resulting stack must be considered invalid.
 696      *
 697      * @param tenantId The Openstack ID of the tenant in which to perform the delete
 698      * @param cloudSiteId The cloud identifier (may be a region) from which to delete the stack.
 699      * @param stackName The name/id of the stack to delete. May be simple or canonical
 700      * @param pollForCompletion Indicator that polling should be handled in Java vs. in the client
 701      * @return A StackInfo object
 702      * @throws MsoOpenstackException Thrown if the Openstack API call returns an exception.
 703      * @throws MsoCloudSiteNotFound
 704      */
 705     public StackInfo deleteStack (String tenantId,
 706                                   String cloudSiteId,
 707                                   String stackName,
 708                                   boolean pollForCompletion) throws MsoException {
 709         // Obtain the cloud site information where we will create the stack
 710         CloudSite cloudSite = cloudConfig.getCloudSite(cloudSiteId).orElseThrow(
 711                 () -> new MsoCloudSiteNotFound(cloudSiteId));
 712         LOGGER.debug("Found: " + cloudSite.toString());
 713
 714         // Get a Heat client. They are cached between calls (keyed by tenantId:cloudId)
 715         Heat heatClient = null;
 716         try {
 717             heatClient = getHeatClient (cloudSite, tenantId);
 718             if (heatClient != null) {
 719                 LOGGER.debug("Found: " + heatClient.toString());
 720             }
 721         } catch (MsoTenantNotFound e) {
 722             // Tenant doesn't exist, so stack doesn't either
 723             LOGGER.debug ("Tenant with id " + tenantId + "not found.", e);
 724             return new StackInfo (stackName, HeatStatus.NOTFOUND);
 725         } catch (MsoException me) {
 726             // Got an Openstack error. Propagate it
 727             LOGGER.error (MessageEnum.RA_CONNECTION_EXCEPTION, "Openstack", "Openstack Exception on Token request: " + me, "Openstack", "", MsoLogger.ErrorCode.AvailabilityError, "Connection Exception");
 728             me.addContext (DELETE_STACK);
 729             throw me;
 730         }
 731
 732         // OK if stack not found, perform a query first
 733         Stack heatStack = queryHeatStack (heatClient, stackName);
 734         if (heatStack == null || "DELETE_COMPLETE".equals (heatStack.getStackStatus ())) {
 735             // Not found. Return a StackInfo with status NOTFOUND
 736             return new StackInfo (stackName, HeatStatus.NOTFOUND);
 737         }
 738
 739         // Delete the stack.
 740
 741         // Use canonical name "<stack name>/<stack-id>" to delete.
 742         // Otherwise, deletion by name returns a 302 redirect.
 743         // NOTE: This is specific to the v1 Orchestration API.
 744         String canonicalName = heatStack.getStackName () + "/" + heatStack.getId ();
 745
 746         try {
 747             OpenStackRequest <Void> request = null;
 748             if(null != heatClient) {
 749                 request = heatClient.getStacks ().deleteByName (canonicalName);
 750             }
 751             else {
 752                 LOGGER.debug ("Heat Client is NULL" );
 753             }
 754
 755             executeAndRecordOpenstackRequest (request, msoProps);
 756         } catch (OpenStackResponseException e) {
 757             if (e.getStatus () == 404) {
 758                 // Not found. We are OK with this. Return a StackInfo with status NOTFOUND
 759                 return new StackInfo (stackName, HeatStatus.NOTFOUND);
 760             } else {
 761                 // Convert the OpenStackResponseException to an MsoOpenstackException
 762                 throw heatExceptionToMsoException (e, DELETE_STACK);
 763             }
 764         } catch (OpenStackConnectException e) {
 765             // Error connecting to Openstack instance. Convert to an MsoException
 766             throw heatExceptionToMsoException (e, DELETE_STACK);
 767         } catch (RuntimeException e) {
 768             // Catch-all
 769             throw runtimeExceptionToMsoException (e, DELETE_STACK);
 770         }
 771
 772         // Requery the stack for current status.
 773         // It will probably still exist with "DELETE_IN_PROGRESS" status.
 774         heatStack = queryHeatStack (heatClient, canonicalName);
 775
 776         if (pollForCompletion) {
 777             // Set a timeout on polling
 778             int pollInterval = msoProps.getIntProperty (deletePollIntervalProp, deletePollIntervalDefault);
 779             int pollTimeout = msoProps.getIntProperty (deletePollTimeoutProp, deletePollTimeoutDefault);
 780
 781             // When querying by canonical name, Openstack returns DELETE_COMPLETE status
 782             // instead of "404" (which would result from query by stack name).
 783             while (heatStack != null && !"DELETE_COMPLETE".equals (heatStack.getStackStatus ())) {
 784                 LOGGER.debug ("Stack status: " + heatStack.getStackStatus ());
 785
 786                 if ("DELETE_FAILED".equals (heatStack.getStackStatus ())) {
 787                     // Throw a 'special case' of MsoOpenstackException to report the Heat status
 788                     String error = "Stack delete error (" + heatStack.getStackStatus ()
 789                                    + "): "
 790                                    + heatStack.getStackStatusReason ();
 791                     MsoOpenstackException me = new MsoOpenstackException (0, "", error);
 792                     me.addContext (DELETE_STACK);
 793
 794                     // Alarm this condition, stack deletion failed
 795                     alarmLogger.sendAlarm (HEAT_ERROR, MsoAlarmLogger.CRITICAL, me.getContextMessage ());
 796
 797                     throw me;
 798                 }
 799
 800                 if (pollTimeout <= 0) {
 801                     LOGGER.error (MessageEnum.RA_DELETE_STACK_TIMEOUT, cloudSiteId, tenantId, stackName, heatStack.getStackStatus (), "", "", MsoLogger.ErrorCode.AvailabilityError, "Delete Stack Timeout");
 802
 803                     // Throw a 'special case' of MsoOpenstackException to report the Heat status
 804                     MsoOpenstackException me = new MsoOpenstackException (0, "", "Stack Deletion Timeout");
 805                     me.addContext (DELETE_STACK);
 806
 807                     // Alarm this condition, stack deletion failed
 808                     alarmLogger.sendAlarm (HEAT_ERROR, MsoAlarmLogger.CRITICAL, me.getContextMessage ());
 809
 810                     throw me;
 811                 }
 812
 813                 try {
 814                     Thread.sleep (pollInterval * 1000L);
 815                 } catch (InterruptedException e) {
 816                     LOGGER.debug ("Thread interrupted while sleeping", e);
 817                 }
 818
 819                 pollTimeout -= pollInterval;
 820
 821                 heatStack = queryHeatStack (heatClient, canonicalName);
 822             }
 823
 824             // The stack is gone when this point is reached
 825             return new StackInfo (stackName, HeatStatus.NOTFOUND);
 826         }
 827
 828         // Return the current status (if not polling, the delete may still be in progress)
 829         StackInfo stackInfo = new StackInfo (heatStack);
 830         stackInfo.setName (stackName);
 831
 832         return stackInfo;
 833     }
 834
 835     /**
 836      * Query for all stacks in a tenant site. This call will return a List of StackInfo
 837      * objects, one for each deployed stack.
 838      *
 839      * Note that this is limited to a single site. To ensure that a tenant is truly
 840      * empty would require looping across all tenant endpoints.
 841      *
 842      * @param tenantId The Openstack ID of the tenant to query
 843      * @param cloudSiteId The cloud identifier (may be a region) in which to query.
 844      * @return A List of StackInfo objects
 845      * @throws MsoOpenstackException Thrown if the Openstack API call returns an exception.
 846      * @throws MsoCloudSiteNotFound
 847      */
 848     public List <StackInfo> queryAllStacks (String tenantId, String cloudSiteId) throws MsoException {
 849         // Obtain the cloud site information where we will create the stack
 850         CloudSite cloudSite = cloudConfig.getCloudSite(cloudSiteId).orElseThrow(
 851                 () -> new MsoCloudSiteNotFound(cloudSiteId));
 852         // Get a Heat client. They are cached between calls (keyed by tenantId:cloudId)
 853         Heat heatClient = getHeatClient (cloudSite, tenantId);
 854
 855         try {
 856             OpenStackRequest <Stacks> request = heatClient.getStacks ().list ();
 857             Stacks stacks = executeAndRecordOpenstackRequest (request, msoProps);
 858
 859             List <StackInfo> stackList = new ArrayList <> ();
 860
 861             // Not sure if returns an empty list or null if no stacks exist
 862             if (stacks != null) {
 863                 for (Stack stack : stacks) {
 864                     stackList.add (new StackInfo (stack));
 865                 }
 866             }
 867
 868             return stackList;
 869         } catch (OpenStackResponseException e) {
 870             if (e.getStatus () == 404) {
 871                 // Not sure if this can happen, but return an empty list
 872                 LOGGER.debug ("queryAllStacks - stack not found: ");
 873                 return new ArrayList <> ();
 874             } else {
 875                 // Convert the OpenStackResponseException to an MsoOpenstackException
 876                 throw heatExceptionToMsoException (e, QUERY_ALL_STACKS);
 877             }
 878         } catch (OpenStackConnectException e) {
 879             // Error connecting to Openstack instance. Convert to an MsoException
 880             throw heatExceptionToMsoException (e, QUERY_ALL_STACKS);
 881         } catch (RuntimeException e) {
 882             // Catch-all
 883             throw runtimeExceptionToMsoException (e, QUERY_ALL_STACKS);
 884         }
 885     }
 886
 887     /**
 888      * Validate parameters to be passed to Heat template. This method performs
 889      * three functions:
 890      * 1. Apply default values to parameters which have them defined
 891      * 2. Report any required parameters that are missing. This will generate an
 892      * exception in the caller, since stack create/update operations would fail.
 893      * 3. Report and remove any extraneous parameters. This will allow clients to
 894      * pass supersets of parameters and not get errors.
 895      *
 896      * These functions depend on the HeatTemplate definition from the MSO Catalog DB,
 897      * along with the input parameter Map. The output is an updated parameter map.
 898      * If the parameters are invalid for the template, an IllegalArgumentException
 899      * is thrown.
 900      */
 901     public Map <String, Object> validateStackParams (Map <String, Object> inputParams,
 902                                                      HeatTemplate heatTemplate) throws IllegalArgumentException {
 903         // Check that required parameters have been supplied for this template type
 904         StringBuilder missingParams = null;
 905         List <String> paramList = new ArrayList <> ();
 906
 907         // TODO: Enhance DB to support defaults for Heat Template parameters
 908
 909         for (HeatTemplateParam parm : heatTemplate.getParameters ()) {
 910             if (parm.isRequired () && !inputParams.containsKey (parm.getParamName ())) {
 911                 if (missingParams == null) {
 912                     missingParams = new StringBuilder(parm.getParamName());
 913                 } else {
 914                     missingParams.append("," + parm.getParamName());
 915                 }
 916             }
 917             paramList.add (parm.getParamName ());
 918         }
 919         if (missingParams != null) {
 920             // Problem - missing one or more required parameters
 921             String error = "Missing Required inputs for HEAT Template: " + missingParams;
 922             LOGGER.error (MessageEnum.RA_MISSING_PARAM, missingParams + " for HEAT Template", "", "", MsoLogger.ErrorCode.SchemaError, "Missing Required inputs for HEAT Template: " + missingParams);
 923             throw new IllegalArgumentException (error);
 924         }
 925
 926         // Remove any extraneous parameters (don't throw an error)
 927         Map <String, Object> updatedParams = new HashMap <> ();
 928         List <String> extraParams = new ArrayList <> ();
 929         for (String key : inputParams.keySet ()) {
 930             if (!paramList.contains (key)) {
 931                 // This is not a valid parameter for this template
 932                 extraParams.add (key);
 933             } else {
 934                 updatedParams.put (key, inputParams.get (key));
 935             }
 936         }
 937         if (!extraParams.isEmpty ()) {
 938             LOGGER.warn (MessageEnum.RA_GENERAL_WARNING, "Heat Stack (" + heatTemplate.getTemplateName ()
 939                          + ") extra input params received: "
 940                          + extraParams, "", "", MsoLogger.ErrorCode.DataError, "Heat Stack (" + heatTemplate.getTemplateName () + ") extra input params received: "+ extraParams);
 941         }
 942
 943         return updatedParams;
 944     }
 945
 946     // ---------------------------------------------------------------
 947     // PRIVATE FUNCTIONS FOR USE WITHIN THIS CLASS
 948
 949     /**
 950      * Get a Heat client for the Openstack Identity service.
 951      * This requires a 'member'-level userId + password, which will be retrieved from
 952      * properties based on the specified cloud Id. The tenant in which to operate
 953      * must also be provided.
 954      * <p>
 955      * On successful authentication, the Heat object will be cached for the
 956      * tenantID + cloudId so that it can be reused without reauthenticating with
 957      * Openstack every time.
 958      *
 959      * @return an authenticated Heat object
 960      */
 961     public Heat getHeatClient (CloudSite cloudSite, String tenantId) throws MsoException {
 962         String cloudId = cloudSite.getId ();
 963
 964         // Check first in the cache of previously authorized clients
 965         String cacheKey = cloudId + ":" + tenantId;
 966         if (heatClientCache.containsKey (cacheKey)) {
 967             if (!heatClientCache.get (cacheKey).isExpired ()) {
 968                 LOGGER.debug ("Using Cached HEAT Client for " + cacheKey);
 969                 return heatClientCache.get (cacheKey).getHeatClient ();
 970             } else {
 971                 // Token is expired. Remove it from cache.
 972                 heatClientCache.remove (cacheKey);
 973                 LOGGER.debug ("Expired Cached HEAT Client for " + cacheKey);
 974             }
 975         }
 976
 977         // Obtain an MSO token for the tenant
 978         CloudIdentity cloudIdentity = cloudSite.getIdentityService ();
 979         LOGGER.debug("Found: " + cloudIdentity.toString());
 980         String keystoneUrl = cloudIdentity.getKeystoneUrl (cloudId, msoPropID);
 981         LOGGER.debug("keystoneUrl=" + keystoneUrl);
 982         Keystone keystoneTenantClient = new Keystone (keystoneUrl);
 983         Access access = null;
 984         try {
 985                 Authentication credentials = cloudIdentity.getAuthentication ();
 986
 987                 OpenStackRequest <Access> request = keystoneTenantClient.tokens ()
 988                        .authenticate (credentials).withTenantId (tenantId);
 989
 990             access = executeAndRecordOpenstackRequest (request, msoProps);
 991         } catch (OpenStackResponseException e) {
 992             if (e.getStatus () == 401) {
 993                 // Authentication error.
 994                 String error = "Authentication Failure: tenant=" + tenantId + ",cloud=" + cloudIdentity.getId ();
 995                 alarmLogger.sendAlarm ("MsoAuthenticationError", MsoAlarmLogger.CRITICAL, error);
 996                 throw new MsoAdapterException (error);
 997             } else {
 998                 throw keystoneErrorToMsoException (e, TOKEN_AUTH);
 999             }
1000         } catch (OpenStackConnectException e) {
1001             // Connection to Openstack failed
1002             MsoIOException me = new MsoIOException (e.getMessage (), e);
1003             me.addContext (TOKEN_AUTH);
1004             throw me;
1005         } catch (RuntimeException e) {
1006             // Catch-all
1007             throw runtimeExceptionToMsoException (e, TOKEN_AUTH);
1008         }
1009
1010         // For DCP/LCP, the region should be the cloudId.
1011         String region = cloudSite.getRegionId ();
1012         String heatUrl = null;
1013         try {
1014             heatUrl = KeystoneUtils.findEndpointURL (access.getServiceCatalog (), "orchestration", region, "public");
1015             LOGGER.debug("heatUrl=" + heatUrl + ", region=" + region);
1016         } catch (RuntimeException e) {
1017             // This comes back for not found (probably an incorrect region ID)
1018             String error = "Orchestration service not found: region=" + region + ",cloud=" + cloudIdentity.getId ();
1019             alarmLogger.sendAlarm ("MsoConfigurationError", MsoAlarmLogger.CRITICAL, error);
1020             throw new MsoAdapterException (error, e);
1021         }
1022
1023         Heat heatClient = new Heat (heatUrl);
1024         heatClient.token (access.getToken ().getId ());
1025
1026         heatClientCache.put (cacheKey,
1027                              new HeatCacheEntry (heatUrl,
1028                                                  access.getToken ().getId (),
1029                                                  access.getToken ().getExpires ()));
1030         LOGGER.debug ("Caching HEAT Client for " + cacheKey);
1031
1032         return heatClient;
1033     }
1034
1035     /**
1036      * Forcibly expire a HEAT client from the cache. This call is for use by
1037      * the KeystoneClient in case where a tenant is deleted. In that case,
1038      * all cached credentials must be purged so that fresh authentication is
1039      * done if a similarly named tenant is re-created.
1040      * <p>
1041      * Note: This is probably only applicable to dev/test environments where
1042      * the same Tenant Name is repeatedly used for creation/deletion.
1043      * <p>
1044      *
1045      */
1046     public static void expireHeatClient (String tenantId, String cloudId) {
1047         String cacheKey = cloudId + ":" + tenantId;
1048         if (heatClientCache.containsKey (cacheKey)) {
1049             heatClientCache.remove (cacheKey);
1050             LOGGER.debug ("Deleted Cached HEAT Client for " + cacheKey);
1051         }
1052     }
1053
1054     /*
1055      * Query for a Heat Stack. This function is needed in several places, so
1056      * a common method is useful. This method takes an authenticated Heat Client
1057      * (which internally identifies the cloud & tenant to search), and returns
1058      * a Stack object if found, Null if not found, or an MsoOpenstackException
1059      * if the Openstack API call fails.
1060      *
1061      * The stack name may be a simple name or a canonical name ("{name}/{id}").
1062      * When simple name is used, Openstack always returns a 302 redirect which
1063      * results in a 2nd request (to the canonical name). Note that query by
1064      * canonical name for a deleted stack returns a Stack object with status
1065      * "DELETE_COMPLETE" while query by simple name for a deleted stack returns
1066      * HTTP 404.
1067      *
1068      * @param heatClient an authenticated Heat client
1069      *
1070      * @param stackName the stack name to query
1071      *
1072      * @return a Stack object that describes the current stack or null if the
1073      * requested stack doesn't exist.
1074      *
1075      * @throws MsoOpenstackException Thrown if the Openstack API call returns an exception
1076      */
1077     protected Stack queryHeatStack (Heat heatClient, String stackName) throws MsoException {
1078         if (stackName == null) {
1079             return null;
1080         }
1081         try {
1082             OpenStackRequest <Stack> request = heatClient.getStacks ().byName (stackName);
1083             return executeAndRecordOpenstackRequest (request, msoProps);
1084         } catch (OpenStackResponseException e) {
1085             if (e.getStatus () == 404) {
1086                 LOGGER.debug ("queryHeatStack - stack not found: " + stackName);
1087                 return null;
1088             } else {
1089                 // Convert the OpenStackResponseException to an MsoOpenstackException
1090                 throw heatExceptionToMsoException (e, "QueryStack");
1091             }
1092         } catch (OpenStackConnectException e) {
1093             // Connection to Openstack failed
1094             throw heatExceptionToMsoException (e, "QueryAllStack");
1095         }
1096     }
1097
1098     /*
1099      * An entry in the Heat Client Cache. It saves the Heat client object
1100      * along with the token expiration. After this interval, this cache
1101      * item will no longer be used.
1102      */
1103     private static class HeatCacheEntry implements Serializable {
1104
1105         private static final long serialVersionUID = 1L;
1106
1107         private String heatUrl;
1108         private String token;
1109         private Calendar expires;
1110
1111         public HeatCacheEntry (String heatUrl, String token, Calendar expires) {
1112             this.heatUrl = heatUrl;
1113             this.token = token;
1114             this.expires = expires;
1115         }
1116
1117         public Heat getHeatClient () {
1118             Heat heatClient = new Heat (heatUrl);
1119             heatClient.token (token);
1120             return heatClient;
1121         }
1122
1123         public boolean isExpired () {
1124             return expires == null || System.currentTimeMillis() > expires.getTimeInMillis();
1125
1126         }
1127     }
1128
1129     /**
1130      * Clean up the Heat client cache to remove expired entries.
1131      */
1132     public static void heatCacheCleanup () {
1133         for (String cacheKey : heatClientCache.keySet ()) {
1134             if (heatClientCache.get (cacheKey).isExpired ()) {
1135                 heatClientCache.remove (cacheKey);
1136                 LOGGER.debug ("Cleaned Up Cached Heat Client for " + cacheKey);
1137             }
1138         }
1139     }
1140
1141     /**
1142      * Reset the Heat client cache.
1143      * This may be useful if cached credentials get out of sync.
1144      */
1145     public static void heatCacheReset () {
1146         heatClientCache = new HashMap <> ();
1147     }
1148
1149         public Map<String, Object> queryStackForOutputs(String cloudSiteId,
1150                         String tenantId, String stackName) throws MsoException {
1151                 LOGGER.debug("MsoHeatUtils.queryStackForOutputs)");
1152                 StackInfo heatStack = this.queryStack(cloudSiteId, tenantId, stackName);
1153                 if (heatStack == null || heatStack.getStatus() == HeatStatus.NOTFOUND) {
1154                         return null;
1155                 }
1156                 Map<String, Object> outputs = heatStack.getOutputs();
1157                 return outputs;
1158         }
1159
1160         public void queryAndCopyOutputsToInputs(String cloudSiteId,
1161                         String tenantId, String stackName, Map<String, String> inputs,
1162                         boolean overWrite) throws MsoException {
1163                 LOGGER.debug("MsoHeatUtils.queryAndCopyOutputsToInputs");
1164                 Map<String, Object> outputs = this.queryStackForOutputs(cloudSiteId,
1165                                 tenantId, stackName);
1166                 this.copyStringOutputsToInputs(inputs, outputs, overWrite);
1167                 return;
1168         }
1169
1170         public void copyStringOutputsToInputs(Map<String, String> inputs,
1171                         Map<String, Object> otherStackOutputs, boolean overWrite) {
1172                 if (inputs == null || otherStackOutputs == null)
1173                         return;
1174                 for (String key : otherStackOutputs.keySet()) {
1175                         if (!inputs.containsKey(key)) {
1176                                 Object obj = otherStackOutputs.get(key);
1177                                 if (obj instanceof String) {
1178                                         inputs.put(key, (String) otherStackOutputs.get(key));
1179                                 } else if (obj instanceof JsonNode ){
1180                                         // This is a bit of mess - but I think it's the least impacting
1181                                         // let's convert it BACK to a string - then it will get converted back later
1182                                         try {
1183                                                 String str = this.convertNode((JsonNode) obj);
1184                                                 inputs.put(key, str);
1185                                         } catch (Exception e) {
1186                                                 LOGGER.debug("DANGER WILL ROBINSON: unable to convert value for JsonNode "+ key, e);
1187                                                 //effect here is this value will not have been copied to the inputs - and therefore will error out downstream
1188                                         }
1189                                 } else if (obj instanceof java.util.LinkedHashMap) {
1190                                         LOGGER.debug("LinkedHashMap - this is showing up as a LinkedHashMap instead of JsonNode");
1191                                         try {
1192                                                 String str = JSON_MAPPER.writeValueAsString(obj);
1193                                                 inputs.put(key, str);
1194                                         } catch (Exception e) {
1195                                                 LOGGER.debug("DANGER WILL ROBINSON: unable to convert value for LinkedHashMap "+ key, e);
1196                                         }
1197                                 } else if (obj instanceof Integer) {
1198                                         try {
1199                                                 String str = "" + obj;
1200                                                 inputs.put(key, str);
1201                                         } catch (Exception e) {
1202                                                 LOGGER.debug("DANGER WILL ROBINSON: unable to convert value for Integer "+ key, e);
1203                                         }
1204                                 } else {
1205                                         try {
1206                                                 String str = obj.toString();
1207                                                 inputs.put(key, str);
1208                                         } catch (Exception e) {
1209                                                 LOGGER.debug("DANGER WILL ROBINSON: unable to convert value for Other "+ key +" (" + e.getMessage() + ")", e);
1210                                                 //effect here is this value will not have been copied to the inputs - and therefore will error out downstream
1211                                         }
1212                                 }
1213                         }
1214                 }
1215                 return;
1216         }
1217         public StringBuilder requestToStringBuilder(CreateStackParam stack) {
1218                 StringBuilder sb = new StringBuilder();
1219                 sb.append("Stack:\n");
1220                 sb.append("\tStackName: " + stack.getStackName());
1221                 sb.append("\tTemplateUrl: " + stack.getTemplateUrl());
1222                 sb.append("\tTemplate: " + stack.getTemplate());
1223                 sb.append("\tEnvironment: " + stack.getEnvironment());
1224                 sb.append("\tTimeout: " + stack.getTimeoutMinutes());
1225                 sb.append("\tParameters:\n");
1226                 Map<String, Object> params = stack.getParameters();
1227                 if (params == null || params.size() < 1) {
1228                         sb.append("\nNONE");
1229                 } else {
1230                         for (String key : params.keySet()) {
1231                                 if (params.get(key) instanceof String) {
1232                                         sb.append("\n").append(key).append("=").append((String) params.get(key));
1233                                 } else if (params.get(key) instanceof JsonNode) {
1234                                         String jsonStringOut = this.convertNode((JsonNode)params.get(key));
1235                                         sb.append("\n").append(key).append("=").append(jsonStringOut);
1236                                 } else if (params.get(key) instanceof Integer) {
1237                                         String integerOut = "" + params.get(key);
1238                                         sb.append("\n").append(key).append("=").append(integerOut);
1239
1240                                 } else {
1241                                         try {
1242                                                 String str = params.get(key).toString();
1243                                                 sb.append("\n").append(key).append("=").append(str);
1244                                         } catch (Exception e) {
1245                                                 LOGGER.debug("Exception :",e);
1246                                         }
1247                                 }
1248                         }
1249                 }
1250                 return sb;
1251         }
1252
1253         private String convertNode(final JsonNode node) {
1254                 try {
1255                         final Object obj = JSON_MAPPER.treeToValue(node, Object.class);
1256                         final String json = JSON_MAPPER.writeValueAsString(obj);
1257                         return json;
1258                 } catch (Exception e) {
1259                         LOGGER.debug("Error converting json to string " + e.getMessage(), e);
1260                 }
1261                 return "[Error converting json to string]";
1262         }
1263
1264
1265         private StringBuilder getOutputsAsStringBuilder(Stack heatStack) {
1266                 // This should only be used as a utility to print out the stack outputs
1267                 // to the log
1268                 StringBuilder sb = new StringBuilder("");
1269                 if (heatStack == null) {
1270                         sb.append("(heatStack is null)");
1271                         return sb;
1272                 }
1273                 List<Output> outputList = heatStack.getOutputs();
1274                 if (outputList == null || outputList.isEmpty()) {
1275                         sb.append("(outputs is empty)");
1276                         return sb;
1277                 }
1278                 Map<String, Object> outputs = new HashMap<>();
1279                 for (Output outputItem : outputList) {
1280                         outputs.put(outputItem.getOutputKey(), outputItem.getOutputValue());
1281                 }
1282                 int counter = 0;
1283                 sb.append("OUTPUTS:\n");
1284                 for (String key : outputs.keySet()) {
1285                         sb.append("outputs[").append(counter++).append("]: ").append(key).append("=");
1286                         Object obj = outputs.get(key);
1287                         if (obj instanceof String) {
1288                                 sb.append((String) obj).append(" (a string)");
1289                         } else if (obj instanceof JsonNode) {
1290                                 sb.append(this.convertNode((JsonNode) obj)).append(" (a JsonNode)");
1291                         } else if (obj instanceof java.util.LinkedHashMap) {
1292                                 try {
1293                                         String str = JSON_MAPPER.writeValueAsString(obj);
1294                                         sb.append(str).append(" (a java.util.LinkedHashMap)");
1295                                 } catch (Exception e) {
1296                                         LOGGER.debug("Exception :",e);
1297                                         sb.append("(a LinkedHashMap value that would not convert nicely)");
1298                                 }
1299                         } else if (obj instanceof Integer) {
1300                                 String str = "";
1301                                 try {
1302                                         str = obj.toString() + " (an Integer)\n";
1303                                 } catch (Exception e) {
1304                                         LOGGER.debug("Exception :",e);
1305                                         str = "(an Integer unable to call .toString() on)";
1306                                 }
1307                                 sb.append(str);
1308                         } else if (obj instanceof ArrayList) {
1309                                 String str = "";
1310                                 try {
1311                                         str = obj.toString() + " (an ArrayList)";
1312                                 } catch (Exception e) {
1313                                         LOGGER.debug("Exception :",e);
1314                                         str = "(an ArrayList unable to call .toString() on?)";
1315                                 }
1316                                 sb.append(str);
1317                         } else if (obj instanceof Boolean) {
1318                                 String str = "";
1319                                 try {
1320                                         str = obj.toString() + " (a Boolean)";
1321                                 } catch (Exception e) {
1322                                         LOGGER.debug("Exception :",e);
1323                                         str = "(an Boolean unable to call .toString() on?)";
1324                                 }
1325                                 sb.append(str);
1326                         }
1327                         else {
1328                                 String str = "";
1329                                 try {
1330                                         str = obj.toString() + " (unknown Object type)";
1331                                 } catch (Exception e) {
1332                                         LOGGER.debug("Exception :",e);
1333                                         str = "(a value unable to call .toString() on?)";
1334                                 }
1335                                 sb.append(str);
1336                         }
1337                         sb.append("\n");
1338                 }
1339                 sb.append("[END]");
1340                 return sb;
1341         }
1342
1343
1344         public void copyBaseOutputsToInputs(Map<String, Object> inputs,
1345                         Map<String, Object> otherStackOutputs, ArrayList<String> paramNames, HashMap<String, String> aliases) {
1346                 if (inputs == null || otherStackOutputs == null)
1347                         return;
1348                 for (String key : otherStackOutputs.keySet()) {
1349                         if (paramNames != null) {
1350                                 if (!paramNames.contains(key) && !aliases.containsKey(key)) {
1351                                         LOGGER.debug("\tParameter " + key + " is NOT defined to be in the template - do not copy to inputs");
1352                                         continue;
1353                                 }
1354                                 if (aliases.containsKey(key)) {
1355                                         LOGGER.debug("Found an alias! Will move " + key + " to " + aliases.get(key));
1356                                         Object obj = otherStackOutputs.get(key);
1357                                         key = aliases.get(key);
1358                                         otherStackOutputs.put(key, obj);
1359                                 }
1360                         }
1361                         if (!inputs.containsKey(key)) {
1362                                 Object obj = otherStackOutputs.get(key);
1363                                 LOGGER.debug("\t**Adding " + key + " to inputs (.toString()=" + obj.toString());
1364                                 if (obj instanceof String) {
1365                                         LOGGER.debug("\t\t**A String");
1366                                         inputs.put(key, obj);
1367                                 } else if (obj instanceof Integer) {
1368                                         LOGGER.debug("\t\t**An Integer");
1369                                         inputs.put(key, obj);
1370                                 } else if (obj instanceof JsonNode) {
1371                                         LOGGER.debug("\t\t**A JsonNode");
1372                                         inputs.put(key, obj);
1373                                 } else if (obj instanceof Boolean) {
1374                                         LOGGER.debug("\t\t**A Boolean");
1375                                         inputs.put(key, obj);
1376                                 } else if (obj instanceof java.util.LinkedHashMap) {
1377                                         LOGGER.debug("\t\t**A java.util.LinkedHashMap **");
1378                                         //Object objJson = this.convertObjectToJsonNode(obj.toString());
1379                                         //if (objJson == null) {
1380                                         //      LOGGER.debug("\t\tFAILED!! Will just put LinkedHashMap on the inputs");
1381                                         inputs.put(key, obj);
1382                                         //}
1383                                         //else {
1384                                         //      LOGGER.debug("\t\tSuccessfully converted to JsonNode: " + objJson.toString());
1385                                         //      inputs.put(key, objJson);
1386                                         //}
1387                                 } else if (obj instanceof java.util.ArrayList) {
1388                                         LOGGER.debug("\t\t**An ArrayList");
1389                                         inputs.put(key, obj);
1390                                 } else {
1391                                         LOGGER.debug("\t\t**UNKNOWN OBJECT TYPE");
1392                                         inputs.put(key, obj);
1393                                 }
1394                         } else {
1395                                 LOGGER.debug("key=" + key + " is already in the inputs - will not overwrite");
1396                         }
1397                 }
1398                 return;
1399         }
1400
1401         public JsonNode convertObjectToJsonNode(Object lhm) {
1402                 if (lhm == null) {
1403                         return null;
1404                 }
1405                 JsonNode jsonNode = null;
1406                 try {
1407                         String jsonString = lhm.toString();
1408                         jsonNode = new ObjectMapper().readTree(jsonString);
1409                 } catch (Exception e) {
1410                         LOGGER.debug("Unable to convert " + lhm.toString() + " to a JsonNode " + e.getMessage(), e);
1411                         jsonNode = null;
1412                 }
1413                 return jsonNode;
1414         }
1415
1416         public ArrayList<String> convertCdlToArrayList(String cdl) {
1417                 String cdl2 = cdl.trim();
1418                 String cdl3;
1419                 if (cdl2.startsWith("[") && cdl2.endsWith("]")) {
1420                         cdl3 = cdl2.substring(1, cdl2.lastIndexOf("]"));
1421                 } else {
1422                         cdl3 = cdl2;
1423                 }
1424                 ArrayList<String> list = new ArrayList<>(Arrays.asList(cdl3.split(",")));
1425                 return list;
1426         }
1427
1428     /**
1429      * New with 1707 - this method will convert all the String *values* of the inputs
1430      * to their "actual" object type (based on the param type: in the db - which comes from the template):
1431      * (heat variable type) -> java Object type
1432      * string -> String
1433      * number -> Integer
1434      * json -> JsonNode XXX Removed with MSO-1475 / 1802
1435      * comma_delimited_list -> ArrayList
1436      * boolean -> Boolean
1437      * if any of the conversions should fail, we will default to adding it to the inputs
1438      * as a string - see if Openstack can handle it.
1439      * Also, will remove any params that are extra.
1440      * Any aliases will be converted to their appropriate name (anyone use this feature?)
1441      * @param inputs - the Map<String, String> of the inputs received on the request
1442      * @param template the HeatTemplate object - this is so we can also verify if the param is valid for this template
1443      * @return HashMap<String, Object> of the inputs, cleaned and converted
1444      */
1445         public HashMap<String, Object> convertInputMap(Map<String, String> inputs, HeatTemplate template) {
1446                 HashMap<String, Object> newInputs = new HashMap<>();
1447                 HashMap<String, HeatTemplateParam> params = new HashMap<>();
1448                 HashMap<String, HeatTemplateParam> paramAliases = new HashMap<>();
1449
1450                 if (inputs == null) {
1451                         LOGGER.debug("convertInputMap - inputs is null - nothing to do here");
1452                         return new HashMap<>();
1453                 }
1454
1455                 LOGGER.debug("convertInputMap in MsoHeatUtils called, with " + inputs.size() + " inputs, and template " + template.getArtifactUuid());
1456                 try {
1457                         LOGGER.debug(template.toString());
1458                         Set<HeatTemplateParam> paramSet = template.getParameters();
1459                         LOGGER.debug("paramSet has " + paramSet.size() + " entries");
1460                 } catch (Exception e) {
1461                         LOGGER.debug("Exception occurred in convertInputMap:" + e.getMessage(), e);
1462                 }
1463
1464                 for (HeatTemplateParam htp : template.getParameters()) {
1465                         LOGGER.debug("Adding " + htp.getParamName());
1466                         params.put(htp.getParamName(), htp);
1467                         if (htp.getParamAlias() != null && !"".equals(htp.getParamAlias())) {
1468                                 LOGGER.debug("\tFound ALIAS " + htp.getParamName() + "->" + htp.getParamAlias());
1469                                 paramAliases.put(htp.getParamAlias(), htp);
1470                         }
1471                 }
1472                 LOGGER.debug("Now iterate through the inputs...");
1473                 for (String key : inputs.keySet()) {
1474                         LOGGER.debug("key=" + key);
1475                         boolean alias = false;
1476                         String realName = null;
1477                         if (!params.containsKey(key)) {
1478                                 LOGGER.debug(key + " is not a parameter in the template! - check for an alias");
1479                                 // add check here for an alias
1480                                 if (!paramAliases.containsKey(key)) {
1481                                         LOGGER.debug("The parameter " + key + " is in the inputs, but it's not a parameter for this template - omit");
1482                                         continue;
1483                                 } else {
1484                                         alias = true;
1485                                         realName = paramAliases.get(key).getParamName();
1486                                         LOGGER.debug("FOUND AN ALIAS! Will use " + realName + " in lieu of give key/alias " + key);
1487                                 }
1488                         }
1489                         String type = params.get(key).getParamType();
1490                         if (type == null || "".equals(type)) {
1491                                 LOGGER.debug("**PARAM_TYPE is null/empty for " + key + ", will default to string");
1492                                 type = "string";
1493                         }
1494                         LOGGER.debug("Parameter: " + key + " is of type " + type);
1495                         if ("string".equalsIgnoreCase(type)) {
1496                                 // Easiest!
1497                                 String str = inputs.get(key);
1498                                 if (alias)
1499                                         newInputs.put(realName, str);
1500                                 else
1501                                         newInputs.put(key, str);
1502                         } else if ("number".equalsIgnoreCase(type)) {
1503                                 String integerString = inputs.get(key);
1504                                 Integer anInteger = null;
1505                                 try {
1506                                         anInteger = Integer.parseInt(integerString);
1507                                 } catch (Exception e) {
1508                                         LOGGER.debug("Unable to convert " + integerString + " to an integer!!", e);
1509                                         anInteger = null;
1510                                 }
1511                                 if (anInteger != null) {
1512                                         if (alias)
1513                                                 newInputs.put(realName, anInteger);
1514                                         else
1515                                                 newInputs.put(key, anInteger);
1516                                 }
1517                                 else {
1518                                         if (alias)
1519                                                 newInputs.put(realName, integerString);
1520                                         else
1521                                                 newInputs.put(key, integerString);
1522                                 }
1523                         } else if ("json".equalsIgnoreCase(type)) {
1524                                 // MSO-1475 - Leave this as a string now
1525                                 String jsonString = inputs.get(key);
1526                                 LOGGER.debug("Skipping conversion to jsonNode...");
1527                         if (alias)
1528                                 newInputs.put(realName, jsonString);
1529                         else
1530                                 newInputs.put(key, jsonString);
1531                         //}
1532                         } else if ("comma_delimited_list".equalsIgnoreCase(type)) {
1533                                 String commaSeparated = inputs.get(key);
1534                                 try {
1535                                         ArrayList<String> anArrayList = this.convertCdlToArrayList(commaSeparated);
1536                                         if (alias)
1537                                                 newInputs.put(realName, anArrayList);
1538                                         else
1539                                                 newInputs.put(key, anArrayList);
1540                                 } catch (Exception e) {
1541                                         LOGGER.debug("Unable to convert " + commaSeparated + " to an ArrayList!!", e);
1542                                         if (alias)
1543                                                 newInputs.put(realName, commaSeparated);
1544                                         else
1545                                                 newInputs.put(key, commaSeparated);
1546                                 }
1547                         } else if ("boolean".equalsIgnoreCase(type)) {
1548                                 String booleanString = inputs.get(key);
1549                                 Boolean aBool = Boolean.valueOf(booleanString);
1550                                 if (alias)
1551                                         newInputs.put(realName, aBool);
1552                                 else
1553                                         newInputs.put(key, aBool);
1554                         } else {
1555                                 // it's null or something undefined - just add it back as a String
1556                                 String str = inputs.get(key);
1557                                 if (alias)
1558                                         newInputs.put(realName, str);
1559                                 else
1560                                         newInputs.put(key, str);
1561                         }
1562                 }
1563                 return newInputs;
1564         }
1565
1566
1567         /*
1568          * Create a string suitable for being dumped to a debug log that creates a
1569          * pseudo-JSON request dumping what's being sent to Openstack API in the create or update request
1570          */
1571
1572         private String printStackRequest(String tenantId,
1573                         Map<String, Object> heatFiles,
1574                         Map<String, Object> nestedTemplates,
1575                         String environment,
1576                         Map<String, Object> inputs,
1577                         String vfModuleName,
1578                         String template,
1579                         int timeoutMinutes,
1580                         boolean backout,
1581                         String cloudSiteId) {
1582                 StringBuffer sb = new StringBuffer();
1583                 sb.append("CREATE STACK REQUEST (formatted for readability)\n");
1584                 sb.append("tenant=" + tenantId + ", cloud=" + cloudSiteId);
1585                 sb.append("{\n");
1586                 sb.append("  \"stack_name\": \"" + vfModuleName + "\",\n");
1587                 sb.append("  \"disable_rollback\": " + backout + ",\n");
1588                 sb.append("  \"timeout_mins\": " + timeoutMinutes + ",\n");
1589                 sb.append("  \"template\": {\n");
1590                 sb.append(template);
1591                 sb.append("  },\n");
1592                 sb.append("  \"environment\": {\n");
1593                 if (environment == null)
1594                         sb.append("<none>");
1595                 else
1596                         sb.append(environment);
1597                 sb.append("  },\n");
1598                 sb.append("  \"files\": {\n");
1599                 int filesCounter = 0;
1600                 if (heatFiles != null) {
1601                         for (String key : heatFiles.keySet()) {
1602                                 filesCounter++;
1603                                 if (filesCounter > 1) {
1604                                         sb.append(",\n");
1605                                 }
1606                                 sb.append("    \"" + key + "\": {\n");
1607                                 sb.append(heatFiles.get(key).toString() + "\n    }");
1608                         }
1609                 }
1610                 if (nestedTemplates != null) {
1611                         for (String key : nestedTemplates.keySet()) {
1612                                 filesCounter++;
1613                                 if (filesCounter > 1) {
1614                                         sb.append(",\n");
1615                                 }
1616                                 sb.append("    \"" + key + "\": {\n");
1617                                 sb.append(nestedTemplates.get(key).toString() + "\n    }");
1618                         }
1619                 }
1620                 sb.append("\n  },\n");
1621                 sb.append("  \"parameters\": {\n");
1622                 int paramCounter = 0;
1623                 for (String name : inputs.keySet()) {
1624                         paramCounter++;
1625                         if (paramCounter > 1) {
1626                                 sb.append(",\n");
1627                         }
1628                         Object o = inputs.get(name);
1629                         if (o instanceof java.lang.String) {
1630                                 sb.append("    \"" + name + "\": \"" + inputs.get(name).toString() + "\"");
1631                         } else if (o instanceof Integer) {
1632                                 sb.append("    \"" + name + "\": " + inputs.get(name).toString() );
1633                         } else if (o instanceof ArrayList) {
1634                                 sb.append("    \"" + name + "\": " + inputs.get(name).toString() );
1635                         } else if (o instanceof Boolean) {
1636                                 sb.append("    \"" + name + "\": " + inputs.get(name).toString() );
1637                         } else {
1638                                 sb.append("    \"" + name + "\": " + "\"(there was an issue trying to dump this value...)\"" );
1639                         }
1640                 }
1641                 sb.append("\n  }\n}\n");
1642
1643                 return sb.toString();
1644         }
1645
1646 }