adapters/mso-adapter-utils/src/main/java/org/openecomp/mso/openstack/utils/MsoHeatUtils.java

   1 /*-
   2  * ============LICENSE_START=======================================================
   3  * ONAP - SO
   4  * ================================================================================
   5  * Copyright (C) 2017 AT&T Intellectual Property. All rights reserved.
   6  * ================================================================================
   7  * Licensed under the Apache License, Version 2.0 (the "License");
   8  * you may not use this file except in compliance with the License.
   9  * You may obtain a copy of the License at
  10  *
  11  *      http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  * ============LICENSE_END=========================================================
  19  */
  20
  21 package org.openecomp.mso.openstack.utils;
  22
  23 import java.io.Serializable;
  24 import java.rmi.server.ObjID;
  25 import java.util.ArrayList;
  26 import java.util.Arrays;
  27 import java.util.Calendar;
  28 import java.util.HashMap;
  29 import java.util.HashSet;
  30 import java.util.List;
  31 import java.util.Map;
  32 import java.util.Set;
  33
  34 import org.codehaus.jackson.map.ObjectMapper;
  35 import org.codehaus.jackson.JsonNode;
  36 import org.codehaus.jackson.JsonParseException;
  37
  38 import org.openecomp.mso.cloud.CloudConfig;
  39 import org.openecomp.mso.cloud.CloudConfigFactory;
  40 import org.openecomp.mso.cloud.CloudIdentity;
  41 import org.openecomp.mso.cloud.CloudSite;
  42 import org.openecomp.mso.db.catalog.beans.HeatTemplate;
  43 import org.openecomp.mso.db.catalog.beans.HeatTemplateParam;
  44 import org.openecomp.mso.logger.MessageEnum;
  45 import org.openecomp.mso.logger.MsoAlarmLogger;
  46 import org.openecomp.mso.logger.MsoLogger;
  47 import org.openecomp.mso.openstack.beans.HeatStatus;
  48 import org.openecomp.mso.openstack.beans.StackInfo;
  49 import org.openecomp.mso.openstack.exceptions.MsoAdapterException;
  50 import org.openecomp.mso.openstack.exceptions.MsoCloudSiteNotFound;
  51 import org.openecomp.mso.openstack.exceptions.MsoException;
  52 import org.openecomp.mso.openstack.exceptions.MsoIOException;
  53 import org.openecomp.mso.openstack.exceptions.MsoOpenstackException;
  54 import org.openecomp.mso.openstack.exceptions.MsoStackAlreadyExists;
  55 import org.openecomp.mso.openstack.exceptions.MsoTenantNotFound;
  56 import org.openecomp.mso.properties.MsoJavaProperties;
  57 import org.openecomp.mso.properties.MsoPropertiesException;
  58 import org.openecomp.mso.properties.MsoPropertiesFactory;
  59 import com.woorea.openstack.base.client.OpenStackConnectException;
  60 import com.woorea.openstack.base.client.OpenStackRequest;
  61 import com.woorea.openstack.base.client.OpenStackResponseException;
  62 import com.woorea.openstack.heat.Heat;
  63 import com.woorea.openstack.heat.model.CreateStackParam;
  64 import com.woorea.openstack.heat.model.Stack;
  65 import com.woorea.openstack.heat.model.Stack.Output;
  66 import com.woorea.openstack.heat.model.Stacks;
  67 import com.woorea.openstack.keystone.Keystone;
  68 import com.woorea.openstack.keystone.model.Access;
  69 import com.woorea.openstack.keystone.model.Authentication;
  70 import com.woorea.openstack.keystone.utils.KeystoneUtils;
  71
  72 public class MsoHeatUtils extends MsoCommonUtils {
  73
  74         private MsoPropertiesFactory msoPropertiesFactory;
  75
  76         private CloudConfigFactory cloudConfigFactory;
  77
  78     private static final String TOKEN_AUTH = "TokenAuth";
  79
  80     private static final String QUERY_ALL_STACKS = "QueryAllStacks";
  81
  82     private static final String DELETE_STACK = "DeleteStack";
  83
  84     private static final String HEAT_ERROR = "HeatError";
  85
  86     private static final String CREATE_STACK = "CreateStack";
  87
  88     // Cache Heat Clients statically. Since there is just one MSO user, there is no
  89     // benefit to re-authentication on every request (or across different flows). The
  90     // token will be used until it expires.
  91     //
  92     // The cache key is "tenantId:cloudId"
  93     private static Map <String, HeatCacheEntry> heatClientCache = new HashMap <String, HeatCacheEntry> ();
  94
  95     // Fetch cloud configuration each time (may be cached in CloudConfig class)
  96     protected CloudConfig cloudConfig;
  97
  98     private static MsoLogger LOGGER = MsoLogger.getMsoLogger (MsoLogger.Catalog.RA);
  99
 100     protected MsoJavaProperties msoProps = null;
 101
 102     // Properties names and variables (with default values)
 103     protected String createPollIntervalProp = "ecomp.mso.adapters.heat.create.pollInterval";
 104     private String deletePollIntervalProp = "ecomp.mso.adapters.heat.delete.pollInterval";
 105     private String deletePollTimeoutProp = "ecomp.mso.adapters.heat.delete.pollTimeout";
 106
 107     protected int createPollIntervalDefault = 15;
 108     private int deletePollIntervalDefault = 15;
 109     private int deletePollTimeoutDefault = 300;
 110     private String msoPropID;
 111
 112     private static final ObjectMapper JSON_MAPPER = new ObjectMapper();
 113
 114     /**
 115      * This constructor MUST be used ONLY in the JUNIT tests, not for real code.
 116      * The MsoPropertiesFactory will be added by EJB injection.
 117      *
 118      * @param msoPropID ID of the mso pro config as defined in web.xml
 119      * @param msoPropFactory The mso properties factory instanciated by EJB injection
 120      * @param cloudConfFactory the Cloud Config instantiated by EJB injection
 121      */
 122     public MsoHeatUtils (String msoPropID, MsoPropertiesFactory msoPropFactory, CloudConfigFactory cloudConfFactory) {
 123         msoPropertiesFactory = msoPropFactory;
 124         cloudConfigFactory = cloudConfFactory;
 125         this.msoPropID = msoPropID;
 126         // Dynamically get properties each time (in case reloaded).
 127
 128         try {
 129                         msoProps = msoPropertiesFactory.getMsoJavaProperties (msoPropID);
 130                 } catch (MsoPropertiesException e) {
 131                         LOGGER.error (MessageEnum.LOAD_PROPERTIES_FAIL, "Unknown. Mso Properties ID not found in cache: " + msoPropID, "", "", MsoLogger.ErrorCode.DataError, "Exception - Mso Properties ID not found in cache", e);
 132                 }
 133         cloudConfig = cloudConfigFactory.getCloudConfig ();
 134         LOGGER.debug("MsoHeatUtils:" + msoPropID);
 135
 136     }
 137
 138
 139     /**
 140      * keep this old method signature here to maintain backwards compatibility. keep others as well.
 141      * this method does not include environment, files, or heatFiles
 142      */
 143     public StackInfo createStack (String cloudSiteId,
 144                                   String tenantId,
 145                                   String stackName,
 146                                   String heatTemplate,
 147                                   Map <String, ? extends Object> stackInputs,
 148                                   boolean pollForCompletion,
 149                                   int timeoutMinutes) throws MsoException {
 150         // Just call the new method with the environment & files variable set to null
 151         return this.createStack (cloudSiteId,
 152                                  tenantId,
 153                                  stackName,
 154                                  heatTemplate,
 155                                  stackInputs,
 156                                  pollForCompletion,
 157                                  timeoutMinutes,
 158                                  null,
 159                                  null,
 160                                  null,
 161                                  true);
 162     }
 163
 164     // This method has environment, but not files or heatFiles
 165     public StackInfo createStack (String cloudSiteId,
 166                                   String tenantId,
 167                                   String stackName,
 168                                   String heatTemplate,
 169                                   Map <String, ? extends Object> stackInputs,
 170                                   boolean pollForCompletion,
 171                                   int timeoutMinutes,
 172                                   String environment) throws MsoException {
 173         // Just call the new method with the files/heatFiles variables set to null
 174         return this.createStack (cloudSiteId,
 175                                  tenantId,
 176                                  stackName,
 177                                  heatTemplate,
 178                                  stackInputs,
 179                                  pollForCompletion,
 180                                  timeoutMinutes,
 181                                  environment,
 182                                  null,
 183                                  null,
 184                                  true);
 185     }
 186
 187     // This method has environment and files, but not heatFiles.
 188     public StackInfo createStack (String cloudSiteId,
 189                                   String tenantId,
 190                                   String stackName,
 191                                   String heatTemplate,
 192                                   Map <String, ? extends Object> stackInputs,
 193                                   boolean pollForCompletion,
 194                                   int timeoutMinutes,
 195                                   String environment,
 196                                   Map <String, Object> files) throws MsoException {
 197         return this.createStack (cloudSiteId,
 198                                  tenantId,
 199                                  stackName,
 200                                  heatTemplate,
 201                                  stackInputs,
 202                                  pollForCompletion,
 203                                  timeoutMinutes,
 204                                  environment,
 205                                  files,
 206                                  null,
 207                                  true);
 208     }
 209
 210     // This method has environment, files, heatfiles
 211     public StackInfo createStack (String cloudSiteId,
 212                                   String tenantId,
 213                                   String stackName,
 214                                   String heatTemplate,
 215                                   Map <String, ? extends Object> stackInputs,
 216                                   boolean pollForCompletion,
 217                                   int timeoutMinutes,
 218                                   String environment,
 219                                   Map <String, Object> files,
 220                                   Map <String, Object> heatFiles) throws MsoException {
 221         return this.createStack (cloudSiteId,
 222                                  tenantId,
 223                                  stackName,
 224                                  heatTemplate,
 225                                  stackInputs,
 226                                  pollForCompletion,
 227                                  timeoutMinutes,
 228                                  environment,
 229                                  files,
 230                                  heatFiles,
 231                                  true);
 232     }
 233
 234     /**
 235      * Create a new Stack in the specified cloud location and tenant. The Heat template
 236      * and parameter map are passed in as arguments, along with the cloud access credentials.
 237      * It is expected that parameters have been validated and contain at minimum the required
 238      * parameters for the given template with no extra (undefined) parameters..
 239      *
 240      * The Stack name supplied by the caller must be unique in the scope of this tenant.
 241      * However, it should also be globally unique, as it will be the identifier for the
 242      * resource going forward in Inventory. This latter is managed by the higher levels
 243      * invoking this function.
 244      *
 245      * The caller may choose to let this function poll Openstack for completion of the
 246      * stack creation, or may handle polling itself via separate calls to query the status.
 247      * In either case, a StackInfo object will be returned containing the current status.
 248      * When polling is enabled, a status of CREATED is expected. When not polling, a
 249      * status of BUILDING is expected.
 250      *
 251      * An error will be thrown if the requested Stack already exists in the specified
 252      * Tenant and Cloud.
 253      *
 254      * For 1510 - add "environment", "files" (nested templates), and "heatFiles" (get_files) as
 255      * parameters for createStack. If environment is non-null, it will be added to the stack.
 256      * The nested templates and get_file entries both end up being added to the "files" on the
 257      * stack. We must combine them before we add them to the stack if they're both non-null.
 258      *
 259      * @param cloudSiteId The cloud (may be a region) in which to create the stack.
 260      * @param tenantId The Openstack ID of the tenant in which to create the Stack
 261      * @param stackName The name of the stack to create
 262      * @param stackTemplate The Heat template
 263      * @param stackInputs A map of key/value inputs
 264      * @param pollForCompletion Indicator that polling should be handled in Java vs. in the client
 265      * @param environment An optional yaml-format string to specify environmental parameters
 266      * @param files a Map<String, Object> that lists the child template IDs (file is the string, object is an int of
 267      *        Template id)
 268      * @param heatFiles a Map<String, Object> that lists the get_file entries (fileName, fileBody)
 269      * @param backout Donot delete stack on create Failure - defaulted to True
 270      * @return A StackInfo object
 271      * @throws MsoOpenstackException Thrown if the Openstack API call returns an exception.
 272      */
 273
 274     @SuppressWarnings("unchecked")
 275     public StackInfo createStack (String cloudSiteId,
 276                                   String tenantId,
 277                                   String stackName,
 278                                   String heatTemplate,
 279                                   Map <String, ? extends Object> stackInputs,
 280                                   boolean pollForCompletion,
 281                                   int timeoutMinutes,
 282                                   String environment,
 283                                   Map <String, Object> files,
 284                                   Map <String, Object> heatFiles,
 285                                   boolean backout) throws MsoException {
 286         // Create local variables checking to see if we have an environment, nested, get_files
 287         // Could later add some checks to see if it's valid.
 288         boolean haveEnvtVariable = true;
 289         if (environment == null || "".equalsIgnoreCase (environment.trim ())) {
 290             haveEnvtVariable = false;
 291             LOGGER.debug ("createStack called with no environment variable");
 292         } else {
 293             LOGGER.debug ("createStack called with an environment variable: " + environment);
 294         }
 295
 296         boolean haveFiles = true;
 297         if (files == null || files.isEmpty ()) {
 298             haveFiles = false;
 299             LOGGER.debug ("createStack called with no files / child template ids");
 300         } else {
 301             LOGGER.debug ("createStack called with " + files.size () + " files / child template ids");
 302         }
 303
 304         boolean haveHeatFiles = true;
 305         if (heatFiles == null || heatFiles.isEmpty ()) {
 306             haveHeatFiles = false;
 307             LOGGER.debug ("createStack called with no heatFiles");
 308         } else {
 309             LOGGER.debug ("createStack called with " + heatFiles.size () + " heatFiles");
 310         }
 311
 312         // Obtain the cloud site information where we will create the stack
 313         CloudSite cloudSite = cloudConfig.getCloudSite (cloudSiteId);
 314         if (cloudSite == null) {
 315             throw new MsoCloudSiteNotFound (cloudSiteId);
 316         }
 317         LOGGER.debug("Found: " + cloudSite.toString());
 318         // Get a Heat client. They are cached between calls (keyed by tenantId:cloudId)
 319         // This could throw MsoTenantNotFound or MsoOpenstackException (both propagated)
 320         Heat heatClient = getHeatClient (cloudSite, tenantId);
 321         if (heatClient != null) {
 322                 LOGGER.debug("Found: " + heatClient.toString());
 323         }
 324
 325         LOGGER.debug ("Ready to Create Stack (" + heatTemplate + ") with input params: " + stackInputs);
 326
 327         // Build up the stack to create
 328         // Disable auto-rollback, because error reason is lost. Always rollback in the code.
 329         CreateStackParam stack = new CreateStackParam ();
 330         stack.setStackName (stackName);
 331         stack.setTimeoutMinutes (timeoutMinutes);
 332         stack.setParameters ((Map <String, Object>) stackInputs);
 333         stack.setTemplate (heatTemplate);
 334         stack.setDisableRollback (true);
 335         // TJM New for PO Adapter - add envt variable
 336         if (haveEnvtVariable) {
 337             LOGGER.debug ("Found an environment variable - value: " + environment);
 338             stack.setEnvironment (environment);
 339         }
 340         // Now handle nested templates or get_files - have to combine if we have both
 341         // as they're both treated as "files:" on the stack.
 342         if (haveFiles && haveHeatFiles) {
 343             // Let's do this here - not in the bean
 344             LOGGER.debug ("Found files AND heatFiles - combine and add!");
 345             Map <String, Object> combinedFiles = new HashMap <String, Object> ();
 346             for (String keyString : files.keySet ()) {
 347                 combinedFiles.put (keyString, files.get (keyString));
 348             }
 349             for (String keyString : heatFiles.keySet ()) {
 350                 combinedFiles.put (keyString, heatFiles.get (keyString));
 351             }
 352             stack.setFiles (combinedFiles);
 353         } else {
 354             // Handle if we only have one or neither:
 355             if (haveFiles) {
 356                 LOGGER.debug ("Found files - adding to stack");
 357                 stack.setFiles (files);
 358             }
 359             if (haveHeatFiles) {
 360                 LOGGER.debug ("Found heatFiles - adding to stack");
 361                 // the setFiles was modified to handle adding the entries
 362                 stack.setFiles (heatFiles);
 363             }
 364         }
 365
 366         Stack heatStack = null;
 367         try {
 368             // Execute the actual Openstack command to create the Heat stack
 369             OpenStackRequest <Stack> request = heatClient.getStacks ().create (stack);
 370             // Begin X-Auth-User
 371             // Obtain an MSO token for the tenant
 372             CloudIdentity cloudIdentity = cloudSite.getIdentityService ();
 373             // cloudIdentity.getMsoId(), cloudIdentity.getMsoPass()
 374             //req
 375             request.header ("X-Auth-User", cloudIdentity.getMsoId ());
 376             request.header ("X-Auth-Key", cloudIdentity.getMsoPass ());
 377             LOGGER.debug ("headers added, about to executeAndRecordOpenstackRequest");
 378             LOGGER.debug(this.requestToStringBuilder(stack).toString());
 379             // END - try to fix X-Auth-User
 380             heatStack = executeAndRecordOpenstackRequest (request, msoProps);
 381         } catch (OpenStackResponseException e) {
 382             // Since this came on the 'Create Stack' command, nothing was changed
 383             // in the cloud. Return the error as an exception.
 384             if (e.getStatus () == 409) {
 385                 // Stack already exists. Return a specific error for this case
 386                 MsoStackAlreadyExists me = new MsoStackAlreadyExists (stackName, tenantId, cloudSiteId);
 387                 me.addContext (CREATE_STACK);
 388                 throw me;
 389             } else {
 390                 // Convert the OpenStackResponseException to an MsoOpenstackException
 391                 LOGGER.debug("ERROR STATUS = " + e.getStatus() + ",\n" + e.getMessage() + "\n" + e.getLocalizedMessage());
 392                 throw heatExceptionToMsoException (e, CREATE_STACK);
 393             }
 394         } catch (OpenStackConnectException e) {
 395             // Error connecting to Openstack instance. Convert to an MsoException
 396             throw heatExceptionToMsoException (e, CREATE_STACK);
 397         } catch (RuntimeException e) {
 398             // Catch-all
 399             throw runtimeExceptionToMsoException (e, CREATE_STACK);
 400         }
 401
 402         // Subsequent access by the canonical name "<stack name>/<stack-id>".
 403         // Otherwise, simple query by name returns a 302 redirect.
 404         // NOTE: This is specific to the v1 Orchestration API.
 405         String canonicalName = stackName + "/" + heatStack.getId ();
 406
 407         // If client has requested a final response, poll for stack completion
 408         if (pollForCompletion) {
 409             // Set a time limit on overall polling.
 410             // Use the resource (template) timeout for Openstack (expressed in minutes)
 411             // and add one poll interval to give Openstack a chance to fail on its own.
 412             int createPollInterval = msoProps.getIntProperty (createPollIntervalProp, createPollIntervalDefault);
 413             int pollTimeout = (timeoutMinutes * 60) + createPollInterval;
 414             // New 1610 - poll on delete if we rollback - use same values for now
 415             int deletePollInterval = createPollInterval;
 416             int deletePollTimeout = pollTimeout;
 417             boolean createTimedOut = false;
 418             StringBuilder stackErrorStatusReason = new StringBuilder("");
 419             LOGGER.debug("createPollInterval=" + createPollInterval + ", pollTimeout=" + pollTimeout);
 420
 421             while (true) {
 422                 try {
 423                     heatStack = queryHeatStack (heatClient, canonicalName);
 424                     LOGGER.debug (heatStack.getStackStatus () + " (" + canonicalName + ")");
 425                     try {
 426                         LOGGER.debug("Current stack " + this.getOutputsAsStringBuilder(heatStack).toString());
 427                     } catch (Exception e) {
 428                         LOGGER.debug("an error occurred trying to print out the current outputs of the stack");
 429                     }
 430
 431                     if ("CREATE_IN_PROGRESS".equals (heatStack.getStackStatus ())) {
 432                         // Stack creation is still running.
 433                         // Sleep and try again unless timeout has been reached
 434                         if (pollTimeout <= 0) {
 435                             // Note that this should not occur, since there is a timeout specified
 436                             // in the Openstack call.
 437                             LOGGER.error (MessageEnum.RA_CREATE_STACK_TIMEOUT, cloudSiteId, tenantId, stackName, heatStack.getStackStatus (), "", "", MsoLogger.ErrorCode.AvailabilityError, "Create stack timeout");
 438                             createTimedOut = true;
 439                             break;
 440                         }
 441                         try {
 442                             Thread.sleep (createPollInterval * 1000L);
 443                         } catch (InterruptedException e) {
 444                             LOGGER.debug ("Thread interrupted while sleeping", e);
 445                         }
 446
 447                         pollTimeout -= createPollInterval;
 448                                 LOGGER.debug("pollTimeout remaining: " + pollTimeout);
 449                     } else {
 450                         //save off the status & reason msg before we attempt delete
 451                         stackErrorStatusReason.append("Stack error (" + heatStack.getStackStatus() + "): " + heatStack.getStackStatusReason());
 452                         break;
 453                     }
 454                 } catch (MsoException me) {
 455                         // Cannot query the stack status. Something is wrong.
 456                         // Try to roll back the stack
 457                         if (!backout)
 458                         {
 459                                 LOGGER.warn(MessageEnum.RA_CREATE_STACK_ERR, "Create Stack errored, stack deletion suppressed", "", "", MsoLogger.ErrorCode.BusinessProcesssError, "Exception in Create Stack, stack deletion suppressed");
 460                         }
 461                         else
 462                         {
 463                                 try {
 464                                         LOGGER.debug("Create Stack error - unable to query for stack status - attempting to delete stack: " + canonicalName + " - This will likely fail and/or we won't be able to query to see if delete worked");
 465                                         OpenStackRequest <Void> request = heatClient.getStacks ().deleteByName (canonicalName);
 466                                         executeAndRecordOpenstackRequest (request, msoProps);
 467                                         // this may be a waste of time - if we just got an exception trying to query the stack - we'll just
 468                                         // get another one, n'est-ce pas?
 469                                         boolean deleted = false;
 470                                         while (!deleted) {
 471                                                 try {
 472                                                         heatStack = queryHeatStack(heatClient, canonicalName);
 473                                                         if (heatStack != null) {
 474                                                         LOGGER.debug(heatStack.getStackStatus());
 475                                                         if ("DELETE_IN_PROGRESS".equals(heatStack.getStackStatus())) {
 476                                                                 if (deletePollTimeout <= 0) {
 477                                                                         LOGGER.error (MessageEnum.RA_CREATE_STACK_TIMEOUT, cloudSiteId, tenantId, stackName,
 478                                                                                         heatStack.getStackStatus (), "", "", MsoLogger.ErrorCode.AvailabilityError,
 479                                                                                         "Rollback: DELETE stack timeout");
 480                                                                         break;
 481                                                                 } else {
 482                                                                         try {
 483                                                                                 Thread.sleep(deletePollInterval * 1000L);
 484                                                                         } catch (InterruptedException ie) {
 485                                                                                 LOGGER.debug("Thread interrupted while sleeping", ie);
 486                                                                         }
 487                                                                         deletePollTimeout -= deletePollInterval;
 488                                                                 }
 489                                                         } else if ("DELETE_COMPLETE".equals(heatStack.getStackStatus())){
 490                                                                 LOGGER.debug("DELETE_COMPLETE for " + canonicalName);
 491                                                                 deleted = true;
 492                                                                 continue;
 493                                                         } else {
 494                                                                 //got a status other than DELETE_IN_PROGRESS or DELETE_COMPLETE - so break and evaluate
 495                                                                 break;
 496                                                         }
 497                                                 } else {
 498                                                         // assume if we can't find it - it's deleted
 499                                                         LOGGER.debug("heatStack returned null - assume the stack " + canonicalName + " has been deleted");
 500                                                         deleted = true;
 501                                                         continue;
 502                                                         }
 503
 504                                                 } catch (Exception e3) {
 505                                                         // Just log this one. We will report the original exception.
 506                                                         LOGGER.error (MessageEnum.RA_CREATE_STACK_ERR, "Create Stack: Nested exception rolling back stack: " + e3, "", "", MsoLogger.ErrorCode.BusinessProcesssError, "Create Stack: Nested exception rolling back stack on error on query");
 507
 508                                                 }
 509                                         }
 510                                 } catch (Exception e2) {
 511                                         // Just log this one. We will report the original exception.
 512                                         LOGGER.error (MessageEnum.RA_CREATE_STACK_ERR, "Create Stack: Nested exception rolling back stack: " + e2, "", "", MsoLogger.ErrorCode.BusinessProcesssError, "Create Stack: Nested exception rolling back stack");
 513                                 }
 514                         }
 515
 516                     // Propagate the original exception from Stack Query.
 517                     me.addContext (CREATE_STACK);
 518                     throw me;
 519                 }
 520             }
 521
 522             if (!"CREATE_COMPLETE".equals (heatStack.getStackStatus ())) {
 523                 LOGGER.error (MessageEnum.RA_CREATE_STACK_ERR, "Create Stack error:  Polling complete with non-success status: "
 524                               + heatStack.getStackStatus () + ", " + heatStack.getStackStatusReason (), "", "", MsoLogger.ErrorCode.BusinessProcesssError, "Create Stack error");
 525
 526                 // Rollback the stack creation, since it is in an indeterminate state.
 527                 if (!backout)
 528                 {
 529                         LOGGER.warn(MessageEnum.RA_CREATE_STACK_ERR, "Create Stack errored, stack deletion suppressed", "", "", MsoLogger.ErrorCode.BusinessProcesssError, "Create Stack error, stack deletion suppressed");
 530                 }
 531                 else
 532                 {
 533                         try {
 534                                 LOGGER.debug("Create Stack errored - attempting to DELETE stack: " + canonicalName);
 535                                 LOGGER.debug("deletePollInterval=" + deletePollInterval + ", deletePollTimeout=" + deletePollTimeout);
 536                                 OpenStackRequest <Void> request = heatClient.getStacks ().deleteByName (canonicalName);
 537                                 executeAndRecordOpenstackRequest (request, msoProps);
 538                                 boolean deleted = false;
 539                                 while (!deleted) {
 540                                         try {
 541                                                 heatStack = queryHeatStack(heatClient, canonicalName);
 542                                                 if (heatStack != null) {
 543                                                         LOGGER.debug(heatStack.getStackStatus() + " (" + canonicalName + ")");
 544                                                         if ("DELETE_IN_PROGRESS".equals(heatStack.getStackStatus())) {
 545                                                                 if (deletePollTimeout <= 0) {
 546                                                                         LOGGER.error (MessageEnum.RA_CREATE_STACK_TIMEOUT, cloudSiteId, tenantId, stackName,
 547                                                                                         heatStack.getStackStatus (), "", "", MsoLogger.ErrorCode.AvailabilityError,
 548                                                                                         "Rollback: DELETE stack timeout");
 549                                                                         break;
 550                                                                 } else {
 551                                                                         try {
 552                                                                                 Thread.sleep(deletePollInterval * 1000L);
 553                                                                         } catch (InterruptedException ie) {
 554                                                                                 LOGGER.debug("Thread interrupted while sleeping", ie);
 555                                                                         }
 556                                                                         deletePollTimeout -= deletePollInterval;
 557                                                                         LOGGER.debug("deletePollTimeout remaining: " + deletePollTimeout);
 558                                                                 }
 559                                                         } else if ("DELETE_COMPLETE".equals(heatStack.getStackStatus())){
 560                                                                 LOGGER.debug("DELETE_COMPLETE for " + canonicalName);
 561                                                                 deleted = true;
 562                                                                 continue;
 563                                                         } else if ("DELETE_FAILED".equals(heatStack.getStackStatus())) {
 564                                                                 // Warn about this (?) - but still throw the original exception
 565                                                                 LOGGER.warn(MessageEnum.RA_CREATE_STACK_ERR, "Create Stack errored, stack deletion FAILED", "", "", MsoLogger.ErrorCode.BusinessProcesssError, "Create Stack error, stack deletion FAILED");
 566                                                                 LOGGER.debug("Stack deletion FAILED on a rollback of a create - " + canonicalName + ", status=" + heatStack.getStackStatus() + ", reason=" + heatStack.getStackStatusReason());
 567                                                                 break;
 568                                                         } else {
 569                                                                 //got a status other than DELETE_IN_PROGRESS or DELETE_COMPLETE - so break and evaluate
 570                                                                 break;
 571                                                         }
 572                                                 } else {
 573                                                         // assume if we can't find it - it's deleted
 574                                                         LOGGER.debug("heatStack returned null - assume the stack " + canonicalName + " has been deleted");
 575                                                         deleted = true;
 576                                                         continue;
 577                                                 }
 578
 579                                         } catch (MsoException me2) {
 580                                                 // We got an exception on the delete - don't throw this exception - throw the original - just log.
 581                                                 LOGGER.debug("Exception thrown trying to delete " + canonicalName + " on a create->rollback: " + me2.getContextMessage());
 582                                                 LOGGER.warn(MessageEnum.RA_CREATE_STACK_ERR, "Create Stack errored, then stack deletion FAILED - exception thrown", "", "", MsoLogger.ErrorCode.BusinessProcesssError, me2.getContextMessage());
 583                                         }
 584
 585                                 } // end while !deleted
 586                                 StringBuilder errorContextMessage = null;
 587                                 if (createTimedOut) {
 588                                         errorContextMessage = new StringBuilder("Stack Creation Timeout");
 589                                 } else {
 590                                         errorContextMessage  = stackErrorStatusReason;
 591                                 }
 592                                 if (deleted) {
 593                                         errorContextMessage.append(" - stack successfully deleted");
 594                                 } else {
 595                                         errorContextMessage.append(" - encountered an error trying to delete the stack");
 596                                 }
 597 //                              MsoOpenstackException me = new MsoOpenstackException(0, "", stackErrorStatusReason.toString());
 598  //                             me.addContext(CREATE_STACK);
 599   //                            alarmLogger.sendAlarm(HEAT_ERROR, MsoAlarmLogger.CRITICAL, me.getContextMessage());
 600    //                           throw me;
 601                         } catch (Exception e2) {
 602                                 // shouldn't happen - but handle
 603                                 LOGGER.error (MessageEnum.RA_CREATE_STACK_ERR, "Create Stack: Nested exception rolling back stack: " + e2, "", "", MsoLogger.ErrorCode.BusinessProcesssError, "Exception in Create Stack: rolling back stack");
 604                         }
 605                 }
 606                 MsoOpenstackException me = new MsoOpenstackException(0, "", stackErrorStatusReason.toString());
 607                 me.addContext(CREATE_STACK);
 608                 alarmLogger.sendAlarm(HEAT_ERROR, MsoAlarmLogger.CRITICAL, me.getContextMessage());
 609                 throw me;
 610             }
 611
 612         } else {
 613             // Get initial status, since it will have been null after the create.
 614             heatStack = queryHeatStack (heatClient, canonicalName);
 615             LOGGER.debug (heatStack.getStackStatus ());
 616         }
 617
 618         return new StackInfo (heatStack);
 619     }
 620
 621     /**
 622      * Query for a single stack (by Name) in a tenant. This call will always return a
 623      * StackInfo object. If the stack does not exist, an "empty" StackInfo will be
 624      * returned - containing only the stack name and a status of NOTFOUND.
 625      *
 626      * @param tenantId The Openstack ID of the tenant in which to query
 627      * @param cloudSiteId The cloud identifier (may be a region) in which to query
 628      * @param stackName The name of the stack to query (may be simple or canonical)
 629      * @return A StackInfo object
 630      * @throws MsoOpenstackException Thrown if the Openstack API call returns an exception.
 631      */
 632     public StackInfo queryStack (String cloudSiteId, String tenantId, String stackName) throws MsoException {
 633         LOGGER.debug ("Query HEAT stack: " + stackName + " in tenant " + tenantId);
 634
 635         // Obtain the cloud site information where we will create the stack
 636         CloudSite cloudSite = cloudConfig.getCloudSite (cloudSiteId);
 637         if (cloudSite == null) {
 638             throw new MsoCloudSiteNotFound (cloudSiteId);
 639         }
 640         LOGGER.debug("Found: " + cloudSite.toString());
 641
 642         // Get a Heat client. They are cached between calls (keyed by tenantId:cloudId)
 643         Heat heatClient = null;
 644         try {
 645             heatClient = getHeatClient (cloudSite, tenantId);
 646             if (heatClient != null) {
 647                 LOGGER.debug("Found: " + heatClient.toString());
 648             }
 649         } catch (MsoTenantNotFound e) {
 650             // Tenant doesn't exist, so stack doesn't either
 651             LOGGER.debug ("Tenant with id " + tenantId + "not found.", e);
 652             return new StackInfo (stackName, HeatStatus.NOTFOUND, null, null);
 653         } catch (MsoException me) {
 654             // Got an Openstack error. Propagate it
 655             LOGGER.error (MessageEnum.RA_CONNECTION_EXCEPTION, "OpenStack", "Openstack Exception on Token request: " + me, "Openstack", "", MsoLogger.ErrorCode.AvailabilityError, "Connection Exception");
 656             me.addContext ("QueryStack");
 657             throw me;
 658         }
 659
 660         // Query the Stack.
 661         // An MsoException will propagate transparently to the caller.
 662         Stack heatStack = queryHeatStack (heatClient, stackName);
 663
 664         if (heatStack == null) {
 665             // Stack does not exist. Return a StackInfo with status NOTFOUND
 666             StackInfo stackInfo = new StackInfo (stackName, HeatStatus.NOTFOUND, null, null);
 667             return stackInfo;
 668         }
 669
 670         return new StackInfo (heatStack);
 671     }
 672
 673     /**
 674      * Delete a stack (by Name/ID) in a tenant. If the stack is not found, it will be
 675      * considered a successful deletion. The return value is a StackInfo object which
 676      * contains the current stack status.
 677      *
 678      * The client may choose to let the adapter poll Openstack for completion of the
 679      * stack deletion, or may handle polling itself via separate query calls. In either
 680      * case, a StackInfo object will be returned. When polling is enabled, a final
 681      * status of NOTFOUND is expected. When not polling, a status of DELETING is expected.
 682      *
 683      * There is no rollback from a successful stack deletion. A deletion failure will
 684      * also result in an undefined stack state - the components may or may not have been
 685      * all or partially deleted, so the resulting stack must be considered invalid.
 686      *
 687      * @param tenantId The Openstack ID of the tenant in which to perform the delete
 688      * @param cloudSiteId The cloud identifier (may be a region) from which to delete the stack.
 689      * @param stackName The name/id of the stack to delete. May be simple or canonical
 690      * @param pollForCompletion Indicator that polling should be handled in Java vs. in the client
 691      * @return A StackInfo object
 692      * @throws MsoOpenstackException Thrown if the Openstack API call returns an exception.
 693      * @throws MsoCloudSiteNotFound
 694      */
 695     public StackInfo deleteStack (String tenantId,
 696                                   String cloudSiteId,
 697                                   String stackName,
 698                                   boolean pollForCompletion) throws MsoException {
 699         // Obtain the cloud site information where we will create the stack
 700         CloudSite cloudSite = cloudConfig.getCloudSite (cloudSiteId);
 701         if (cloudSite == null) {
 702             throw new MsoCloudSiteNotFound (cloudSiteId);
 703         }
 704         LOGGER.debug("Found: " + cloudSite.toString());
 705
 706         // Get a Heat client. They are cached between calls (keyed by tenantId:cloudId)
 707         Heat heatClient = null;
 708         try {
 709             heatClient = getHeatClient (cloudSite, tenantId);
 710             if (heatClient != null) {
 711                 LOGGER.debug("Found: " + heatClient.toString());
 712             }
 713         } catch (MsoTenantNotFound e) {
 714             // Tenant doesn't exist, so stack doesn't either
 715             LOGGER.debug ("Tenant with id " + tenantId + "not found.", e);
 716             return new StackInfo (stackName, HeatStatus.NOTFOUND, null, null);
 717         } catch (MsoException me) {
 718             // Got an Openstack error. Propagate it
 719             LOGGER.error (MessageEnum.RA_CONNECTION_EXCEPTION, "Openstack", "Openstack Exception on Token request: " + me, "Openstack", "", MsoLogger.ErrorCode.AvailabilityError, "Connection Exception");
 720             me.addContext (DELETE_STACK);
 721             throw me;
 722         }
 723
 724         // OK if stack not found, perform a query first
 725         Stack heatStack = queryHeatStack (heatClient, stackName);
 726         if (heatStack == null || "DELETE_COMPLETE".equals (heatStack.getStackStatus ())) {
 727             // Not found. Return a StackInfo with status NOTFOUND
 728             return new StackInfo (stackName, HeatStatus.NOTFOUND, null, null);
 729         }
 730
 731         // Delete the stack.
 732
 733         // Use canonical name "<stack name>/<stack-id>" to delete.
 734         // Otherwise, deletion by name returns a 302 redirect.
 735         // NOTE: This is specific to the v1 Orchestration API.
 736         String canonicalName = heatStack.getStackName () + "/" + heatStack.getId ();
 737
 738         try {
 739             OpenStackRequest <Void> request = heatClient.getStacks ().deleteByName (canonicalName);
 740             executeAndRecordOpenstackRequest (request, msoProps);
 741         } catch (OpenStackResponseException e) {
 742             if (e.getStatus () == 404) {
 743                 // Not found. We are OK with this. Return a StackInfo with status NOTFOUND
 744                 return new StackInfo (stackName, HeatStatus.NOTFOUND, null, null);
 745             } else {
 746                 // Convert the OpenStackResponseException to an MsoOpenstackException
 747                 throw heatExceptionToMsoException (e, DELETE_STACK);
 748             }
 749         } catch (OpenStackConnectException e) {
 750             // Error connecting to Openstack instance. Convert to an MsoException
 751             throw heatExceptionToMsoException (e, DELETE_STACK);
 752         } catch (RuntimeException e) {
 753             // Catch-all
 754             throw runtimeExceptionToMsoException (e, DELETE_STACK);
 755         }
 756
 757         // Requery the stack for current status.
 758         // It will probably still exist with "DELETE_IN_PROGRESS" status.
 759         heatStack = queryHeatStack (heatClient, canonicalName);
 760
 761         if (pollForCompletion) {
 762             // Set a timeout on polling
 763             int pollInterval = msoProps.getIntProperty (deletePollIntervalProp, deletePollIntervalDefault);
 764             int pollTimeout = msoProps.getIntProperty (deletePollTimeoutProp, deletePollTimeoutDefault);
 765
 766             // When querying by canonical name, Openstack returns DELETE_COMPLETE status
 767             // instead of "404" (which would result from query by stack name).
 768             while (heatStack != null && !"DELETE_COMPLETE".equals (heatStack.getStackStatus ())) {
 769                 LOGGER.debug ("Stack status: " + heatStack.getStackStatus ());
 770
 771                 if ("DELETE_FAILED".equals (heatStack.getStackStatus ())) {
 772                     // Throw a 'special case' of MsoOpenstackException to report the Heat status
 773                     String error = "Stack delete error (" + heatStack.getStackStatus ()
 774                                    + "): "
 775                                    + heatStack.getStackStatusReason ();
 776                     MsoOpenstackException me = new MsoOpenstackException (0, "", error);
 777                     me.addContext (DELETE_STACK);
 778
 779                     // Alarm this condition, stack deletion failed
 780                     alarmLogger.sendAlarm (HEAT_ERROR, MsoAlarmLogger.CRITICAL, me.getContextMessage ());
 781
 782                     throw me;
 783                 }
 784
 785                 if (pollTimeout <= 0) {
 786                     LOGGER.error (MessageEnum.RA_DELETE_STACK_TIMEOUT, cloudSiteId, tenantId, stackName, heatStack.getStackStatus (), "", "", MsoLogger.ErrorCode.AvailabilityError, "Delete Stack Timeout");
 787
 788                     // Throw a 'special case' of MsoOpenstackException to report the Heat status
 789                     MsoOpenstackException me = new MsoOpenstackException (0, "", "Stack Deletion Timeout");
 790                     me.addContext (DELETE_STACK);
 791
 792                     // Alarm this condition, stack deletion failed
 793                     alarmLogger.sendAlarm (HEAT_ERROR, MsoAlarmLogger.CRITICAL, me.getContextMessage ());
 794
 795                     throw me;
 796                 }
 797
 798                 try {
 799                     Thread.sleep (pollInterval * 1000L);
 800                 } catch (InterruptedException e) {
 801                     LOGGER.debug ("Thread interrupted while sleeping", e);
 802                 }
 803
 804                 pollTimeout -= pollInterval;
 805
 806                 heatStack = queryHeatStack (heatClient, canonicalName);
 807             }
 808
 809             // The stack is gone when this point is reached
 810             return new StackInfo (stackName, HeatStatus.NOTFOUND, null, null);
 811         }
 812
 813         // Return the current status (if not polling, the delete may still be in progress)
 814         StackInfo stackInfo = new StackInfo (heatStack);
 815         stackInfo.setName (stackName);
 816
 817         return stackInfo;
 818     }
 819
 820     /**
 821      * Query for all stacks in a tenant site. This call will return a List of StackInfo
 822      * objects, one for each deployed stack.
 823      *
 824      * Note that this is limited to a single site. To ensure that a tenant is truly
 825      * empty would require looping across all tenant endpoints.
 826      *
 827      * @param tenantId The Openstack ID of the tenant to query
 828      * @param cloudSiteId The cloud identifier (may be a region) in which to query.
 829      * @return A List of StackInfo objects
 830      * @throws MsoOpenstackException Thrown if the Openstack API call returns an exception.
 831      * @throws MsoCloudSiteNotFound
 832      */
 833     public List <StackInfo> queryAllStacks (String tenantId, String cloudSiteId) throws MsoException {
 834         // Obtain the cloud site information where we will create the stack
 835         CloudSite cloudSite = cloudConfig.getCloudSite (cloudSiteId);
 836         if (cloudSite == null) {
 837             throw new MsoCloudSiteNotFound (cloudSiteId);
 838         }
 839
 840         // Get a Heat client. They are cached between calls (keyed by tenantId:cloudId)
 841         Heat heatClient = getHeatClient (cloudSite, tenantId);
 842
 843         try {
 844             OpenStackRequest <Stacks> request = heatClient.getStacks ().list ();
 845             Stacks stacks = executeAndRecordOpenstackRequest (request, msoProps);
 846
 847             List <StackInfo> stackList = new ArrayList <StackInfo> ();
 848
 849             // Not sure if returns an empty list or null if no stacks exist
 850             if (stacks != null) {
 851                 for (Stack stack : stacks) {
 852                     stackList.add (new StackInfo (stack));
 853                 }
 854             }
 855
 856             return stackList;
 857         } catch (OpenStackResponseException e) {
 858             if (e.getStatus () == 404) {
 859                 // Not sure if this can happen, but return an empty list
 860                 LOGGER.debug ("queryAllStacks - stack not found: ");
 861                 return new ArrayList <StackInfo> ();
 862             } else {
 863                 // Convert the OpenStackResponseException to an MsoOpenstackException
 864                 throw heatExceptionToMsoException (e, QUERY_ALL_STACKS);
 865             }
 866         } catch (OpenStackConnectException e) {
 867             // Error connecting to Openstack instance. Convert to an MsoException
 868             throw heatExceptionToMsoException (e, QUERY_ALL_STACKS);
 869         } catch (RuntimeException e) {
 870             // Catch-all
 871             throw runtimeExceptionToMsoException (e, QUERY_ALL_STACKS);
 872         }
 873     }
 874
 875     /**
 876      * Validate parameters to be passed to Heat template. This method performs
 877      * three functions:
 878      * 1. Apply default values to parameters which have them defined
 879      * 2. Report any required parameters that are missing. This will generate an
 880      * exception in the caller, since stack create/update operations would fail.
 881      * 3. Report and remove any extraneous parameters. This will allow clients to
 882      * pass supersets of parameters and not get errors.
 883      *
 884      * These functions depend on the HeatTemplate definition from the MSO Catalog DB,
 885      * along with the input parameter Map. The output is an updated parameter map.
 886      * If the parameters are invalid for the template, an IllegalArgumentException
 887      * is thrown.
 888      */
 889     public Map <String, Object> validateStackParams (Map <String, Object> inputParams,
 890                                                      HeatTemplate heatTemplate) throws IllegalArgumentException {
 891         // Check that required parameters have been supplied for this template type
 892         String missingParams = null;
 893         List <String> paramList = new ArrayList <String> ();
 894
 895         // TODO: Enhance DB to support defaults for Heat Template parameters
 896
 897         for (HeatTemplateParam parm : heatTemplate.getParameters ()) {
 898             if (parm.isRequired () && !inputParams.containsKey (parm.getParamName ())) {
 899                 if (missingParams == null) {
 900                     missingParams = parm.getParamName ();
 901                 } else {
 902                     missingParams += "," + parm.getParamName ();
 903                 }
 904             }
 905             paramList.add (parm.getParamName ());
 906         }
 907         if (missingParams != null) {
 908             // Problem - missing one or more required parameters
 909             String error = "Missing Required inputs for HEAT Template: " + missingParams;
 910             LOGGER.error (MessageEnum.RA_MISSING_PARAM, missingParams + " for HEAT Template", "", "", MsoLogger.ErrorCode.SchemaError, "Missing Required inputs for HEAT Template: " + missingParams);
 911             throw new IllegalArgumentException (error);
 912         }
 913
 914         // Remove any extraneous parameters (don't throw an error)
 915         Map <String, Object> updatedParams = new HashMap <String, Object> ();
 916         List <String> extraParams = new ArrayList <String> ();
 917         for (String key : inputParams.keySet ()) {
 918             if (!paramList.contains (key)) {
 919                 // This is not a valid parameter for this template
 920                 extraParams.add (key);
 921             } else {
 922                 updatedParams.put (key, inputParams.get (key));
 923             }
 924         }
 925         if (!extraParams.isEmpty ()) {
 926             LOGGER.warn (MessageEnum.RA_GENERAL_WARNING, "Heat Stack (" + heatTemplate.getTemplateName ()
 927                          + ") extra input params received: "
 928                          + extraParams, "", "", MsoLogger.ErrorCode.DataError, "Heat Stack (" + heatTemplate.getTemplateName () + ") extra input params received: "+ extraParams);
 929         }
 930
 931         return updatedParams;
 932     }
 933
 934     // ---------------------------------------------------------------
 935     // PRIVATE FUNCTIONS FOR USE WITHIN THIS CLASS
 936
 937     /**
 938      * Get a Heat client for the Openstack Identity service.
 939      * This requires a 'member'-level userId + password, which will be retrieved from
 940      * properties based on the specified cloud Id. The tenant in which to operate
 941      * must also be provided.
 942      * <p>
 943      * On successful authentication, the Heat object will be cached for the
 944      * tenantID + cloudId so that it can be reused without reauthenticating with
 945      * Openstack every time.
 946      *
 947      * @param tenantName
 948      * @param cloudId
 949      * @return an authenticated Heat object
 950      */
 951     public Heat getHeatClient (CloudSite cloudSite, String tenantId) throws MsoException {
 952         String cloudId = cloudSite.getId ();
 953
 954         // Check first in the cache of previously authorized clients
 955         String cacheKey = cloudId + ":" + tenantId;
 956         if (heatClientCache.containsKey (cacheKey)) {
 957             if (!heatClientCache.get (cacheKey).isExpired ()) {
 958                 LOGGER.debug ("Using Cached HEAT Client for " + cacheKey);
 959                 return heatClientCache.get (cacheKey).getHeatClient ();
 960             } else {
 961                 // Token is expired. Remove it from cache.
 962                 heatClientCache.remove (cacheKey);
 963                 LOGGER.debug ("Expired Cached HEAT Client for " + cacheKey);
 964             }
 965         }
 966
 967         // Obtain an MSO token for the tenant
 968         CloudIdentity cloudIdentity = cloudSite.getIdentityService ();
 969         LOGGER.debug("Found: " + cloudIdentity.toString());
 970         String keystoneUrl = cloudIdentity.getKeystoneUrl (cloudId, msoPropID);
 971         LOGGER.debug("keystoneUrl=" + keystoneUrl);
 972         Keystone keystoneTenantClient = new Keystone (keystoneUrl);
 973         Access access = null;
 974         try {
 975                 Authentication credentials = cloudIdentity.getAuthentication ();
 976
 977                 OpenStackRequest <Access> request = keystoneTenantClient.tokens ()
 978                        .authenticate (credentials).withTenantId (tenantId);
 979
 980             access = executeAndRecordOpenstackRequest (request, msoProps);
 981         } catch (OpenStackResponseException e) {
 982             if (e.getStatus () == 401) {
 983                 // Authentication error.
 984                 String error = "Authentication Failure: tenant=" + tenantId + ",cloud=" + cloudIdentity.getId ();
 985                 alarmLogger.sendAlarm ("MsoAuthenticationError", MsoAlarmLogger.CRITICAL, error);
 986                 throw new MsoAdapterException (error);
 987             } else {
 988                 throw keystoneErrorToMsoException (e, TOKEN_AUTH);
 989             }
 990         } catch (OpenStackConnectException e) {
 991             // Connection to Openstack failed
 992             MsoIOException me = new MsoIOException (e.getMessage (), e);
 993             me.addContext (TOKEN_AUTH);
 994             throw me;
 995         } catch (RuntimeException e) {
 996             // Catch-all
 997             throw runtimeExceptionToMsoException (e, TOKEN_AUTH);
 998         }
 999
1000         // For DCP/LCP, the region should be the cloudId.
1001         String region = cloudSite.getRegionId ();
1002         String heatUrl = null;
1003         try {
1004             heatUrl = KeystoneUtils.findEndpointURL (access.getServiceCatalog (), "orchestration", region, "public");
1005             LOGGER.debug("heatUrl=" + heatUrl + ", region=" + region);
1006         } catch (RuntimeException e) {
1007             // This comes back for not found (probably an incorrect region ID)
1008             String error = "Orchestration service not found: region=" + region + ",cloud=" + cloudIdentity.getId ();
1009             alarmLogger.sendAlarm ("MsoConfigurationError", MsoAlarmLogger.CRITICAL, error);
1010             throw new MsoAdapterException (error, e);
1011         }
1012
1013         Heat heatClient = new Heat (heatUrl);
1014         heatClient.token (access.getToken ().getId ());
1015
1016         heatClientCache.put (cacheKey,
1017                              new HeatCacheEntry (heatUrl,
1018                                                  access.getToken ().getId (),
1019                                                  access.getToken ().getExpires ()));
1020         LOGGER.debug ("Caching HEAT Client for " + cacheKey);
1021
1022         return heatClient;
1023     }
1024
1025     /**
1026      * Forcibly expire a HEAT client from the cache. This call is for use by
1027      * the KeystoneClient in case where a tenant is deleted. In that case,
1028      * all cached credentials must be purged so that fresh authentication is
1029      * done if a similarly named tenant is re-created.
1030      * <p>
1031      * Note: This is probably only applicable to dev/test environments where
1032      * the same Tenant Name is repeatedly used for creation/deletion.
1033      * <p>
1034      *
1035      * @param tenantName
1036      * @param cloudId
1037      */
1038     public static void expireHeatClient (String tenantId, String cloudId) {
1039         String cacheKey = cloudId + ":" + tenantId;
1040         if (heatClientCache.containsKey (cacheKey)) {
1041             heatClientCache.remove (cacheKey);
1042             LOGGER.debug ("Deleted Cached HEAT Client for " + cacheKey);
1043         }
1044     }
1045
1046     /*
1047      * Query for a Heat Stack. This function is needed in several places, so
1048      * a common method is useful. This method takes an authenticated Heat Client
1049      * (which internally identifies the cloud & tenant to search), and returns
1050      * a Stack object if found, Null if not found, or an MsoOpenstackException
1051      * if the Openstack API call fails.
1052      *
1053      * The stack name may be a simple name or a canonical name ("{name}/{id}").
1054      * When simple name is used, Openstack always returns a 302 redirect which
1055      * results in a 2nd request (to the canonical name). Note that query by
1056      * canonical name for a deleted stack returns a Stack object with status
1057      * "DELETE_COMPLETE" while query by simple name for a deleted stack returns
1058      * HTTP 404.
1059      *
1060      * @param heatClient an authenticated Heat client
1061      *
1062      * @param stackName the stack name to query
1063      *
1064      * @return a Stack object that describes the current stack or null if the
1065      * requested stack doesn't exist.
1066      *
1067      * @throws MsoOpenstackException Thrown if the Openstack API call returns an exception
1068      */
1069     protected Stack queryHeatStack (Heat heatClient, String stackName) throws MsoException {
1070         if (stackName == null) {
1071             return null;
1072         }
1073         try {
1074             OpenStackRequest <Stack> request = heatClient.getStacks ().byName (stackName);
1075             return executeAndRecordOpenstackRequest (request, msoProps);
1076         } catch (OpenStackResponseException e) {
1077             if (e.getStatus () == 404) {
1078                 LOGGER.debug ("queryHeatStack - stack not found: " + stackName);
1079                 return null;
1080             } else {
1081                 // Convert the OpenStackResponseException to an MsoOpenstackException
1082                 throw heatExceptionToMsoException (e, "QueryStack");
1083             }
1084         } catch (OpenStackConnectException e) {
1085             // Connection to Openstack failed
1086             throw heatExceptionToMsoException (e, "QueryAllStack");
1087         }
1088     }
1089
1090     /*
1091      * An entry in the Heat Client Cache. It saves the Heat client object
1092      * along with the token expiration. After this interval, this cache
1093      * item will no longer be used.
1094      */
1095     private static class HeatCacheEntry implements Serializable {
1096
1097         private static final long serialVersionUID = 1L;
1098
1099         private String heatUrl;
1100         private String token;
1101         private Calendar expires;
1102
1103         public HeatCacheEntry (String heatUrl, String token, Calendar expires) {
1104             this.heatUrl = heatUrl;
1105             this.token = token;
1106             this.expires = expires;
1107         }
1108
1109         public Heat getHeatClient () {
1110             Heat heatClient = new Heat (heatUrl);
1111             heatClient.token (token);
1112             return heatClient;
1113         }
1114
1115         public boolean isExpired () {
1116             if (expires == null) {
1117                 return true;
1118             }
1119
1120             return System.currentTimeMillis() > expires.getTimeInMillis();
1121         }
1122     }
1123
1124     /**
1125      * Clean up the Heat client cache to remove expired entries.
1126      */
1127     public static void heatCacheCleanup () {
1128         for (String cacheKey : heatClientCache.keySet ()) {
1129             if (heatClientCache.get (cacheKey).isExpired ()) {
1130                 heatClientCache.remove (cacheKey);
1131                 LOGGER.debug ("Cleaned Up Cached Heat Client for " + cacheKey);
1132             }
1133         }
1134     }
1135
1136     /**
1137      * Reset the Heat client cache.
1138      * This may be useful if cached credentials get out of sync.
1139      */
1140     public static void heatCacheReset () {
1141         heatClientCache = new HashMap <String, HeatCacheEntry> ();
1142     }
1143
1144         public Map<String, Object> queryStackForOutputs(String cloudSiteId,
1145                         String tenantId, String stackName) throws MsoException {
1146                 LOGGER.debug("MsoHeatUtils.queryStackForOutputs)");
1147                 StackInfo heatStack = this.queryStack(cloudSiteId, tenantId, stackName);
1148                 if (heatStack == null || heatStack.getStatus() == HeatStatus.NOTFOUND) {
1149                         return null;
1150                 }
1151                 Map<String, Object> outputs = heatStack.getOutputs();
1152                 return outputs;
1153         }
1154
1155         public void queryAndCopyOutputsToInputs(String cloudSiteId,
1156                         String tenantId, String stackName, Map<String, String> inputs,
1157                         boolean overWrite) throws MsoException {
1158                 LOGGER.debug("MsoHeatUtils.queryAndCopyOutputsToInputs");
1159                 Map<String, Object> outputs = this.queryStackForOutputs(cloudSiteId,
1160                                 tenantId, stackName);
1161                 this.copyStringOutputsToInputs(inputs, outputs, overWrite);
1162                 return;
1163         }
1164
1165         public void copyStringOutputsToInputs(Map<String, String> inputs,
1166                         Map<String, Object> otherStackOutputs, boolean overWrite) {
1167                 if (inputs == null || otherStackOutputs == null)
1168                         return;
1169                 for (String key : otherStackOutputs.keySet()) {
1170                         if (!inputs.containsKey(key)) {
1171                                 Object obj = otherStackOutputs.get(key);
1172                                 if (obj instanceof String) {
1173                                         inputs.put(key, (String) otherStackOutputs.get(key));
1174                                 } else if (obj instanceof JsonNode ){
1175                                         // This is a bit of mess - but I think it's the least impacting
1176                                         // let's convert it BACK to a string - then it will get converted back later
1177                                         try {
1178                                                 String str = this.convertNode((JsonNode) obj);
1179                                                 inputs.put(key, str);
1180                                         } catch (Exception e) {
1181                                                 LOGGER.debug("DANGER WILL ROBINSON: unable to convert value for JsonNode "+ key);
1182                                                 //effect here is this value will not have been copied to the inputs - and therefore will error out downstream
1183                                         }
1184                                 } else if (obj instanceof java.util.LinkedHashMap) {
1185                                         LOGGER.debug("LinkedHashMap - this is showing up as a LinkedHashMap instead of JsonNode");
1186                                         try {
1187                                                 String str = JSON_MAPPER.writeValueAsString(obj);
1188                                                 inputs.put(key, str);
1189                                         } catch (Exception e) {
1190                                                 LOGGER.debug("DANGER WILL ROBINSON: unable to convert value for LinkedHashMap "+ key);
1191                                         }
1192                                 } else if (obj instanceof Integer) {
1193                                         try {
1194                                                 String str = "" + obj;
1195                                                 inputs.put(key, str);
1196                                         } catch (Exception e) {
1197                                                 LOGGER.debug("DANGER WILL ROBINSON: unable to convert value for Integer "+ key);
1198                                         }
1199                                 } else {
1200                                         try {
1201                                                 String str = obj.toString();
1202                                                 inputs.put(key, str);
1203                                         } catch (Exception e) {
1204                                                 LOGGER.debug("DANGER WILL ROBINSON: unable to convert value for Other "+ key +" (" + e.getMessage() + ")");
1205                                                 //effect here is this value will not have been copied to the inputs - and therefore will error out downstream
1206                                         }
1207                                 }
1208                         }
1209                 }
1210                 return;
1211         }
1212         public StringBuilder requestToStringBuilder(CreateStackParam stack) {
1213                 StringBuilder sb = new StringBuilder();
1214                 sb.append("Stack:\n");
1215                 sb.append("\tStackName: " + stack.getStackName());
1216                 sb.append("\tTemplateUrl: " + stack.getTemplateUrl());
1217                 sb.append("\tTemplate: " + stack.getTemplate());
1218                 sb.append("\tEnvironment: " + stack.getEnvironment());
1219                 sb.append("\tTimeout: " + stack.getTimeoutMinutes());
1220                 sb.append("\tParameters:\n");
1221                 Map<String, Object> params = stack.getParameters();
1222                 if (params == null || params.size() < 1) {
1223                         sb.append("\nNONE");
1224                 } else {
1225                         for (String key : params.keySet()) {
1226                                 if (params.get(key) instanceof String) {
1227                                         sb.append("\n" + key + "=" + (String) params.get(key));
1228                                 } else if (params.get(key) instanceof JsonNode) {
1229                                         String jsonStringOut = this.convertNode((JsonNode)params.get(key));
1230                                         sb.append("\n" + key + "=" + jsonStringOut);
1231                                 } else if (params.get(key) instanceof Integer) {
1232                                         String integerOut = "" + params.get(key);
1233                                         sb.append("\n" + key + "=" + integerOut);
1234
1235                                 } else {
1236                                         try {
1237                                                 String str = params.get(key).toString();
1238                                                 sb.append("\n" + key + "=" + str);
1239                                         } catch (Exception e) {
1240                                                 //non fatal
1241                                         }
1242                                 }
1243                         }
1244                 }
1245                 return sb;
1246         }
1247
1248         private String convertNode(final JsonNode node) {
1249                 try {
1250                         final Object obj = JSON_MAPPER.treeToValue(node, Object.class);
1251                         final String json = JSON_MAPPER.writeValueAsString(obj);
1252                         return json;
1253                 } catch (JsonParseException jpe) {
1254                         LOGGER.debug("Error converting json to string " + jpe.getMessage());
1255                 } catch (Exception e) {
1256                         LOGGER.debug("Error converting json to string " + e.getMessage());
1257                 }
1258                 return "[Error converting json to string]";
1259         }
1260
1261
1262         private StringBuilder getOutputsAsStringBuilder(Stack heatStack) {
1263                 // This should only be used as a utility to print out the stack outputs
1264                 // to the log
1265                 StringBuilder sb = new StringBuilder("");
1266                 if (heatStack == null) {
1267                         sb.append("(heatStack is null)");
1268                         return sb;
1269                 }
1270                 List<Output> outputList = heatStack.getOutputs();
1271                 if (outputList == null || outputList.isEmpty()) {
1272                         sb.append("(outputs is empty)");
1273                         return sb;
1274                 }
1275                 Map<String, Object> outputs = new HashMap<String,Object>();
1276                 for (Output outputItem : outputList) {
1277                         outputs.put(outputItem.getOutputKey(), outputItem.getOutputValue());
1278                 }
1279                 int counter = 0;
1280                 sb.append("OUTPUTS:\n");
1281                 for (String key : outputs.keySet()) {
1282                         sb.append("outputs[" + counter++ + "]: " + key + "=");
1283                         Object obj = outputs.get(key);
1284                         if (obj instanceof String) {
1285                                 sb.append((String)obj +" (a string)");
1286                         } else if (obj instanceof JsonNode) {
1287                                 sb.append(this.convertNode((JsonNode)obj) + " (a JsonNode)");
1288                         } else if (obj instanceof java.util.LinkedHashMap) {
1289                                 try {
1290                                         String str = JSON_MAPPER.writeValueAsString(obj);
1291                                         sb.append(str + " (a java.util.LinkedHashMap)");
1292                                 } catch (Exception e) {
1293                                         sb.append("(a LinkedHashMap value that would not convert nicely)");
1294                                 }
1295                         } else if (obj instanceof Integer) {
1296                                 String str = "";
1297                                 try {
1298                                         str = obj.toString() + " (an Integer)\n";
1299                                 } catch (Exception e) {
1300                                         str = "(an Integer unable to call .toString() on)";
1301                                 }
1302                                 sb.append(str);
1303                         } else if (obj instanceof ArrayList) {
1304                                 String str = "";
1305                                 try {
1306                                         str = obj.toString() + " (an ArrayList)";
1307                                 } catch (Exception e) {
1308                                         str = "(an ArrayList unable to call .toString() on?)";
1309                                 }
1310                                 sb.append(str);
1311                         } else if (obj instanceof Boolean) {
1312                                 String str = "";
1313                                 try {
1314                                         str = obj.toString() + " (a Boolean)";
1315                                 } catch (Exception e) {
1316                                         str = "(an Boolean unable to call .toString() on?)";
1317                                 }
1318                                 sb.append(str);
1319                         }
1320                         else {
1321                                 String str = "";
1322                                 try {
1323                                         str = obj.toString() + " (unknown Object type)";
1324                                 } catch (Exception e) {
1325                                         str = "(a value unable to call .toString() on?)";
1326                                 }
1327                                 sb.append(str);
1328                         }
1329                         sb.append("\n");
1330                 }
1331                 sb.append("[END]");
1332                 return sb;
1333         }
1334
1335
1336         public void copyBaseOutputsToInputs(Map<String, Object> inputs,
1337                         Map<String, Object> otherStackOutputs, ArrayList<String> paramNames, HashMap<String, String> aliases) {
1338                 if (inputs == null || otherStackOutputs == null)
1339                         return;
1340                 for (String key : otherStackOutputs.keySet()) {
1341                         if (paramNames != null) {
1342                                 if (!paramNames.contains(key) && !aliases.containsKey(key)) {
1343                                         LOGGER.debug("\tParameter " + key + " is NOT defined to be in the template - do not copy to inputs");
1344                                         continue;
1345                                 }
1346                                 if (aliases.containsKey(key)) {
1347                                         LOGGER.debug("Found an alias! Will move " + key + " to " + aliases.get(key));
1348                                         Object obj = otherStackOutputs.get(key);
1349                                         key = aliases.get(key);
1350                                         otherStackOutputs.put(key, obj);
1351                                 }
1352                         }
1353                         if (!inputs.containsKey(key)) {
1354                                 Object obj = otherStackOutputs.get(key);
1355                                 LOGGER.debug("\t**Adding " + key + " to inputs (.toString()=" + obj.toString());
1356                                 if (obj instanceof String) {
1357                                         LOGGER.debug("\t\t**A String");
1358                                         inputs.put(key, obj);
1359                                 } else if (obj instanceof Integer) {
1360                                         LOGGER.debug("\t\t**An Integer");
1361                                         inputs.put(key, obj);
1362                                 } else if (obj instanceof JsonNode) {
1363                                         LOGGER.debug("\t\t**A JsonNode");
1364                                         inputs.put(key, obj);
1365                                 } else if (obj instanceof Boolean) {
1366                                         LOGGER.debug("\t\t**A Boolean");
1367                                         inputs.put(key, obj);
1368                                 } else if (obj instanceof java.util.LinkedHashMap) {
1369                                         LOGGER.debug("\t\t**A java.util.LinkedHashMap **");
1370                                         //Object objJson = this.convertObjectToJsonNode(obj.toString());
1371                                         //if (objJson == null) {
1372                                         //      LOGGER.debug("\t\tFAILED!! Will just put LinkedHashMap on the inputs");
1373                                         inputs.put(key, obj);
1374                                         //}
1375                                         //else {
1376                                         //      LOGGER.debug("\t\tSuccessfully converted to JsonNode: " + objJson.toString());
1377                                         //      inputs.put(key, objJson);
1378                                         //}
1379                                 } else if (obj instanceof java.util.ArrayList) {
1380                                         LOGGER.debug("\t\t**An ArrayList");
1381                                         inputs.put(key, obj);
1382                                 } else {
1383                                         LOGGER.debug("\t\t**UNKNOWN OBJECT TYPE");
1384                                         inputs.put(key, obj);
1385                                 }
1386                         } else {
1387                                 LOGGER.debug("key=" + key + " is already in the inputs - will not overwrite");
1388                         }
1389                 }
1390                 return;
1391         }
1392
1393         public JsonNode convertObjectToJsonNode(Object lhm) {
1394                 if (lhm == null) {
1395                         return null;
1396                 }
1397                 JsonNode jsonNode = null;
1398                 try {
1399                         String jsonString = lhm.toString();
1400                         jsonNode = new ObjectMapper().readTree(jsonString);
1401                 } catch (Exception e) {
1402                         LOGGER.debug("Unable to convert " + lhm.toString() + " to a JsonNode " + e.getMessage());
1403                         jsonNode = null;
1404                 }
1405                 return jsonNode;
1406         }
1407
1408         public ArrayList<String> convertCdlToArrayList(String cdl) {
1409                 String cdl2 = cdl.trim();
1410                 String cdl3;
1411                 if (cdl2.startsWith("[") && cdl2.endsWith("]")) {
1412                         cdl3 = cdl2.substring(1, cdl2.lastIndexOf("]"));
1413                 } else {
1414                         cdl3 = cdl2;
1415                 }
1416                 ArrayList<String> list = new ArrayList<String>(Arrays.asList(cdl3.split(",")));
1417                 return list;
1418         }
1419
1420     /**
1421      * New with 1707 - this method will convert all the String *values* of the inputs
1422      * to their "actual" object type (based on the param type: in the db - which comes from the template):
1423      * (heat variable type) -> java Object type
1424      * string -> String
1425      * number -> Integer
1426      * json -> JsonNode
1427      * comma_delimited_list -> ArrayList
1428      * boolean -> Boolean
1429      * if any of the conversions should fail, we will default to adding it to the inputs
1430      * as a string - see if Openstack can handle it.
1431      * Also, will remove any params that are extra.
1432      * Any aliases will be converted to their appropriate name (anyone use this feature?)
1433      * @param inputs - the Map<String, String> of the inputs received on the request
1434      * @param template the HeatTemplate object - this is so we can also verify if the param is valid for this template
1435      * @return HashMap<String, Object> of the inputs, cleaned and converted
1436      */
1437         public HashMap<String, Object> convertInputMap(Map<String, String> inputs, HeatTemplate template) {
1438                 HashMap<String, Object> newInputs = new HashMap<String, Object>();
1439                 HashMap<String, HeatTemplateParam> params = new HashMap<String, HeatTemplateParam>();
1440                 HashMap<String, HeatTemplateParam> paramAliases = new HashMap<String, HeatTemplateParam>();
1441
1442                 if (inputs == null) {
1443                         LOGGER.debug("convertInputMap - inputs is null - nothing to do here");
1444                         return new HashMap<String, Object>();
1445                 }
1446
1447                 LOGGER.debug("convertInputMap in MsoHeatUtils called, with " + inputs.size() + " inputs, and template " + template.getArtifactUuid());
1448                 try {
1449                         LOGGER.debug(template.toString());
1450                         Set<HeatTemplateParam> paramSet = template.getParameters();
1451                         LOGGER.debug("paramSet has " + paramSet.size() + " entries");
1452                 } catch (Exception e) {
1453                         LOGGER.debug("Exception occurred in convertInputMap:" + e.getMessage());
1454                 }
1455
1456                 for (HeatTemplateParam htp : template.getParameters()) {
1457                         LOGGER.debug("Adding " + htp.getParamName());
1458                         params.put(htp.getParamName(), htp);
1459                         if (htp.getParamAlias() != null && !htp.getParamAlias().equals("")) {
1460                                 LOGGER.debug("\tFound ALIAS " + htp.getParamName() + "->" + htp.getParamAlias());
1461                                 paramAliases.put(htp.getParamAlias(), htp);
1462                         }
1463                 }
1464                 LOGGER.debug("Now iterate through the inputs...");
1465                 for (String key : inputs.keySet()) {
1466                         LOGGER.debug("key=" + key);
1467                         boolean alias = false;
1468                         String realName = null;
1469                         if (!params.containsKey(key)) {
1470                                 LOGGER.debug(key + " is not a parameter in the template! - check for an alias");
1471                                 // add check here for an alias
1472                                 if (!paramAliases.containsKey(key)) {
1473                                         LOGGER.debug("The parameter " + key + " is in the inputs, but it's not a parameter for this template - omit");
1474                                         continue;
1475                                 } else {
1476                                         alias = true;
1477                                         realName = paramAliases.get(key).getParamName();
1478                                         LOGGER.debug("FOUND AN ALIAS! Will use " + realName + " in lieu of give key/alias " + key);
1479                                 }
1480                         }
1481                         String type = params.get(key).getParamType();
1482                         if (type == null || type.equals("")) {
1483                                 LOGGER.debug("**PARAM_TYPE is null/empty for " + key + ", will default to string");
1484                                 type = "string";
1485                         }
1486                         LOGGER.debug("Parameter: " + key + " is of type " + type);
1487                         if (type.equalsIgnoreCase("string")) {
1488                                 // Easiest!
1489                                 String str = inputs.get(key);
1490                                 if (alias)
1491                                         newInputs.put(realName, str);
1492                                 else
1493                                         newInputs.put(key, str);
1494                         } else if (type.equalsIgnoreCase("number")) {
1495                                 String integerString = inputs.get(key).toString();
1496                                 Integer anInteger = null;
1497                                 try {
1498                                         anInteger = Integer.parseInt(integerString);
1499                                 } catch (Exception e) {
1500                                         LOGGER.debug("Unable to convert " + integerString + " to an integer!!");
1501                                         anInteger = null;
1502                                 }
1503                                 if (anInteger != null) {
1504                                         if (alias)
1505                                                 newInputs.put(realName, anInteger);
1506                                         else
1507                                                 newInputs.put(key, anInteger);
1508                                 }
1509                                 else {
1510                                         if (alias)
1511                                                 newInputs.put(realName, integerString);
1512                                         else
1513                                                 newInputs.put(key, integerString);
1514                                 }
1515                         } else if (type.equalsIgnoreCase("json")) {
1516                                 String jsonString = inputs.get(key).toString();
1517                         JsonNode jsonNode = null;
1518                         try {
1519                                 jsonNode = new ObjectMapper().readTree(jsonString);
1520                         } catch (Exception e) {
1521                                         LOGGER.debug("Unable to convert " + jsonString + " to a JsonNode!!");
1522                                         jsonNode = null;
1523                         }
1524                         if (jsonNode != null) {
1525                                 if (alias)
1526                                         newInputs.put(realName, jsonNode);
1527                                 else
1528                                         newInputs.put(key, jsonNode);
1529                         }
1530                         else {
1531                                 if (alias)
1532                                         newInputs.put(realName, jsonString);
1533                                 else
1534                                         newInputs.put(key, jsonString);
1535                         }
1536                         } else if (type.equalsIgnoreCase("comma_delimited_list")) {
1537                                 String commaSeparated = inputs.get(key).toString();
1538                                 try {
1539                                         ArrayList<String> anArrayList = this.convertCdlToArrayList(commaSeparated);
1540                                         if (alias)
1541                                                 newInputs.put(realName, anArrayList);
1542                                         else
1543                                                 newInputs.put(key, anArrayList);
1544                                 } catch (Exception e) {
1545                                         LOGGER.debug("Unable to convert " + commaSeparated + " to an ArrayList!!");
1546                                         if (alias)
1547                                                 newInputs.put(realName, commaSeparated);
1548                                         else
1549                                                 newInputs.put(key, commaSeparated);
1550                                 }
1551                         } else if (type.equalsIgnoreCase("boolean")) {
1552                                 String booleanString = inputs.get(key).toString();
1553                                 Boolean aBool = new Boolean(booleanString);
1554                                 if (alias)
1555                                         newInputs.put(realName, aBool);
1556                                 else
1557                                         newInputs.put(key, aBool);
1558                         } else {
1559                                 // it's null or something undefined - just add it back as a String
1560                                 String str = inputs.get(key);
1561                                 if (alias)
1562                                         newInputs.put(realName, str);
1563                                 else
1564                                         newInputs.put(key, str);
1565                         }
1566                 }
1567                 return newInputs;
1568         }
1569
1570 }