adapters/mso-adapter-utils/src/main/java/org/onap/so/openstack/utils/MsoHeatUtils.java

   1 /*-
   2  * ============LICENSE_START=======================================================
   3  * ONAP - SO
   4  * ================================================================================
   5  * Copyright (C) 2017 AT&T Intellectual Property. All rights reserved.
   6  * Copyright (C) 2017 Huawei Technologies Co., Ltd. All rights reserved.
   7  * ================================================================================
   8  * Licensed under the Apache License, Version 2.0 (the "License");
   9  * you may not use this file except in compliance with the License.
  10  * You may obtain a copy of the License at
  11  *
  12  *      http://www.apache.org/licenses/LICENSE-2.0
  13  *
  14  * Unless required by applicable law or agreed to in writing, software
  15  * distributed under the License is distributed on an "AS IS" BASIS,
  16  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  17  * See the License for the specific language governing permissions and
  18  * limitations under the License.
  19  * ============LICENSE_END=========================================================
  20  */
  21
  22 package org.onap.so.openstack.utils;
  23
  24 import java.io.IOException;
  25 import java.util.ArrayList;
  26 import java.util.Arrays;
  27 import java.util.HashMap;
  28 import java.util.List;
  29 import java.util.Map;
  30 import java.util.Map.Entry;
  31 import java.util.Set;
  32
  33 import org.onap.so.adapters.vdu.CloudInfo;
  34 import org.onap.so.adapters.vdu.PluginAction;
  35 import org.onap.so.adapters.vdu.VduArtifact;
  36 import org.onap.so.adapters.vdu.VduArtifact.ArtifactType;
  37 import org.onap.so.adapters.vdu.VduException;
  38 import org.onap.so.adapters.vdu.VduInstance;
  39 import org.onap.so.adapters.vdu.VduModelInfo;
  40 import org.onap.so.adapters.vdu.VduPlugin;
  41 import org.onap.so.adapters.vdu.VduStateType;
  42 import org.onap.so.adapters.vdu.VduStatus;
  43 import org.onap.so.cloud.CloudConfig;
  44 import org.onap.so.cloud.CloudIdentity;
  45 import org.onap.so.cloud.CloudSite;
  46 import org.onap.so.cloud.authentication.AuthenticationMethodFactory;
  47 import org.onap.so.db.catalog.beans.HeatTemplate;
  48 import org.onap.so.db.catalog.beans.HeatTemplateParam;
  49 import org.onap.so.logger.MessageEnum;
  50 import org.onap.so.logger.MsoAlarmLogger;
  51 import org.onap.so.logger.MsoLogger;
  52 import org.onap.so.openstack.beans.HeatCacheEntry;
  53 import org.onap.so.openstack.beans.HeatStatus;
  54 import org.onap.so.openstack.beans.StackInfo;
  55 import org.onap.so.openstack.exceptions.MsoAdapterException;
  56 import org.onap.so.openstack.exceptions.MsoCloudSiteNotFound;
  57 import org.onap.so.openstack.exceptions.MsoException;
  58 import org.onap.so.openstack.exceptions.MsoIOException;
  59 import org.onap.so.openstack.exceptions.MsoOpenstackException;
  60 import org.onap.so.openstack.exceptions.MsoStackAlreadyExists;
  61 import org.onap.so.openstack.exceptions.MsoTenantNotFound;
  62 import org.onap.so.openstack.mappers.StackInfoMapper;
  63 import org.onap.so.utils.CryptoUtils;
  64 import org.springframework.beans.factory.annotation.Autowired;
  65 import org.springframework.context.annotation.Primary;
  66 import org.springframework.core.env.Environment;
  67 import org.springframework.stereotype.Component;
  68
  69 import com.fasterxml.jackson.core.type.TypeReference;
  70 import com.fasterxml.jackson.databind.JsonNode;
  71 import com.fasterxml.jackson.databind.ObjectMapper;
  72 import com.woorea.openstack.base.client.OpenStackConnectException;
  73 import com.woorea.openstack.base.client.OpenStackRequest;
  74 import com.woorea.openstack.base.client.OpenStackResponseException;
  75 import com.woorea.openstack.heat.Heat;
  76 import com.woorea.openstack.heat.model.CreateStackParam;
  77 import com.woorea.openstack.heat.model.Stack;
  78 import com.woorea.openstack.heat.model.Stack.Output;
  79 import com.woorea.openstack.heat.model.Stacks;
  80 import com.woorea.openstack.keystone.Keystone;
  81 import com.woorea.openstack.keystone.model.Access;
  82 import com.woorea.openstack.keystone.model.Authentication;
  83 import com.woorea.openstack.keystone.utils.KeystoneUtils;
  84
  85 @Primary
  86 @Component
  87 public class MsoHeatUtils extends MsoCommonUtils implements VduPlugin{
  88
  89     private static final String TOKEN_AUTH = "TokenAuth";
  90
  91     private static final String QUERY_ALL_STACKS = "QueryAllStacks";
  92
  93     private static final String DELETE_STACK = "DeleteStack";
  94
  95     private static final String HEAT_ERROR = "HeatError";
  96
  97     private static final String CREATE_STACK = "CreateStack";
  98
  99     // Cache Heat Clients statically. Since there is just one MSO user, there is no
 100     // benefit to re-authentication on every request (or across different flows). The
 101     // token will be used until it expires.
 102     //
 103     // The cache key is "tenantId:cloudId"
 104     private static Map <String, HeatCacheEntry> heatClientCache = new HashMap <> ();
 105
 106     // Fetch cloud configuration each time (may be cached in CloudConfig class)
 107     @Autowired
 108     protected CloudConfig cloudConfig;
 109
 110     @Autowired
 111     private Environment environment;
 112
 113     @Autowired
 114     private AuthenticationMethodFactory authenticationMethodFactory;
 115
 116     @Autowired
 117     private MsoTenantUtilsFactory tenantUtilsFactory;
 118
 119     private static final MsoLogger LOGGER = MsoLogger.getMsoLogger (MsoLogger.Catalog.RA, MsoHeatUtils.class);
 120
 121     // Properties names and variables (with default values)
 122     protected String createPollIntervalProp = "ecomp.mso.adapters.po.pollInterval";
 123     private String deletePollIntervalProp = "ecomp.mso.adapters.po.pollInterval";
 124     private String deletePollTimeoutProp = "ecomp.mso.adapters.po.pollTimeout";
 125
 126     protected static final String createPollIntervalDefault = "15";
 127     private static final String deletePollIntervalDefault = "15";
 128
 129     private static final ObjectMapper JSON_MAPPER = new ObjectMapper();
 130
 131     /**
 132      * keep this old method signature here to maintain backwards compatibility. keep others as well.
 133      * this method does not include environment, files, or heatFiles
 134      */
 135     public StackInfo createStack (String cloudSiteId,
 136                                   String tenantId,
 137                                   String stackName,
 138                                   String heatTemplate,
 139                                   Map <String, ?> stackInputs,
 140                                   boolean pollForCompletion,
 141                                   int timeoutMinutes) throws MsoException {
 142         // Just call the new method with the environment & files variable set to null
 143         return this.createStack (cloudSiteId,
 144                                  tenantId,
 145                                  stackName,
 146                                  heatTemplate,
 147                                  stackInputs,
 148                                  pollForCompletion,
 149                                  timeoutMinutes,
 150                                  null,
 151                                  null,
 152                                  null,
 153                                  true);
 154     }
 155
 156     // This method has environment, but not files or heatFiles
 157     public StackInfo createStack (String cloudSiteId,
 158                                   String tenantId,
 159                                   String stackName,
 160                                   String heatTemplate,
 161                                   Map <String, ?> stackInputs,
 162                                   boolean pollForCompletion,
 163                                   int timeoutMinutes,
 164                                   String environment) throws MsoException {
 165         // Just call the new method with the files/heatFiles variables set to null
 166         return this.createStack (cloudSiteId,
 167                                  tenantId,
 168                                  stackName,
 169                                  heatTemplate,
 170                                  stackInputs,
 171                                  pollForCompletion,
 172                                  timeoutMinutes,
 173                                  environment,
 174                                  null,
 175                                  null,
 176                                  true);
 177     }
 178
 179     // This method has environment and files, but not heatFiles.
 180     public StackInfo createStack (String cloudSiteId,
 181                                   String tenantId,
 182                                   String stackName,
 183                                   String heatTemplate,
 184                                   Map <String, ?> stackInputs,
 185                                   boolean pollForCompletion,
 186                                   int timeoutMinutes,
 187                                   String environment,
 188                                   Map <String, Object> files) throws MsoException {
 189         return this.createStack (cloudSiteId,
 190                                  tenantId,
 191                                  stackName,
 192                                  heatTemplate,
 193                                  stackInputs,
 194                                  pollForCompletion,
 195                                  timeoutMinutes,
 196                                  environment,
 197                                  files,
 198                                  null,
 199                                  true);
 200     }
 201
 202     // This method has environment, files, heatfiles
 203     public StackInfo createStack (String cloudSiteId,
 204                                   String tenantId,
 205                                   String stackName,
 206                                   String heatTemplate,
 207                                   Map <String, ?> stackInputs,
 208                                   boolean pollForCompletion,
 209                                   int timeoutMinutes,
 210                                   String environment,
 211                                   Map <String, Object> files,
 212                                   Map <String, Object> heatFiles) throws MsoException {
 213         return this.createStack (cloudSiteId,
 214                                  tenantId,
 215                                  stackName,
 216                                  heatTemplate,
 217                                  stackInputs,
 218                                  pollForCompletion,
 219                                  timeoutMinutes,
 220                                  environment,
 221                                  files,
 222                                  heatFiles,
 223                                  true);
 224     }
 225
 226     /**
 227      * Create a new Stack in the specified cloud location and tenant. The Heat template
 228      * and parameter map are passed in as arguments, along with the cloud access credentials.
 229      * It is expected that parameters have been validated and contain at minimum the required
 230      * parameters for the given template with no extra (undefined) parameters..
 231      *
 232      * The Stack name supplied by the caller must be unique in the scope of this tenant.
 233      * However, it should also be globally unique, as it will be the identifier for the
 234      * resource going forward in Inventory. This latter is managed by the higher levels
 235      * invoking this function.
 236      *
 237      * The caller may choose to let this function poll Openstack for completion of the
 238      * stack creation, or may handle polling itself via separate calls to query the status.
 239      * In either case, a StackInfo object will be returned containing the current status.
 240      * When polling is enabled, a status of CREATED is expected. When not polling, a
 241      * status of BUILDING is expected.
 242      *
 243      * An error will be thrown if the requested Stack already exists in the specified
 244      * Tenant and Cloud.
 245      *
 246      * For 1510 - add "environment", "files" (nested templates), and "heatFiles" (get_files) as
 247      * parameters for createStack. If environment is non-null, it will be added to the stack.
 248      * The nested templates and get_file entries both end up being added to the "files" on the
 249      * stack. We must combine them before we add them to the stack if they're both non-null.
 250      *
 251      * @param cloudSiteId The cloud (may be a region) in which to create the stack.
 252      * @param tenantId The Openstack ID of the tenant in which to create the Stack
 253      * @param stackName The name of the stack to create
 254      * @param heatTemplate The Heat template
 255      * @param stackInputs A map of key/value inputs
 256      * @param pollForCompletion Indicator that polling should be handled in Java vs. in the client
 257      * @param environment An optional yaml-format string to specify environmental parameters
 258      * @param files a Map<String, Object> that lists the child template IDs (file is the string, object is an int of
 259      *        Template id)
 260      * @param heatFiles a Map<String, Object> that lists the get_file entries (fileName, fileBody)
 261      * @param backout Donot delete stack on create Failure - defaulted to True
 262      * @return A StackInfo object
 263      * @throws MsoOpenstackException Thrown if the Openstack API call returns an exception.
 264      */
 265
 266     @SuppressWarnings("unchecked")
 267     public StackInfo createStack (String cloudSiteId,
 268                                   String tenantId,
 269                                   String stackName,
 270                                   String heatTemplate,
 271                                   Map <String, ?> stackInputs,
 272                                   boolean pollForCompletion,
 273                                   int timeoutMinutes,
 274                                   String environment,
 275                                   Map <String, Object> files,
 276                                   Map <String, Object> heatFiles,
 277                                   boolean backout) throws MsoException {
 278         // Create local variables checking to see if we have an environment, nested, get_files
 279         // Could later add some checks to see if it's valid.
 280         boolean haveEnvtVariable = true;
 281         if (environment == null || "".equalsIgnoreCase (environment.trim ())) {
 282             haveEnvtVariable = false;
 283             LOGGER.debug ("createStack called with no environment variable");
 284         } else {
 285             LOGGER.debug ("createStack called with an environment variable: " + environment);
 286         }
 287
 288         boolean haveFiles = true;
 289         if (files == null || files.isEmpty ()) {
 290             haveFiles = false;
 291             LOGGER.debug ("createStack called with no files / child template ids");
 292         } else {
 293             LOGGER.debug ("createStack called with " + files.size () + " files / child template ids");
 294         }
 295
 296         boolean haveHeatFiles = true;
 297         if (heatFiles == null || heatFiles.isEmpty ()) {
 298             haveHeatFiles = false;
 299             LOGGER.debug ("createStack called with no heatFiles");
 300         } else {
 301             LOGGER.debug ("createStack called with " + heatFiles.size () + " heatFiles");
 302         }
 303
 304         // Obtain the cloud site information where we will create the stack
 305         CloudSite cloudSite = cloudConfig.getCloudSite(cloudSiteId).orElseThrow(
 306                 () -> new MsoCloudSiteNotFound(cloudSiteId));
 307         LOGGER.debug("Found: " + cloudSite.toString());
 308         // Get a Heat client. They are cached between calls (keyed by tenantId:cloudId)
 309         // This could throw MsoTenantNotFound or MsoOpenstackException (both propagated)
 310         Heat heatClient = getHeatClient (cloudSite, tenantId);
 311         if (heatClient != null) {
 312                 LOGGER.debug("Found: " + heatClient.toString());
 313         }
 314
 315         LOGGER.debug ("Ready to Create Stack (" + heatTemplate + ") with input params: " + stackInputs);
 316
 317         //force entire stackInput object to generic Map<String, Object> for openstack compatibility
 318                 ObjectMapper mapper = new ObjectMapper();
 319                 Map<String, Object> normalized = new HashMap<>();
 320                 try {
 321                         normalized = mapper.readValue(mapper.writeValueAsString(stackInputs), new TypeReference<HashMap<String,Object>>() {});
 322                 } catch (IOException e1) {
 323                         LOGGER.debug("could not map json", e1);
 324                 }
 325
 326         // Build up the stack to create
 327         // Disable auto-rollback, because error reason is lost. Always rollback in the code.
 328         CreateStackParam stack = new CreateStackParam ();
 329         stack.setStackName (stackName);
 330         stack.setTimeoutMinutes (timeoutMinutes);
 331         stack.setParameters (normalized);
 332         stack.setTemplate (heatTemplate);
 333         stack.setDisableRollback (true);
 334         // TJM New for PO Adapter - add envt variable
 335         if (haveEnvtVariable) {
 336             LOGGER.debug ("Found an environment variable - value: " + environment);
 337             stack.setEnvironment (environment);
 338         }
 339         // Now handle nested templates or get_files - have to combine if we have both
 340         // as they're both treated as "files:" on the stack.
 341         if (haveFiles && haveHeatFiles) {
 342             // Let's do this here - not in the bean
 343             LOGGER.debug ("Found files AND heatFiles - combine and add!");
 344             Map <String, Object> combinedFiles = new HashMap <> ();
 345             for (Entry<String, Object> entry : files.entrySet()) {
 346                 combinedFiles.put(entry.getKey(), entry.getValue());
 347             }
 348             for (Entry<String, Object> entry : heatFiles.entrySet()) {
 349                 combinedFiles.put(entry.getKey(), entry.getValue());
 350             }
 351             stack.setFiles (combinedFiles);
 352         } else {
 353             // Handle if we only have one or neither:
 354             if (haveFiles) {
 355                 LOGGER.debug ("Found files - adding to stack");
 356                 stack.setFiles (files);
 357             }
 358             if (haveHeatFiles) {
 359                 LOGGER.debug ("Found heatFiles - adding to stack");
 360                 // the setFiles was modified to handle adding the entries
 361                 stack.setFiles (heatFiles);
 362             }
 363         }
 364
 365         // 1802 - attempt to add better formatted printout of request to openstack
 366         try {
 367                 Map<String, Object> inputs = new HashMap<>();
 368                 for (Entry<String, ?> entry : stackInputs.entrySet()) {
 369                         if (entry.getValue() != null) {
 370                                 inputs.put(entry.getKey(), entry.getValue());
 371                         }
 372                 }
 373                 LOGGER.debug(this.printStackRequest(tenantId, heatFiles, files, environment, inputs, stackName, heatTemplate, timeoutMinutes, backout, cloudSiteId));
 374         } catch (Exception e) {
 375                 // that's okay - this is a nice-to-have
 376                 LOGGER.debug("(had an issue printing nicely formatted request to debuglog) " + e.getMessage());
 377         }
 378
 379         Stack heatStack = null;
 380         try {
 381             // Execute the actual Openstack command to create the Heat stack
 382             OpenStackRequest <Stack> request = heatClient.getStacks ().create (stack);
 383             // Begin X-Auth-User
 384             // Obtain an MSO token for the tenant
 385             CloudIdentity cloudIdentity = cloudConfig.getIdentityService(cloudSite.getIdentityServiceId());
 386             // cloudIdentity.getMsoId(), cloudIdentity.getMsoPass()
 387             //req
 388             request.header ("X-Auth-User", cloudIdentity.getMsoId ());
 389             request.header ("X-Auth-Key", CryptoUtils.decryptCloudConfigPassword(cloudIdentity.getMsoPass ()));
 390             LOGGER.debug ("headers added, about to executeAndRecordOpenstackRequest");
 391             //LOGGER.debug(this.requestToStringBuilder(stack).toString());
 392             // END - try to fix X-Auth-User
 393             heatStack = executeAndRecordOpenstackRequest (request);
 394         } catch (OpenStackResponseException e) {
 395             // Since this came on the 'Create Stack' command, nothing was changed
 396             // in the cloud. Return the error as an exception.
 397             if (e.getStatus () == 409) {
 398                 // Stack already exists. Return a specific error for this case
 399                 MsoStackAlreadyExists me = new MsoStackAlreadyExists (stackName, tenantId, cloudSiteId);
 400                 me.addContext (CREATE_STACK);
 401                 throw me;
 402             } else {
 403                 // Convert the OpenStackResponseException to an MsoOpenstackException
 404                 LOGGER.debug("ERROR STATUS = " + e.getStatus() + ",\n" + e.getMessage() + "\n" + e.getLocalizedMessage());
 405                 throw heatExceptionToMsoException (e, CREATE_STACK);
 406             }
 407         } catch (OpenStackConnectException e) {
 408             // Error connecting to Openstack instance. Convert to an MsoException
 409             throw heatExceptionToMsoException (e, CREATE_STACK);
 410         } catch (RuntimeException e) {
 411             // Catch-all
 412             throw runtimeExceptionToMsoException (e, CREATE_STACK);
 413         }
 414
 415         // Subsequent access by the canonical name "<stack name>/<stack-id>".
 416         // Otherwise, simple query by name returns a 302 redirect.
 417         // NOTE: This is specific to the v1 Orchestration API.
 418         String canonicalName = stackName + "/" + heatStack.getId ();
 419
 420         // If client has requested a final response, poll for stack completion
 421         if (pollForCompletion) {
 422             // Set a time limit on overall polling.
 423             // Use the resource (template) timeout for Openstack (expressed in minutes)
 424             // and add one poll interval to give Openstack a chance to fail on its own.s
 425
 426                 int createPollInterval = Integer.parseInt(this.environment.getProperty(createPollIntervalProp, createPollIntervalDefault));
 427             int pollTimeout = (timeoutMinutes * 60) + createPollInterval;
 428             // New 1610 - poll on delete if we rollback - use same values for now
 429             int deletePollInterval = createPollInterval;
 430             int deletePollTimeout = pollTimeout;
 431             boolean createTimedOut = false;
 432             StringBuilder stackErrorStatusReason = new StringBuilder("");
 433             LOGGER.debug("createPollInterval=" + createPollInterval + ", pollTimeout=" + pollTimeout);
 434
 435             while (true) {
 436                 try {
 437                     heatStack = queryHeatStack (heatClient, canonicalName);
 438                     LOGGER.debug (heatStack.getStackStatus () + " (" + canonicalName + ")");
 439                     try {
 440                         LOGGER.debug("Current stack " + this.getOutputsAsStringBuilder(heatStack).toString());
 441                     } catch (Exception e) {
 442                         LOGGER.debug("an error occurred trying to print out the current outputs of the stack", e);
 443                     }
 444
 445                     if ("CREATE_IN_PROGRESS".equals (heatStack.getStackStatus ())) {
 446                         // Stack creation is still running.
 447                         // Sleep and try again unless timeout has been reached
 448                         if (pollTimeout <= 0) {
 449                             // Note that this should not occur, since there is a timeout specified
 450                             // in the Openstack call.
 451                             LOGGER.error (MessageEnum.RA_CREATE_STACK_TIMEOUT, cloudSiteId, tenantId, stackName, heatStack.getStackStatus (), "", "", MsoLogger.ErrorCode.AvailabilityError, "Create stack timeout");
 452                             createTimedOut = true;
 453                             break;
 454                         }
 455
 456                         sleep(createPollInterval * 1000L);
 457
 458                         pollTimeout -= createPollInterval;
 459                                 LOGGER.debug("pollTimeout remaining: " + pollTimeout);
 460                     } else {
 461                         //save off the status & reason msg before we attempt delete
 462                         stackErrorStatusReason.append("Stack error (" + heatStack.getStackStatus() + "): " + heatStack.getStackStatusReason());
 463                         break;
 464                     }
 465                 } catch (MsoException me) {
 466                         // Cannot query the stack status. Something is wrong.
 467                         // Try to roll back the stack
 468                         if (!backout)
 469                         {
 470                                 LOGGER.warn(MessageEnum.RA_CREATE_STACK_ERR, "Create Stack errored, stack deletion suppressed", "", "", MsoLogger.ErrorCode.BusinessProcesssError, "Exception in Create Stack, stack deletion suppressed");
 471                         }
 472                         else
 473                         {
 474                                 try {
 475                                         LOGGER.debug("Create Stack error - unable to query for stack status - attempting to delete stack: " + canonicalName + " - This will likely fail and/or we won't be able to query to see if delete worked");
 476                                         OpenStackRequest <Void> request = heatClient.getStacks ().deleteByName (canonicalName);
 477                                         executeAndRecordOpenstackRequest (request);
 478                                         // this may be a waste of time - if we just got an exception trying to query the stack - we'll just
 479                                         // get another one, n'est-ce pas?
 480                                         boolean deleted = false;
 481                                         while (!deleted) {
 482                                                 try {
 483                                                         heatStack = queryHeatStack(heatClient, canonicalName);
 484                                                         if (heatStack != null) {
 485                                                         LOGGER.debug(heatStack.getStackStatus());
 486                                                         if ("DELETE_IN_PROGRESS".equals(heatStack.getStackStatus())) {
 487                                                                 if (deletePollTimeout <= 0) {
 488                                                                         LOGGER.error (MessageEnum.RA_CREATE_STACK_TIMEOUT, cloudSiteId, tenantId, stackName,
 489                                                                                         heatStack.getStackStatus (), "", "", MsoLogger.ErrorCode.AvailabilityError,
 490                                                                                         "Rollback: DELETE stack timeout");
 491                                                                         break;
 492                                                                 } else {
 493                                                                         sleep(deletePollInterval * 1000L);
 494                                                                         deletePollTimeout -= deletePollInterval;
 495                                                                 }
 496                                                         } else if ("DELETE_COMPLETE".equals(heatStack.getStackStatus())){
 497                                                                 LOGGER.debug("DELETE_COMPLETE for " + canonicalName);
 498                                                                 deleted = true;
 499                                                                 continue;
 500                                                         } else {
 501                                                                 //got a status other than DELETE_IN_PROGRESS or DELETE_COMPLETE - so break and evaluate
 502                                                                 break;
 503                                                         }
 504                                                 } else {
 505                                                         // assume if we can't find it - it's deleted
 506                                                         LOGGER.debug("heatStack returned null - assume the stack " + canonicalName + " has been deleted");
 507                                                         deleted = true;
 508                                                         continue;
 509                                                         }
 510
 511                                                 } catch (Exception e3) {
 512                                                         // Just log this one. We will report the original exception.
 513                                                         LOGGER.error (MessageEnum.RA_CREATE_STACK_ERR, "Create Stack: Nested exception rolling back stack: " + e3, "", "", MsoLogger.ErrorCode.BusinessProcesssError, "Create Stack: Nested exception rolling back stack on error on query");
 514
 515                                                 }
 516                                         }
 517                                 } catch (Exception e2) {
 518                                         // Just log this one. We will report the original exception.
 519                                         LOGGER.error (MessageEnum.RA_CREATE_STACK_ERR, "Create Stack: Nested exception rolling back stack: " + e2, "", "", MsoLogger.ErrorCode.BusinessProcesssError, "Create Stack: Nested exception rolling back stack");
 520                                 }
 521                         }
 522
 523                     // Propagate the original exception from Stack Query.
 524                     me.addContext (CREATE_STACK);
 525                     throw me;
 526                 }
 527             }
 528
 529             if (!"CREATE_COMPLETE".equals (heatStack.getStackStatus ())) {
 530                 LOGGER.error (MessageEnum.RA_CREATE_STACK_ERR, "Create Stack error:  Polling complete with non-success status: "
 531                               + heatStack.getStackStatus () + ", " + heatStack.getStackStatusReason (), "", "", MsoLogger.ErrorCode.BusinessProcesssError, "Create Stack error");
 532
 533                 // Rollback the stack creation, since it is in an indeterminate state.
 534                 if (!backout)
 535                 {
 536                         LOGGER.warn(MessageEnum.RA_CREATE_STACK_ERR, "Create Stack errored, stack deletion suppressed", "", "", MsoLogger.ErrorCode.BusinessProcesssError, "Create Stack error, stack deletion suppressed");
 537                 }
 538                 else
 539                 {
 540                         try {
 541                                 LOGGER.debug("Create Stack errored - attempting to DELETE stack: " + canonicalName);
 542                                 LOGGER.debug("deletePollInterval=" + deletePollInterval + ", deletePollTimeout=" + deletePollTimeout);
 543                                 OpenStackRequest <Void> request = heatClient.getStacks ().deleteByName (canonicalName);
 544                                 executeAndRecordOpenstackRequest (request);
 545                                 boolean deleted = false;
 546                                 while (!deleted) {
 547                                         try {
 548                                                 heatStack = queryHeatStack(heatClient, canonicalName);
 549                                                 if (heatStack != null) {
 550                                                         LOGGER.debug(heatStack.getStackStatus() + " (" + canonicalName + ")");
 551                                                         if ("DELETE_IN_PROGRESS".equals(heatStack.getStackStatus())) {
 552                                                                 if (deletePollTimeout <= 0) {
 553                                                                         LOGGER.error (MessageEnum.RA_CREATE_STACK_TIMEOUT, cloudSiteId, tenantId, stackName,
 554                                                                                         heatStack.getStackStatus (), "", "", MsoLogger.ErrorCode.AvailabilityError,
 555                                                                                         "Rollback: DELETE stack timeout");
 556                                                                         break;
 557                                                                 } else {
 558                                                                         sleep(deletePollInterval * 1000L);
 559                                                                         deletePollTimeout -= deletePollInterval;
 560                                                                         LOGGER.debug("deletePollTimeout remaining: " + deletePollTimeout);
 561                                                                 }
 562                                                         } else if ("DELETE_COMPLETE".equals(heatStack.getStackStatus())){
 563                                                                 LOGGER.debug("DELETE_COMPLETE for " + canonicalName);
 564                                                                 deleted = true;
 565                                                                 continue;
 566                                                         } else if ("DELETE_FAILED".equals(heatStack.getStackStatus())) {
 567                                                                 // Warn about this (?) - but still throw the original exception
 568                                                                 LOGGER.warn(MessageEnum.RA_CREATE_STACK_ERR, "Create Stack errored, stack deletion FAILED", "", "", MsoLogger.ErrorCode.BusinessProcesssError, "Create Stack error, stack deletion FAILED");
 569                                                                 LOGGER.debug("Stack deletion FAILED on a rollback of a create - " + canonicalName + ", status=" + heatStack.getStackStatus() + ", reason=" + heatStack.getStackStatusReason());
 570                                                                 break;
 571                                                         } else {
 572                                                                 //got a status other than DELETE_IN_PROGRESS or DELETE_COMPLETE - so break and evaluate
 573                                                                 break;
 574                                                         }
 575                                                 } else {
 576                                                         // assume if we can't find it - it's deleted
 577                                                         LOGGER.debug("heatStack returned null - assume the stack " + canonicalName + " has been deleted");
 578                                                         deleted = true;
 579                                                         continue;
 580                                                 }
 581
 582                                         } catch (MsoException me2) {
 583                                                 // We got an exception on the delete - don't throw this exception - throw the original - just log.
 584                                                 LOGGER.debug("Exception thrown trying to delete " + canonicalName + " on a create->rollback: " + me2.getContextMessage(), me2);
 585                                                 LOGGER.warn(MessageEnum.RA_CREATE_STACK_ERR, "Create Stack errored, then stack deletion FAILED - exception thrown", "", "", MsoLogger.ErrorCode.BusinessProcesssError, me2.getContextMessage());
 586                                         }
 587
 588                                 } // end while !deleted
 589                                 StringBuilder errorContextMessage;
 590                                 if (createTimedOut) {
 591                                         errorContextMessage = new StringBuilder("Stack Creation Timeout");
 592                                 } else {
 593                                         errorContextMessage  = stackErrorStatusReason;
 594                                 }
 595                                 if (deleted) {
 596                                         errorContextMessage.append(" - stack successfully deleted");
 597                                 } else {
 598                                         errorContextMessage.append(" - encountered an error trying to delete the stack");
 599                                 }
 600 //                              MsoOpenstackException me = new MsoOpenstackException(0, "", stackErrorStatusReason.toString());
 601  //                             me.addContext(CREATE_STACK);
 602   //                            alarmLogger.sendAlarm(HEAT_ERROR, MsoAlarmLogger.CRITICAL, me.getContextMessage());
 603    //                           throw me;
 604                         } catch (Exception e2) {
 605                                 // shouldn't happen - but handle
 606                                 LOGGER.error (MessageEnum.RA_CREATE_STACK_ERR, "Create Stack: Nested exception rolling back stack: " + e2, "", "", MsoLogger.ErrorCode.BusinessProcesssError, "Exception in Create Stack: rolling back stack");
 607                         }
 608                 }
 609                 MsoOpenstackException me = new MsoOpenstackException(0, "", stackErrorStatusReason.toString());
 610                 me.addContext(CREATE_STACK);
 611                 alarmLogger.sendAlarm(HEAT_ERROR, MsoAlarmLogger.CRITICAL, me.getContextMessage());
 612                 throw me;
 613             }
 614
 615         } else {
 616             // Get initial status, since it will have been null after the create.
 617             heatStack = queryHeatStack (heatClient, canonicalName);
 618             LOGGER.debug (heatStack.getStackStatus ());
 619         }
 620
 621         return new StackInfoMapper(heatStack).map();
 622     }
 623
 624     /**
 625      * Query for a single stack (by Name) in a tenant. This call will always return a
 626      * StackInfo object. If the stack does not exist, an "empty" StackInfo will be
 627      * returned - containing only the stack name and a status of NOTFOUND.
 628      *
 629      * @param tenantId The Openstack ID of the tenant in which to query
 630      * @param cloudSiteId The cloud identifier (may be a region) in which to query
 631      * @param stackName The name of the stack to query (may be simple or canonical)
 632      * @return A StackInfo object
 633      * @throws MsoOpenstackException Thrown if the Openstack API call returns an exception.
 634      */
 635     public StackInfo queryStack (String cloudSiteId, String tenantId, String stackName) throws MsoException {
 636         LOGGER.debug ("Query HEAT stack: " + stackName + " in tenant " + tenantId);
 637
 638         // Obtain the cloud site information where we will create the stack
 639         CloudSite cloudSite = cloudConfig.getCloudSite(cloudSiteId).orElseThrow(
 640                 () -> new MsoCloudSiteNotFound(cloudSiteId));
 641         LOGGER.debug("Found: " + cloudSite.toString());
 642
 643         // Get a Heat client. They are cached between calls (keyed by tenantId:cloudId)
 644         Heat heatClient = null;
 645         try {
 646             heatClient = getHeatClient (cloudSite, tenantId);
 647             if (heatClient != null) {
 648                 LOGGER.debug("Found: " + heatClient.toString());
 649             }
 650         } catch (MsoTenantNotFound e) {
 651             // Tenant doesn't exist, so stack doesn't either
 652             LOGGER.debug ("Tenant with id " + tenantId + "not found.", e);
 653             return new StackInfo (stackName, HeatStatus.NOTFOUND);
 654         } catch (MsoException me) {
 655             // Got an Openstack error. Propagate it
 656             LOGGER.error (MessageEnum.RA_CONNECTION_EXCEPTION, "OpenStack", "Openstack Exception on Token request: " + me, "Openstack", "", MsoLogger.ErrorCode.AvailabilityError, "Connection Exception");
 657             me.addContext ("QueryStack");
 658             throw me;
 659         }
 660
 661         // Query the Stack.
 662         // An MsoException will propagate transparently to the caller.
 663         Stack heatStack = queryHeatStack (heatClient, stackName);
 664
 665         if (heatStack == null) {
 666             // Stack does not exist. Return a StackInfo with status NOTFOUND
 667             return new StackInfo (stackName, HeatStatus.NOTFOUND);
 668         }
 669
 670         return new StackInfoMapper(heatStack).map();
 671     }
 672
 673     /**
 674      * Delete a stack (by Name/ID) in a tenant. If the stack is not found, it will be
 675      * considered a successful deletion. The return value is a StackInfo object which
 676      * contains the current stack status.
 677      *
 678      * The client may choose to let the adapter poll Openstack for completion of the
 679      * stack deletion, or may handle polling itself via separate query calls. In either
 680      * case, a StackInfo object will be returned. When polling is enabled, a final
 681      * status of NOTFOUND is expected. When not polling, a status of DELETING is expected.
 682      *
 683      * There is no rollback from a successful stack deletion. A deletion failure will
 684      * also result in an undefined stack state - the components may or may not have been
 685      * all or partially deleted, so the resulting stack must be considered invalid.
 686      *
 687      * @param tenantId The Openstack ID of the tenant in which to perform the delete
 688      * @param cloudSiteId The cloud identifier (may be a region) from which to delete the stack.
 689      * @param stackName The name/id of the stack to delete. May be simple or canonical
 690      * @param pollForCompletion Indicator that polling should be handled in Java vs. in the client
 691      * @return A StackInfo object
 692      * @throws MsoOpenstackException Thrown if the Openstack API call returns an exception.
 693      * @throws MsoCloudSiteNotFound
 694      */
 695     public StackInfo deleteStack (String tenantId,
 696                                   String cloudSiteId,
 697                                   String stackName,
 698                                   boolean pollForCompletion) throws MsoException {
 699         // Obtain the cloud site information where we will create the stack
 700         CloudSite cloudSite = cloudConfig.getCloudSite(cloudSiteId).orElseThrow(
 701                 () -> new MsoCloudSiteNotFound(cloudSiteId));
 702         LOGGER.debug("Found: " + cloudSite.toString());
 703
 704         // Get a Heat client. They are cached between calls (keyed by tenantId:cloudId)
 705         Heat heatClient = null;
 706         try {
 707             heatClient = getHeatClient (cloudSite, tenantId);
 708             if (heatClient != null) {
 709                 LOGGER.debug("Found: " + heatClient.toString());
 710             }
 711         } catch (MsoTenantNotFound e) {
 712             // Tenant doesn't exist, so stack doesn't either
 713             LOGGER.debug ("Tenant with id " + tenantId + "not found.", e);
 714             return new StackInfo (stackName, HeatStatus.NOTFOUND);
 715         } catch (MsoException me) {
 716             // Got an Openstack error. Propagate it
 717             LOGGER.error (MessageEnum.RA_CONNECTION_EXCEPTION, "Openstack", "Openstack Exception on Token request: " + me, "Openstack", "", MsoLogger.ErrorCode.AvailabilityError, "Connection Exception");
 718             me.addContext (DELETE_STACK);
 719             throw me;
 720         }
 721
 722         // OK if stack not found, perform a query first
 723         Stack heatStack = queryHeatStack (heatClient, stackName);
 724         if (heatStack == null || "DELETE_COMPLETE".equals (heatStack.getStackStatus ())) {
 725             // Not found. Return a StackInfo with status NOTFOUND
 726             return new StackInfo (stackName, HeatStatus.NOTFOUND);
 727         }
 728
 729         // Delete the stack.
 730
 731         // Use canonical name "<stack name>/<stack-id>" to delete.
 732         // Otherwise, deletion by name returns a 302 redirect.
 733         // NOTE: This is specific to the v1 Orchestration API.
 734         String canonicalName = heatStack.getStackName () + "/" + heatStack.getId ();
 735
 736         try {
 737             OpenStackRequest <Void> request = null;
 738             if(null != heatClient) {
 739                 request = heatClient.getStacks ().deleteByName (canonicalName);
 740             }
 741             else {
 742                 LOGGER.debug ("Heat Client is NULL" );
 743             }
 744
 745             executeAndRecordOpenstackRequest (request);
 746         } catch (OpenStackResponseException e) {
 747             if (e.getStatus () == 404) {
 748                 // Not found. We are OK with this. Return a StackInfo with status NOTFOUND
 749                 return new StackInfo (stackName, HeatStatus.NOTFOUND);
 750             } else {
 751                 // Convert the OpenStackResponseException to an MsoOpenstackException
 752                 throw heatExceptionToMsoException (e, DELETE_STACK);
 753             }
 754         } catch (OpenStackConnectException e) {
 755             // Error connecting to Openstack instance. Convert to an MsoException
 756             throw heatExceptionToMsoException (e, DELETE_STACK);
 757         } catch (RuntimeException e) {
 758             // Catch-all
 759             throw runtimeExceptionToMsoException (e, DELETE_STACK);
 760         }
 761
 762         // Requery the stack for current status.
 763         // It will probably still exist with "DELETE_IN_PROGRESS" status.
 764         heatStack = queryHeatStack (heatClient, canonicalName);
 765
 766         if (pollForCompletion) {
 767             // Set a timeout on polling
 768
 769             int pollInterval = Integer.parseInt(this.environment.getProperty(deletePollIntervalProp, "" + deletePollIntervalDefault));
 770             int pollTimeout = Integer.parseInt(this.environment.getProperty(deletePollTimeoutProp, "" + deletePollIntervalDefault));
 771
 772             // When querying by canonical name, Openstack returns DELETE_COMPLETE status
 773             // instead of "404" (which would result from query by stack name).
 774             while (heatStack != null && !"DELETE_COMPLETE".equals (heatStack.getStackStatus ())) {
 775                 LOGGER.debug ("Stack status: " + heatStack.getStackStatus ());
 776
 777                 if ("DELETE_FAILED".equals (heatStack.getStackStatus ())) {
 778                     // Throw a 'special case' of MsoOpenstackException to report the Heat status
 779                     String error = "Stack delete error (" + heatStack.getStackStatus ()
 780                                    + "): "
 781                                    + heatStack.getStackStatusReason ();
 782                     MsoOpenstackException me = new MsoOpenstackException (0, "", error);
 783                     me.addContext (DELETE_STACK);
 784
 785                     // Alarm this condition, stack deletion failed
 786                     alarmLogger.sendAlarm (HEAT_ERROR, MsoAlarmLogger.CRITICAL, me.getContextMessage ());
 787
 788                     throw me;
 789                 }
 790
 791                 if (pollTimeout <= 0) {
 792                     LOGGER.error (MessageEnum.RA_DELETE_STACK_TIMEOUT, cloudSiteId, tenantId, stackName, heatStack.getStackStatus (), "", "", MsoLogger.ErrorCode.AvailabilityError, "Delete Stack Timeout");
 793
 794                     // Throw a 'special case' of MsoOpenstackException to report the Heat status
 795                     MsoOpenstackException me = new MsoOpenstackException (0, "", "Stack Deletion Timeout");
 796                     me.addContext (DELETE_STACK);
 797
 798                     // Alarm this condition, stack deletion failed
 799                     alarmLogger.sendAlarm (HEAT_ERROR, MsoAlarmLogger.CRITICAL, me.getContextMessage ());
 800
 801                     throw me;
 802                 }
 803
 804                 sleep(pollInterval * 1000L);
 805
 806                 pollTimeout -= pollInterval;
 807                 LOGGER.debug("pollTimeout remaining: " + pollTimeout);
 808
 809                 heatStack = queryHeatStack (heatClient, canonicalName);
 810             }
 811
 812             // The stack is gone when this point is reached
 813             return new StackInfo (stackName, HeatStatus.NOTFOUND);
 814         }
 815
 816         // Return the current status (if not polling, the delete may still be in progress)
 817         StackInfo stackInfo = new StackInfoMapper(heatStack).map();
 818         stackInfo.setName (stackName);
 819
 820         return stackInfo;
 821     }
 822
 823     /**
 824      * Query for all stacks in a tenant site. This call will return a List of StackInfo
 825      * objects, one for each deployed stack.
 826      *
 827      * Note that this is limited to a single site. To ensure that a tenant is truly
 828      * empty would require looping across all tenant endpoints.
 829      *
 830      * @param tenantId The Openstack ID of the tenant to query
 831      * @param cloudSiteId The cloud identifier (may be a region) in which to query.
 832      * @return A List of StackInfo objects
 833      * @throws MsoOpenstackException Thrown if the Openstack API call returns an exception.
 834      * @throws MsoCloudSiteNotFound
 835      */
 836     public List <StackInfo> queryAllStacks (String tenantId, String cloudSiteId) throws MsoException {
 837         // Obtain the cloud site information where we will create the stack
 838         CloudSite cloudSite = cloudConfig.getCloudSite(cloudSiteId).orElseThrow(
 839                 () -> new MsoCloudSiteNotFound(cloudSiteId));
 840         // Get a Heat client. They are cached between calls (keyed by tenantId:cloudId)
 841         Heat heatClient = getHeatClient (cloudSite, tenantId);
 842
 843         try {
 844             OpenStackRequest <Stacks> request = heatClient.getStacks ().list ();
 845             Stacks stacks = executeAndRecordOpenstackRequest (request);
 846
 847             List <StackInfo> stackList = new ArrayList <> ();
 848
 849             // Not sure if returns an empty list or null if no stacks exist
 850             if (stacks != null) {
 851                 for (Stack stack : stacks) {
 852                     stackList.add (new StackInfoMapper(stack).map());
 853                 }
 854             }
 855
 856             return stackList;
 857         } catch (OpenStackResponseException e) {
 858             if (e.getStatus () == 404) {
 859                 // Not sure if this can happen, but return an empty list
 860                 LOGGER.debug ("queryAllStacks - stack not found: ");
 861                 return new ArrayList <> ();
 862             } else {
 863                 // Convert the OpenStackResponseException to an MsoOpenstackException
 864                 throw heatExceptionToMsoException (e, QUERY_ALL_STACKS);
 865             }
 866         } catch (OpenStackConnectException e) {
 867             // Error connecting to Openstack instance. Convert to an MsoException
 868             throw heatExceptionToMsoException (e, QUERY_ALL_STACKS);
 869         } catch (RuntimeException e) {
 870             // Catch-all
 871             throw runtimeExceptionToMsoException (e, QUERY_ALL_STACKS);
 872         }
 873     }
 874
 875     /**
 876      * Validate parameters to be passed to Heat template. This method performs
 877      * three functions:
 878      * 1. Apply default values to parameters which have them defined
 879      * 2. Report any required parameters that are missing. This will generate an
 880      * exception in the caller, since stack create/update operations would fail.
 881      * 3. Report and remove any extraneous parameters. This will allow clients to
 882      * pass supersets of parameters and not get errors.
 883      *
 884      * These functions depend on the HeatTemplate definition from the MSO Catalog DB,
 885      * along with the input parameter Map. The output is an updated parameter map.
 886      * If the parameters are invalid for the template, an IllegalArgumentException
 887      * is thrown.
 888      */
 889     public Map <String, Object> validateStackParams (Map <String, Object> inputParams,
 890                                                      HeatTemplate heatTemplate) {
 891         // Check that required parameters have been supplied for this template type
 892         StringBuilder missingParams = null;
 893         List <String> paramList = new ArrayList <> ();
 894
 895         // TODO: Enhance DB to support defaults for Heat Template parameters
 896
 897         for (HeatTemplateParam parm : heatTemplate.getParameters ()) {
 898             if (parm.isRequired () && !inputParams.containsKey (parm.getParamName ())) {
 899                 if (missingParams == null) {
 900                     missingParams = new StringBuilder(parm.getParamName());
 901                 } else {
 902                     missingParams.append("," + parm.getParamName());
 903                 }
 904             }
 905             paramList.add (parm.getParamName ());
 906         }
 907         if (missingParams != null) {
 908             // Problem - missing one or more required parameters
 909             String error = "Missing Required inputs for HEAT Template: " + missingParams;
 910             LOGGER.error (MessageEnum.RA_MISSING_PARAM, missingParams + " for HEAT Template", "", "", MsoLogger.ErrorCode.SchemaError, "Missing Required inputs for HEAT Template: " + missingParams);
 911             throw new IllegalArgumentException (error);
 912         }
 913
 914         // Remove any extraneous parameters (don't throw an error)
 915         Map <String, Object> updatedParams = new HashMap <> ();
 916         List <String> extraParams = new ArrayList <> ();
 917
 918         for (Entry<String, Object> entry : inputParams.entrySet()) {
 919                 if (!paramList.contains(entry.getKey())) {
 920                         // This is not a valid parameter for this template
 921                         extraParams.add(entry.getKey());
 922                 } else {
 923                         updatedParams.put(entry.getKey(), entry.getValue());
 924                 }
 925         }
 926
 927         if (!extraParams.isEmpty ()) {
 928             LOGGER.warn (MessageEnum.RA_GENERAL_WARNING, "Heat Stack (" + heatTemplate.getTemplateName ()
 929                          + ") extra input params received: "
 930                          + extraParams, "", "", MsoLogger.ErrorCode.DataError, "Heat Stack (" + heatTemplate.getTemplateName () + ") extra input params received: "+ extraParams);
 931         }
 932
 933         return updatedParams;
 934     }
 935
 936     // ---------------------------------------------------------------
 937     // PRIVATE FUNCTIONS FOR USE WITHIN THIS CLASS
 938
 939     /**
 940      * Get a Heat client for the Openstack Identity service.
 941      * This requires a 'member'-level userId + password, which will be retrieved from
 942      * properties based on the specified cloud Id. The tenant in which to operate
 943      * must also be provided.
 944      * <p>
 945      * On successful authentication, the Heat object will be cached for the
 946      * tenantID + cloudId so that it can be reused without reauthenticating with
 947      * Openstack every time.
 948      *
 949      * @return an authenticated Heat object
 950      */
 951     public Heat getHeatClient (CloudSite cloudSite, String tenantId) throws MsoException {
 952         String cloudId = cloudConfig.getCloudSiteId(cloudSite);
 953
 954         // Check first in the cache of previously authorized clients
 955         String cacheKey = cloudId + ":" + tenantId;
 956         if (heatClientCache.containsKey (cacheKey)) {
 957             if (!heatClientCache.get (cacheKey).isExpired ()) {
 958                 LOGGER.debug ("Using Cached HEAT Client for " + cacheKey);
 959                 return heatClientCache.get (cacheKey).getHeatClient ();
 960             } else {
 961                 // Token is expired. Remove it from cache.
 962                 heatClientCache.remove (cacheKey);
 963                 LOGGER.debug ("Expired Cached HEAT Client for " + cacheKey);
 964             }
 965         }
 966
 967         // Obtain an MSO token for the tenant
 968         CloudIdentity cloudIdentity = cloudConfig.getIdentityService(cloudSite.getIdentityServiceId());
 969         LOGGER.debug("Found: " + cloudIdentity.toString());
 970         MsoTenantUtils tenantUtils = tenantUtilsFactory.getTenantUtilsByServerType(cloudIdentity.getIdentityServerType());
 971         String keystoneUrl = tenantUtils.getKeystoneUrl(cloudId, cloudIdentity);
 972         LOGGER.debug("keystoneUrl=" + keystoneUrl);
 973         Keystone keystoneTenantClient = new Keystone (keystoneUrl);
 974         Access access = null;
 975         try {
 976                 Authentication credentials = authenticationMethodFactory.getAuthenticationFor(cloudIdentity);
 977
 978                 OpenStackRequest <Access> request = keystoneTenantClient.tokens ()
 979                        .authenticate (credentials).withTenantId (tenantId);
 980
 981             access = executeAndRecordOpenstackRequest (request);
 982         } catch (OpenStackResponseException e) {
 983             if (e.getStatus () == 401) {
 984                 // Authentication error.
 985                 String error = "Authentication Failure: tenant=" + tenantId + ",cloud=" + cloudIdentity.getId ();
 986                 alarmLogger.sendAlarm ("MsoAuthenticationError", MsoAlarmLogger.CRITICAL, error);
 987                 throw new MsoAdapterException (error);
 988             } else {
 989                 throw keystoneErrorToMsoException (e, TOKEN_AUTH);
 990             }
 991         } catch (OpenStackConnectException e) {
 992             // Connection to Openstack failed
 993             MsoIOException me = new MsoIOException (e.getMessage (), e);
 994             me.addContext (TOKEN_AUTH);
 995             throw me;
 996         } catch (RuntimeException e) {
 997             // Catch-all
 998             throw runtimeExceptionToMsoException (e, TOKEN_AUTH);
 999         }
1000
1001         // For DCP/LCP, the region should be the cloudId.
1002         String region = cloudSite.getRegionId ();
1003         String heatUrl = null;
1004         try {
1005                 // Isolate trying to printout the region IDs
1006                 try {
1007                         LOGGER.debug("access=" + access.toString());
1008                         for (Access.Service service : access.getServiceCatalog()) {
1009                                 List<Access.Service.Endpoint> endpoints = service.getEndpoints();
1010                                 for (Access.Service.Endpoint endpoint : endpoints) {
1011                                         LOGGER.debug("AIC returned region=" + endpoint.getRegion());
1012                                 }
1013                         }
1014                 } catch (Exception e) {
1015                         LOGGER.debug("Encountered an error trying to printout Access object returned from AIC. " + e.getMessage());
1016                 }
1017             heatUrl = KeystoneUtils.findEndpointURL (access.getServiceCatalog (), "orchestration", region, "public");
1018             LOGGER.debug("heatUrl=" + heatUrl + ", region=" + region);
1019         } catch (RuntimeException e) {
1020             // This comes back for not found (probably an incorrect region ID)
1021             String error = "AIC did not match an orchestration service for: region=" + region + ",cloud=" + cloudIdentity.getIdentityUrl();
1022             alarmLogger.sendAlarm ("MsoConfigurationError", MsoAlarmLogger.CRITICAL, error);
1023             throw new MsoAdapterException (error, e);
1024         }
1025
1026         Heat heatClient = new Heat (heatUrl);
1027         heatClient.token (access.getToken ().getId ());
1028
1029         heatClientCache.put (cacheKey,
1030                              new HeatCacheEntry (heatUrl,
1031                                                  access.getToken ().getId (),
1032                                                  access.getToken ().getExpires ()));
1033         LOGGER.debug ("Caching HEAT Client for " + cacheKey);
1034
1035         return heatClient;
1036     }
1037
1038     /**
1039      * Forcibly expire a HEAT client from the cache. This call is for use by
1040      * the KeystoneClient in case where a tenant is deleted. In that case,
1041      * all cached credentials must be purged so that fresh authentication is
1042      * done if a similarly named tenant is re-created.
1043      * <p>
1044      * Note: This is probably only applicable to dev/test environments where
1045      * the same Tenant Name is repeatedly used for creation/deletion.
1046      * <p>
1047      *
1048      */
1049     public void expireHeatClient (String tenantId, String cloudId) {
1050         String cacheKey = cloudId + ":" + tenantId;
1051         if (heatClientCache.containsKey (cacheKey)) {
1052             heatClientCache.remove (cacheKey);
1053             LOGGER.debug ("Deleted Cached HEAT Client for " + cacheKey);
1054         }
1055     }
1056
1057     /*
1058      * Query for a Heat Stack. This function is needed in several places, so
1059      * a common method is useful. This method takes an authenticated Heat Client
1060      * (which internally identifies the cloud & tenant to search), and returns
1061      * a Stack object if found, Null if not found, or an MsoOpenstackException
1062      * if the Openstack API call fails.
1063      *
1064      * The stack name may be a simple name or a canonical name ("{name}/{id}").
1065      * When simple name is used, Openstack always returns a 302 redirect which
1066      * results in a 2nd request (to the canonical name). Note that query by
1067      * canonical name for a deleted stack returns a Stack object with status
1068      * "DELETE_COMPLETE" while query by simple name for a deleted stack returns
1069      * HTTP 404.
1070      *
1071      * @param heatClient an authenticated Heat client
1072      *
1073      * @param stackName the stack name to query
1074      *
1075      * @return a Stack object that describes the current stack or null if the
1076      * requested stack doesn't exist.
1077      *
1078      * @throws MsoOpenstackException Thrown if the Openstack API call returns an exception
1079      */
1080     protected Stack queryHeatStack (Heat heatClient, String stackName) throws MsoException {
1081         if (stackName == null) {
1082             return null;
1083         }
1084         try {
1085             OpenStackRequest <Stack> request = heatClient.getStacks ().byName (stackName);
1086             return executeAndRecordOpenstackRequest (request);
1087         } catch (OpenStackResponseException e) {
1088             if (e.getStatus () == 404) {
1089                 LOGGER.debug ("queryHeatStack - stack not found: " + stackName);
1090                 return null;
1091             } else {
1092                 // Convert the OpenStackResponseException to an MsoOpenstackException
1093                 throw heatExceptionToMsoException (e, "QueryStack");
1094             }
1095         } catch (OpenStackConnectException e) {
1096             // Connection to Openstack failed
1097             throw heatExceptionToMsoException (e, "QueryAllStack");
1098         }
1099     }
1100
1101
1102         public Map<String, Object> queryStackForOutputs(String cloudSiteId,
1103                         String tenantId, String stackName) throws MsoException {
1104                 LOGGER.debug("MsoHeatUtils.queryStackForOutputs)");
1105                 StackInfo heatStack = this.queryStack(cloudSiteId, tenantId, stackName);
1106                 if (heatStack == null || heatStack.getStatus() == HeatStatus.NOTFOUND) {
1107                         return null;
1108                 }
1109                 return heatStack.getOutputs();
1110         }
1111
1112         public void copyStringOutputsToInputs(Map<String, String> inputs,
1113                         Map<String, Object> otherStackOutputs, boolean overWrite) {
1114                 if (inputs == null || otherStackOutputs == null)
1115                         return;
1116                 for (String key : otherStackOutputs.keySet()) {
1117                         if (!inputs.containsKey(key)) {
1118                                 Object obj = otherStackOutputs.get(key);
1119                                 if (obj instanceof String) {
1120                                         inputs.put(key, (String) otherStackOutputs.get(key));
1121                                 } else if (obj instanceof JsonNode ){
1122                                         // This is a bit of mess - but I think it's the least impacting
1123                                         // let's convert it BACK to a string - then it will get converted back later
1124                                         try {
1125                                                 String str = this.convertNode((JsonNode) obj);
1126                                                 inputs.put(key, str);
1127                                         } catch (Exception e) {
1128                                                 LOGGER.debug("DANGER WILL ROBINSON: unable to convert value for JsonNode "+ key, e);
1129                                                 //effect here is this value will not have been copied to the inputs - and therefore will error out downstream
1130                                         }
1131                                 } else if (obj instanceof java.util.LinkedHashMap) {
1132                                         LOGGER.debug("LinkedHashMap - this is showing up as a LinkedHashMap instead of JsonNode");
1133                                         try {
1134                                                 String str = JSON_MAPPER.writeValueAsString(obj);
1135                                                 inputs.put(key, str);
1136                                         } catch (Exception e) {
1137                                                 LOGGER.debug("DANGER WILL ROBINSON: unable to convert value for LinkedHashMap "+ key, e);
1138                                         }
1139                                 } else if (obj instanceof Integer) {
1140                                         try {
1141                                                 String str = "" + obj;
1142                                                 inputs.put(key, str);
1143                                         } catch (Exception e) {
1144                                                 LOGGER.debug("DANGER WILL ROBINSON: unable to convert value for Integer "+ key, e);
1145                                         }
1146                                 } else {
1147                                         try {
1148                                                 String str = obj.toString();
1149                                                 inputs.put(key, str);
1150                                         } catch (Exception e) {
1151                                                 LOGGER.debug("DANGER WILL ROBINSON: unable to convert value for Other "+ key +" (" + e.getMessage() + ")", e);
1152                                                 //effect here is this value will not have been copied to the inputs - and therefore will error out downstream
1153                                         }
1154                                 }
1155                         }
1156                 }
1157                 return;
1158         }
1159         public StringBuilder requestToStringBuilder(CreateStackParam stack) {
1160                 StringBuilder sb = new StringBuilder();
1161                 sb.append("Stack:\n");
1162                 sb.append("\tStackName: " + stack.getStackName());
1163                 sb.append("\tTemplateUrl: " + stack.getTemplateUrl());
1164                 sb.append("\tTemplate: " + stack.getTemplate());
1165                 sb.append("\tEnvironment: " + stack.getEnvironment());
1166                 sb.append("\tTimeout: " + stack.getTimeoutMinutes());
1167                 sb.append("\tParameters:\n");
1168                 Map<String, Object> params = stack.getParameters();
1169                 if (params == null || params.size() < 1) {
1170                         sb.append("\nNONE");
1171                 } else {
1172                         for (String key : params.keySet()) {
1173                                 if (params.get(key) instanceof String) {
1174                                         sb.append("\n").append(key).append("=").append((String) params.get(key));
1175                                 } else if (params.get(key) instanceof JsonNode) {
1176                                         String jsonStringOut = this.convertNode((JsonNode)params.get(key));
1177                                         sb.append("\n").append(key).append("=").append(jsonStringOut);
1178                                 } else if (params.get(key) instanceof Integer) {
1179                                         String integerOut = "" + params.get(key);
1180                                         sb.append("\n").append(key).append("=").append(integerOut);
1181
1182                                 } else {
1183                                         try {
1184                                                 String str = params.get(key).toString();
1185                                                 sb.append("\n").append(key).append("=").append(str);
1186                                         } catch (Exception e) {
1187                                                 LOGGER.debug("Exception :",e);
1188                                         }
1189                                 }
1190                         }
1191                 }
1192                 return sb;
1193         }
1194
1195         private String convertNode(final JsonNode node) {
1196                 try {
1197                         final Object obj = JSON_MAPPER.treeToValue(node, Object.class);
1198                         final String json = JSON_MAPPER.writeValueAsString(obj);
1199                         return json;
1200                 } catch (Exception e) {
1201                         LOGGER.debug("Error converting json to string " + e.getMessage(), e);
1202                 }
1203                 return "[Error converting json to string]";
1204         }
1205
1206
1207         private StringBuilder getOutputsAsStringBuilder(Stack heatStack) {
1208                 // This should only be used as a utility to print out the stack outputs
1209                 // to the log
1210                 StringBuilder sb = new StringBuilder("");
1211                 if (heatStack == null) {
1212                         sb.append("(heatStack is null)");
1213                         return sb;
1214                 }
1215                 List<Output> outputList = heatStack.getOutputs();
1216                 if (outputList == null || outputList.isEmpty()) {
1217                         sb.append("(outputs is empty)");
1218                         return sb;
1219                 }
1220                 Map<String, Object> outputs = new HashMap<>();
1221                 for (Output outputItem : outputList) {
1222                         outputs.put(outputItem.getOutputKey(), outputItem.getOutputValue());
1223                 }
1224                 int counter = 0;
1225                 sb.append("OUTPUTS:\n");
1226                 for (String key : outputs.keySet()) {
1227                         sb.append("outputs[").append(counter++).append("]: ").append(key).append("=");
1228                         Object obj = outputs.get(key);
1229                         if (obj instanceof String) {
1230                                 sb.append((String) obj).append(" (a string)");
1231                         } else if (obj instanceof JsonNode) {
1232                                 sb.append(this.convertNode((JsonNode) obj)).append(" (a JsonNode)");
1233                         } else if (obj instanceof java.util.LinkedHashMap) {
1234                                 try {
1235                                         String str = JSON_MAPPER.writeValueAsString(obj);
1236                                         sb.append(str).append(" (a java.util.LinkedHashMap)");
1237                                 } catch (Exception e) {
1238                                         LOGGER.debug("Exception :",e);
1239                                         sb.append("(a LinkedHashMap value that would not convert nicely)");
1240                                 }
1241                         } else if (obj instanceof Integer) {
1242                                 String str = "";
1243                                 try {
1244                                         str = obj.toString() + " (an Integer)\n";
1245                                 } catch (Exception e) {
1246                                         LOGGER.debug("Exception :",e);
1247                                         str = "(an Integer unable to call .toString() on)";
1248                                 }
1249                                 sb.append(str);
1250                         } else if (obj instanceof ArrayList) {
1251                                 String str = "";
1252                                 try {
1253                                         str = obj.toString() + " (an ArrayList)";
1254                                 } catch (Exception e) {
1255                                         LOGGER.debug("Exception :",e);
1256                                         str = "(an ArrayList unable to call .toString() on?)";
1257                                 }
1258                                 sb.append(str);
1259                         } else if (obj instanceof Boolean) {
1260                                 String str = "";
1261                                 try {
1262                                         str = obj.toString() + " (a Boolean)";
1263                                 } catch (Exception e) {
1264                                         LOGGER.debug("Exception :",e);
1265                                         str = "(an Boolean unable to call .toString() on?)";
1266                                 }
1267                                 sb.append(str);
1268                         }
1269                         else {
1270                                 String str = "";
1271                                 try {
1272                                         str = obj.toString() + " (unknown Object type)";
1273                                 } catch (Exception e) {
1274                                         LOGGER.debug("Exception :",e);
1275                                         str = "(a value unable to call .toString() on?)";
1276                                 }
1277                                 sb.append(str);
1278                         }
1279                         sb.append("\n");
1280                 }
1281                 sb.append("[END]");
1282                 return sb;
1283         }
1284
1285
1286         public void copyBaseOutputsToInputs(Map<String, Object> inputs,
1287                         Map<String, Object> otherStackOutputs, List<String> paramNames, Map<String, String> aliases) {
1288                 if (inputs == null || otherStackOutputs == null)
1289                         return;
1290                 for (String key : otherStackOutputs.keySet()) {
1291                         if (paramNames != null) {
1292                                 if (!paramNames.contains(key) && !aliases.containsKey(key)) {
1293                                         LOGGER.debug("\tParameter " + key + " is NOT defined to be in the template - do not copy to inputs");
1294                                         continue;
1295                                 }
1296                                 if (aliases.containsKey(key)) {
1297                                         LOGGER.debug("Found an alias! Will move " + key + " to " + aliases.get(key));
1298                                         Object obj = otherStackOutputs.get(key);
1299                                         key = aliases.get(key);
1300                                         otherStackOutputs.put(key, obj);
1301                                 }
1302                         }
1303                         if (!inputs.containsKey(key)) {
1304                                 Object obj = otherStackOutputs.get(key);
1305                                 LOGGER.debug("\t**Adding " + key + " to inputs (.toString()=" + obj.toString());
1306                                 if (obj instanceof String) {
1307                                         LOGGER.debug("\t\t**A String");
1308                                         inputs.put(key, obj);
1309                                 } else if (obj instanceof Integer) {
1310                                         LOGGER.debug("\t\t**An Integer");
1311                                         inputs.put(key, obj);
1312                                 } else if (obj instanceof JsonNode) {
1313                                         LOGGER.debug("\t\t**A JsonNode");
1314                                         inputs.put(key, obj);
1315                                 } else if (obj instanceof Boolean) {
1316                                         LOGGER.debug("\t\t**A Boolean");
1317                                         inputs.put(key, obj);
1318                                 } else if (obj instanceof java.util.LinkedHashMap) {
1319                                         LOGGER.debug("\t\t**A java.util.LinkedHashMap **");
1320                                         inputs.put(key, obj);
1321                                 } else if (obj instanceof java.util.ArrayList) {
1322                                         LOGGER.debug("\t\t**An ArrayList");
1323                                         inputs.put(key, obj);
1324                                 } else {
1325                                         LOGGER.debug("\t\t**UNKNOWN OBJECT TYPE");
1326                                         inputs.put(key, obj);
1327                                 }
1328                         } else {
1329                                 LOGGER.debug("key=" + key + " is already in the inputs - will not overwrite");
1330                         }
1331                 }
1332                 return;
1333         }
1334
1335         public List<String> convertCdlToArrayList(String cdl) {
1336                 String cdl2 = cdl.trim();
1337                 String cdl3;
1338                 if (cdl2.startsWith("[") && cdl2.endsWith("]")) {
1339                         cdl3 = cdl2.substring(1, cdl2.lastIndexOf("]"));
1340                 } else {
1341                         cdl3 = cdl2;
1342                 }
1343                 return new ArrayList<>(Arrays.asList(cdl3.split(",")));
1344         }
1345
1346     /**
1347      * New with 1707 - this method will convert all the String *values* of the inputs
1348      * to their "actual" object type (based on the param type: in the db - which comes from the template):
1349      * (heat variable type) -> java Object type
1350      * string -> String
1351      * number -> Integer
1352      * json -> JsonNode XXX Removed with MSO-1475 / 1802
1353      * comma_delimited_list -> ArrayList
1354      * boolean -> Boolean
1355      * if any of the conversions should fail, we will default to adding it to the inputs
1356      * as a string - see if Openstack can handle it.
1357      * Also, will remove any params that are extra.
1358      * Any aliases will be converted to their appropriate name (anyone use this feature?)
1359      * @param inputs - the Map<String, String> of the inputs received on the request
1360      * @param template the HeatTemplate object - this is so we can also verify if the param is valid for this template
1361      * @return HashMap<String, Object> of the inputs, cleaned and converted
1362      */
1363         public Map<String, Object> convertInputMap(Map<String, String> inputs, HeatTemplate template) {
1364                 HashMap<String, Object> newInputs = new HashMap<>();
1365                 HashMap<String, HeatTemplateParam> params = new HashMap<>();
1366                 HashMap<String, HeatTemplateParam> paramAliases = new HashMap<>();
1367
1368                 if (inputs == null) {
1369                         LOGGER.debug("convertInputMap - inputs is null - nothing to do here");
1370                         return new HashMap<>();
1371                 }
1372
1373                 LOGGER.debug("convertInputMap in MsoHeatUtils called, with " + inputs.size() + " inputs, and template " + template.getArtifactUuid());
1374                 try {
1375                         LOGGER.debug(template.toString());
1376                         Set<HeatTemplateParam> paramSet = template.getParameters();
1377                         LOGGER.debug("paramSet has " + paramSet.size() + " entries");
1378                 } catch (Exception e) {
1379                         LOGGER.debug("Exception occurred in convertInputMap:" + e.getMessage(), e);
1380                 }
1381
1382                 for (HeatTemplateParam htp : template.getParameters()) {
1383                         LOGGER.debug("Adding " + htp.getParamName());
1384                         params.put(htp.getParamName(), htp);
1385                         if (htp.getParamAlias() != null && !"".equals(htp.getParamAlias())) {
1386                                 LOGGER.debug("\tFound ALIAS " + htp.getParamName() + "->" + htp.getParamAlias());
1387                                 paramAliases.put(htp.getParamAlias(), htp);
1388                         }
1389                 }
1390                 LOGGER.debug("Now iterate through the inputs...");
1391                 for (String key : inputs.keySet()) {
1392                         LOGGER.debug("key=" + key);
1393                         boolean alias = false;
1394                         String realName = null;
1395                         if (!params.containsKey(key)) {
1396                                 LOGGER.debug(key + " is not a parameter in the template! - check for an alias");
1397                                 // add check here for an alias
1398                                 if (!paramAliases.containsKey(key)) {
1399                                         LOGGER.debug("The parameter " + key + " is in the inputs, but it's not a parameter for this template - omit");
1400                                         continue;
1401                                 } else {
1402                                         alias = true;
1403                                         realName = paramAliases.get(key).getParamName();
1404                                         LOGGER.debug("FOUND AN ALIAS! Will use " + realName + " in lieu of give key/alias " + key);
1405                                 }
1406                         }
1407                         String type = params.get(key).getParamType();
1408                         if (type == null || "".equals(type)) {
1409                                 LOGGER.debug("**PARAM_TYPE is null/empty for " + key + ", will default to string");
1410                                 type = "string";
1411                         }
1412                         LOGGER.debug("Parameter: " + key + " is of type " + type);
1413                         if ("string".equalsIgnoreCase(type)) {
1414                                 // Easiest!
1415                                 String str = inputs.get(key);
1416                                 if (alias)
1417                                         newInputs.put(realName, str);
1418                                 else
1419                                         newInputs.put(key, str);
1420                         } else if ("number".equalsIgnoreCase(type)) {
1421                                 String integerString = inputs.get(key);
1422                                 Integer anInteger = null;
1423                                 try {
1424                                         anInteger = Integer.parseInt(integerString);
1425                                 } catch (Exception e) {
1426                                         LOGGER.debug("Unable to convert " + integerString + " to an integer!!", e);
1427                                         anInteger = null;
1428                                 }
1429                                 if (anInteger != null) {
1430                                         if (alias)
1431                                                 newInputs.put(realName, anInteger);
1432                                         else
1433                                                 newInputs.put(key, anInteger);
1434                                 }
1435                                 else {
1436                                         if (alias)
1437                                                 newInputs.put(realName, integerString);
1438                                         else
1439                                                 newInputs.put(key, integerString);
1440                                 }
1441                         } else if ("json".equalsIgnoreCase(type)) {
1442                                 // MSO-1475 - Leave this as a string now
1443                                 String jsonString = inputs.get(key);
1444                                 LOGGER.debug("Skipping conversion to jsonNode...");
1445                         if (alias)
1446                                 newInputs.put(realName, jsonString);
1447                         else
1448                                 newInputs.put(key, jsonString);
1449                         //}
1450                         } else if ("comma_delimited_list".equalsIgnoreCase(type)) {
1451                                 String commaSeparated = inputs.get(key);
1452                                 try {
1453                                         List<String> anArrayList = this.convertCdlToArrayList(commaSeparated);
1454                                         if (alias)
1455                                                 newInputs.put(realName, anArrayList);
1456                                         else
1457                                                 newInputs.put(key, anArrayList);
1458                                 } catch (Exception e) {
1459                                         LOGGER.debug("Unable to convert " + commaSeparated + " to an ArrayList!!", e);
1460                                         if (alias)
1461                                                 newInputs.put(realName, commaSeparated);
1462                                         else
1463                                                 newInputs.put(key, commaSeparated);
1464                                 }
1465                         } else if ("boolean".equalsIgnoreCase(type)) {
1466                                 String booleanString = inputs.get(key);
1467                                 Boolean aBool = Boolean.valueOf(booleanString);
1468                                 if (alias)
1469                                         newInputs.put(realName, aBool);
1470                                 else
1471                                         newInputs.put(key, aBool);
1472                         } else {
1473                                 // it's null or something undefined - just add it back as a String
1474                                 String str = inputs.get(key);
1475                                 if (alias)
1476                                         newInputs.put(realName, str);
1477                                 else
1478                                         newInputs.put(key, str);
1479                         }
1480                 }
1481                 return newInputs;
1482         }
1483
1484         /*
1485          * This helpful method added for Valet
1486          */
1487         public String getCloudSiteKeystoneUrl(String cloudSiteId) throws MsoCloudSiteNotFound {
1488                 String keystone_url = null;
1489                 try {
1490                         CloudSite cloudSite = cloudConfig.getCloudSite(cloudSiteId).orElseThrow(() -> new MsoCloudSiteNotFound(cloudSiteId));
1491                         CloudIdentity cloudIdentity = cloudConfig.getIdentityService(cloudSite.getIdentityServiceId());
1492                         keystone_url = cloudIdentity.getIdentityUrl();
1493                 } catch (Exception e) {
1494                         throw new MsoCloudSiteNotFound(cloudSiteId);
1495                 }
1496                 if (keystone_url == null || keystone_url.isEmpty()) {
1497                         throw new MsoCloudSiteNotFound(cloudSiteId);
1498                 }
1499                 return keystone_url;
1500         }
1501
1502         /*
1503          * Create a string suitable for being dumped to a debug log that creates a
1504          * pseudo-JSON request dumping what's being sent to Openstack API in the create or update request
1505          */
1506
1507         private String printStackRequest(String tenantId,
1508                         Map<String, Object> heatFiles,
1509                         Map<String, Object> nestedTemplates,
1510                         String environment,
1511                         Map<String, Object> inputs,
1512                         String vfModuleName,
1513                         String template,
1514                         int timeoutMinutes,
1515                         boolean backout,
1516                         String cloudSiteId) {
1517                 StringBuilder sb = new StringBuilder();
1518                 sb.append("CREATE STACK REQUEST (formatted for readability)\n");
1519                 sb.append("tenant=" + tenantId + ", cloud=" + cloudSiteId);
1520                 sb.append("{\n");
1521                 sb.append("  \"stack_name\": \"" + vfModuleName + "\",\n");
1522                 sb.append("  \"disable_rollback\": " + backout + ",\n");
1523                 sb.append("  \"timeout_mins\": " + timeoutMinutes + ",\n");
1524                 sb.append("  \"template\": {\n");
1525                 sb.append(template);
1526                 sb.append("  },\n");
1527                 sb.append("  \"environment\": {\n");
1528                 if (environment == null)
1529                         sb.append("<none>");
1530                 else
1531                         sb.append(environment);
1532                 sb.append("  },\n");
1533                 sb.append("  \"files\": {\n");
1534                 int filesCounter = 0;
1535                 if (heatFiles != null) {
1536                         for (String key : heatFiles.keySet()) {
1537                                 filesCounter++;
1538                                 if (filesCounter > 1) {
1539                                         sb.append(",\n");
1540                                 }
1541                                 sb.append("    \"" + key + "\": {\n");
1542                                 sb.append(heatFiles.get(key).toString() + "\n    }");
1543                         }
1544                 }
1545                 if (nestedTemplates != null) {
1546                         for (String key : nestedTemplates.keySet()) {
1547                                 filesCounter++;
1548                                 if (filesCounter > 1) {
1549                                         sb.append(",\n");
1550                                 }
1551                                 sb.append("    \"" + key + "\": {\n");
1552                                 sb.append(nestedTemplates.get(key).toString() + "\n    }");
1553                         }
1554                 }
1555                 sb.append("\n  },\n");
1556                 sb.append("  \"parameters\": {\n");
1557                 int paramCounter = 0;
1558                 for (String name : inputs.keySet()) {
1559                         paramCounter++;
1560                         if (paramCounter > 1) {
1561                                 sb.append(",\n");
1562                         }
1563                         Object o = inputs.get(name);
1564                         if (o instanceof java.lang.String) {
1565                                 sb.append("    \"" + name + "\": \"" + inputs.get(name).toString() + "\"");
1566                         } else if (o instanceof Integer) {
1567                                 sb.append("    \"" + name + "\": " + inputs.get(name).toString() );
1568                         } else if (o instanceof ArrayList) {
1569                                 sb.append("    \"" + name + "\": " + inputs.get(name).toString() );
1570                         } else if (o instanceof Boolean) {
1571                                 sb.append("    \"" + name + "\": " + inputs.get(name).toString() );
1572                         } else {
1573                                 sb.append("    \"" + name + "\": " + "\"(there was an issue trying to dump this value...)\"" );
1574                         }
1575                 }
1576                 sb.append("\n  }\n}\n");
1577
1578                 return sb.toString();
1579         }
1580
1581         /*******************************************************************************
1582      *
1583      * Methods (and associated utilities) to implement the VduPlugin interface
1584      *
1585      *******************************************************************************/
1586
1587     /**
1588      * VduPlugin interface for instantiate function.
1589      *
1590      * Translate the VduPlugin parameters to the corresponding 'createStack' parameters,
1591      * and then invoke the existing function.
1592      */
1593     @Override
1594         public VduInstance instantiateVdu (
1595                         CloudInfo cloudInfo,
1596                         String instanceName,
1597                         Map<String,Object> inputs,
1598                         VduModelInfo vduModel,
1599                         boolean rollbackOnFailure)
1600         throws VduException
1601     {
1602         String cloudSiteId = cloudInfo.getCloudSiteId();
1603         String tenantId = cloudInfo.getTenantId();
1604
1605         // Translate the VDU ModelInformation structure to that which is needed for
1606         // creating the Heat stack.  Loop through the artifacts, looking specifically
1607         // for MAIN_TEMPLATE and ENVIRONMENT.  Any other artifact will
1608         // be attached as a FILE.
1609         String heatTemplate = null;
1610         Map<String,Object> nestedTemplates = new HashMap<>();
1611         Map<String,Object> files = new HashMap<>();
1612         String heatEnvironment = null;
1613
1614         for (VduArtifact vduArtifact: vduModel.getArtifacts()) {
1615                 if (vduArtifact.getType() == ArtifactType.MAIN_TEMPLATE) {
1616                         heatTemplate = new String(vduArtifact.getContent());
1617                 }
1618                 else if (vduArtifact.getType() == ArtifactType.NESTED_TEMPLATE) {
1619                         nestedTemplates.put(vduArtifact.getName(), new String(vduArtifact.getContent()));
1620                 }
1621                 else if (vduArtifact.getType() == ArtifactType.ENVIRONMENT) {
1622                         heatEnvironment = new String(vduArtifact.getContent());
1623                 }
1624         }
1625
1626         try {
1627             StackInfo stackInfo = createStack (cloudSiteId,
1628                     tenantId,
1629                     instanceName,
1630                     heatTemplate,
1631                     inputs,
1632                     true,       // poll for completion
1633                     vduModel.getTimeoutMinutes(),
1634                     heatEnvironment,
1635                     nestedTemplates,
1636                     files,
1637                     rollbackOnFailure);
1638
1639             // Populate a vduInstance from the StackInfo
1640                 return stackInfoToVduInstance(stackInfo);
1641         }
1642         catch (Exception e) {
1643                 throw new VduException ("MsoHeatUtils (instantiateVDU): createStack Exception", e);
1644         }
1645     }
1646
1647
1648     /**
1649      * VduPlugin interface for query function.
1650      */
1651     @Override
1652         public VduInstance queryVdu (CloudInfo cloudInfo, String instanceId)
1653         throws VduException
1654     {
1655         String cloudSiteId = cloudInfo.getCloudSiteId();
1656         String tenantId = cloudInfo.getTenantId();
1657
1658         try {
1659                 // Query the Cloudify Deployment object and  populate a VduInstance
1660                 StackInfo stackInfo = queryStack (cloudSiteId, tenantId, instanceId);
1661
1662                 return stackInfoToVduInstance(stackInfo);
1663         }
1664         catch (Exception e) {
1665                 throw new VduException ("MsoHeatUtile (queryVdu): queryStack Exception ", e);
1666         }
1667     }
1668
1669
1670     /**
1671      * VduPlugin interface for delete function.
1672      */
1673     @Override
1674         public VduInstance deleteVdu (CloudInfo cloudInfo, String instanceId, int timeoutMinutes)
1675         throws VduException
1676     {
1677         String cloudSiteId = cloudInfo.getCloudSiteId();
1678         String tenantId = cloudInfo.getTenantId();
1679
1680         try {
1681                 // Delete the Heat stack
1682                 StackInfo stackInfo = deleteStack (tenantId, cloudSiteId, instanceId, true);
1683
1684                 // Populate a VduInstance based on the deleted Cloudify Deployment object
1685                 VduInstance vduInstance = stackInfoToVduInstance(stackInfo);
1686
1687                 // Override return state to DELETED (HeatUtils sets to NOTFOUND)
1688                 vduInstance.getStatus().setState(VduStateType.DELETED);
1689
1690                 return vduInstance;
1691         }
1692         catch (Exception e) {
1693                 throw new VduException ("Delete VDU Exception", e);
1694         }
1695     }
1696
1697
1698     /**
1699      * VduPlugin interface for update function.
1700      *
1701      * Update is currently not supported in the MsoHeatUtils implementation of VduPlugin.
1702      * Just return a VduException.
1703      *
1704      */
1705     @Override
1706         public VduInstance updateVdu (
1707                         CloudInfo cloudInfo,
1708                         String instanceId,
1709                         Map<String,Object> inputs,
1710                         VduModelInfo vduModel,
1711                         boolean rollbackOnFailure)
1712         throws VduException
1713     {
1714         throw new VduException ("MsoHeatUtils: updateVdu interface not supported");
1715     }
1716
1717
1718     /*
1719      * Convert the local DeploymentInfo object (Cloudify-specific) to a generic VduInstance object
1720      */
1721     private VduInstance stackInfoToVduInstance (StackInfo stackInfo)
1722     {
1723         VduInstance vduInstance = new VduInstance();
1724
1725         // The full canonical name as the instance UUID
1726         vduInstance.setVduInstanceId(stackInfo.getCanonicalName());
1727         vduInstance.setVduInstanceName(stackInfo.getName());
1728
1729         // Copy inputs and outputs
1730         vduInstance.setInputs(stackInfo.getParameters());
1731         vduInstance.setOutputs(stackInfo.getOutputs());
1732
1733         // Translate the status elements
1734         vduInstance.setStatus(stackStatusToVduStatus (stackInfo));
1735
1736         return vduInstance;
1737     }
1738
1739     private VduStatus stackStatusToVduStatus (StackInfo stackInfo)
1740     {
1741         VduStatus vduStatus = new VduStatus();
1742
1743         // Map the status fields to more generic VduStatus.
1744         // There are lots of HeatStatus values, so this is a bit long...
1745         HeatStatus heatStatus = stackInfo.getStatus();
1746         String statusMessage = stackInfo.getStatusMessage();
1747
1748         if (heatStatus == HeatStatus.INIT  ||  heatStatus == HeatStatus.BUILDING) {
1749                 vduStatus.setState(VduStateType.INSTANTIATING);
1750                 vduStatus.setLastAction((new PluginAction ("create", "in_progress", statusMessage)));
1751         }
1752         else if (heatStatus == HeatStatus.NOTFOUND) {
1753                 vduStatus.setState(VduStateType.NOTFOUND);
1754         }
1755         else if (heatStatus == HeatStatus.CREATED) {
1756                 vduStatus.setState(VduStateType.INSTANTIATED);
1757                 vduStatus.setLastAction((new PluginAction ("create", "complete", statusMessage)));
1758         }
1759         else if (heatStatus == HeatStatus.UPDATED) {
1760                 vduStatus.setState(VduStateType.INSTANTIATED);
1761                 vduStatus.setLastAction((new PluginAction ("update", "complete", statusMessage)));
1762         }
1763         else if (heatStatus == HeatStatus.UPDATING) {
1764                 vduStatus.setState(VduStateType.UPDATING);
1765                 vduStatus.setLastAction((new PluginAction ("update", "in_progress", statusMessage)));
1766         }
1767         else if (heatStatus == HeatStatus.DELETING) {
1768                 vduStatus.setState(VduStateType.DELETING);
1769                 vduStatus.setLastAction((new PluginAction ("delete", "in_progress", statusMessage)));
1770         }
1771         else if (heatStatus == HeatStatus.FAILED) {
1772                 vduStatus.setState(VduStateType.FAILED);
1773                 vduStatus.setErrorMessage(stackInfo.getStatusMessage());
1774         } else {
1775                 vduStatus.setState(VduStateType.UNKNOWN);
1776         }
1777
1778         return vduStatus;
1779     }
1780
1781     private void sleep(long time) {
1782         try {
1783             Thread.sleep(time);
1784         } catch (InterruptedException e) {
1785             LOGGER.debug ("Thread interrupted while sleeping", e);
1786             Thread.currentThread().interrupt();
1787         }
1788     }
1789
1790 }