adapters/mso-adapter-utils/src/main/java/org/onap/so/openstack/utils/MsoHeatUtils.java

   1 /*-
   2  * ============LICENSE_START=======================================================
   3  * ONAP - SO
   4  * ================================================================================
   5  * Copyright (C) 2017 AT&T Intellectual Property. All rights reserved.
   6  * Copyright (C) 2017 Huawei Technologies Co., Ltd. All rights reserved.
   7  * ================================================================================
   8  * Licensed under the Apache License, Version 2.0 (the "License");
   9  * you may not use this file except in compliance with the License.
  10  * You may obtain a copy of the License at
  11  *
  12  *      http://www.apache.org/licenses/LICENSE-2.0
  13  *
  14  * Unless required by applicable law or agreed to in writing, software
  15  * distributed under the License is distributed on an "AS IS" BASIS,
  16  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  17  * See the License for the specific language governing permissions and
  18  * limitations under the License.
  19  * ============LICENSE_END=========================================================
  20  */
  21
  22 package org.onap.so.openstack.utils;
  23
  24 import java.io.IOException;
  25 import java.util.ArrayList;
  26 import java.util.Arrays;
  27 import java.util.HashMap;
  28 import java.util.List;
  29 import java.util.Map;
  30 import java.util.Map.Entry;
  31 import java.util.Set;
  32
  33 import org.onap.so.adapters.vdu.CloudInfo;
  34 import org.onap.so.adapters.vdu.PluginAction;
  35 import org.onap.so.adapters.vdu.VduArtifact;
  36 import org.onap.so.adapters.vdu.VduArtifact.ArtifactType;
  37 import org.onap.so.adapters.vdu.VduException;
  38 import org.onap.so.adapters.vdu.VduInstance;
  39 import org.onap.so.adapters.vdu.VduModelInfo;
  40 import org.onap.so.adapters.vdu.VduPlugin;
  41 import org.onap.so.adapters.vdu.VduStateType;
  42 import org.onap.so.adapters.vdu.VduStatus;
  43 import org.onap.so.cloud.CloudConfig;
  44 import org.onap.so.db.catalog.beans.CloudIdentity;
  45 import org.onap.so.db.catalog.beans.CloudSite;
  46 import org.onap.so.cloud.authentication.AuthenticationMethodFactory;
  47 import org.onap.so.db.catalog.beans.HeatTemplate;
  48 import org.onap.so.db.catalog.beans.HeatTemplateParam;
  49 import org.onap.so.logger.MessageEnum;
  50 import org.onap.so.logger.MsoAlarmLogger;
  51 import org.onap.so.logger.MsoLogger;
  52 import org.onap.so.openstack.beans.HeatCacheEntry;
  53 import org.onap.so.openstack.beans.HeatStatus;
  54 import org.onap.so.openstack.beans.StackInfo;
  55 import org.onap.so.openstack.exceptions.MsoAdapterException;
  56 import org.onap.so.openstack.exceptions.MsoCloudSiteNotFound;
  57 import org.onap.so.openstack.exceptions.MsoException;
  58 import org.onap.so.openstack.exceptions.MsoIOException;
  59 import org.onap.so.openstack.exceptions.MsoOpenstackException;
  60 import org.onap.so.openstack.exceptions.MsoStackAlreadyExists;
  61 import org.onap.so.openstack.exceptions.MsoTenantNotFound;
  62 import org.onap.so.openstack.mappers.StackInfoMapper;
  63 import org.onap.so.utils.CryptoUtils;
  64 import org.springframework.beans.factory.annotation.Autowired;
  65 import org.springframework.context.annotation.Primary;
  66 import org.springframework.core.env.Environment;
  67 import org.springframework.stereotype.Component;
  68
  69 import com.fasterxml.jackson.core.type.TypeReference;
  70 import com.fasterxml.jackson.databind.JsonNode;
  71 import com.fasterxml.jackson.databind.ObjectMapper;
  72 import com.woorea.openstack.base.client.OpenStackConnectException;
  73 import com.woorea.openstack.base.client.OpenStackRequest;
  74 import com.woorea.openstack.base.client.OpenStackResponseException;
  75 import com.woorea.openstack.heat.Heat;
  76 import com.woorea.openstack.heat.model.CreateStackParam;
  77 import com.woorea.openstack.heat.model.Stack;
  78 import com.woorea.openstack.heat.model.Stack.Output;
  79 import com.woorea.openstack.heat.model.Stacks;
  80 import com.woorea.openstack.keystone.Keystone;
  81 import com.woorea.openstack.keystone.model.Access;
  82 import com.woorea.openstack.keystone.model.Authentication;
  83 import com.woorea.openstack.keystone.utils.KeystoneUtils;
  84
  85 @Primary
  86 @Component
  87 public class MsoHeatUtils extends MsoCommonUtils implements VduPlugin{
  88
  89     private static final String TOKEN_AUTH = "TokenAuth";
  90
  91     private static final String QUERY_ALL_STACKS = "QueryAllStacks";
  92
  93     private static final String DELETE_STACK = "DeleteStack";
  94
  95     private static final String HEAT_ERROR = "HeatError";
  96
  97     private static final String CREATE_STACK = "CreateStack";
  98
  99     // Cache Heat Clients statically. Since there is just one MSO user, there is no
 100     // benefit to re-authentication on every request (or across different flows). The
 101     // token will be used until it expires.
 102     //
 103     // The cache key is "tenantId:cloudId"
 104     private static Map <String, HeatCacheEntry> heatClientCache = new HashMap <> ();
 105
 106     // Fetch cloud configuration each time (may be cached in CloudConfig class)
 107     @Autowired
 108     protected CloudConfig cloudConfig;
 109
 110     @Autowired
 111     private Environment environment;
 112
 113     @Autowired
 114     private AuthenticationMethodFactory authenticationMethodFactory;
 115
 116     @Autowired
 117     private MsoTenantUtilsFactory tenantUtilsFactory;
 118
 119     private static final MsoLogger LOGGER = MsoLogger.getMsoLogger (MsoLogger.Catalog.RA, MsoHeatUtils.class);
 120
 121     // Properties names and variables (with default values)
 122     protected String createPollIntervalProp = "ecomp.mso.adapters.po.pollInterval";
 123     private String deletePollIntervalProp = "ecomp.mso.adapters.po.pollInterval";
 124     private String deletePollTimeoutProp = "ecomp.mso.adapters.po.pollTimeout";
 125
 126     protected static final String createPollIntervalDefault = "15";
 127     private static final String deletePollIntervalDefault = "15";
 128
 129     private static final ObjectMapper JSON_MAPPER = new ObjectMapper();
 130
 131     /**
 132      * keep this old method signature here to maintain backwards compatibility. keep others as well.
 133      * this method does not include environment, files, or heatFiles
 134      */
 135     public StackInfo createStack (String cloudSiteId,
 136                                   String tenantId,
 137                                   String stackName,
 138                                   String heatTemplate,
 139                                   Map <String, ?> stackInputs,
 140                                   boolean pollForCompletion,
 141                                   int timeoutMinutes) throws MsoException {
 142         // Just call the new method with the environment & files variable set to null
 143         return this.createStack (cloudSiteId,
 144                                  tenantId,
 145                                  stackName,
 146                                  heatTemplate,
 147                                  stackInputs,
 148                                  pollForCompletion,
 149                                  timeoutMinutes,
 150                                  null,
 151                                  null,
 152                                  null,
 153                                  true);
 154     }
 155
 156     // This method has environment, but not files or heatFiles
 157     public StackInfo createStack (String cloudSiteId,
 158                                   String tenantId,
 159                                   String stackName,
 160                                   String heatTemplate,
 161                                   Map <String, ?> stackInputs,
 162                                   boolean pollForCompletion,
 163                                   int timeoutMinutes,
 164                                   String environment) throws MsoException {
 165         // Just call the new method with the files/heatFiles variables set to null
 166         return this.createStack (cloudSiteId,
 167                                  tenantId,
 168                                  stackName,
 169                                  heatTemplate,
 170                                  stackInputs,
 171                                  pollForCompletion,
 172                                  timeoutMinutes,
 173                                  environment,
 174                                  null,
 175                                  null,
 176                                  true);
 177     }
 178
 179     // This method has environment and files, but not heatFiles.
 180     public StackInfo createStack (String cloudSiteId,
 181                                   String tenantId,
 182                                   String stackName,
 183                                   String heatTemplate,
 184                                   Map <String, ?> stackInputs,
 185                                   boolean pollForCompletion,
 186                                   int timeoutMinutes,
 187                                   String environment,
 188                                   Map <String, Object> files) throws MsoException {
 189         return this.createStack (cloudSiteId,
 190                                  tenantId,
 191                                  stackName,
 192                                  heatTemplate,
 193                                  stackInputs,
 194                                  pollForCompletion,
 195                                  timeoutMinutes,
 196                                  environment,
 197                                  files,
 198                                  null,
 199                                  true);
 200     }
 201
 202     // This method has environment, files, heatfiles
 203     public StackInfo createStack (String cloudSiteId,
 204                                   String tenantId,
 205                                   String stackName,
 206                                   String heatTemplate,
 207                                   Map <String, ?> stackInputs,
 208                                   boolean pollForCompletion,
 209                                   int timeoutMinutes,
 210                                   String environment,
 211                                   Map <String, Object> files,
 212                                   Map <String, Object> heatFiles) throws MsoException {
 213         return this.createStack (cloudSiteId,
 214                                  tenantId,
 215                                  stackName,
 216                                  heatTemplate,
 217                                  stackInputs,
 218                                  pollForCompletion,
 219                                  timeoutMinutes,
 220                                  environment,
 221                                  files,
 222                                  heatFiles,
 223                                  true);
 224     }
 225
 226     /**
 227      * Create a new Stack in the specified cloud location and tenant. The Heat template
 228      * and parameter map are passed in as arguments, along with the cloud access credentials.
 229      * It is expected that parameters have been validated and contain at minimum the required
 230      * parameters for the given template with no extra (undefined) parameters..
 231      *
 232      * The Stack name supplied by the caller must be unique in the scope of this tenant.
 233      * However, it should also be globally unique, as it will be the identifier for the
 234      * resource going forward in Inventory. This latter is managed by the higher levels
 235      * invoking this function.
 236      *
 237      * The caller may choose to let this function poll Openstack for completion of the
 238      * stack creation, or may handle polling itself via separate calls to query the status.
 239      * In either case, a StackInfo object will be returned containing the current status.
 240      * When polling is enabled, a status of CREATED is expected. When not polling, a
 241      * status of BUILDING is expected.
 242      *
 243      * An error will be thrown if the requested Stack already exists in the specified
 244      * Tenant and Cloud.
 245      *
 246      * For 1510 - add "environment", "files" (nested templates), and "heatFiles" (get_files) as
 247      * parameters for createStack. If environment is non-null, it will be added to the stack.
 248      * The nested templates and get_file entries both end up being added to the "files" on the
 249      * stack. We must combine them before we add them to the stack if they're both non-null.
 250      *
 251      * @param cloudSiteId The cloud (may be a region) in which to create the stack.
 252      * @param tenantId The Openstack ID of the tenant in which to create the Stack
 253      * @param stackName The name of the stack to create
 254      * @param heatTemplate The Heat template
 255      * @param stackInputs A map of key/value inputs
 256      * @param pollForCompletion Indicator that polling should be handled in Java vs. in the client
 257      * @param environment An optional yaml-format string to specify environmental parameters
 258      * @param files a Map<String, Object> that lists the child template IDs (file is the string, object is an int of
 259      *        Template id)
 260      * @param heatFiles a Map<String, Object> that lists the get_file entries (fileName, fileBody)
 261      * @param backout Donot delete stack on create Failure - defaulted to True
 262      * @return A StackInfo object
 263      * @throws MsoOpenstackException Thrown if the Openstack API call returns an exception.
 264      */
 265
 266     @SuppressWarnings("unchecked")
 267     public StackInfo createStack (String cloudSiteId,
 268                                   String tenantId,
 269                                   String stackName,
 270                                   String heatTemplate,
 271                                   Map <String, ?> stackInputs,
 272                                   boolean pollForCompletion,
 273                                   int timeoutMinutes,
 274                                   String environment,
 275                                   Map <String, Object> files,
 276                                   Map <String, Object> heatFiles,
 277                                   boolean backout) throws MsoException {
 278
 279         // Take out the multicloud inputs, if present.
 280         String[] directives = { "oof_directives", "sdnc_directives", "generic_vnf_id", "vf_module_id" };
 281         for (String key : directives) {
 282             if (stackInputs.containsKey(key)) {
 283                 stackInputs.remove(key);
 284                 if (stackInputs.isEmpty()) {
 285                     break;
 286                 }
 287             }
 288         }
 289
 290         CreateStackParam stack = createStackParam(stackName, heatTemplate, stackInputs, timeoutMinutes, environment, files, heatFiles);
 291
 292         // Obtain the cloud site information where we will create the stack
 293         CloudSite cloudSite = cloudConfig.getCloudSite(cloudSiteId).orElseThrow(
 294                 () -> new MsoCloudSiteNotFound(cloudSiteId));
 295         LOGGER.debug("Found: " + cloudSite.toString());
 296         // Get a Heat client. They are cached between calls (keyed by tenantId:cloudId)
 297         // This could throw MsoTenantNotFound or MsoOpenstackException (both propagated)
 298         Heat heatClient = getHeatClient (cloudSite, tenantId);
 299         if (heatClient != null) {
 300             LOGGER.debug("Found: " + heatClient.toString());
 301         }
 302
 303         LOGGER.debug ("Ready to Create Stack (" + heatTemplate + ") with input params: " + stackInputs);
 304
 305         Stack heatStack = null;
 306         try {
 307             // Execute the actual Openstack command to create the Heat stack
 308             OpenStackRequest <Stack> request = heatClient.getStacks ().create (stack);
 309             // Begin X-Auth-User
 310             // Obtain an MSO token for the tenant
 311             CloudIdentity cloudIdentity = cloudSite.getIdentityService();
 312             // cloudIdentity.getMsoId(), cloudIdentity.getMsoPass()
 313             //req
 314             request.header ("X-Auth-User", cloudIdentity.getMsoId ());
 315             request.header ("X-Auth-Key", CryptoUtils.decryptCloudConfigPassword(cloudIdentity.getMsoPass ()));
 316             LOGGER.debug ("headers added, about to executeAndRecordOpenstackRequest");
 317             //LOGGER.debug(this.requestToStringBuilder(stack).toString());
 318             // END - try to fix X-Auth-User
 319             heatStack = executeAndRecordOpenstackRequest (request);
 320         } catch (OpenStackResponseException e) {
 321             // Since this came on the 'Create Stack' command, nothing was changed
 322             // in the cloud. Return the error as an exception.
 323             if (e.getStatus () == 409) {
 324                 // Stack already exists. Return a specific error for this case
 325                 MsoStackAlreadyExists me = new MsoStackAlreadyExists (stackName, tenantId, cloudSiteId);
 326                 me.addContext (CREATE_STACK);
 327                 throw me;
 328             } else {
 329                 // Convert the OpenStackResponseException to an MsoOpenstackException
 330                 LOGGER.debug("ERROR STATUS = " + e.getStatus() + ",\n" + e.getMessage() + "\n" + e.getLocalizedMessage());
 331                 throw heatExceptionToMsoException (e, CREATE_STACK);
 332             }
 333         } catch (OpenStackConnectException e) {
 334             // Error connecting to Openstack instance. Convert to an MsoException
 335             throw heatExceptionToMsoException (e, CREATE_STACK);
 336         } catch (RuntimeException e) {
 337             // Catch-all
 338             throw runtimeExceptionToMsoException (e, CREATE_STACK);
 339         }
 340
 341         // Subsequent access by the canonical name "<stack name>/<stack-id>".
 342         // Otherwise, simple query by name returns a 302 redirect.
 343         // NOTE: This is specific to the v1 Orchestration API.
 344         String canonicalName = stackName + "/" + heatStack.getId ();
 345
 346         // If client has requested a final response, poll for stack completion
 347         if (pollForCompletion) {
 348             // Set a time limit on overall polling.
 349             // Use the resource (template) timeout for Openstack (expressed in minutes)
 350             // and add one poll interval to give Openstack a chance to fail on its own.s
 351
 352             int createPollInterval = Integer.parseInt(this.environment.getProperty(createPollIntervalProp, createPollIntervalDefault));
 353             int pollTimeout = (timeoutMinutes * 60) + createPollInterval;
 354             // New 1610 - poll on delete if we rollback - use same values for now
 355             int deletePollInterval = createPollInterval;
 356             int deletePollTimeout = pollTimeout;
 357             boolean createTimedOut = false;
 358             StringBuilder stackErrorStatusReason = new StringBuilder("");
 359             LOGGER.debug("createPollInterval=" + createPollInterval + ", pollTimeout=" + pollTimeout);
 360
 361             while (true) {
 362                 try {
 363                     heatStack = queryHeatStack (heatClient, canonicalName);
 364                     LOGGER.debug (heatStack.getStackStatus () + " (" + canonicalName + ")");
 365                     try {
 366                         LOGGER.debug("Current stack " + this.getOutputsAsStringBuilder(heatStack).toString());
 367                     } catch (Exception e) {
 368                         LOGGER.debug("an error occurred trying to print out the current outputs of the stack", e);
 369                     }
 370
 371                     if ("CREATE_IN_PROGRESS".equals (heatStack.getStackStatus ())) {
 372                         // Stack creation is still running.
 373                         // Sleep and try again unless timeout has been reached
 374                         if (pollTimeout <= 0) {
 375                             // Note that this should not occur, since there is a timeout specified
 376                             // in the Openstack call.
 377                             LOGGER.error (MessageEnum.RA_CREATE_STACK_TIMEOUT, cloudSiteId, tenantId, stackName, heatStack.getStackStatus (), "", "", MsoLogger.ErrorCode.AvailabilityError, "Create stack timeout");
 378                             createTimedOut = true;
 379                             break;
 380                         }
 381
 382                         sleep(createPollInterval * 1000L);
 383
 384                         pollTimeout -= createPollInterval;
 385                                 LOGGER.debug("pollTimeout remaining: " + pollTimeout);
 386                     } else {
 387                         //save off the status & reason msg before we attempt delete
 388                         stackErrorStatusReason.append("Stack error (" + heatStack.getStackStatus() + "): " + heatStack.getStackStatusReason());
 389                         break;
 390                     }
 391                 } catch (MsoException me) {
 392                         // Cannot query the stack status. Something is wrong.
 393                         // Try to roll back the stack
 394                         if (!backout)
 395                         {
 396                                 LOGGER.warn(MessageEnum.RA_CREATE_STACK_ERR, "Create Stack errored, stack deletion suppressed", "", "", MsoLogger.ErrorCode.BusinessProcesssError, "Exception in Create Stack, stack deletion suppressed");
 397                         }
 398                         else
 399                         {
 400                                 try {
 401                                         LOGGER.debug("Create Stack error - unable to query for stack status - attempting to delete stack: " + canonicalName + " - This will likely fail and/or we won't be able to query to see if delete worked");
 402                                         OpenStackRequest <Void> request = heatClient.getStacks ().deleteByName (canonicalName);
 403                                         executeAndRecordOpenstackRequest (request);
 404                                         // this may be a waste of time - if we just got an exception trying to query the stack - we'll just
 405                                         // get another one, n'est-ce pas?
 406                                         boolean deleted = false;
 407                                         while (!deleted) {
 408                                                 try {
 409                                                         heatStack = queryHeatStack(heatClient, canonicalName);
 410                                                         if (heatStack != null) {
 411                                                         LOGGER.debug(heatStack.getStackStatus());
 412                                                         if ("DELETE_IN_PROGRESS".equals(heatStack.getStackStatus())) {
 413                                                                 if (deletePollTimeout <= 0) {
 414                                                                         LOGGER.error (MessageEnum.RA_CREATE_STACK_TIMEOUT, cloudSiteId, tenantId, stackName,
 415                                                                                         heatStack.getStackStatus (), "", "", MsoLogger.ErrorCode.AvailabilityError,
 416                                                                                         "Rollback: DELETE stack timeout");
 417                                                                         break;
 418                                                                 } else {
 419                                                                         sleep(deletePollInterval * 1000L);
 420                                                                         deletePollTimeout -= deletePollInterval;
 421                                                                 }
 422                                                         } else if ("DELETE_COMPLETE".equals(heatStack.getStackStatus())){
 423                                                                 LOGGER.debug("DELETE_COMPLETE for " + canonicalName);
 424                                                                 deleted = true;
 425                                                                 continue;
 426                                                         } else {
 427                                                                 //got a status other than DELETE_IN_PROGRESS or DELETE_COMPLETE - so break and evaluate
 428                                                                 break;
 429                                                         }
 430                                                 } else {
 431                                                         // assume if we can't find it - it's deleted
 432                                                         LOGGER.debug("heatStack returned null - assume the stack " + canonicalName + " has been deleted");
 433                                                         deleted = true;
 434                                                         continue;
 435                                                         }
 436
 437                                                 } catch (Exception e3) {
 438                                                         // Just log this one. We will report the original exception.
 439                                                         LOGGER.error (MessageEnum.RA_CREATE_STACK_ERR, "Create Stack: Nested exception rolling back stack: " + e3, "", "", MsoLogger.ErrorCode.BusinessProcesssError, "Create Stack: Nested exception rolling back stack on error on query");
 440
 441                                                 }
 442                                         }
 443                                 } catch (Exception e2) {
 444                                         // Just log this one. We will report the original exception.
 445                                         LOGGER.error (MessageEnum.RA_CREATE_STACK_ERR, "Create Stack: Nested exception rolling back stack: " + e2, "", "", MsoLogger.ErrorCode.BusinessProcesssError, "Create Stack: Nested exception rolling back stack");
 446                                 }
 447                         }
 448
 449                     // Propagate the original exception from Stack Query.
 450                     me.addContext (CREATE_STACK);
 451                     throw me;
 452                 }
 453             }
 454
 455             if (!"CREATE_COMPLETE".equals (heatStack.getStackStatus ())) {
 456                 LOGGER.error (MessageEnum.RA_CREATE_STACK_ERR, "Create Stack error:  Polling complete with non-success status: "
 457                               + heatStack.getStackStatus () + ", " + heatStack.getStackStatusReason (), "", "", MsoLogger.ErrorCode.BusinessProcesssError, "Create Stack error");
 458
 459                 // Rollback the stack creation, since it is in an indeterminate state.
 460                 if (!backout)
 461                 {
 462                         LOGGER.warn(MessageEnum.RA_CREATE_STACK_ERR, "Create Stack errored, stack deletion suppressed", "", "", MsoLogger.ErrorCode.BusinessProcesssError, "Create Stack error, stack deletion suppressed");
 463                 }
 464                 else
 465                 {
 466                         try {
 467                                 LOGGER.debug("Create Stack errored - attempting to DELETE stack: " + canonicalName);
 468                                 LOGGER.debug("deletePollInterval=" + deletePollInterval + ", deletePollTimeout=" + deletePollTimeout);
 469                                 OpenStackRequest <Void> request = heatClient.getStacks ().deleteByName (canonicalName);
 470                                 executeAndRecordOpenstackRequest (request);
 471                                 boolean deleted = false;
 472                                 while (!deleted) {
 473                                         try {
 474                                                 heatStack = queryHeatStack(heatClient, canonicalName);
 475                                                 if (heatStack != null) {
 476                                                         LOGGER.debug(heatStack.getStackStatus() + " (" + canonicalName + ")");
 477                                                         if ("DELETE_IN_PROGRESS".equals(heatStack.getStackStatus())) {
 478                                                                 if (deletePollTimeout <= 0) {
 479                                                                         LOGGER.error (MessageEnum.RA_CREATE_STACK_TIMEOUT, cloudSiteId, tenantId, stackName,
 480                                                                                         heatStack.getStackStatus (), "", "", MsoLogger.ErrorCode.AvailabilityError,
 481                                                                                         "Rollback: DELETE stack timeout");
 482                                                                         break;
 483                                                                 } else {
 484                                                                         sleep(deletePollInterval * 1000L);
 485                                                                         deletePollTimeout -= deletePollInterval;
 486                                                                         LOGGER.debug("deletePollTimeout remaining: " + deletePollTimeout);
 487                                                                 }
 488                                                         } else if ("DELETE_COMPLETE".equals(heatStack.getStackStatus())){
 489                                                                 LOGGER.debug("DELETE_COMPLETE for " + canonicalName);
 490                                                                 deleted = true;
 491                                                                 continue;
 492                                                         } else if ("DELETE_FAILED".equals(heatStack.getStackStatus())) {
 493                                                                 // Warn about this (?) - but still throw the original exception
 494                                                                 LOGGER.warn(MessageEnum.RA_CREATE_STACK_ERR, "Create Stack errored, stack deletion FAILED", "", "", MsoLogger.ErrorCode.BusinessProcesssError, "Create Stack error, stack deletion FAILED");
 495                                                                 LOGGER.debug("Stack deletion FAILED on a rollback of a create - " + canonicalName + ", status=" + heatStack.getStackStatus() + ", reason=" + heatStack.getStackStatusReason());
 496                                                                 break;
 497                                                         } else {
 498                                                                 //got a status other than DELETE_IN_PROGRESS or DELETE_COMPLETE - so break and evaluate
 499                                                                 break;
 500                                                         }
 501                                                 } else {
 502                                                         // assume if we can't find it - it's deleted
 503                                                         LOGGER.debug("heatStack returned null - assume the stack " + canonicalName + " has been deleted");
 504                                                         deleted = true;
 505                                                         continue;
 506                                                 }
 507
 508                                         } catch (MsoException me2) {
 509                                                 // We got an exception on the delete - don't throw this exception - throw the original - just log.
 510                                                 LOGGER.debug("Exception thrown trying to delete " + canonicalName + " on a create->rollback: " + me2.getContextMessage(), me2);
 511                                                 LOGGER.warn(MessageEnum.RA_CREATE_STACK_ERR, "Create Stack errored, then stack deletion FAILED - exception thrown", "", "", MsoLogger.ErrorCode.BusinessProcesssError, me2.getContextMessage());
 512                                         }
 513
 514                                 } // end while !deleted
 515                                 StringBuilder errorContextMessage;
 516                                 if (createTimedOut) {
 517                                         errorContextMessage = new StringBuilder("Stack Creation Timeout");
 518                                 } else {
 519                                         errorContextMessage  = stackErrorStatusReason;
 520                                 }
 521                                 if (deleted) {
 522                                         errorContextMessage.append(" - stack successfully deleted");
 523                                 } else {
 524                                         errorContextMessage.append(" - encountered an error trying to delete the stack");
 525                                 }
 526 //                              MsoOpenstackException me = new MsoOpenstackException(0, "", stackErrorStatusReason.toString());
 527  //                             me.addContext(CREATE_STACK);
 528   //                            alarmLogger.sendAlarm(HEAT_ERROR, MsoAlarmLogger.CRITICAL, me.getContextMessage());
 529    //                           throw me;
 530                         } catch (Exception e2) {
 531                                 // shouldn't happen - but handle
 532                                 LOGGER.error (MessageEnum.RA_CREATE_STACK_ERR, "Create Stack: Nested exception rolling back stack: " + e2, "", "", MsoLogger.ErrorCode.BusinessProcesssError, "Exception in Create Stack: rolling back stack");
 533                         }
 534                 }
 535                 MsoOpenstackException me = new MsoOpenstackException(0, "", stackErrorStatusReason.toString());
 536                 me.addContext(CREATE_STACK);
 537                 alarmLogger.sendAlarm(HEAT_ERROR, MsoAlarmLogger.CRITICAL, me.getContextMessage());
 538                 throw me;
 539             }
 540
 541         } else {
 542             // Get initial status, since it will have been null after the create.
 543             heatStack = queryHeatStack (heatClient, canonicalName);
 544             LOGGER.debug (heatStack.getStackStatus ());
 545         }
 546
 547         return new StackInfoMapper(heatStack).map();
 548     }
 549
 550     /**
 551      * Query for a single stack (by Name) in a tenant. This call will always return a
 552      * StackInfo object. If the stack does not exist, an "empty" StackInfo will be
 553      * returned - containing only the stack name and a status of NOTFOUND.
 554      *
 555      * @param tenantId The Openstack ID of the tenant in which to query
 556      * @param cloudSiteId The cloud identifier (may be a region) in which to query
 557      * @param stackName The name of the stack to query (may be simple or canonical)
 558      * @return A StackInfo object
 559      * @throws MsoOpenstackException Thrown if the Openstack API call returns an exception.
 560      */
 561     public StackInfo queryStack (String cloudSiteId, String tenantId, String stackName) throws MsoException {
 562         LOGGER.debug ("Query HEAT stack: " + stackName + " in tenant " + tenantId);
 563
 564         // Obtain the cloud site information where we will create the stack
 565         CloudSite cloudSite = cloudConfig.getCloudSite(cloudSiteId).orElseThrow(
 566                 () -> new MsoCloudSiteNotFound(cloudSiteId));
 567         LOGGER.debug("Found: " + cloudSite.toString());
 568
 569         // Get a Heat client. They are cached between calls (keyed by tenantId:cloudId)
 570         Heat heatClient = null;
 571         try {
 572             heatClient = getHeatClient (cloudSite, tenantId);
 573             if (heatClient != null) {
 574                 LOGGER.debug("Found: " + heatClient.toString());
 575             }
 576         } catch (MsoTenantNotFound e) {
 577             // Tenant doesn't exist, so stack doesn't either
 578             LOGGER.debug ("Tenant with id " + tenantId + "not found.", e);
 579             return new StackInfo (stackName, HeatStatus.NOTFOUND);
 580         } catch (MsoException me) {
 581             // Got an Openstack error. Propagate it
 582             LOGGER.error (MessageEnum.RA_CONNECTION_EXCEPTION, "OpenStack", "Openstack Exception on Token request: " + me, "Openstack", "", MsoLogger.ErrorCode.AvailabilityError, "Connection Exception");
 583             me.addContext ("QueryStack");
 584             throw me;
 585         }
 586
 587         // Query the Stack.
 588         // An MsoException will propagate transparently to the caller.
 589         Stack heatStack = queryHeatStack (heatClient, stackName);
 590
 591         if (heatStack == null) {
 592             // Stack does not exist. Return a StackInfo with status NOTFOUND
 593             return new StackInfo (stackName, HeatStatus.NOTFOUND);
 594         }
 595
 596         return new StackInfoMapper(heatStack).map();
 597     }
 598
 599     /**
 600      * Delete a stack (by Name/ID) in a tenant. If the stack is not found, it will be
 601      * considered a successful deletion. The return value is a StackInfo object which
 602      * contains the current stack status.
 603      *
 604      * The client may choose to let the adapter poll Openstack for completion of the
 605      * stack deletion, or may handle polling itself via separate query calls. In either
 606      * case, a StackInfo object will be returned. When polling is enabled, a final
 607      * status of NOTFOUND is expected. When not polling, a status of DELETING is expected.
 608      *
 609      * There is no rollback from a successful stack deletion. A deletion failure will
 610      * also result in an undefined stack state - the components may or may not have been
 611      * all or partially deleted, so the resulting stack must be considered invalid.
 612      *
 613      * @param tenantId The Openstack ID of the tenant in which to perform the delete
 614      * @param cloudSiteId The cloud identifier (may be a region) from which to delete the stack.
 615      * @param stackName The name/id of the stack to delete. May be simple or canonical
 616      * @param pollForCompletion Indicator that polling should be handled in Java vs. in the client
 617      * @return A StackInfo object
 618      * @throws MsoOpenstackException Thrown if the Openstack API call returns an exception.
 619      * @throws MsoCloudSiteNotFound
 620      */
 621     public StackInfo deleteStack (String tenantId,
 622                                   String cloudSiteId,
 623                                   String stackName,
 624                                   boolean pollForCompletion) throws MsoException {
 625         // Obtain the cloud site information where we will create the stack
 626         CloudSite cloudSite = cloudConfig.getCloudSite(cloudSiteId).orElseThrow(
 627                 () -> new MsoCloudSiteNotFound(cloudSiteId));
 628         LOGGER.debug("Found: " + cloudSite.toString());
 629
 630         // Get a Heat client. They are cached between calls (keyed by tenantId:cloudId)
 631         Heat heatClient = null;
 632         try {
 633             heatClient = getHeatClient (cloudSite, tenantId);
 634             if (heatClient != null) {
 635                 LOGGER.debug("Found: " + heatClient.toString());
 636             }
 637         } catch (MsoTenantNotFound e) {
 638             // Tenant doesn't exist, so stack doesn't either
 639             LOGGER.debug ("Tenant with id " + tenantId + "not found.", e);
 640             return new StackInfo (stackName, HeatStatus.NOTFOUND);
 641         } catch (MsoException me) {
 642             // Got an Openstack error. Propagate it
 643             LOGGER.error (MessageEnum.RA_CONNECTION_EXCEPTION, "Openstack", "Openstack Exception on Token request: " + me, "Openstack", "", MsoLogger.ErrorCode.AvailabilityError, "Connection Exception");
 644             me.addContext (DELETE_STACK);
 645             throw me;
 646         }
 647
 648         // OK if stack not found, perform a query first
 649         Stack heatStack = queryHeatStack (heatClient, stackName);
 650         if (heatStack == null || "DELETE_COMPLETE".equals (heatStack.getStackStatus ())) {
 651             // Not found. Return a StackInfo with status NOTFOUND
 652             return new StackInfo (stackName, HeatStatus.NOTFOUND);
 653         }
 654
 655         // Delete the stack.
 656
 657         // Use canonical name "<stack name>/<stack-id>" to delete.
 658         // Otherwise, deletion by name returns a 302 redirect.
 659         // NOTE: This is specific to the v1 Orchestration API.
 660         String canonicalName = heatStack.getStackName () + "/" + heatStack.getId ();
 661
 662         try {
 663             OpenStackRequest <Void> request = null;
 664             if(null != heatClient) {
 665                 request = heatClient.getStacks ().deleteByName (canonicalName);
 666             }
 667             else {
 668                 LOGGER.debug ("Heat Client is NULL" );
 669             }
 670
 671             executeAndRecordOpenstackRequest (request);
 672         } catch (OpenStackResponseException e) {
 673             if (e.getStatus () == 404) {
 674                 // Not found. We are OK with this. Return a StackInfo with status NOTFOUND
 675                 return new StackInfo (stackName, HeatStatus.NOTFOUND);
 676             } else {
 677                 // Convert the OpenStackResponseException to an MsoOpenstackException
 678                 throw heatExceptionToMsoException (e, DELETE_STACK);
 679             }
 680         } catch (OpenStackConnectException e) {
 681             // Error connecting to Openstack instance. Convert to an MsoException
 682             throw heatExceptionToMsoException (e, DELETE_STACK);
 683         } catch (RuntimeException e) {
 684             // Catch-all
 685             throw runtimeExceptionToMsoException (e, DELETE_STACK);
 686         }
 687
 688         // Requery the stack for current status.
 689         // It will probably still exist with "DELETE_IN_PROGRESS" status.
 690         heatStack = queryHeatStack (heatClient, canonicalName);
 691
 692         if (pollForCompletion) {
 693             // Set a timeout on polling
 694
 695             int pollInterval = Integer.parseInt(this.environment.getProperty(deletePollIntervalProp, "" + deletePollIntervalDefault));
 696             int pollTimeout = Integer.parseInt(this.environment.getProperty(deletePollTimeoutProp, "" + deletePollIntervalDefault));
 697
 698             // When querying by canonical name, Openstack returns DELETE_COMPLETE status
 699             // instead of "404" (which would result from query by stack name).
 700             while (heatStack != null && !"DELETE_COMPLETE".equals (heatStack.getStackStatus ())) {
 701                 LOGGER.debug ("Stack status: " + heatStack.getStackStatus ());
 702
 703                 if ("DELETE_FAILED".equals (heatStack.getStackStatus ())) {
 704                     // Throw a 'special case' of MsoOpenstackException to report the Heat status
 705                     String error = "Stack delete error (" + heatStack.getStackStatus ()
 706                                    + "): "
 707                                    + heatStack.getStackStatusReason ();
 708                     MsoOpenstackException me = new MsoOpenstackException (0, "", error);
 709                     me.addContext (DELETE_STACK);
 710
 711                     // Alarm this condition, stack deletion failed
 712                     alarmLogger.sendAlarm (HEAT_ERROR, MsoAlarmLogger.CRITICAL, me.getContextMessage ());
 713
 714                     throw me;
 715                 }
 716
 717                 if (pollTimeout <= 0) {
 718                     LOGGER.error (MessageEnum.RA_DELETE_STACK_TIMEOUT, cloudSiteId, tenantId, stackName, heatStack.getStackStatus (), "", "", MsoLogger.ErrorCode.AvailabilityError, "Delete Stack Timeout");
 719
 720                     // Throw a 'special case' of MsoOpenstackException to report the Heat status
 721                     MsoOpenstackException me = new MsoOpenstackException (0, "", "Stack Deletion Timeout");
 722                     me.addContext (DELETE_STACK);
 723
 724                     // Alarm this condition, stack deletion failed
 725                     alarmLogger.sendAlarm (HEAT_ERROR, MsoAlarmLogger.CRITICAL, me.getContextMessage ());
 726
 727                     throw me;
 728                 }
 729
 730                 sleep(pollInterval * 1000L);
 731
 732                 pollTimeout -= pollInterval;
 733                 LOGGER.debug("pollTimeout remaining: " + pollTimeout);
 734
 735                 heatStack = queryHeatStack (heatClient, canonicalName);
 736             }
 737
 738             // The stack is gone when this point is reached
 739             return new StackInfo (stackName, HeatStatus.NOTFOUND);
 740         }
 741
 742         // Return the current status (if not polling, the delete may still be in progress)
 743         StackInfo stackInfo = new StackInfoMapper(heatStack).map();
 744         stackInfo.setName (stackName);
 745
 746         return stackInfo;
 747     }
 748
 749     /**
 750      * Query for all stacks in a tenant site. This call will return a List of StackInfo
 751      * objects, one for each deployed stack.
 752      *
 753      * Note that this is limited to a single site. To ensure that a tenant is truly
 754      * empty would require looping across all tenant endpoints.
 755      *
 756      * @param tenantId The Openstack ID of the tenant to query
 757      * @param cloudSiteId The cloud identifier (may be a region) in which to query.
 758      * @return A List of StackInfo objects
 759      * @throws MsoOpenstackException Thrown if the Openstack API call returns an exception.
 760      * @throws MsoCloudSiteNotFound
 761      */
 762     public List <StackInfo> queryAllStacks (String tenantId, String cloudSiteId) throws MsoException {
 763         // Obtain the cloud site information where we will create the stack
 764         CloudSite cloudSite = cloudConfig.getCloudSite(cloudSiteId).orElseThrow(
 765                 () -> new MsoCloudSiteNotFound(cloudSiteId));
 766         // Get a Heat client. They are cached between calls (keyed by tenantId:cloudId)
 767         Heat heatClient = getHeatClient (cloudSite, tenantId);
 768
 769         try {
 770             OpenStackRequest <Stacks> request = heatClient.getStacks ().list ();
 771             Stacks stacks = executeAndRecordOpenstackRequest (request);
 772
 773             List <StackInfo> stackList = new ArrayList <> ();
 774
 775             // Not sure if returns an empty list or null if no stacks exist
 776             if (stacks != null) {
 777                 for (Stack stack : stacks) {
 778                     stackList.add (new StackInfoMapper(stack).map());
 779                 }
 780             }
 781
 782             return stackList;
 783         } catch (OpenStackResponseException e) {
 784             if (e.getStatus () == 404) {
 785                 // Not sure if this can happen, but return an empty list
 786                 LOGGER.debug ("queryAllStacks - stack not found: ");
 787                 return new ArrayList <> ();
 788             } else {
 789                 // Convert the OpenStackResponseException to an MsoOpenstackException
 790                 throw heatExceptionToMsoException (e, QUERY_ALL_STACKS);
 791             }
 792         } catch (OpenStackConnectException e) {
 793             // Error connecting to Openstack instance. Convert to an MsoException
 794             throw heatExceptionToMsoException (e, QUERY_ALL_STACKS);
 795         } catch (RuntimeException e) {
 796             // Catch-all
 797             throw runtimeExceptionToMsoException (e, QUERY_ALL_STACKS);
 798         }
 799     }
 800
 801     /**
 802      * Validate parameters to be passed to Heat template. This method performs
 803      * three functions:
 804      * 1. Apply default values to parameters which have them defined
 805      * 2. Report any required parameters that are missing. This will generate an
 806      * exception in the caller, since stack create/update operations would fail.
 807      * 3. Report and remove any extraneous parameters. This will allow clients to
 808      * pass supersets of parameters and not get errors.
 809      *
 810      * These functions depend on the HeatTemplate definition from the MSO Catalog DB,
 811      * along with the input parameter Map. The output is an updated parameter map.
 812      * If the parameters are invalid for the template, an IllegalArgumentException
 813      * is thrown.
 814      */
 815     public Map <String, Object> validateStackParams (Map <String, Object> inputParams,
 816                                                      HeatTemplate heatTemplate) {
 817         // Check that required parameters have been supplied for this template type
 818         StringBuilder missingParams = null;
 819         List <String> paramList = new ArrayList <> ();
 820
 821         // TODO: Enhance DB to support defaults for Heat Template parameters
 822
 823         for (HeatTemplateParam parm : heatTemplate.getParameters ()) {
 824             if (parm.isRequired () && !inputParams.containsKey (parm.getParamName ())) {
 825                 if (missingParams == null) {
 826                     missingParams = new StringBuilder(parm.getParamName());
 827                 } else {
 828                     missingParams.append("," + parm.getParamName());
 829                 }
 830             }
 831             paramList.add (parm.getParamName ());
 832         }
 833         if (missingParams != null) {
 834             // Problem - missing one or more required parameters
 835             String error = "Missing Required inputs for HEAT Template: " + missingParams;
 836             LOGGER.error (MessageEnum.RA_MISSING_PARAM, missingParams + " for HEAT Template", "", "", MsoLogger.ErrorCode.SchemaError, "Missing Required inputs for HEAT Template: " + missingParams);
 837             throw new IllegalArgumentException (error);
 838         }
 839
 840         // Remove any extraneous parameters (don't throw an error)
 841         Map <String, Object> updatedParams = new HashMap <> ();
 842         List <String> extraParams = new ArrayList <> ();
 843
 844         for (Entry<String, Object> entry : inputParams.entrySet()) {
 845                 if (!paramList.contains(entry.getKey())) {
 846                         // This is not a valid parameter for this template
 847                         extraParams.add(entry.getKey());
 848                 } else {
 849                         updatedParams.put(entry.getKey(), entry.getValue());
 850                 }
 851         }
 852
 853         if (!extraParams.isEmpty ()) {
 854             LOGGER.warn (MessageEnum.RA_GENERAL_WARNING, "Heat Stack (" + heatTemplate.getTemplateName ()
 855                          + ") extra input params received: "
 856                          + extraParams, "", "", MsoLogger.ErrorCode.DataError, "Heat Stack (" + heatTemplate.getTemplateName () + ") extra input params received: "+ extraParams);
 857         }
 858
 859         return updatedParams;
 860     }
 861
 862     // ---------------------------------------------------------------
 863     // PRIVATE FUNCTIONS FOR USE WITHIN THIS CLASS
 864
 865     /**
 866      * Get a Heat client for the Openstack Identity service.
 867      * This requires a 'member'-level userId + password, which will be retrieved from
 868      * properties based on the specified cloud Id. The tenant in which to operate
 869      * must also be provided.
 870      * <p>
 871      * On successful authentication, the Heat object will be cached for the
 872      * tenantID + cloudId so that it can be reused without reauthenticating with
 873      * Openstack every time.
 874      *
 875      * @return an authenticated Heat object
 876      */
 877     public Heat getHeatClient (CloudSite cloudSite, String tenantId) throws MsoException {
 878         String cloudId = cloudSite.getId();
 879
 880         // Check first in the cache of previously authorized clients
 881         String cacheKey = cloudId + ":" + tenantId;
 882         if (heatClientCache.containsKey (cacheKey)) {
 883             if (!heatClientCache.get (cacheKey).isExpired ()) {
 884                 LOGGER.debug ("Using Cached HEAT Client for " + cacheKey);
 885                 return heatClientCache.get (cacheKey).getHeatClient ();
 886             } else {
 887                 // Token is expired. Remove it from cache.
 888                 heatClientCache.remove (cacheKey);
 889                 LOGGER.debug ("Expired Cached HEAT Client for " + cacheKey);
 890             }
 891         }
 892
 893         // Obtain an MSO token for the tenant
 894         CloudIdentity cloudIdentity = cloudSite.getIdentityService();
 895         LOGGER.debug("Found: " + cloudIdentity.toString());
 896         MsoTenantUtils tenantUtils = tenantUtilsFactory.getTenantUtilsByServerType(cloudIdentity.getIdentityServerType());
 897         String keystoneUrl = tenantUtils.getKeystoneUrl(cloudId, cloudIdentity);
 898         LOGGER.debug("keystoneUrl=" + keystoneUrl);
 899         Keystone keystoneTenantClient = new Keystone (keystoneUrl);
 900         Access access = null;
 901         try {
 902                 Authentication credentials = authenticationMethodFactory.getAuthenticationFor(cloudIdentity);
 903
 904                 OpenStackRequest <Access> request = keystoneTenantClient.tokens ()
 905                        .authenticate (credentials).withTenantId (tenantId);
 906
 907             access = executeAndRecordOpenstackRequest (request);
 908         } catch (OpenStackResponseException e) {
 909             if (e.getStatus () == 401) {
 910                 // Authentication error.
 911                 String error = "Authentication Failure: tenant=" + tenantId + ",cloud=" + cloudIdentity.getId ();
 912                 alarmLogger.sendAlarm ("MsoAuthenticationError", MsoAlarmLogger.CRITICAL, error);
 913                 throw new MsoAdapterException (error);
 914             } else {
 915                 throw keystoneErrorToMsoException (e, TOKEN_AUTH);
 916             }
 917         } catch (OpenStackConnectException e) {
 918             // Connection to Openstack failed
 919             MsoIOException me = new MsoIOException (e.getMessage (), e);
 920             me.addContext (TOKEN_AUTH);
 921             throw me;
 922         } catch (RuntimeException e) {
 923             // Catch-all
 924             throw runtimeExceptionToMsoException (e, TOKEN_AUTH);
 925         }
 926
 927         // For DCP/LCP, the region should be the cloudId.
 928         String region = cloudSite.getRegionId ();
 929         String heatUrl = null;
 930         try {
 931                 // Isolate trying to printout the region IDs
 932                 try {
 933                         LOGGER.debug("access=" + access.toString());
 934                         for (Access.Service service : access.getServiceCatalog()) {
 935                                 List<Access.Service.Endpoint> endpoints = service.getEndpoints();
 936                                 for (Access.Service.Endpoint endpoint : endpoints) {
 937                                         LOGGER.debug("AIC returned region=" + endpoint.getRegion());
 938                                 }
 939                         }
 940                 } catch (Exception e) {
 941                         LOGGER.debug("Encountered an error trying to printout Access object returned from AIC. " + e.getMessage());
 942                 }
 943             heatUrl = KeystoneUtils.findEndpointURL (access.getServiceCatalog (), "orchestration", region, "public");
 944             LOGGER.debug("heatUrl=" + heatUrl + ", region=" + region);
 945         } catch (RuntimeException e) {
 946             // This comes back for not found (probably an incorrect region ID)
 947             String error = "AIC did not match an orchestration service for: region=" + region + ",cloud=" + cloudIdentity.getIdentityUrl();
 948             alarmLogger.sendAlarm ("MsoConfigurationError", MsoAlarmLogger.CRITICAL, error);
 949             throw new MsoAdapterException (error, e);
 950         }
 951
 952         Heat heatClient = new Heat (heatUrl);
 953         heatClient.token (access.getToken ().getId ());
 954
 955         heatClientCache.put (cacheKey,
 956                              new HeatCacheEntry (heatUrl,
 957                                                  access.getToken ().getId (),
 958                                                  access.getToken ().getExpires ()));
 959         LOGGER.debug ("Caching HEAT Client for " + cacheKey);
 960
 961         return heatClient;
 962     }
 963
 964     /**
 965      * Forcibly expire a HEAT client from the cache. This call is for use by
 966      * the KeystoneClient in case where a tenant is deleted. In that case,
 967      * all cached credentials must be purged so that fresh authentication is
 968      * done if a similarly named tenant is re-created.
 969      * <p>
 970      * Note: This is probably only applicable to dev/test environments where
 971      * the same Tenant Name is repeatedly used for creation/deletion.
 972      * <p>
 973      *
 974      */
 975     public void expireHeatClient (String tenantId, String cloudId) {
 976         String cacheKey = cloudId + ":" + tenantId;
 977         if (heatClientCache.containsKey (cacheKey)) {
 978             heatClientCache.remove (cacheKey);
 979             LOGGER.debug ("Deleted Cached HEAT Client for " + cacheKey);
 980         }
 981     }
 982
 983     /*
 984      * Query for a Heat Stack. This function is needed in several places, so
 985      * a common method is useful. This method takes an authenticated Heat Client
 986      * (which internally identifies the cloud & tenant to search), and returns
 987      * a Stack object if found, Null if not found, or an MsoOpenstackException
 988      * if the Openstack API call fails.
 989      *
 990      * The stack name may be a simple name or a canonical name ("{name}/{id}").
 991      * When simple name is used, Openstack always returns a 302 redirect which
 992      * results in a 2nd request (to the canonical name). Note that query by
 993      * canonical name for a deleted stack returns a Stack object with status
 994      * "DELETE_COMPLETE" while query by simple name for a deleted stack returns
 995      * HTTP 404.
 996      *
 997      * @param heatClient an authenticated Heat client
 998      *
 999      * @param stackName the stack name to query
1000      *
1001      * @return a Stack object that describes the current stack or null if the
1002      * requested stack doesn't exist.
1003      *
1004      * @throws MsoOpenstackException Thrown if the Openstack API call returns an exception
1005      */
1006     protected Stack queryHeatStack (Heat heatClient, String stackName) throws MsoException {
1007         if (stackName == null) {
1008             return null;
1009         }
1010         try {
1011             OpenStackRequest <Stack> request = heatClient.getStacks ().byName (stackName);
1012             return executeAndRecordOpenstackRequest (request);
1013         } catch (OpenStackResponseException e) {
1014             if (e.getStatus () == 404) {
1015                 LOGGER.debug ("queryHeatStack - stack not found: " + stackName);
1016                 return null;
1017             } else {
1018                 // Convert the OpenStackResponseException to an MsoOpenstackException
1019                 throw heatExceptionToMsoException (e, "QueryStack");
1020             }
1021         } catch (OpenStackConnectException e) {
1022             // Connection to Openstack failed
1023             throw heatExceptionToMsoException (e, "QueryAllStack");
1024         }
1025     }
1026
1027
1028         public Map<String, Object> queryStackForOutputs(String cloudSiteId,
1029                         String tenantId, String stackName) throws MsoException {
1030                 LOGGER.debug("MsoHeatUtils.queryStackForOutputs)");
1031                 StackInfo heatStack = this.queryStack(cloudSiteId, tenantId, stackName);
1032                 if (heatStack == null || heatStack.getStatus() == HeatStatus.NOTFOUND) {
1033                         return null;
1034                 }
1035                 return heatStack.getOutputs();
1036         }
1037
1038         public void copyStringOutputsToInputs(Map<String, String> inputs,
1039                         Map<String, Object> otherStackOutputs, boolean overWrite) {
1040                 if (inputs == null || otherStackOutputs == null)
1041                         return;
1042                 for (String key : otherStackOutputs.keySet()) {
1043                         if (!inputs.containsKey(key)) {
1044                                 Object obj = otherStackOutputs.get(key);
1045                                 if (obj instanceof String) {
1046                                         inputs.put(key, (String) otherStackOutputs.get(key));
1047                                 } else if (obj instanceof JsonNode ){
1048                                         // This is a bit of mess - but I think it's the least impacting
1049                                         // let's convert it BACK to a string - then it will get converted back later
1050                                         try {
1051                                                 String str = this.convertNode((JsonNode) obj);
1052                                                 inputs.put(key, str);
1053                                         } catch (Exception e) {
1054                                                 LOGGER.debug("DANGER WILL ROBINSON: unable to convert value for JsonNode "+ key, e);
1055                                                 //effect here is this value will not have been copied to the inputs - and therefore will error out downstream
1056                                         }
1057                                 } else if (obj instanceof java.util.LinkedHashMap) {
1058                                         LOGGER.debug("LinkedHashMap - this is showing up as a LinkedHashMap instead of JsonNode");
1059                                         try {
1060                                                 String str = JSON_MAPPER.writeValueAsString(obj);
1061                                                 inputs.put(key, str);
1062                                         } catch (Exception e) {
1063                                                 LOGGER.debug("DANGER WILL ROBINSON: unable to convert value for LinkedHashMap "+ key, e);
1064                                         }
1065                                 } else if (obj instanceof Integer) {
1066                                         try {
1067                                                 String str = "" + obj;
1068                                                 inputs.put(key, str);
1069                                         } catch (Exception e) {
1070                                                 LOGGER.debug("DANGER WILL ROBINSON: unable to convert value for Integer "+ key, e);
1071                                         }
1072                                 } else {
1073                                         try {
1074                                                 String str = obj.toString();
1075                                                 inputs.put(key, str);
1076                                         } catch (Exception e) {
1077                                                 LOGGER.debug("DANGER WILL ROBINSON: unable to convert value for Other "+ key +" (" + e.getMessage() + ")", e);
1078                                                 //effect here is this value will not have been copied to the inputs - and therefore will error out downstream
1079                                         }
1080                                 }
1081                         }
1082                 }
1083                 return;
1084         }
1085         public StringBuilder requestToStringBuilder(CreateStackParam stack) {
1086                 StringBuilder sb = new StringBuilder();
1087                 sb.append("Stack:\n");
1088                 sb.append("\tStackName: " + stack.getStackName());
1089                 sb.append("\tTemplateUrl: " + stack.getTemplateUrl());
1090                 sb.append("\tTemplate: " + stack.getTemplate());
1091                 sb.append("\tEnvironment: " + stack.getEnvironment());
1092                 sb.append("\tTimeout: " + stack.getTimeoutMinutes());
1093                 sb.append("\tParameters:\n");
1094                 Map<String, Object> params = stack.getParameters();
1095                 if (params == null || params.size() < 1) {
1096                         sb.append("\nNONE");
1097                 } else {
1098                         for (String key : params.keySet()) {
1099                                 if (params.get(key) instanceof String) {
1100                                         sb.append("\n").append(key).append("=").append((String) params.get(key));
1101                                 } else if (params.get(key) instanceof JsonNode) {
1102                                         String jsonStringOut = this.convertNode((JsonNode)params.get(key));
1103                                         sb.append("\n").append(key).append("=").append(jsonStringOut);
1104                                 } else if (params.get(key) instanceof Integer) {
1105                                         String integerOut = "" + params.get(key);
1106                                         sb.append("\n").append(key).append("=").append(integerOut);
1107
1108                                 } else {
1109                                         try {
1110                                                 String str = params.get(key).toString();
1111                                                 sb.append("\n").append(key).append("=").append(str);
1112                                         } catch (Exception e) {
1113                                                 LOGGER.debug("Exception :",e);
1114                                         }
1115                                 }
1116                         }
1117                 }
1118                 return sb;
1119         }
1120
1121         private String convertNode(final JsonNode node) {
1122                 try {
1123                         final Object obj = JSON_MAPPER.treeToValue(node, Object.class);
1124                         final String json = JSON_MAPPER.writeValueAsString(obj);
1125                         return json;
1126                 } catch (Exception e) {
1127                         LOGGER.debug("Error converting json to string " + e.getMessage(), e);
1128                 }
1129                 return "[Error converting json to string]";
1130         }
1131
1132
1133         protected StringBuilder getOutputsAsStringBuilder(Stack heatStack) {
1134                 // This should only be used as a utility to print out the stack outputs
1135                 // to the log
1136                 StringBuilder sb = new StringBuilder("");
1137                 if (heatStack == null) {
1138                         sb.append("(heatStack is null)");
1139                         return sb;
1140                 }
1141                 List<Output> outputList = heatStack.getOutputs();
1142                 if (outputList == null || outputList.isEmpty()) {
1143                         sb.append("(outputs is empty)");
1144                         return sb;
1145                 }
1146                 Map<String, Object> outputs = new HashMap<>();
1147                 for (Output outputItem : outputList) {
1148                         outputs.put(outputItem.getOutputKey(), outputItem.getOutputValue());
1149                 }
1150                 int counter = 0;
1151                 sb.append("OUTPUTS:\n");
1152                 for (String key : outputs.keySet()) {
1153                         sb.append("outputs[").append(counter++).append("]: ").append(key).append("=");
1154                         Object obj = outputs.get(key);
1155                         if (obj instanceof String) {
1156                                 sb.append((String) obj).append(" (a string)");
1157                         } else if (obj instanceof JsonNode) {
1158                                 sb.append(this.convertNode((JsonNode) obj)).append(" (a JsonNode)");
1159                         } else if (obj instanceof java.util.LinkedHashMap) {
1160                                 try {
1161                                         String str = JSON_MAPPER.writeValueAsString(obj);
1162                                         sb.append(str).append(" (a java.util.LinkedHashMap)");
1163                                 } catch (Exception e) {
1164                                         LOGGER.debug("Exception :",e);
1165                                         sb.append("(a LinkedHashMap value that would not convert nicely)");
1166                                 }
1167                         } else if (obj instanceof Integer) {
1168                                 String str = "";
1169                                 try {
1170                                         str = obj.toString() + " (an Integer)\n";
1171                                 } catch (Exception e) {
1172                                         LOGGER.debug("Exception :",e);
1173                                         str = "(an Integer unable to call .toString() on)";
1174                                 }
1175                                 sb.append(str);
1176                         } else if (obj instanceof ArrayList) {
1177                                 String str = "";
1178                                 try {
1179                                         str = obj.toString() + " (an ArrayList)";
1180                                 } catch (Exception e) {
1181                                         LOGGER.debug("Exception :",e);
1182                                         str = "(an ArrayList unable to call .toString() on?)";
1183                                 }
1184                                 sb.append(str);
1185                         } else if (obj instanceof Boolean) {
1186                                 String str = "";
1187                                 try {
1188                                         str = obj.toString() + " (a Boolean)";
1189                                 } catch (Exception e) {
1190                                         LOGGER.debug("Exception :",e);
1191                                         str = "(an Boolean unable to call .toString() on?)";
1192                                 }
1193                                 sb.append(str);
1194                         }
1195                         else {
1196                                 String str = "";
1197                                 try {
1198                                         str = obj.toString() + " (unknown Object type)";
1199                                 } catch (Exception e) {
1200                                         LOGGER.debug("Exception :",e);
1201                                         str = "(a value unable to call .toString() on?)";
1202                                 }
1203                                 sb.append(str);
1204                         }
1205                         sb.append("\n");
1206                 }
1207                 sb.append("[END]");
1208                 return sb;
1209         }
1210
1211
1212         public void copyBaseOutputsToInputs(Map<String, Object> inputs,
1213                         Map<String, Object> otherStackOutputs, List<String> paramNames, Map<String, String> aliases) {
1214                 if (inputs == null || otherStackOutputs == null)
1215                         return;
1216                 for (String key : otherStackOutputs.keySet()) {
1217                         if (paramNames != null) {
1218                                 if (!paramNames.contains(key) && !aliases.containsKey(key)) {
1219                                         LOGGER.debug("\tParameter " + key + " is NOT defined to be in the template - do not copy to inputs");
1220                                         continue;
1221                                 }
1222                                 if (aliases.containsKey(key)) {
1223                                         LOGGER.debug("Found an alias! Will move " + key + " to " + aliases.get(key));
1224                                         Object obj = otherStackOutputs.get(key);
1225                                         key = aliases.get(key);
1226                                         otherStackOutputs.put(key, obj);
1227                                 }
1228                         }
1229                         if (!inputs.containsKey(key)) {
1230                                 Object obj = otherStackOutputs.get(key);
1231                                 LOGGER.debug("\t**Adding " + key + " to inputs (.toString()=" + obj.toString());
1232                                 if (obj instanceof String) {
1233                                         LOGGER.debug("\t\t**A String");
1234                                         inputs.put(key, obj);
1235                                 } else if (obj instanceof Integer) {
1236                                         LOGGER.debug("\t\t**An Integer");
1237                                         inputs.put(key, obj);
1238                                 } else if (obj instanceof JsonNode) {
1239                                         LOGGER.debug("\t\t**A JsonNode");
1240                                         inputs.put(key, obj);
1241                                 } else if (obj instanceof Boolean) {
1242                                         LOGGER.debug("\t\t**A Boolean");
1243                                         inputs.put(key, obj);
1244                                 } else if (obj instanceof java.util.LinkedHashMap) {
1245                                         LOGGER.debug("\t\t**A java.util.LinkedHashMap **");
1246                                         inputs.put(key, obj);
1247                                 } else if (obj instanceof java.util.ArrayList) {
1248                                         LOGGER.debug("\t\t**An ArrayList");
1249                                         inputs.put(key, obj);
1250                                 } else {
1251                                         LOGGER.debug("\t\t**UNKNOWN OBJECT TYPE");
1252                                         inputs.put(key, obj);
1253                                 }
1254                         } else {
1255                                 LOGGER.debug("key=" + key + " is already in the inputs - will not overwrite");
1256                         }
1257                 }
1258                 return;
1259         }
1260
1261         public List<String> convertCdlToArrayList(String cdl) {
1262                 String cdl2 = cdl.trim();
1263                 String cdl3;
1264                 if (cdl2.startsWith("[") && cdl2.endsWith("]")) {
1265                         cdl3 = cdl2.substring(1, cdl2.lastIndexOf("]"));
1266                 } else {
1267                         cdl3 = cdl2;
1268                 }
1269                 return new ArrayList<>(Arrays.asList(cdl3.split(",")));
1270         }
1271
1272     /**
1273      * New with 1707 - this method will convert all the String *values* of the inputs
1274      * to their "actual" object type (based on the param type: in the db - which comes from the template):
1275      * (heat variable type) -> java Object type
1276      * string -> String
1277      * number -> Integer
1278      * json -> JsonNode XXX Removed with MSO-1475 / 1802
1279      * comma_delimited_list -> ArrayList
1280      * boolean -> Boolean
1281      * if any of the conversions should fail, we will default to adding it to the inputs
1282      * as a string - see if Openstack can handle it.
1283      * Also, will remove any params that are extra.
1284      * Any aliases will be converted to their appropriate name (anyone use this feature?)
1285      * @param inputs - the Map<String, String> of the inputs received on the request
1286      * @param template the HeatTemplate object - this is so we can also verify if the param is valid for this template
1287      * @return HashMap<String, Object> of the inputs, cleaned and converted
1288      */
1289         public Map<String, Object> convertInputMap(Map<String, String> inputs, HeatTemplate template) {
1290                 HashMap<String, Object> newInputs = new HashMap<>();
1291                 HashMap<String, HeatTemplateParam> params = new HashMap<>();
1292                 HashMap<String, HeatTemplateParam> paramAliases = new HashMap<>();
1293
1294                 if (inputs == null) {
1295                         LOGGER.debug("convertInputMap - inputs is null - nothing to do here");
1296                         return new HashMap<>();
1297                 }
1298
1299                 LOGGER.debug("convertInputMap in MsoHeatUtils called, with " + inputs.size() + " inputs, and template " + template.getArtifactUuid());
1300                 try {
1301                         LOGGER.debug(template.toString());
1302                         Set<HeatTemplateParam> paramSet = template.getParameters();
1303                         LOGGER.debug("paramSet has " + paramSet.size() + " entries");
1304                 } catch (Exception e) {
1305                         LOGGER.debug("Exception occurred in convertInputMap:" + e.getMessage(), e);
1306                 }
1307
1308                 for (HeatTemplateParam htp : template.getParameters()) {
1309                         LOGGER.debug("Adding " + htp.getParamName());
1310                         params.put(htp.getParamName(), htp);
1311                         if (htp.getParamAlias() != null && !"".equals(htp.getParamAlias())) {
1312                                 LOGGER.debug("\tFound ALIAS " + htp.getParamName() + "->" + htp.getParamAlias());
1313                                 paramAliases.put(htp.getParamAlias(), htp);
1314                         }
1315                 }
1316                 LOGGER.debug("Now iterate through the inputs...");
1317                 for (String key : inputs.keySet()) {
1318                         LOGGER.debug("key=" + key);
1319                         boolean alias = false;
1320                         String realName = null;
1321                         if (!params.containsKey(key)) {
1322                                 LOGGER.debug(key + " is not a parameter in the template! - check for an alias");
1323                                 // add check here for an alias
1324                                 if (!paramAliases.containsKey(key)) {
1325                                         LOGGER.debug("The parameter " + key + " is in the inputs, but it's not a parameter for this template - omit");
1326                                         continue;
1327                                 } else {
1328                                         alias = true;
1329                                         realName = paramAliases.get(key).getParamName();
1330                                         LOGGER.debug("FOUND AN ALIAS! Will use " + realName + " in lieu of give key/alias " + key);
1331                                 }
1332                         }
1333                         String type = params.get(key).getParamType();
1334                         if (type == null || "".equals(type)) {
1335                                 LOGGER.debug("**PARAM_TYPE is null/empty for " + key + ", will default to string");
1336                                 type = "string";
1337                         }
1338                         LOGGER.debug("Parameter: " + key + " is of type " + type);
1339                         if ("string".equalsIgnoreCase(type)) {
1340                                 // Easiest!
1341                                 String str = inputs.get(key);
1342                                 if (alias)
1343                                         newInputs.put(realName, str);
1344                                 else
1345                                         newInputs.put(key, str);
1346                         } else if ("number".equalsIgnoreCase(type)) {
1347                                 String integerString = inputs.get(key);
1348                                 Integer anInteger = null;
1349                                 try {
1350                                         anInteger = Integer.parseInt(integerString);
1351                                 } catch (Exception e) {
1352                                         LOGGER.debug("Unable to convert " + integerString + " to an integer!!", e);
1353                                         anInteger = null;
1354                                 }
1355                                 if (anInteger != null) {
1356                                         if (alias)
1357                                                 newInputs.put(realName, anInteger);
1358                                         else
1359                                                 newInputs.put(key, anInteger);
1360                                 }
1361                                 else {
1362                                         if (alias)
1363                                                 newInputs.put(realName, integerString);
1364                                         else
1365                                                 newInputs.put(key, integerString);
1366                                 }
1367                         } else if ("json".equalsIgnoreCase(type)) {
1368                                 // MSO-1475 - Leave this as a string now
1369                                 String jsonString = inputs.get(key);
1370                                 LOGGER.debug("Skipping conversion to jsonNode...");
1371                         if (alias)
1372                                 newInputs.put(realName, jsonString);
1373                         else
1374                                 newInputs.put(key, jsonString);
1375                         //}
1376                         } else if ("comma_delimited_list".equalsIgnoreCase(type)) {
1377                                 String commaSeparated = inputs.get(key);
1378                                 try {
1379                                         List<String> anArrayList = this.convertCdlToArrayList(commaSeparated);
1380                                         if (alias)
1381                                                 newInputs.put(realName, anArrayList);
1382                                         else
1383                                                 newInputs.put(key, anArrayList);
1384                                 } catch (Exception e) {
1385                                         LOGGER.debug("Unable to convert " + commaSeparated + " to an ArrayList!!", e);
1386                                         if (alias)
1387                                                 newInputs.put(realName, commaSeparated);
1388                                         else
1389                                                 newInputs.put(key, commaSeparated);
1390                                 }
1391                         } else if ("boolean".equalsIgnoreCase(type)) {
1392                                 String booleanString = inputs.get(key);
1393                                 Boolean aBool = Boolean.valueOf(booleanString);
1394                                 if (alias)
1395                                         newInputs.put(realName, aBool);
1396                                 else
1397                                         newInputs.put(key, aBool);
1398                         } else {
1399                                 // it's null or something undefined - just add it back as a String
1400                                 String str = inputs.get(key);
1401                                 if (alias)
1402                                         newInputs.put(realName, str);
1403                                 else
1404                                         newInputs.put(key, str);
1405                         }
1406                 }
1407                 return newInputs;
1408         }
1409
1410         /*
1411          * This helpful method added for Valet
1412          */
1413         public String getCloudSiteKeystoneUrl(String cloudSiteId) throws MsoCloudSiteNotFound {
1414                 String keystone_url = null;
1415                 try {
1416                         CloudSite cloudSite = cloudConfig.getCloudSite(cloudSiteId).orElseThrow(() -> new MsoCloudSiteNotFound(cloudSiteId));
1417                         CloudIdentity cloudIdentity = cloudSite.getIdentityService();
1418                         keystone_url = cloudIdentity.getIdentityUrl();
1419                 } catch (Exception e) {
1420                         throw new MsoCloudSiteNotFound(cloudSiteId);
1421                 }
1422                 if (keystone_url == null || keystone_url.isEmpty()) {
1423                         throw new MsoCloudSiteNotFound(cloudSiteId);
1424                 }
1425                 return keystone_url;
1426         }
1427
1428         /*
1429          * Create a string suitable for being dumped to a debug log that creates a
1430          * pseudo-JSON request dumping what's being sent to Openstack API in the create or update request
1431          */
1432
1433         private String printStackRequest(String tenantId,
1434                         Map<String, Object> heatFiles,
1435                         Map<String, Object> nestedTemplates,
1436                         String environment,
1437                         Map<String, Object> inputs,
1438                         String vfModuleName,
1439                         String template,
1440                         int timeoutMinutes,
1441                         boolean backout,
1442                         String cloudSiteId) {
1443                 StringBuilder sb = new StringBuilder();
1444                 sb.append("CREATE STACK REQUEST (formatted for readability)\n");
1445                 sb.append("tenant=" + tenantId + ", cloud=" + cloudSiteId);
1446                 sb.append("{\n");
1447                 sb.append("  \"stack_name\": \"" + vfModuleName + "\",\n");
1448                 sb.append("  \"disable_rollback\": " + backout + ",\n");
1449                 sb.append("  \"timeout_mins\": " + timeoutMinutes + ",\n");
1450                 sb.append("  \"template\": {\n");
1451                 sb.append(template);
1452                 sb.append("  },\n");
1453                 sb.append("  \"environment\": {\n");
1454                 if (environment == null)
1455                         sb.append("<none>");
1456                 else
1457                         sb.append(environment);
1458                 sb.append("  },\n");
1459                 sb.append("  \"files\": {\n");
1460                 int filesCounter = 0;
1461                 if (heatFiles != null) {
1462                         for (String key : heatFiles.keySet()) {
1463                                 filesCounter++;
1464                                 if (filesCounter > 1) {
1465                                         sb.append(",\n");
1466                                 }
1467                                 sb.append("    \"" + key + "\": {\n");
1468                                 sb.append(heatFiles.get(key).toString() + "\n    }");
1469                         }
1470                 }
1471                 if (nestedTemplates != null) {
1472                         for (String key : nestedTemplates.keySet()) {
1473                                 filesCounter++;
1474                                 if (filesCounter > 1) {
1475                                         sb.append(",\n");
1476                                 }
1477                                 sb.append("    \"" + key + "\": {\n");
1478                                 sb.append(nestedTemplates.get(key).toString() + "\n    }");
1479                         }
1480                 }
1481                 sb.append("\n  },\n");
1482                 sb.append("  \"parameters\": {\n");
1483                 int paramCounter = 0;
1484                 for (String name : inputs.keySet()) {
1485                         paramCounter++;
1486                         if (paramCounter > 1) {
1487                                 sb.append(",\n");
1488                         }
1489                         Object o = inputs.get(name);
1490                         if (o instanceof java.lang.String) {
1491                                 sb.append("    \"" + name + "\": \"" + inputs.get(name).toString() + "\"");
1492                         } else if (o instanceof Integer) {
1493                                 sb.append("    \"" + name + "\": " + inputs.get(name).toString() );
1494                         } else if (o instanceof ArrayList) {
1495                                 sb.append("    \"" + name + "\": " + inputs.get(name).toString() );
1496                         } else if (o instanceof Boolean) {
1497                                 sb.append("    \"" + name + "\": " + inputs.get(name).toString() );
1498                         } else {
1499                                 sb.append("    \"" + name + "\": " + "\"(there was an issue trying to dump this value...)\"" );
1500                         }
1501                 }
1502                 sb.append("\n  }\n}\n");
1503
1504                 return sb.toString();
1505         }
1506
1507         /*******************************************************************************
1508      *
1509      * Methods (and associated utilities) to implement the VduPlugin interface
1510      *
1511      *******************************************************************************/
1512
1513     /**
1514      * VduPlugin interface for instantiate function.
1515      *
1516      * Translate the VduPlugin parameters to the corresponding 'createStack' parameters,
1517      * and then invoke the existing function.
1518      */
1519     @Override
1520         public VduInstance instantiateVdu (
1521                         CloudInfo cloudInfo,
1522                         String instanceName,
1523                         Map<String,Object> inputs,
1524                         VduModelInfo vduModel,
1525                         boolean rollbackOnFailure)
1526         throws VduException
1527     {
1528         String cloudSiteId = cloudInfo.getCloudSiteId();
1529         String tenantId = cloudInfo.getTenantId();
1530
1531         // Translate the VDU ModelInformation structure to that which is needed for
1532         // creating the Heat stack.  Loop through the artifacts, looking specifically
1533         // for MAIN_TEMPLATE and ENVIRONMENT.  Any other artifact will
1534         // be attached as a FILE.
1535         String heatTemplate = null;
1536         Map<String,Object> nestedTemplates = new HashMap<>();
1537         Map<String,Object> files = new HashMap<>();
1538         String heatEnvironment = null;
1539
1540         for (VduArtifact vduArtifact: vduModel.getArtifacts()) {
1541                 if (vduArtifact.getType() == ArtifactType.MAIN_TEMPLATE) {
1542                         heatTemplate = new String(vduArtifact.getContent());
1543                 }
1544                 else if (vduArtifact.getType() == ArtifactType.NESTED_TEMPLATE) {
1545                         nestedTemplates.put(vduArtifact.getName(), new String(vduArtifact.getContent()));
1546                 }
1547                 else if (vduArtifact.getType() == ArtifactType.ENVIRONMENT) {
1548                         heatEnvironment = new String(vduArtifact.getContent());
1549                 }
1550         }
1551
1552         try {
1553             StackInfo stackInfo = createStack (cloudSiteId,
1554                     tenantId,
1555                     instanceName,
1556                     heatTemplate,
1557                     inputs,
1558                     true,       // poll for completion
1559                     vduModel.getTimeoutMinutes(),
1560                     heatEnvironment,
1561                     nestedTemplates,
1562                     files,
1563                     rollbackOnFailure);
1564
1565             // Populate a vduInstance from the StackInfo
1566                 return stackInfoToVduInstance(stackInfo);
1567         }
1568         catch (Exception e) {
1569                 throw new VduException ("MsoHeatUtils (instantiateVDU): createStack Exception", e);
1570         }
1571     }
1572
1573
1574     /**
1575      * VduPlugin interface for query function.
1576      */
1577     @Override
1578         public VduInstance queryVdu (CloudInfo cloudInfo, String instanceId)
1579         throws VduException
1580     {
1581         String cloudSiteId = cloudInfo.getCloudSiteId();
1582         String tenantId = cloudInfo.getTenantId();
1583
1584         try {
1585                 // Query the Cloudify Deployment object and  populate a VduInstance
1586                 StackInfo stackInfo = queryStack (cloudSiteId, tenantId, instanceId);
1587
1588                 return stackInfoToVduInstance(stackInfo);
1589         }
1590         catch (Exception e) {
1591                 throw new VduException ("MsoHeatUtile (queryVdu): queryStack Exception ", e);
1592         }
1593     }
1594
1595
1596     /**
1597      * VduPlugin interface for delete function.
1598      */
1599     @Override
1600         public VduInstance deleteVdu (CloudInfo cloudInfo, String instanceId, int timeoutMinutes)
1601         throws VduException
1602     {
1603         String cloudSiteId = cloudInfo.getCloudSiteId();
1604         String tenantId = cloudInfo.getTenantId();
1605
1606         try {
1607                 // Delete the Heat stack
1608                 StackInfo stackInfo = deleteStack (tenantId, cloudSiteId, instanceId, true);
1609
1610                 // Populate a VduInstance based on the deleted Cloudify Deployment object
1611                 VduInstance vduInstance = stackInfoToVduInstance(stackInfo);
1612
1613                 // Override return state to DELETED (HeatUtils sets to NOTFOUND)
1614                 vduInstance.getStatus().setState(VduStateType.DELETED);
1615
1616                 return vduInstance;
1617         }
1618         catch (Exception e) {
1619                 throw new VduException ("Delete VDU Exception", e);
1620         }
1621     }
1622
1623
1624     /**
1625      * VduPlugin interface for update function.
1626      *
1627      * Update is currently not supported in the MsoHeatUtils implementation of VduPlugin.
1628      * Just return a VduException.
1629      *
1630      */
1631     @Override
1632         public VduInstance updateVdu (
1633                         CloudInfo cloudInfo,
1634                         String instanceId,
1635                         Map<String,Object> inputs,
1636                         VduModelInfo vduModel,
1637                         boolean rollbackOnFailure)
1638         throws VduException
1639     {
1640         throw new VduException ("MsoHeatUtils: updateVdu interface not supported");
1641     }
1642
1643
1644     /*
1645      * Convert the local DeploymentInfo object (Cloudify-specific) to a generic VduInstance object
1646      */
1647     protected VduInstance stackInfoToVduInstance (StackInfo stackInfo)
1648     {
1649         VduInstance vduInstance = new VduInstance();
1650
1651         // The full canonical name as the instance UUID
1652         vduInstance.setVduInstanceId(stackInfo.getCanonicalName());
1653         vduInstance.setVduInstanceName(stackInfo.getName());
1654
1655         // Copy inputs and outputs
1656         vduInstance.setInputs(stackInfo.getParameters());
1657         vduInstance.setOutputs(stackInfo.getOutputs());
1658
1659         // Translate the status elements
1660         vduInstance.setStatus(stackStatusToVduStatus (stackInfo));
1661
1662         return vduInstance;
1663     }
1664
1665     private VduStatus stackStatusToVduStatus (StackInfo stackInfo)
1666     {
1667         VduStatus vduStatus = new VduStatus();
1668
1669         // Map the status fields to more generic VduStatus.
1670         // There are lots of HeatStatus values, so this is a bit long...
1671         HeatStatus heatStatus = stackInfo.getStatus();
1672         String statusMessage = stackInfo.getStatusMessage();
1673
1674         if (heatStatus == HeatStatus.INIT  ||  heatStatus == HeatStatus.BUILDING) {
1675                 vduStatus.setState(VduStateType.INSTANTIATING);
1676                 vduStatus.setLastAction((new PluginAction ("create", "in_progress", statusMessage)));
1677         }
1678         else if (heatStatus == HeatStatus.NOTFOUND) {
1679                 vduStatus.setState(VduStateType.NOTFOUND);
1680         }
1681         else if (heatStatus == HeatStatus.CREATED) {
1682                 vduStatus.setState(VduStateType.INSTANTIATED);
1683                 vduStatus.setLastAction((new PluginAction ("create", "complete", statusMessage)));
1684         }
1685         else if (heatStatus == HeatStatus.UPDATED) {
1686                 vduStatus.setState(VduStateType.INSTANTIATED);
1687                 vduStatus.setLastAction((new PluginAction ("update", "complete", statusMessage)));
1688         }
1689         else if (heatStatus == HeatStatus.UPDATING) {
1690                 vduStatus.setState(VduStateType.UPDATING);
1691                 vduStatus.setLastAction((new PluginAction ("update", "in_progress", statusMessage)));
1692         }
1693         else if (heatStatus == HeatStatus.DELETING) {
1694                 vduStatus.setState(VduStateType.DELETING);
1695                 vduStatus.setLastAction((new PluginAction ("delete", "in_progress", statusMessage)));
1696         }
1697         else if (heatStatus == HeatStatus.FAILED) {
1698                 vduStatus.setState(VduStateType.FAILED);
1699                 vduStatus.setErrorMessage(stackInfo.getStatusMessage());
1700         } else {
1701                 vduStatus.setState(VduStateType.UNKNOWN);
1702         }
1703
1704         return vduStatus;
1705     }
1706
1707     private void sleep(long time) {
1708         try {
1709             Thread.sleep(time);
1710         } catch (InterruptedException e) {
1711             LOGGER.debug ("Thread interrupted while sleeping", e);
1712             Thread.currentThread().interrupt();
1713         }
1714     }
1715
1716 }