Openstack adapter can't resolve HEAT parameter
[so.git] / adapters / mso-adapter-utils / src / main / java / org / onap / so / openstack / utils / MsoHeatUtils.java
index 476bff3..47ff6c1 100644 (file)
@@ -5,6 +5,8 @@
  * Copyright (C) 2017 AT&T Intellectual Property. All rights reserved.
  * Copyright (C) 2017 Huawei Technologies Co., Ltd. All rights reserved.
  * ================================================================================
+ * Modifications Copyright (c) 2019 Samsung
+ * ================================================================================
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
 
 package org.onap.so.openstack.utils;
 
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.woorea.openstack.base.client.OpenStackConnectException;
+import com.woorea.openstack.base.client.OpenStackRequest;
+import com.woorea.openstack.base.client.OpenStackResponseException;
+import com.woorea.openstack.heat.Heat;
+import com.woorea.openstack.heat.model.CreateStackParam;
+import com.woorea.openstack.heat.model.Resources;
+import com.woorea.openstack.heat.model.Stack;
+import com.woorea.openstack.heat.model.Stack.Output;
+import com.woorea.openstack.heat.model.Stacks;
+import com.woorea.openstack.keystone.Keystone;
+import com.woorea.openstack.keystone.model.Access;
+import com.woorea.openstack.keystone.model.Authentication;
+import com.woorea.openstack.keystone.utils.KeystoneUtils;
+
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Calendar;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Set;
-
 import org.onap.so.adapters.vdu.CloudInfo;
 import org.onap.so.adapters.vdu.PluginAction;
 import org.onap.so.adapters.vdu.VduArtifact;
@@ -41,15 +59,17 @@ import org.onap.so.adapters.vdu.VduPlugin;
 import org.onap.so.adapters.vdu.VduStateType;
 import org.onap.so.adapters.vdu.VduStatus;
 import org.onap.so.cloud.CloudConfig;
+import org.onap.so.cloud.authentication.AuthenticationMethodFactory;
+import org.onap.so.cloud.authentication.KeystoneAuthHolder;
+import org.onap.so.cloud.authentication.KeystoneV3Authentication;
+import org.onap.so.cloud.authentication.ServiceEndpointNotFoundException;
 import org.onap.so.db.catalog.beans.CloudIdentity;
 import org.onap.so.db.catalog.beans.CloudSite;
-import org.onap.so.cloud.authentication.AuthenticationMethodFactory;
 import org.onap.so.db.catalog.beans.HeatTemplate;
 import org.onap.so.db.catalog.beans.HeatTemplateParam;
+import org.onap.so.db.catalog.beans.ServerType;
 import org.onap.so.logger.MessageEnum;
-import org.onap.so.logger.MsoAlarmLogger;
 import org.onap.so.logger.MsoLogger;
-import org.onap.so.openstack.beans.HeatCacheEntry;
 import org.onap.so.openstack.beans.HeatStatus;
 import org.onap.so.openstack.beans.StackInfo;
 import org.onap.so.openstack.exceptions.MsoAdapterException;
@@ -61,11 +81,14 @@ import org.onap.so.openstack.exceptions.MsoStackAlreadyExists;
 import org.onap.so.openstack.exceptions.MsoTenantNotFound;
 import org.onap.so.openstack.mappers.StackInfoMapper;
 import org.onap.so.utils.CryptoUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.context.annotation.Primary;
 import org.springframework.core.env.Environment;
 import org.springframework.stereotype.Component;
 
+import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.core.type.TypeReference;
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
@@ -74,6 +97,7 @@ import com.woorea.openstack.base.client.OpenStackRequest;
 import com.woorea.openstack.base.client.OpenStackResponseException;
 import com.woorea.openstack.heat.Heat;
 import com.woorea.openstack.heat.model.CreateStackParam;
+import com.woorea.openstack.heat.model.Resources;
 import com.woorea.openstack.heat.model.Stack;
 import com.woorea.openstack.heat.model.Stack.Output;
 import com.woorea.openstack.heat.model.Stacks;
@@ -92,16 +116,9 @@ public class MsoHeatUtils extends MsoCommonUtils implements VduPlugin{
 
     private static final String DELETE_STACK = "DeleteStack";
 
-    private static final String HEAT_ERROR = "HeatError";
+    protected static final String HEAT_ERROR = "HeatError";
 
-    private static final String CREATE_STACK = "CreateStack";
-
-    // Cache Heat Clients statically. Since there is just one MSO user, there is no
-    // benefit to re-authentication on every request (or across different flows). The
-    // token will be used until it expires.
-    //
-    // The cache key is "tenantId:cloudId"
-    private static Map <String, HeatCacheEntry> heatClientCache = new HashMap <> ();
+    protected static final String CREATE_STACK = "CreateStack";
 
     // Fetch cloud configuration each time (may be cached in CloudConfig class)
     @Autowired
@@ -115,13 +132,16 @@ public class MsoHeatUtils extends MsoCommonUtils implements VduPlugin{
 
     @Autowired
     private MsoTenantUtilsFactory tenantUtilsFactory;
-
-    private static final MsoLogger LOGGER = MsoLogger.getMsoLogger (MsoLogger.Catalog.RA, MsoHeatUtils.class);
+    
+    @Autowired
+    private KeystoneV3Authentication keystoneV3Authentication;
+    
+    private static final Logger logger = LoggerFactory.getLogger(MsoHeatUtils.class);
 
     // Properties names and variables (with default values)
-    protected String createPollIntervalProp = "ecomp.mso.adapters.po.pollInterval";
-    private String deletePollIntervalProp = "ecomp.mso.adapters.po.pollInterval";
-    private String deletePollTimeoutProp = "ecomp.mso.adapters.po.pollTimeout";
+    protected String createPollIntervalProp = "org.onap.so.adapters.po.pollInterval";
+    private String deletePollIntervalProp = "org.onap.so.adapters.po.pollInterval";
+    private String deletePollTimeoutProp = "org.onap.so.adapters.po.pollTimeout";
 
     protected static final String createPollIntervalDefault = "15";
     private static final String deletePollIntervalDefault = "15";
@@ -291,49 +311,33 @@ public class MsoHeatUtils extends MsoCommonUtils implements VduPlugin{
         // Obtain the cloud site information where we will create the stack
         CloudSite cloudSite = cloudConfig.getCloudSite(cloudSiteId).orElseThrow(
                 () -> new MsoCloudSiteNotFound(cloudSiteId));
-        LOGGER.debug("Found: " + cloudSite.toString());
+        logger.debug("Found: {}", cloudSite);
         // Get a Heat client. They are cached between calls (keyed by tenantId:cloudId)
         // This could throw MsoTenantNotFound or MsoOpenstackException (both propagated)
         Heat heatClient = getHeatClient (cloudSite, tenantId);
-        if (heatClient != null) {
-            LOGGER.debug("Found: " + heatClient.toString());
-        }
+        logger.debug("Found: {}", heatClient);
 
-        LOGGER.debug ("Ready to Create Stack (" + heatTemplate + ") with input params: " + stackInputs);
+        logger.debug ("Ready to Create Stack ({}) with input params: {}", heatTemplate, stackInputs);
 
         Stack heatStack = null;
         try {
-            // Execute the actual Openstack command to create the Heat stack
             OpenStackRequest <Stack> request = heatClient.getStacks ().create (stack);
-            // Begin X-Auth-User
-            // Obtain an MSO token for the tenant
             CloudIdentity cloudIdentity = cloudSite.getIdentityService();
-            // cloudIdentity.getMsoId(), cloudIdentity.getMsoPass()
-            //req
             request.header ("X-Auth-User", cloudIdentity.getMsoId ());
             request.header ("X-Auth-Key", CryptoUtils.decryptCloudConfigPassword(cloudIdentity.getMsoPass ()));
-            LOGGER.debug ("headers added, about to executeAndRecordOpenstackRequest");
-            //LOGGER.debug(this.requestToStringBuilder(stack).toString());
-            // END - try to fix X-Auth-User
             heatStack = executeAndRecordOpenstackRequest (request);
         } catch (OpenStackResponseException e) {
-            // Since this came on the 'Create Stack' command, nothing was changed
-            // in the cloud. Return the error as an exception.
-            if (e.getStatus () == 409) {
-                // Stack already exists. Return a specific error for this case
+            if (e.getStatus () == 409) {                
                 MsoStackAlreadyExists me = new MsoStackAlreadyExists (stackName, tenantId, cloudSiteId);
                 me.addContext (CREATE_STACK);
                 throw me;
-            } else {
-                // Convert the OpenStackResponseException to an MsoOpenstackException
-                LOGGER.debug("ERROR STATUS = " + e.getStatus() + ",\n" + e.getMessage() + "\n" + e.getLocalizedMessage());
+        } else {
+                logger.debug("ERROR STATUS = {},\n{}\n{}", e.getStatus(), e.getMessage(), e.getLocalizedMessage());
                 throw heatExceptionToMsoException (e, CREATE_STACK);
             }
-        } catch (OpenStackConnectException e) {
-            // Error connecting to Openstack instance. Convert to an MsoException
+        } catch (OpenStackConnectException e) {          
             throw heatExceptionToMsoException (e, CREATE_STACK);
-        } catch (RuntimeException e) {
-            // Catch-all
+        } catch (RuntimeException e) {           
             throw runtimeExceptionToMsoException (e, CREATE_STACK);
         }
 
@@ -341,211 +345,216 @@ public class MsoHeatUtils extends MsoCommonUtils implements VduPlugin{
         // Otherwise, simple query by name returns a 302 redirect.
         // NOTE: This is specific to the v1 Orchestration API.
         String canonicalName = stackName + "/" + heatStack.getId ();
-
-        // If client has requested a final response, poll for stack completion
+       
         if (pollForCompletion) {
-            // Set a time limit on overall polling.
-            // Use the resource (template) timeout for Openstack (expressed in minutes)
-            // and add one poll interval to give Openstack a chance to fail on its own.s
-
-            int createPollInterval = Integer.parseInt(this.environment.getProperty(createPollIntervalProp, createPollIntervalDefault));
-            int pollTimeout = (timeoutMinutes * 60) + createPollInterval;
-            // New 1610 - poll on delete if we rollback - use same values for now
-            int deletePollInterval = createPollInterval;
-            int deletePollTimeout = pollTimeout;
-            boolean createTimedOut = false;
-            StringBuilder stackErrorStatusReason = new StringBuilder("");
-            LOGGER.debug("createPollInterval=" + createPollInterval + ", pollTimeout=" + pollTimeout);
-
-            while (true) {
-                try {
-                    heatStack = queryHeatStack (heatClient, canonicalName);
-                    LOGGER.debug (heatStack.getStackStatus () + " (" + canonicalName + ")");
-                    try {
-                        LOGGER.debug("Current stack " + this.getOutputsAsStringBuilder(heatStack).toString());
-                    } catch (Exception e) {
-                        LOGGER.debug("an error occurred trying to print out the current outputs of the stack", e);
-                    }
-
-                    if ("CREATE_IN_PROGRESS".equals (heatStack.getStackStatus ())) {
-                        // Stack creation is still running.
-                        // Sleep and try again unless timeout has been reached
-                        if (pollTimeout <= 0) {
-                            // Note that this should not occur, since there is a timeout specified
-                            // in the Openstack call.
-                            LOGGER.error (MessageEnum.RA_CREATE_STACK_TIMEOUT, cloudSiteId, tenantId, stackName, heatStack.getStackStatus (), "", "", MsoLogger.ErrorCode.AvailabilityError, "Create stack timeout");
-                            createTimedOut = true;
-                            break;
-                        }
-
-                        sleep(createPollInterval * 1000L);
-
-                        pollTimeout -= createPollInterval;
-                               LOGGER.debug("pollTimeout remaining: " + pollTimeout);
-                    } else {
-                       //save off the status & reason msg before we attempt delete
-                       stackErrorStatusReason.append("Stack error (" + heatStack.getStackStatus() + "): " + heatStack.getStackStatusReason());
-                        break;
-                    }
-                } catch (MsoException me) {
-                       // Cannot query the stack status. Something is wrong.
-                       // Try to roll back the stack
-                       if (!backout)
-                       {
-                               LOGGER.warn(MessageEnum.RA_CREATE_STACK_ERR, "Create Stack errored, stack deletion suppressed", "", "", MsoLogger.ErrorCode.BusinessProcesssError, "Exception in Create Stack, stack deletion suppressed");
-                       }
-                       else
-                       {
-                               try {
-                                       LOGGER.debug("Create Stack error - unable to query for stack status - attempting to delete stack: " + canonicalName + " - This will likely fail and/or we won't be able to query to see if delete worked");
-                                       OpenStackRequest <Void> request = heatClient.getStacks ().deleteByName (canonicalName);
-                                       executeAndRecordOpenstackRequest (request);
-                                       // this may be a waste of time - if we just got an exception trying to query the stack - we'll just
-                                       // get another one, n'est-ce pas?
-                                       boolean deleted = false;
-                                       while (!deleted) {
-                                               try {
-                                                       heatStack = queryHeatStack(heatClient, canonicalName);
-                                                       if (heatStack != null) {
-                                                       LOGGER.debug(heatStack.getStackStatus());
-                                                       if ("DELETE_IN_PROGRESS".equals(heatStack.getStackStatus())) {
-                                                               if (deletePollTimeout <= 0) {
-                                                                       LOGGER.error (MessageEnum.RA_CREATE_STACK_TIMEOUT, cloudSiteId, tenantId, stackName,
-                                                                                       heatStack.getStackStatus (), "", "", MsoLogger.ErrorCode.AvailabilityError,
-                                                                                       "Rollback: DELETE stack timeout");
-                                                                       break;
-                                                               } else {
-                                                                       sleep(deletePollInterval * 1000L);
-                                                                       deletePollTimeout -= deletePollInterval;
-                                                               }
-                                                       } else if ("DELETE_COMPLETE".equals(heatStack.getStackStatus())){
-                                                               LOGGER.debug("DELETE_COMPLETE for " + canonicalName);
-                                                               deleted = true;
-                                                               continue;
-                                                       } else {
-                                                               //got a status other than DELETE_IN_PROGRESS or DELETE_COMPLETE - so break and evaluate
-                                                               break;
-                                                       }
-                                               } else {
-                                                       // assume if we can't find it - it's deleted
-                                                       LOGGER.debug("heatStack returned null - assume the stack " + canonicalName + " has been deleted");
-                                                       deleted = true;
-                                                       continue;
-                                                       }
-
-                                               } catch (Exception e3) {
-                                                       // Just log this one. We will report the original exception.
-                                                       LOGGER.error (MessageEnum.RA_CREATE_STACK_ERR, "Create Stack: Nested exception rolling back stack: " + e3, "", "", MsoLogger.ErrorCode.BusinessProcesssError, "Create Stack: Nested exception rolling back stack on error on query");
-
-                                               }
-                                       }
-                               } catch (Exception e2) {
-                                       // Just log this one. We will report the original exception.
-                                       LOGGER.error (MessageEnum.RA_CREATE_STACK_ERR, "Create Stack: Nested exception rolling back stack: " + e2, "", "", MsoLogger.ErrorCode.BusinessProcesssError, "Create Stack: Nested exception rolling back stack");
-                               }
-                       }
-
-                    // Propagate the original exception from Stack Query.
-                    me.addContext (CREATE_STACK);
-                    throw me;
-                }
-            }
-
-            if (!"CREATE_COMPLETE".equals (heatStack.getStackStatus ())) {
-                LOGGER.error (MessageEnum.RA_CREATE_STACK_ERR, "Create Stack error:  Polling complete with non-success status: "
-                              + heatStack.getStackStatus () + ", " + heatStack.getStackStatusReason (), "", "", MsoLogger.ErrorCode.BusinessProcesssError, "Create Stack error");
-
-                // Rollback the stack creation, since it is in an indeterminate state.
-                if (!backout)
-                {
-                       LOGGER.warn(MessageEnum.RA_CREATE_STACK_ERR, "Create Stack errored, stack deletion suppressed", "", "", MsoLogger.ErrorCode.BusinessProcesssError, "Create Stack error, stack deletion suppressed");
-                }
-                else
-                {
-                       try {
-                               LOGGER.debug("Create Stack errored - attempting to DELETE stack: " + canonicalName);
-                               LOGGER.debug("deletePollInterval=" + deletePollInterval + ", deletePollTimeout=" + deletePollTimeout);
-                               OpenStackRequest <Void> request = heatClient.getStacks ().deleteByName (canonicalName);
-                               executeAndRecordOpenstackRequest (request);
-                               boolean deleted = false;
-                               while (!deleted) {
-                                       try {
-                                               heatStack = queryHeatStack(heatClient, canonicalName);
-                                               if (heatStack != null) {
-                                                       LOGGER.debug(heatStack.getStackStatus() + " (" + canonicalName + ")");
-                                                       if ("DELETE_IN_PROGRESS".equals(heatStack.getStackStatus())) {
-                                                               if (deletePollTimeout <= 0) {
-                                                                       LOGGER.error (MessageEnum.RA_CREATE_STACK_TIMEOUT, cloudSiteId, tenantId, stackName,
-                                                                                       heatStack.getStackStatus (), "", "", MsoLogger.ErrorCode.AvailabilityError,
-                                                                                       "Rollback: DELETE stack timeout");
-                                                                       break;
-                                                               } else {
-                                                                       sleep(deletePollInterval * 1000L);
-                                                                       deletePollTimeout -= deletePollInterval;
-                                                                       LOGGER.debug("deletePollTimeout remaining: " + deletePollTimeout);
-                                                               }
-                                                       } else if ("DELETE_COMPLETE".equals(heatStack.getStackStatus())){
-                                                               LOGGER.debug("DELETE_COMPLETE for " + canonicalName);
-                                                               deleted = true;
-                                                               continue;
-                                                       } else if ("DELETE_FAILED".equals(heatStack.getStackStatus())) {
-                                                               // Warn about this (?) - but still throw the original exception
-                                                               LOGGER.warn(MessageEnum.RA_CREATE_STACK_ERR, "Create Stack errored, stack deletion FAILED", "", "", MsoLogger.ErrorCode.BusinessProcesssError, "Create Stack error, stack deletion FAILED");
-                                                               LOGGER.debug("Stack deletion FAILED on a rollback of a create - " + canonicalName + ", status=" + heatStack.getStackStatus() + ", reason=" + heatStack.getStackStatusReason());
-                                                               break;
-                                                       } else {
-                                                               //got a status other than DELETE_IN_PROGRESS or DELETE_COMPLETE - so break and evaluate
-                                                               break;
-                                                       }
-                                               } else {
-                                                       // assume if we can't find it - it's deleted
-                                                       LOGGER.debug("heatStack returned null - assume the stack " + canonicalName + " has been deleted");
-                                                       deleted = true;
-                                                       continue;
-                                               }
-
-                                       } catch (MsoException me2) {
-                                               // We got an exception on the delete - don't throw this exception - throw the original - just log.
-                                               LOGGER.debug("Exception thrown trying to delete " + canonicalName + " on a create->rollback: " + me2.getContextMessage(), me2);
-                                               LOGGER.warn(MessageEnum.RA_CREATE_STACK_ERR, "Create Stack errored, then stack deletion FAILED - exception thrown", "", "", MsoLogger.ErrorCode.BusinessProcesssError, me2.getContextMessage());
-                                       }
-
-                               } // end while !deleted
-                               StringBuilder errorContextMessage;
-                               if (createTimedOut) {
-                                       errorContextMessage = new StringBuilder("Stack Creation Timeout");
-                               } else {
-                                       errorContextMessage  = stackErrorStatusReason;
-                               }
-                               if (deleted) {
-                                       errorContextMessage.append(" - stack successfully deleted");
-                               } else {
-                                       errorContextMessage.append(" - encountered an error trying to delete the stack");
-                               }
-//                             MsoOpenstackException me = new MsoOpenstackException(0, "", stackErrorStatusReason.toString());
- //                            me.addContext(CREATE_STACK);
-  //                           alarmLogger.sendAlarm(HEAT_ERROR, MsoAlarmLogger.CRITICAL, me.getContextMessage());
-   //                          throw me;
-                       } catch (Exception e2) {
-                               // shouldn't happen - but handle
-                               LOGGER.error (MessageEnum.RA_CREATE_STACK_ERR, "Create Stack: Nested exception rolling back stack: " + e2, "", "", MsoLogger.ErrorCode.BusinessProcesssError, "Exception in Create Stack: rolling back stack");
-                       }
-                }
-                MsoOpenstackException me = new MsoOpenstackException(0, "", stackErrorStatusReason.toString());
-                me.addContext(CREATE_STACK);
-                alarmLogger.sendAlarm(HEAT_ERROR, MsoAlarmLogger.CRITICAL, me.getContextMessage());
-                throw me;
-            }
-
+            heatStack = pollStackForCompletion(cloudSiteId, tenantId, stackName, timeoutMinutes, backout, heatClient,
+                                       heatStack, canonicalName);
         } else {
             // Get initial status, since it will have been null after the create.
             heatStack = queryHeatStack (heatClient, canonicalName);
-            LOGGER.debug (heatStack.getStackStatus ());
+            logger.debug (heatStack.getStackStatus ());
         }
-
         return new StackInfoMapper(heatStack).map();
     }
 
+       private Stack pollStackForCompletion(String cloudSiteId, String tenantId, String stackName, int timeoutMinutes,
+                       boolean backout, Heat heatClient, Stack heatStack, String canonicalName)
+                       throws MsoException, MsoOpenstackException {
+               int createPollInterval = Integer.parseInt(this.environment.getProperty(createPollIntervalProp, createPollIntervalDefault));
+               int pollTimeout = (timeoutMinutes * 60) + createPollInterval;
+               int deletePollInterval = createPollInterval;
+               int deletePollTimeout = pollTimeout;
+               boolean createTimedOut = false;
+               StringBuilder stackErrorStatusReason = new StringBuilder("");
+      logger.debug("createPollInterval={}, pollTimeout={}", createPollInterval, pollTimeout);
+
+               while (true) {
+                   try {
+                       heatStack = queryHeatStack (heatClient, canonicalName);
+            logger.debug("{} ({})", heatStack.getStackStatus(), canonicalName);
+            try {
+                logger.debug("Current stack {}", this.getOutputsAsStringBuilder(heatStack).toString());
+            } catch (Exception e) {
+                           logger.debug("an error occurred trying to print out the current outputs of the stack", e);
+                       }
+
+                       if ("CREATE_IN_PROGRESS".equals (heatStack.getStackStatus ())) {                       
+                           if (pollTimeout <= 0) {
+                    logger.error("{} Cloud site: {} Tenant: {} Stack: {} Stack status: {} {} Create stack timeout",
+                        MessageEnum.RA_CREATE_STACK_TIMEOUT, cloudSiteId, tenantId, stackName,
+                        heatStack.getStackStatus(), MsoLogger.ErrorCode.AvailabilityError.getValue());
+                    createTimedOut = true;
+                               break;
+                           }
+                           sleep(createPollInterval * 1000L);
+                           pollTimeout -= createPollInterval;
+                logger.debug("pollTimeout remaining: {}", pollTimeout);
+            } else {
+                               stackErrorStatusReason.append("Stack error (" + heatStack.getStackStatus() + "): " + heatStack.getStackStatusReason());
+                           break;
+                       }
+                   } catch (MsoException me) {
+                       // Cannot query the stack status. Something is wrong.
+                       // Try to roll back the stack
+                       if (!backout)
+                       {
+              logger.warn("{} Exception in Create Stack, stack deletion suppressed {}",
+                  MessageEnum.RA_CREATE_STACK_ERR, MsoLogger.ErrorCode.BusinessProcesssError.getValue());
+          }
+                       else
+                       {
+                               try {
+                logger.debug(
+                    "Create Stack error - unable to query for stack status - attempting to delete stack: {}"
+                        + " - This will likely fail and/or we won't be able to query to see if delete worked",
+                    canonicalName);
+                OpenStackRequest <Void> request = heatClient.getStacks ().deleteByName (canonicalName);
+                                       executeAndRecordOpenstackRequest (request);
+                                       boolean deleted = false;
+                                       while (!deleted) {
+                                               try {
+                                                       heatStack = queryHeatStack(heatClient, canonicalName);
+                                                       if (heatStack != null) {
+                      logger.debug(heatStack.getStackStatus());
+                      if ("DELETE_IN_PROGRESS".equals(heatStack.getStackStatus())) {
+                                                                       if (deletePollTimeout <= 0) {
+                            logger.error(
+                                "{} Cloud site: {} Tenant: {} Stack: {} Stack status: {} {} Rollback: DELETE stack timeout",
+                                MessageEnum.RA_CREATE_STACK_TIMEOUT, cloudSiteId, tenantId, stackName,
+                                heatStack.getStackStatus(), MsoLogger.ErrorCode.AvailabilityError.getValue());
+                            break;
+                                                                       } else {
+                                                                               sleep(deletePollInterval * 1000L);
+                                                                               deletePollTimeout -= deletePollInterval;
+                                                                       }
+                                                               } else if ("DELETE_COMPLETE".equals(heatStack.getStackStatus())){
+                          logger.debug("DELETE_COMPLETE for {}", canonicalName);
+                          deleted = true;
+                                                                       continue;
+                                                               } else {
+                                                                       //got a status other than DELETE_IN_PROGRESS or DELETE_COMPLETE - so break and evaluate
+                                                                       break;
+                                                               }
+                                                       } else {
+                                                               // assume if we can't find it - it's deleted
+                      logger.debug("heatStack returned null - assume the stack {} has been deleted", canonicalName);
+                      deleted = true;
+                                                               continue;
+                                                       }
+
+                                               } catch (Exception e3) {
+                                                       // Just log this one. We will report the original exception.
+                    logger.error("{} Create Stack: Nested exception rolling back stack: {} ",
+                        MessageEnum.RA_CREATE_STACK_ERR, MsoLogger.ErrorCode.BusinessProcesssError.getValue(), e3);
+                                               }
+                                       }
+                               } catch (Exception e2) {
+                                       // Just log this one. We will report the original exception.
+                logger.error("{} Create Stack: Nested exception rolling back stack: {} ",
+                    MessageEnum.RA_CREATE_STACK_ERR, MsoLogger.ErrorCode.BusinessProcesssError.getValue(), e2);
+            }
+                       }
+
+                       // Propagate the original exception from Stack Query.
+                       me.addContext (CREATE_STACK);
+                       throw me;
+                   }
+               }
+
+               if (!"CREATE_COMPLETE".equals (heatStack.getStackStatus ())) {
+        logger.error("{} Create Stack error:  Polling complete with non-success status: {}, {} {} ",
+            MessageEnum.RA_CREATE_STACK_ERR, heatStack.getStackStatus(), heatStack.getStackStatusReason(),
+            MsoLogger.ErrorCode.BusinessProcesssError.getValue());
+
+                   // Rollback the stack creation, since it is in an indeterminate state.
+                   if (!backout)
+                   {
+            logger.warn(
+                "{} Create Stack errored, stack deletion suppressed {} Create Stack error, stack deletion suppressed",
+                MessageEnum.RA_CREATE_STACK_ERR, MsoLogger.ErrorCode.BusinessProcesssError.getValue());
+        }
+                   else
+                   {
+                       try {
+              logger.debug("Create Stack errored - attempting to DELETE stack: {}", canonicalName);
+              logger.debug("deletePollInterval={}, deletePollTimeout={}", deletePollInterval, deletePollTimeout);
+              OpenStackRequest <Void> request = heatClient.getStacks ().deleteByName (canonicalName);
+                               executeAndRecordOpenstackRequest (request);
+                               boolean deleted = false;
+                               while (!deleted) {
+                                       try {
+                                               heatStack = queryHeatStack(heatClient, canonicalName);
+                                               if (heatStack != null) {
+                    logger.debug("{} ({})", heatStack.getStackStatus(), canonicalName);
+                    if ("DELETE_IN_PROGRESS".equals(heatStack.getStackStatus())) {
+                                                               if (deletePollTimeout <= 0) {
+                        logger.error(
+                            "{} Cloud site: {} Tenant: {} Stack: {} Stack status: {} {} Rollback: DELETE stack timeout",
+                            MessageEnum.RA_CREATE_STACK_TIMEOUT, cloudSiteId, tenantId, stackName,
+                            heatStack.getStackStatus(), MsoLogger.ErrorCode.AvailabilityError.getValue());
+                        break;
+                                                               } else {
+                                                                       sleep(deletePollInterval * 1000L);
+                                                                       deletePollTimeout -= deletePollInterval;
+                        logger.debug("deletePollTimeout remaining: {}", deletePollTimeout);
+                    }
+                                                       } else if ("DELETE_COMPLETE".equals(heatStack.getStackStatus())){
+                      logger.debug("DELETE_COMPLETE for {}", canonicalName);
+                      deleted = true;
+                                                               continue;
+                                                       } else if ("DELETE_FAILED".equals(heatStack.getStackStatus())) {
+                                                               // Warn about this (?) - but still throw the original exception
+                      logger.warn(
+                          "{} Create Stack errored, stack deletion FAILED {} Create Stack error, stack deletion FAILED",
+                          MessageEnum.RA_CREATE_STACK_ERR, MsoLogger.ErrorCode.BusinessProcesssError.getValue());
+                      logger.debug("Stack deletion FAILED on a rollback of a create - {}, status={}, reason={}",
+                          canonicalName, heatStack.getStackStatus(), heatStack.getStackStatusReason());
+                      break;
+                                                       } else {
+                                                               //got a status other than DELETE_IN_PROGRESS or DELETE_COMPLETE - so break and evaluate
+                                                               break;
+                                                       }
+                                               } else {
+                                                       // assume if we can't find it - it's deleted
+                    logger.debug("heatStack returned null - assume the stack {} has been deleted", canonicalName);
+                    deleted = true;
+                                                       continue;
+                                               }
+
+                                       } catch (MsoException me2) {
+                                               // We got an exception on the delete - don't throw this exception - throw the original - just log.
+                  logger.debug("Exception thrown trying to delete {} on a create->rollback: {} ", canonicalName,
+                      me2.getContextMessage(), me2);
+                  logger.warn("{} Create Stack errored, then stack deletion FAILED - exception thrown {} {}",
+                      MessageEnum.RA_CREATE_STACK_ERR, MsoLogger.ErrorCode.BusinessProcesssError.getValue(),
+                      me2.getContextMessage());
+              }
+
+                               } // end while !deleted
+                               StringBuilder errorContextMessage;
+                               if (createTimedOut) {
+                                       errorContextMessage = new StringBuilder("Stack Creation Timeout");
+                               } else {
+                                       errorContextMessage  = stackErrorStatusReason;
+                               }
+                               if (deleted) {
+                                       errorContextMessage.append(" - stack successfully deleted");
+                               } else {
+                                       errorContextMessage.append(" - encountered an error trying to delete the stack");
+                               }
+                       } catch (Exception e2) {
+                               // shouldn't happen - but handle
+              logger.error("{} Create Stack: Nested exception rolling back stack: {} ", MessageEnum.RA_CREATE_STACK_ERR,
+                  MsoLogger.ErrorCode.BusinessProcesssError.getValue(), e2);
+          }
+                   }
+                   MsoOpenstackException me = new MsoOpenstackException(0, "", stackErrorStatusReason.toString());
+                   me.addContext(CREATE_STACK);
+                   throw me;
+               }
+               return heatStack;
+       }
+
     /**
      * Query for a single stack (by Name) in a tenant. This call will always return a
      * StackInfo object. If the stack does not exist, an "empty" StackInfo will be
@@ -558,27 +567,28 @@ public class MsoHeatUtils extends MsoCommonUtils implements VduPlugin{
      * @throws MsoOpenstackException Thrown if the Openstack API call returns an exception.
      */
     public StackInfo queryStack (String cloudSiteId, String tenantId, String stackName) throws MsoException {
-        LOGGER.debug ("Query HEAT stack: " + stackName + " in tenant " + tenantId);
+        logger.debug ("Query HEAT stack: {} in tenant {}", stackName, tenantId);
 
         // Obtain the cloud site information where we will create the stack
         CloudSite cloudSite = cloudConfig.getCloudSite(cloudSiteId).orElseThrow(
                 () -> new MsoCloudSiteNotFound(cloudSiteId));
-        LOGGER.debug("Found: " + cloudSite.toString());
+        logger.debug("Found: {}", cloudSite.toString());
 
         // Get a Heat client. They are cached between calls (keyed by tenantId:cloudId)
         Heat heatClient = null;
         try {
             heatClient = getHeatClient (cloudSite, tenantId);
             if (heatClient != null) {
-               LOGGER.debug("Found: " + heatClient.toString());
+                logger.debug("Found: {}", heatClient.toString());
             }
         } catch (MsoTenantNotFound e) {
             // Tenant doesn't exist, so stack doesn't either
-            LOGGER.debug ("Tenant with id " + tenantId + "not found.", e);
+            logger.debug ("Tenant with id " + tenantId + "not found.", e);
             return new StackInfo (stackName, HeatStatus.NOTFOUND);
         } catch (MsoException me) {
             // Got an Openstack error. Propagate it
-            LOGGER.error (MessageEnum.RA_CONNECTION_EXCEPTION, "OpenStack", "Openstack Exception on Token request: " + me, "Openstack", "", MsoLogger.ErrorCode.AvailabilityError, "Connection Exception");
+            logger.error("{} {} Openstack Exception on Token request: ", MessageEnum.RA_CONNECTION_EXCEPTION,
+                MsoLogger.ErrorCode.AvailabilityError.getValue(), me);
             me.addContext ("QueryStack");
             throw me;
         }
@@ -624,22 +634,23 @@ public class MsoHeatUtils extends MsoCommonUtils implements VduPlugin{
         // Obtain the cloud site information where we will create the stack
         CloudSite cloudSite = cloudConfig.getCloudSite(cloudSiteId).orElseThrow(
                 () -> new MsoCloudSiteNotFound(cloudSiteId));
-        LOGGER.debug("Found: " + cloudSite.toString());
+        logger.debug("Found: {}", cloudSite.toString());
 
         // Get a Heat client. They are cached between calls (keyed by tenantId:cloudId)
         Heat heatClient = null;
         try {
             heatClient = getHeatClient (cloudSite, tenantId);
             if (heatClient != null) {
-               LOGGER.debug("Found: " + heatClient.toString());
+                logger.debug("Found: {}", heatClient.toString());
             }
         } catch (MsoTenantNotFound e) {
             // Tenant doesn't exist, so stack doesn't either
-            LOGGER.debug ("Tenant with id " + tenantId + "not found.", e);
+            logger.debug ("Tenant with id " + tenantId + "not found.", e);
             return new StackInfo (stackName, HeatStatus.NOTFOUND);
         } catch (MsoException me) {
             // Got an Openstack error. Propagate it
-            LOGGER.error (MessageEnum.RA_CONNECTION_EXCEPTION, "Openstack", "Openstack Exception on Token request: " + me, "Openstack", "", MsoLogger.ErrorCode.AvailabilityError, "Connection Exception");
+            logger.error("{} {} Openstack Exception on Token request: ", MessageEnum.RA_CONNECTION_EXCEPTION,
+                MsoLogger.ErrorCode.AvailabilityError.getValue(), me);
             me.addContext (DELETE_STACK);
             throw me;
         }
@@ -664,7 +675,7 @@ public class MsoHeatUtils extends MsoCommonUtils implements VduPlugin{
                 request = heatClient.getStacks ().deleteByName (canonicalName);
             }
             else {
-                LOGGER.debug ("Heat Client is NULL" );
+                logger.debug ("Heat Client is NULL" );
             }
 
             executeAndRecordOpenstackRequest (request);
@@ -697,7 +708,7 @@ public class MsoHeatUtils extends MsoCommonUtils implements VduPlugin{
             // When querying by canonical name, Openstack returns DELETE_COMPLETE status
             // instead of "404" (which would result from query by stack name).
             while (heatStack != null && !"DELETE_COMPLETE".equals (heatStack.getStackStatus ())) {
-                LOGGER.debug ("Stack status: " + heatStack.getStackStatus ());
+                logger.debug ("Stack status: {}", heatStack.getStackStatus ());
 
                 if ("DELETE_FAILED".equals (heatStack.getStackStatus ())) {
                     // Throw a 'special case' of MsoOpenstackException to report the Heat status
@@ -708,20 +719,22 @@ public class MsoHeatUtils extends MsoCommonUtils implements VduPlugin{
                     me.addContext (DELETE_STACK);
 
                     // Alarm this condition, stack deletion failed
-                    alarmLogger.sendAlarm (HEAT_ERROR, MsoAlarmLogger.CRITICAL, me.getContextMessage ());
+
 
                     throw me;
                 }
 
                 if (pollTimeout <= 0) {
-                    LOGGER.error (MessageEnum.RA_DELETE_STACK_TIMEOUT, cloudSiteId, tenantId, stackName, heatStack.getStackStatus (), "", "", MsoLogger.ErrorCode.AvailabilityError, "Delete Stack Timeout");
+                    logger.error("{} Cloud site: {} Tenant: {} Stack: {} Stack status: {} {} Delete Stack Timeout",
+                        MessageEnum.RA_DELETE_STACK_TIMEOUT, cloudSiteId, tenantId, stackName,
+                        heatStack.getStackStatus(), MsoLogger.ErrorCode.AvailabilityError.getValue());
 
                     // Throw a 'special case' of MsoOpenstackException to report the Heat status
                     MsoOpenstackException me = new MsoOpenstackException (0, "", "Stack Deletion Timeout");
                     me.addContext (DELETE_STACK);
 
                     // Alarm this condition, stack deletion failed
-                    alarmLogger.sendAlarm (HEAT_ERROR, MsoAlarmLogger.CRITICAL, me.getContextMessage ());
+
 
                     throw me;
                 }
@@ -729,7 +742,7 @@ public class MsoHeatUtils extends MsoCommonUtils implements VduPlugin{
                 sleep(pollInterval * 1000L);
 
                 pollTimeout -= pollInterval;
-                LOGGER.debug("pollTimeout remaining: " + pollTimeout);
+                logger.debug("pollTimeout remaining: {}", pollTimeout);
 
                 heatStack = queryHeatStack (heatClient, canonicalName);
             }
@@ -782,7 +795,7 @@ public class MsoHeatUtils extends MsoCommonUtils implements VduPlugin{
         } catch (OpenStackResponseException e) {
             if (e.getStatus () == 404) {
                 // Not sure if this can happen, but return an empty list
-                LOGGER.debug ("queryAllStacks - stack not found: ");
+                logger.debug ("queryAllStacks - stack not found: ");
                 return new ArrayList <> ();
             } else {
                 // Convert the OpenStackResponseException to an MsoOpenstackException
@@ -832,7 +845,8 @@ public class MsoHeatUtils extends MsoCommonUtils implements VduPlugin{
         if (missingParams != null) {
             // Problem - missing one or more required parameters
             String error = "Missing Required inputs for HEAT Template: " + missingParams;
-            LOGGER.error (MessageEnum.RA_MISSING_PARAM, missingParams + " for HEAT Template", "", "", MsoLogger.ErrorCode.SchemaError, "Missing Required inputs for HEAT Template: " + missingParams);
+            logger.error("{} for HEAT Template {} Missing Required inputs for HEAT Template: {}",
+                MessageEnum.RA_MISSING_PARAM, MsoLogger.ErrorCode.SchemaError.getValue(), missingParams);
             throw new IllegalArgumentException (error);
         }
 
@@ -850,9 +864,8 @@ public class MsoHeatUtils extends MsoCommonUtils implements VduPlugin{
         }
 
         if (!extraParams.isEmpty ()) {
-            LOGGER.warn (MessageEnum.RA_GENERAL_WARNING, "Heat Stack (" + heatTemplate.getTemplateName ()
-                         + ") extra input params received: "
-                         + extraParams, "", "", MsoLogger.ErrorCode.DataError, "Heat Stack (" + heatTemplate.getTemplateName () + ") extra input params received: "+ extraParams);
+            logger.warn("{} Heat Stack ({}) extra input params received: {} {}", MessageEnum.RA_GENERAL_WARNING,
+                heatTemplate.getTemplateName(), extraParams, MsoLogger.ErrorCode.DataError.getValue());
         }
 
         return updatedParams;
@@ -875,40 +888,70 @@ public class MsoHeatUtils extends MsoCommonUtils implements VduPlugin{
      */
     public Heat getHeatClient (CloudSite cloudSite, String tenantId) throws MsoException {
         String cloudId = cloudSite.getId();
-
-        // Check first in the cache of previously authorized clients
-        String cacheKey = cloudId + ":" + tenantId;
-        if (heatClientCache.containsKey (cacheKey)) {
-            if (!heatClientCache.get (cacheKey).isExpired ()) {
-                LOGGER.debug ("Using Cached HEAT Client for " + cacheKey);
-                return heatClientCache.get (cacheKey).getHeatClient ();
-            } else {
-                // Token is expired. Remove it from cache.
-                heatClientCache.remove (cacheKey);
-                LOGGER.debug ("Expired Cached HEAT Client for " + cacheKey);
-            }
-        }
+        // For DCP/LCP, the region should be the cloudId.
+        String region = cloudSite.getRegionId ();
 
         // Obtain an MSO token for the tenant
         CloudIdentity cloudIdentity = cloudSite.getIdentityService();
-        LOGGER.debug("Found: " + cloudIdentity.toString());
+        logger.debug("Found: {}", cloudIdentity.toString());
         MsoTenantUtils tenantUtils = tenantUtilsFactory.getTenantUtilsByServerType(cloudIdentity.getIdentityServerType());
         String keystoneUrl = tenantUtils.getKeystoneUrl(cloudId, cloudIdentity);
-        LOGGER.debug("keystoneUrl=" + keystoneUrl);
-        Keystone keystoneTenantClient = new Keystone (keystoneUrl);
-        Access access = null;
-        try {
-               Authentication credentials = authenticationMethodFactory.getAuthenticationFor(cloudIdentity);
-
-               OpenStackRequest <Access> request = keystoneTenantClient.tokens ()
-                       .authenticate (credentials).withTenantId (tenantId);
-
-            access = executeAndRecordOpenstackRequest (request);
-        } catch (OpenStackResponseException e) {
+        logger.debug("keystoneUrl={}", keystoneUrl);
+        String heatUrl = null;
+        String tokenId = null;
+        Calendar expiration = null;
+           try {
+               if (ServerType.KEYSTONE.equals(cloudIdentity.getIdentityServerType())) {
+                       Keystone keystoneTenantClient = new Keystone (keystoneUrl);
+                       Access access = null;
+               
+                       Authentication credentials = authenticationMethodFactory.getAuthenticationFor(cloudIdentity);
+       
+                       OpenStackRequest <Access> request = keystoneTenantClient.tokens ()
+                              .authenticate (credentials).withTenantId (tenantId);
+       
+                   access = executeAndRecordOpenstackRequest (request);
+               
+                       try {
+                               // Isolate trying to printout the region IDs
+                               try {
+                  logger.debug("access={}", access.toString());
+                  for (Access.Service service : access.getServiceCatalog()) {
+                                               List<Access.Service.Endpoint> endpoints = service.getEndpoints();
+                                               for (Access.Service.Endpoint endpoint : endpoints) {
+                      logger.debug("AIC returned region={}", endpoint.getRegion());
+                  }
+                                       }
+                               } catch (Exception e) {
+                  logger.debug("Encountered an error trying to printout Access object returned from AIC. {}",
+                      e.getMessage(), e);
+              }
+                           heatUrl = KeystoneUtils.findEndpointURL (access.getServiceCatalog (), "orchestration", region, "public");
+                logger.debug("heatUrl={}, region={}", heatUrl, region);
+            } catch (RuntimeException e) {
+                           // This comes back for not found (probably an incorrect region ID)
+                           String error = "AIC did not match an orchestration service for: region=" + region + ",cloud=" + cloudIdentity.getIdentityUrl();
+                           throw new MsoAdapterException (error, e);
+                       }
+                       tokenId = access.getToken ().getId ();
+                       expiration = access.getToken ().getExpires ();
+               } else if (ServerType.KEYSTONE_V3.equals(cloudIdentity.getIdentityServerType())) {
+                       try {
+                               KeystoneAuthHolder holder = keystoneV3Authentication.getToken(cloudSite, tenantId, "orchestration");
+                               tokenId = holder.getId();
+                               expiration = holder.getexpiration();
+                               heatUrl = holder.getServiceUrl();
+                       } catch (ServiceEndpointNotFoundException e) {
+                               // This comes back for not found (probably an incorrect region ID)
+                           String error = "cloud did not match an orchestration service for: region=" + region + ",cloud=" + cloudIdentity.getIdentityUrl();
+                           throw new MsoAdapterException (error, e);
+                       }
+               }
+           } catch (OpenStackResponseException e) {
             if (e.getStatus () == 401) {
                 // Authentication error.
                 String error = "Authentication Failure: tenant=" + tenantId + ",cloud=" + cloudIdentity.getId ();
-                alarmLogger.sendAlarm ("MsoAuthenticationError", MsoAlarmLogger.CRITICAL, error);
+
                 throw new MsoAdapterException (error);
             } else {
                 throw keystoneErrorToMsoException (e, TOKEN_AUTH);
@@ -922,63 +965,11 @@ public class MsoHeatUtils extends MsoCommonUtils implements VduPlugin{
             // Catch-all
             throw runtimeExceptionToMsoException (e, TOKEN_AUTH);
         }
-
-        // For DCP/LCP, the region should be the cloudId.
-        String region = cloudSite.getRegionId ();
-        String heatUrl = null;
-        try {
-               // Isolate trying to printout the region IDs
-               try {
-                       LOGGER.debug("access=" + access.toString());
-                       for (Access.Service service : access.getServiceCatalog()) {
-                               List<Access.Service.Endpoint> endpoints = service.getEndpoints();
-                               for (Access.Service.Endpoint endpoint : endpoints) {
-                                       LOGGER.debug("AIC returned region=" + endpoint.getRegion());
-                               }
-                       }
-               } catch (Exception e) {
-                       LOGGER.debug("Encountered an error trying to printout Access object returned from AIC. " + e.getMessage());
-               }
-            heatUrl = KeystoneUtils.findEndpointURL (access.getServiceCatalog (), "orchestration", region, "public");
-            LOGGER.debug("heatUrl=" + heatUrl + ", region=" + region);
-        } catch (RuntimeException e) {
-            // This comes back for not found (probably an incorrect region ID)
-            String error = "AIC did not match an orchestration service for: region=" + region + ",cloud=" + cloudIdentity.getIdentityUrl();
-            alarmLogger.sendAlarm ("MsoConfigurationError", MsoAlarmLogger.CRITICAL, error);
-            throw new MsoAdapterException (error, e);
-        }
-
         Heat heatClient = new Heat (heatUrl);
-        heatClient.token (access.getToken ().getId ());
-
-        heatClientCache.put (cacheKey,
-                             new HeatCacheEntry (heatUrl,
-                                                 access.getToken ().getId (),
-                                                 access.getToken ().getExpires ()));
-        LOGGER.debug ("Caching HEAT Client for " + cacheKey);
-
+        heatClient.token (tokenId);
         return heatClient;
     }
 
-    /**
-     * Forcibly expire a HEAT client from the cache. This call is for use by
-     * the KeystoneClient in case where a tenant is deleted. In that case,
-     * all cached credentials must be purged so that fresh authentication is
-     * done if a similarly named tenant is re-created.
-     * <p>
-     * Note: This is probably only applicable to dev/test environments where
-     * the same Tenant Name is repeatedly used for creation/deletion.
-     * <p>
-     *
-     */
-    public void expireHeatClient (String tenantId, String cloudId) {
-        String cacheKey = cloudId + ":" + tenantId;
-        if (heatClientCache.containsKey (cacheKey)) {
-            heatClientCache.remove (cacheKey);
-            LOGGER.debug ("Deleted Cached HEAT Client for " + cacheKey);
-        }
-    }
-
     /*
      * Query for a Heat Stack. This function is needed in several places, so
      * a common method is useful. This method takes an authenticated Heat Client
@@ -1010,8 +1001,9 @@ public class MsoHeatUtils extends MsoCommonUtils implements VduPlugin{
             OpenStackRequest <Stack> request = heatClient.getStacks ().byName (stackName);
             return executeAndRecordOpenstackRequest (request);
         } catch (OpenStackResponseException e) {
+               logger.error("Error in Query Stack", e);
             if (e.getStatus () == 404) {
-                LOGGER.debug ("queryHeatStack - stack not found: " + stackName);
+                logger.debug ("queryHeatStack - stack not found: {}", stackName);
                 return null;
             } else {
                 // Convert the OpenStackResponseException to an MsoOpenstackException
@@ -1026,15 +1018,15 @@ public class MsoHeatUtils extends MsoCommonUtils implements VduPlugin{
 
        public Map<String, Object> queryStackForOutputs(String cloudSiteId,
                        String tenantId, String stackName) throws MsoException {
-               LOGGER.debug("MsoHeatUtils.queryStackForOutputs)");
-               StackInfo heatStack = this.queryStack(cloudSiteId, tenantId, stackName);
+      logger.debug("MsoHeatUtils.queryStackForOutputs)");
+      StackInfo heatStack = this.queryStack(cloudSiteId, tenantId, stackName);
                if (heatStack == null || heatStack.getStatus() == HeatStatus.NOTFOUND) {
                        return null;
                }
                return heatStack.getOutputs();
        }
 
-       public void copyStringOutputsToInputs(Map<String, String> inputs,
+       public void copyStringOutputsToInputs(Map<String, Object> inputs,
                        Map<String, Object> otherStackOutputs, boolean overWrite) {
                if (inputs == null || otherStackOutputs == null)
                        return;
@@ -1050,31 +1042,31 @@ public class MsoHeatUtils extends MsoCommonUtils implements VduPlugin{
                                                String str = this.convertNode((JsonNode) obj);
                                                inputs.put(key, str);
                                        } catch (Exception e) {
-                                               LOGGER.debug("DANGER WILL ROBINSON: unable to convert value for JsonNode "+ key, e);
-                                               //effect here is this value will not have been copied to the inputs - and therefore will error out downstream
+              logger.debug("DANGER WILL ROBINSON: unable to convert value for JsonNode {} ", key, e);
+              //effect here is this value will not have been copied to the inputs - and therefore will error out downstream
                                        }
                                } else if (obj instanceof java.util.LinkedHashMap) {
-                                       LOGGER.debug("LinkedHashMap - this is showing up as a LinkedHashMap instead of JsonNode");
-                                       try {
+            logger.debug("LinkedHashMap - this is showing up as a LinkedHashMap instead of JsonNode");
+            try {
                                                String str = JSON_MAPPER.writeValueAsString(obj);
                                                inputs.put(key, str);
                                        } catch (Exception e) {
-                                               LOGGER.debug("DANGER WILL ROBINSON: unable to convert value for LinkedHashMap "+ key, e);
-                                       }
+              logger.debug("DANGER WILL ROBINSON: unable to convert value for LinkedHashMap {} ", key, e);
+          }
                                } else if (obj instanceof Integer) {
                                        try {
                                                String str = "" + obj;
                                                inputs.put(key, str);
                                        } catch (Exception e) {
-                                               LOGGER.debug("DANGER WILL ROBINSON: unable to convert value for Integer "+ key, e);
-                                       }
+              logger.debug("DANGER WILL ROBINSON: unable to convert value for Integer {} ", key, e);
+          }
                                } else {
                                        try {
                                                String str = obj.toString();
                                                inputs.put(key, str);
                                        } catch (Exception e) {
-                                               LOGGER.debug("DANGER WILL ROBINSON: unable to convert value for Other "+ key +" (" + e.getMessage() + ")", e);
-                                               //effect here is this value will not have been copied to the inputs - and therefore will error out downstream
+              logger.debug("DANGER WILL ROBINSON: unable to convert value for Other {} ({}) ", key, e.getMessage(), e);
+              //effect here is this value will not have been copied to the inputs - and therefore will error out downstream
                                        }
                                }
                        }
@@ -1109,8 +1101,8 @@ public class MsoHeatUtils extends MsoCommonUtils implements VduPlugin{
                                                String str = params.get(key).toString();
                                                sb.append("\n").append(key).append("=").append(str);
                                        } catch (Exception e) {
-                                               LOGGER.debug("Exception :",e);
-                                       }
+              logger.debug("Exception :", e);
+          }
                                }
                        }
                }
@@ -1123,8 +1115,8 @@ public class MsoHeatUtils extends MsoCommonUtils implements VduPlugin{
                        final String json = JSON_MAPPER.writeValueAsString(obj);
                        return json;
                } catch (Exception e) {
-                       LOGGER.debug("Error converting json to string " + e.getMessage(), e);
-               }
+        logger.debug("Error converting json to string {} ", e.getMessage(), e);
+    }
                return "[Error converting json to string]";
        }
 
@@ -1160,7 +1152,7 @@ public class MsoHeatUtils extends MsoCommonUtils implements VduPlugin{
                                        String str = JSON_MAPPER.writeValueAsString(obj);
                                        sb.append(str).append(" (a java.util.LinkedHashMap)");
                                } catch (Exception e) {
-                                       LOGGER.debug("Exception :",e);
+            logger.debug("Exception :", e);
                                        sb.append("(a LinkedHashMap value that would not convert nicely)");
                                }
                        } else if (obj instanceof Integer) {
@@ -1168,8 +1160,8 @@ public class MsoHeatUtils extends MsoCommonUtils implements VduPlugin{
                                try {
                                        str = obj.toString() + " (an Integer)\n";
                                } catch (Exception e) {
-                                       LOGGER.debug("Exception :",e);
-                                       str = "(an Integer unable to call .toString() on)";
+            logger.debug("Exception :", e);
+            str = "(an Integer unable to call .toString() on)";
                                }
                                sb.append(str);
                        } else if (obj instanceof ArrayList) {
@@ -1177,8 +1169,8 @@ public class MsoHeatUtils extends MsoCommonUtils implements VduPlugin{
                                try {
                                        str = obj.toString() + " (an ArrayList)";
                                } catch (Exception e) {
-                                       LOGGER.debug("Exception :",e);
-                                       str = "(an ArrayList unable to call .toString() on?)";
+            logger.debug("Exception :", e);
+            str = "(an ArrayList unable to call .toString() on?)";
                                }
                                sb.append(str);
                        } else if (obj instanceof Boolean) {
@@ -1186,8 +1178,8 @@ public class MsoHeatUtils extends MsoCommonUtils implements VduPlugin{
                                try {
                                        str = obj.toString() + " (a Boolean)";
                                } catch (Exception e) {
-                                       LOGGER.debug("Exception :",e);
-                                       str = "(an Boolean unable to call .toString() on?)";
+            logger.debug("Exception :", e);
+            str = "(an Boolean unable to call .toString() on?)";
                                }
                                sb.append(str);
                        }
@@ -1196,8 +1188,8 @@ public class MsoHeatUtils extends MsoCommonUtils implements VduPlugin{
                                try {
                                        str = obj.toString() + " (unknown Object type)";
                                } catch (Exception e) {
-                                       LOGGER.debug("Exception :",e);
-                                       str = "(a value unable to call .toString() on?)";
+            logger.debug("Exception :", e);
+            str = "(a value unable to call .toString() on?)";
                                }
                                sb.append(str);
                        }
@@ -1206,7 +1198,7 @@ public class MsoHeatUtils extends MsoCommonUtils implements VduPlugin{
                sb.append("[END]");
                return sb;
        }
-
 
        public void copyBaseOutputsToInputs(Map<String, Object> inputs,
                        Map<String, Object> otherStackOutputs, List<String> paramNames, Map<String, String> aliases) {
@@ -1215,44 +1207,44 @@ public class MsoHeatUtils extends MsoCommonUtils implements VduPlugin{
                for (String key : otherStackOutputs.keySet()) {
                        if (paramNames != null) {
                                if (!paramNames.contains(key) && !aliases.containsKey(key)) {
-                                       LOGGER.debug("\tParameter " + key + " is NOT defined to be in the template - do not copy to inputs");
-                                       continue;
+            logger.debug("\tParameter {} is NOT defined to be in the template - do not copy to inputs", key);
+            continue;
                                }
                                if (aliases.containsKey(key)) {
-                                       LOGGER.debug("Found an alias! Will move " + key + " to " + aliases.get(key));
-                                       Object obj = otherStackOutputs.get(key);
+            logger.debug("Found an alias! Will move {} to {}", key, aliases.get(key));
+            Object obj = otherStackOutputs.get(key);
                                        key = aliases.get(key);
                                        otherStackOutputs.put(key, obj);
                                }
                        }
                        if (!inputs.containsKey(key)) {
                                Object obj = otherStackOutputs.get(key);
-                               LOGGER.debug("\t**Adding " + key + " to inputs (.toString()=" + obj.toString());
-                               if (obj instanceof String) {
-                                       LOGGER.debug("\t\t**A String");
-                                       inputs.put(key, obj);
+          logger.debug("\t**Adding {} to inputs (.toString()={}", key, obj.toString());
+          if (obj instanceof String) {
+              logger.debug("\t\t**A String");
+              inputs.put(key, obj);
                                } else if (obj instanceof Integer) {
-                                       LOGGER.debug("\t\t**An Integer");
-                                       inputs.put(key, obj);
+              logger.debug("\t\t**An Integer");
+              inputs.put(key, obj);
                                } else if (obj instanceof JsonNode) {
-                                       LOGGER.debug("\t\t**A JsonNode");
-                                       inputs.put(key, obj);
+              logger.debug("\t\t**A JsonNode");
+              inputs.put(key, obj);
                                } else if (obj instanceof Boolean) {
-                                       LOGGER.debug("\t\t**A Boolean");
-                                       inputs.put(key, obj);
+              logger.debug("\t\t**A Boolean");
+              inputs.put(key, obj);
                                } else if (obj instanceof java.util.LinkedHashMap) {
-                                       LOGGER.debug("\t\t**A java.util.LinkedHashMap **");
-                                       inputs.put(key, obj);
+              logger.debug("\t\t**A java.util.LinkedHashMap **");
+              inputs.put(key, obj);
                                } else if (obj instanceof java.util.ArrayList) {
-                                       LOGGER.debug("\t\t**An ArrayList");
-                                       inputs.put(key, obj);
+              logger.debug("\t\t**An ArrayList");
+              inputs.put(key, obj);
                                } else {
-                                       LOGGER.debug("\t\t**UNKNOWN OBJECT TYPE");
-                                       inputs.put(key, obj);
+              logger.debug("\t\t**UNKNOWN OBJECT TYPE");
+              inputs.put(key, obj);
                                }
                        } else {
-                               LOGGER.debug("key=" + key + " is already in the inputs - will not overwrite");
-                       }
+          logger.debug("key={} is already in the inputs - will not overwrite", key);
+      }
                }
                return;
        }
@@ -1274,7 +1266,7 @@ public class MsoHeatUtils extends MsoCommonUtils implements VduPlugin{
      * (heat variable type) -> java Object type
      * string -> String
      * number -> Integer
-     * json -> JsonNode XXX Removed with MSO-1475 / 1802
+     * json -> marshal object to json
      * comma_delimited_list -> ArrayList
      * boolean -> Boolean
      * if any of the conversions should fail, we will default to adding it to the inputs
@@ -1285,71 +1277,72 @@ public class MsoHeatUtils extends MsoCommonUtils implements VduPlugin{
      * @param template the HeatTemplate object - this is so we can also verify if the param is valid for this template
      * @return HashMap<String, Object> of the inputs, cleaned and converted
      */
-       public Map<String, Object> convertInputMap(Map<String, String> inputs, HeatTemplate template) {
+       public Map<String, Object> convertInputMap(Map<String, Object> inputs, HeatTemplate template) {
                HashMap<String, Object> newInputs = new HashMap<>();
                HashMap<String, HeatTemplateParam> params = new HashMap<>();
                HashMap<String, HeatTemplateParam> paramAliases = new HashMap<>();
 
                if (inputs == null) {
-                       LOGGER.debug("convertInputMap - inputs is null - nothing to do here");
-                       return new HashMap<>();
+        logger.debug("convertInputMap - inputs is null - nothing to do here");
+        return new HashMap<>();
                }
 
-               LOGGER.debug("convertInputMap in MsoHeatUtils called, with " + inputs.size() + " inputs, and template " + template.getArtifactUuid());
-               try {
-                       LOGGER.debug(template.toString());
-                       Set<HeatTemplateParam> paramSet = template.getParameters();
-                       LOGGER.debug("paramSet has " + paramSet.size() + " entries");
-               } catch (Exception e) {
-                       LOGGER.debug("Exception occurred in convertInputMap:" + e.getMessage(), e);
-               }
+      logger.debug("convertInputMap in MsoHeatUtils called, with {} inputs, and template {}", inputs.size(),
+          template.getArtifactUuid());
+      try {
+          logger.debug(template.toString());
+          Set<HeatTemplateParam> paramSet = template.getParameters();
+          logger.debug("paramSet has {} entries", paramSet.size());
+      } catch (Exception e) {
+          logger.debug("Exception occurred in convertInputMap:" + e.getMessage(), e);
+      }
 
                for (HeatTemplateParam htp : template.getParameters()) {
-                       LOGGER.debug("Adding " + htp.getParamName());
-                       params.put(htp.getParamName(), htp);
+        logger.debug("Adding {}", htp.getParamName());
+        params.put(htp.getParamName(), htp);
                        if (htp.getParamAlias() != null && !"".equals(htp.getParamAlias())) {
-                               LOGGER.debug("\tFound ALIAS " + htp.getParamName() + "->" + htp.getParamAlias());
-                               paramAliases.put(htp.getParamAlias(), htp);
+          logger.debug("\tFound ALIAS {} -> {}", htp.getParamName(), htp.getParamAlias());
+          paramAliases.put(htp.getParamAlias(), htp);
                        }
                }
-               LOGGER.debug("Now iterate through the inputs...");
-               for (String key : inputs.keySet()) {
-                       LOGGER.debug("key=" + key);
-                       boolean alias = false;
+      logger.debug("Now iterate through the inputs...");
+      for (String key : inputs.keySet()) {
+        logger.debug("key={}", key);
+        boolean alias = false;
                        String realName = null;
                        if (!params.containsKey(key)) {
-                               LOGGER.debug(key + " is not a parameter in the template! - check for an alias");
-                               // add check here for an alias
+          logger.debug("{} is not a parameter in the template! - check for an alias", key);
+          // add check here for an alias
                                if (!paramAliases.containsKey(key)) {
-                                       LOGGER.debug("The parameter " + key + " is in the inputs, but it's not a parameter for this template - omit");
-                                       continue;
+            logger.debug("The parameter {} is in the inputs, but it's not a parameter for this template - omit", key);
+            continue;
                                } else {
                                        alias = true;
                                        realName = paramAliases.get(key).getParamName();
-                                       LOGGER.debug("FOUND AN ALIAS! Will use " + realName + " in lieu of give key/alias " + key);
-                               }
+            logger.debug("FOUND AN ALIAS! Will use {} in lieu of give key/alias {}", realName, key);
+        }
                        }
                        String type = params.get(key).getParamType();
                        if (type == null || "".equals(type)) {
-                               LOGGER.debug("**PARAM_TYPE is null/empty for " + key + ", will default to string");
-                               type = "string";
+          logger.debug("**PARAM_TYPE is null/empty for {}, will default to string", key);
+          type = "string";
                        }
-                       LOGGER.debug("Parameter: " + key + " is of type " + type);
-                       if ("string".equalsIgnoreCase(type)) {
+        logger.debug("Parameter: {} is of type {}", key, type);
+        if ("string".equalsIgnoreCase(type)) {
                                // Easiest!
-                               String str = inputs.get(key);
+                               String str = inputs.get(key) != null ? inputs.get(key).toString() : null;
                                if (alias)
                                        newInputs.put(realName, str);
                                else
                                        newInputs.put(key, str);
                        } else if ("number".equalsIgnoreCase(type)) {
-                               String integerString = inputs.get(key);
+                               String integerString = inputs.get(key) != null ? inputs.get(key).toString() : null;
                                Integer anInteger = null;
                                try {
                                        anInteger = Integer.parseInt(integerString);
                                } catch (Exception e) {
-                                       LOGGER.debug("Unable to convert " + integerString + " to an integer!!", e);
-                                       anInteger = null;
+            logger.debug("Unable to convert {} to an integer!!", integerString, e);
+            anInteger = null;
                                }
                                if (anInteger != null) {
                                        if (alias)
@@ -1364,16 +1357,25 @@ public class MsoHeatUtils extends MsoCommonUtils implements VduPlugin{
                                                newInputs.put(key, integerString);
                                }
                        } else if ("json".equalsIgnoreCase(type)) {
-                               // MSO-1475 - Leave this as a string now
-                               String jsonString = inputs.get(key);
-                               LOGGER.debug("Skipping conversion to jsonNode...");
+                               Object jsonObj = inputs.get(key);
+                               Object json;
+                               try {
+                                       if (jsonObj instanceof String) {
+                                               json = JSON_MAPPER.readTree(jsonObj.toString());
+                                       } else {
+                                               //will already marshal to json without intervention
+                                               json = jsonObj;
+                                       }
+                               } catch (IOException e) {
+                                       logger.error("failed to map to json, directly converting to string instead", e);
+                                       json = jsonObj.toString();
+                               }
                        if (alias)
-                               newInputs.put(realName, jsonString);
+                               newInputs.put(realName, json);
                        else
-                               newInputs.put(key, jsonString);
-                       //}
+                               newInputs.put(key, json);
                        } else if ("comma_delimited_list".equalsIgnoreCase(type)) {
-                               String commaSeparated = inputs.get(key);
+                               String commaSeparated = inputs.get(key) != null ? inputs.get(key).toString() : null;
                                try {
                                        List<String> anArrayList = this.convertCdlToArrayList(commaSeparated);
                                        if (alias)
@@ -1381,14 +1383,14 @@ public class MsoHeatUtils extends MsoCommonUtils implements VduPlugin{
                                        else
                                                newInputs.put(key, anArrayList);
                                } catch (Exception e) {
-                                       LOGGER.debug("Unable to convert " + commaSeparated + " to an ArrayList!!", e);
-                                       if (alias)
+            logger.debug("Unable to convert {} to an ArrayList!!", commaSeparated, e);
+            if (alias)
                                                newInputs.put(realName, commaSeparated);
                                        else
                                                newInputs.put(key, commaSeparated);
                                }
                        } else if ("boolean".equalsIgnoreCase(type)) {
-                               String booleanString = inputs.get(key);
+                               String booleanString = inputs.get(key) != null ? inputs.get(key).toString() : null;
                                Boolean aBool = Boolean.valueOf(booleanString);
                                if (alias)
                                        newInputs.put(realName, aBool);
@@ -1396,7 +1398,7 @@ public class MsoHeatUtils extends MsoCommonUtils implements VduPlugin{
                                        newInputs.put(key, aBool);
                        } else {
                                // it's null or something undefined - just add it back as a String
-                               String str = inputs.get(key);
+                               String str = inputs.get(key).toString();
                                if (alias)
                                        newInputs.put(realName, str);
                                else
@@ -1702,12 +1704,28 @@ public class MsoHeatUtils extends MsoCommonUtils implements VduPlugin{
 
        return vduStatus;
     }
+    
+       public Resources queryStackResources(String cloudSiteId, String tenantId, String stackName) throws MsoException {
+               CloudSite cloudSite = cloudConfig.getCloudSite(cloudSiteId)
+                               .orElseThrow(() -> new MsoCloudSiteNotFound(cloudSiteId));
+               Heat heatClient = getHeatClient(cloudSite, tenantId);
+               OpenStackRequest<Resources> request = heatClient.getResources().listResources(stackName);
+               return executeAndRecordOpenstackRequest(request);
+       }
+
+       public <R> R executeHeatClientRequest(String url, String cloudSiteId, String tenantId, Class<R> returnType) throws MsoException {
+               CloudSite cloudSite = cloudConfig.getCloudSite(cloudSiteId)
+                               .orElseThrow(() -> new MsoCloudSiteNotFound(cloudSiteId));
+               Heat heatClient = getHeatClient(cloudSite, tenantId);
+               OpenStackRequest<R> request = heatClient.get(url, returnType);
+               return executeAndRecordOpenstackRequest(request);
+       }
 
-    private void sleep(long time) {
+    protected void sleep(long time) {
        try {
             Thread.sleep(time);
         } catch (InterruptedException e) {
-            LOGGER.debug ("Thread interrupted while sleeping", e);
+            logger.debug ("Thread interrupted while sleeping", e);
             Thread.currentThread().interrupt();
         }
     }