Merge "fix major sonar bug"
[so.git] / adapters / mso-adapter-utils / src / main / java / org / onap / so / openstack / utils / MsoHeatUtils.java
index d44857c..20498cb 100644 (file)
@@ -24,10 +24,12 @@ package org.onap.so.openstack.utils;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Calendar;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
+import java.util.Optional;
 import java.util.Set;
 
 import org.onap.so.adapters.vdu.CloudInfo;
@@ -41,13 +43,17 @@ import org.onap.so.adapters.vdu.VduPlugin;
 import org.onap.so.adapters.vdu.VduStateType;
 import org.onap.so.adapters.vdu.VduStatus;
 import org.onap.so.cloud.CloudConfig;
+import org.onap.so.cloud.authentication.AuthenticationMethodFactory;
+import org.onap.so.cloud.authentication.KeystoneAuthHolder;
+import org.onap.so.cloud.authentication.KeystoneV3Authentication;
+import org.onap.so.cloud.authentication.ServiceEndpointNotFoundException;
 import org.onap.so.db.catalog.beans.CloudIdentity;
 import org.onap.so.db.catalog.beans.CloudSite;
-import org.onap.so.cloud.authentication.AuthenticationMethodFactory;
 import org.onap.so.db.catalog.beans.HeatTemplate;
 import org.onap.so.db.catalog.beans.HeatTemplateParam;
+import org.onap.so.db.catalog.beans.ServerType;
 import org.onap.so.logger.MessageEnum;
-import org.onap.so.logger.MsoAlarmLogger;
+
 import org.onap.so.logger.MsoLogger;
 import org.onap.so.openstack.beans.HeatCacheEntry;
 import org.onap.so.openstack.beans.HeatStatus;
@@ -74,6 +80,7 @@ import com.woorea.openstack.base.client.OpenStackRequest;
 import com.woorea.openstack.base.client.OpenStackResponseException;
 import com.woorea.openstack.heat.Heat;
 import com.woorea.openstack.heat.model.CreateStackParam;
+import com.woorea.openstack.heat.model.Resources;
 import com.woorea.openstack.heat.model.Stack;
 import com.woorea.openstack.heat.model.Stack.Output;
 import com.woorea.openstack.heat.model.Stacks;
@@ -96,13 +103,6 @@ public class MsoHeatUtils extends MsoCommonUtils implements VduPlugin{
 
     protected static final String CREATE_STACK = "CreateStack";
 
-    // Cache Heat Clients statically. Since there is just one MSO user, there is no
-    // benefit to re-authentication on every request (or across different flows). The
-    // token will be used until it expires.
-    //
-    // The cache key is "tenantId:cloudId"
-    private static Map <String, HeatCacheEntry> heatClientCache = new HashMap <> ();
-
     // Fetch cloud configuration each time (may be cached in CloudConfig class)
     @Autowired
     protected CloudConfig cloudConfig;
@@ -115,13 +115,16 @@ public class MsoHeatUtils extends MsoCommonUtils implements VduPlugin{
 
     @Autowired
     private MsoTenantUtilsFactory tenantUtilsFactory;
-
+    
+    @Autowired
+    private KeystoneV3Authentication keystoneV3Authentication;
+    
     private static final MsoLogger LOGGER = MsoLogger.getMsoLogger (MsoLogger.Catalog.RA, MsoHeatUtils.class);
 
     // Properties names and variables (with default values)
-    protected String createPollIntervalProp = "ecomp.mso.adapters.po.pollInterval";
-    private String deletePollIntervalProp = "ecomp.mso.adapters.po.pollInterval";
-    private String deletePollTimeoutProp = "ecomp.mso.adapters.po.pollTimeout";
+    protected String createPollIntervalProp = "org.onap.so.adapters.po.pollInterval";
+    private String deletePollIntervalProp = "org.onap.so.adapters.po.pollInterval";
+    private String deletePollTimeoutProp = "org.onap.so.adapters.po.pollTimeout";
 
     protected static final String createPollIntervalDefault = "15";
     private static final String deletePollIntervalDefault = "15";
@@ -303,37 +306,23 @@ public class MsoHeatUtils extends MsoCommonUtils implements VduPlugin{
 
         Stack heatStack = null;
         try {
-            // Execute the actual Openstack command to create the Heat stack
             OpenStackRequest <Stack> request = heatClient.getStacks ().create (stack);
-            // Begin X-Auth-User
-            // Obtain an MSO token for the tenant
             CloudIdentity cloudIdentity = cloudSite.getIdentityService();
-            // cloudIdentity.getMsoId(), cloudIdentity.getMsoPass()
-            //req
             request.header ("X-Auth-User", cloudIdentity.getMsoId ());
             request.header ("X-Auth-Key", CryptoUtils.decryptCloudConfigPassword(cloudIdentity.getMsoPass ()));
-            LOGGER.debug ("headers added, about to executeAndRecordOpenstackRequest");
-            //LOGGER.debug(this.requestToStringBuilder(stack).toString());
-            // END - try to fix X-Auth-User
             heatStack = executeAndRecordOpenstackRequest (request);
         } catch (OpenStackResponseException e) {
-            // Since this came on the 'Create Stack' command, nothing was changed
-            // in the cloud. Return the error as an exception.
-            if (e.getStatus () == 409) {
-                // Stack already exists. Return a specific error for this case
+            if (e.getStatus () == 409) {                
                 MsoStackAlreadyExists me = new MsoStackAlreadyExists (stackName, tenantId, cloudSiteId);
                 me.addContext (CREATE_STACK);
                 throw me;
-            } else {
-                // Convert the OpenStackResponseException to an MsoOpenstackException
+        } else {
                 LOGGER.debug("ERROR STATUS = " + e.getStatus() + ",\n" + e.getMessage() + "\n" + e.getLocalizedMessage());
                 throw heatExceptionToMsoException (e, CREATE_STACK);
             }
-        } catch (OpenStackConnectException e) {
-            // Error connecting to Openstack instance. Convert to an MsoException
+        } catch (OpenStackConnectException e) {          
             throw heatExceptionToMsoException (e, CREATE_STACK);
-        } catch (RuntimeException e) {
-            // Catch-all
+        } catch (RuntimeException e) {           
             throw runtimeExceptionToMsoException (e, CREATE_STACK);
         }
 
@@ -341,211 +330,197 @@ public class MsoHeatUtils extends MsoCommonUtils implements VduPlugin{
         // Otherwise, simple query by name returns a 302 redirect.
         // NOTE: This is specific to the v1 Orchestration API.
         String canonicalName = stackName + "/" + heatStack.getId ();
-
-        // If client has requested a final response, poll for stack completion
+       
         if (pollForCompletion) {
-            // Set a time limit on overall polling.
-            // Use the resource (template) timeout for Openstack (expressed in minutes)
-            // and add one poll interval to give Openstack a chance to fail on its own.s
-
-            int createPollInterval = Integer.parseInt(this.environment.getProperty(createPollIntervalProp, createPollIntervalDefault));
-            int pollTimeout = (timeoutMinutes * 60) + createPollInterval;
-            // New 1610 - poll on delete if we rollback - use same values for now
-            int deletePollInterval = createPollInterval;
-            int deletePollTimeout = pollTimeout;
-            boolean createTimedOut = false;
-            StringBuilder stackErrorStatusReason = new StringBuilder("");
-            LOGGER.debug("createPollInterval=" + createPollInterval + ", pollTimeout=" + pollTimeout);
-
-            while (true) {
-                try {
-                    heatStack = queryHeatStack (heatClient, canonicalName);
-                    LOGGER.debug (heatStack.getStackStatus () + " (" + canonicalName + ")");
-                    try {
-                        LOGGER.debug("Current stack " + this.getOutputsAsStringBuilder(heatStack).toString());
-                    } catch (Exception e) {
-                        LOGGER.debug("an error occurred trying to print out the current outputs of the stack", e);
-                    }
-
-                    if ("CREATE_IN_PROGRESS".equals (heatStack.getStackStatus ())) {
-                        // Stack creation is still running.
-                        // Sleep and try again unless timeout has been reached
-                        if (pollTimeout <= 0) {
-                            // Note that this should not occur, since there is a timeout specified
-                            // in the Openstack call.
-                            LOGGER.error (MessageEnum.RA_CREATE_STACK_TIMEOUT, cloudSiteId, tenantId, stackName, heatStack.getStackStatus (), "", "", MsoLogger.ErrorCode.AvailabilityError, "Create stack timeout");
-                            createTimedOut = true;
-                            break;
-                        }
-
-                        sleep(createPollInterval * 1000L);
-
-                        pollTimeout -= createPollInterval;
-                               LOGGER.debug("pollTimeout remaining: " + pollTimeout);
-                    } else {
-                       //save off the status & reason msg before we attempt delete
-                       stackErrorStatusReason.append("Stack error (" + heatStack.getStackStatus() + "): " + heatStack.getStackStatusReason());
-                        break;
-                    }
-                } catch (MsoException me) {
-                       // Cannot query the stack status. Something is wrong.
-                       // Try to roll back the stack
-                       if (!backout)
-                       {
-                               LOGGER.warn(MessageEnum.RA_CREATE_STACK_ERR, "Create Stack errored, stack deletion suppressed", "", "", MsoLogger.ErrorCode.BusinessProcesssError, "Exception in Create Stack, stack deletion suppressed");
-                       }
-                       else
-                       {
-                               try {
-                                       LOGGER.debug("Create Stack error - unable to query for stack status - attempting to delete stack: " + canonicalName + " - This will likely fail and/or we won't be able to query to see if delete worked");
-                                       OpenStackRequest <Void> request = heatClient.getStacks ().deleteByName (canonicalName);
-                                       executeAndRecordOpenstackRequest (request);
-                                       // this may be a waste of time - if we just got an exception trying to query the stack - we'll just
-                                       // get another one, n'est-ce pas?
-                                       boolean deleted = false;
-                                       while (!deleted) {
-                                               try {
-                                                       heatStack = queryHeatStack(heatClient, canonicalName);
-                                                       if (heatStack != null) {
-                                                       LOGGER.debug(heatStack.getStackStatus());
-                                                       if ("DELETE_IN_PROGRESS".equals(heatStack.getStackStatus())) {
-                                                               if (deletePollTimeout <= 0) {
-                                                                       LOGGER.error (MessageEnum.RA_CREATE_STACK_TIMEOUT, cloudSiteId, tenantId, stackName,
-                                                                                       heatStack.getStackStatus (), "", "", MsoLogger.ErrorCode.AvailabilityError,
-                                                                                       "Rollback: DELETE stack timeout");
-                                                                       break;
-                                                               } else {
-                                                                       sleep(deletePollInterval * 1000L);
-                                                                       deletePollTimeout -= deletePollInterval;
-                                                               }
-                                                       } else if ("DELETE_COMPLETE".equals(heatStack.getStackStatus())){
-                                                               LOGGER.debug("DELETE_COMPLETE for " + canonicalName);
-                                                               deleted = true;
-                                                               continue;
-                                                       } else {
-                                                               //got a status other than DELETE_IN_PROGRESS or DELETE_COMPLETE - so break and evaluate
-                                                               break;
-                                                       }
-                                               } else {
-                                                       // assume if we can't find it - it's deleted
-                                                       LOGGER.debug("heatStack returned null - assume the stack " + canonicalName + " has been deleted");
-                                                       deleted = true;
-                                                       continue;
-                                                       }
-
-                                               } catch (Exception e3) {
-                                                       // Just log this one. We will report the original exception.
-                                                       LOGGER.error (MessageEnum.RA_CREATE_STACK_ERR, "Create Stack: Nested exception rolling back stack: " + e3, "", "", MsoLogger.ErrorCode.BusinessProcesssError, "Create Stack: Nested exception rolling back stack on error on query");
-
-                                               }
-                                       }
-                               } catch (Exception e2) {
-                                       // Just log this one. We will report the original exception.
-                                       LOGGER.error (MessageEnum.RA_CREATE_STACK_ERR, "Create Stack: Nested exception rolling back stack: " + e2, "", "", MsoLogger.ErrorCode.BusinessProcesssError, "Create Stack: Nested exception rolling back stack");
-                               }
-                       }
-
-                    // Propagate the original exception from Stack Query.
-                    me.addContext (CREATE_STACK);
-                    throw me;
-                }
-            }
-
-            if (!"CREATE_COMPLETE".equals (heatStack.getStackStatus ())) {
-                LOGGER.error (MessageEnum.RA_CREATE_STACK_ERR, "Create Stack error:  Polling complete with non-success status: "
-                              + heatStack.getStackStatus () + ", " + heatStack.getStackStatusReason (), "", "", MsoLogger.ErrorCode.BusinessProcesssError, "Create Stack error");
-
-                // Rollback the stack creation, since it is in an indeterminate state.
-                if (!backout)
-                {
-                       LOGGER.warn(MessageEnum.RA_CREATE_STACK_ERR, "Create Stack errored, stack deletion suppressed", "", "", MsoLogger.ErrorCode.BusinessProcesssError, "Create Stack error, stack deletion suppressed");
-                }
-                else
-                {
-                       try {
-                               LOGGER.debug("Create Stack errored - attempting to DELETE stack: " + canonicalName);
-                               LOGGER.debug("deletePollInterval=" + deletePollInterval + ", deletePollTimeout=" + deletePollTimeout);
-                               OpenStackRequest <Void> request = heatClient.getStacks ().deleteByName (canonicalName);
-                               executeAndRecordOpenstackRequest (request);
-                               boolean deleted = false;
-                               while (!deleted) {
-                                       try {
-                                               heatStack = queryHeatStack(heatClient, canonicalName);
-                                               if (heatStack != null) {
-                                                       LOGGER.debug(heatStack.getStackStatus() + " (" + canonicalName + ")");
-                                                       if ("DELETE_IN_PROGRESS".equals(heatStack.getStackStatus())) {
-                                                               if (deletePollTimeout <= 0) {
-                                                                       LOGGER.error (MessageEnum.RA_CREATE_STACK_TIMEOUT, cloudSiteId, tenantId, stackName,
-                                                                                       heatStack.getStackStatus (), "", "", MsoLogger.ErrorCode.AvailabilityError,
-                                                                                       "Rollback: DELETE stack timeout");
-                                                                       break;
-                                                               } else {
-                                                                       sleep(deletePollInterval * 1000L);
-                                                                       deletePollTimeout -= deletePollInterval;
-                                                                       LOGGER.debug("deletePollTimeout remaining: " + deletePollTimeout);
-                                                               }
-                                                       } else if ("DELETE_COMPLETE".equals(heatStack.getStackStatus())){
-                                                               LOGGER.debug("DELETE_COMPLETE for " + canonicalName);
-                                                               deleted = true;
-                                                               continue;
-                                                       } else if ("DELETE_FAILED".equals(heatStack.getStackStatus())) {
-                                                               // Warn about this (?) - but still throw the original exception
-                                                               LOGGER.warn(MessageEnum.RA_CREATE_STACK_ERR, "Create Stack errored, stack deletion FAILED", "", "", MsoLogger.ErrorCode.BusinessProcesssError, "Create Stack error, stack deletion FAILED");
-                                                               LOGGER.debug("Stack deletion FAILED on a rollback of a create - " + canonicalName + ", status=" + heatStack.getStackStatus() + ", reason=" + heatStack.getStackStatusReason());
-                                                               break;
-                                                       } else {
-                                                               //got a status other than DELETE_IN_PROGRESS or DELETE_COMPLETE - so break and evaluate
-                                                               break;
-                                                       }
-                                               } else {
-                                                       // assume if we can't find it - it's deleted
-                                                       LOGGER.debug("heatStack returned null - assume the stack " + canonicalName + " has been deleted");
-                                                       deleted = true;
-                                                       continue;
-                                               }
-
-                                       } catch (MsoException me2) {
-                                               // We got an exception on the delete - don't throw this exception - throw the original - just log.
-                                               LOGGER.debug("Exception thrown trying to delete " + canonicalName + " on a create->rollback: " + me2.getContextMessage(), me2);
-                                               LOGGER.warn(MessageEnum.RA_CREATE_STACK_ERR, "Create Stack errored, then stack deletion FAILED - exception thrown", "", "", MsoLogger.ErrorCode.BusinessProcesssError, me2.getContextMessage());
-                                       }
-
-                               } // end while !deleted
-                               StringBuilder errorContextMessage;
-                               if (createTimedOut) {
-                                       errorContextMessage = new StringBuilder("Stack Creation Timeout");
-                               } else {
-                                       errorContextMessage  = stackErrorStatusReason;
-                               }
-                               if (deleted) {
-                                       errorContextMessage.append(" - stack successfully deleted");
-                               } else {
-                                       errorContextMessage.append(" - encountered an error trying to delete the stack");
-                               }
-//                             MsoOpenstackException me = new MsoOpenstackException(0, "", stackErrorStatusReason.toString());
- //                            me.addContext(CREATE_STACK);
-  //                           alarmLogger.sendAlarm(HEAT_ERROR, MsoAlarmLogger.CRITICAL, me.getContextMessage());
-   //                          throw me;
-                       } catch (Exception e2) {
-                               // shouldn't happen - but handle
-                               LOGGER.error (MessageEnum.RA_CREATE_STACK_ERR, "Create Stack: Nested exception rolling back stack: " + e2, "", "", MsoLogger.ErrorCode.BusinessProcesssError, "Exception in Create Stack: rolling back stack");
-                       }
-                }
-                MsoOpenstackException me = new MsoOpenstackException(0, "", stackErrorStatusReason.toString());
-                me.addContext(CREATE_STACK);
-                alarmLogger.sendAlarm(HEAT_ERROR, MsoAlarmLogger.CRITICAL, me.getContextMessage());
-                throw me;
-            }
-
+            heatStack = pollStackForCompletion(cloudSiteId, tenantId, stackName, timeoutMinutes, backout, heatClient,
+                                       heatStack, canonicalName);
         } else {
             // Get initial status, since it will have been null after the create.
             heatStack = queryHeatStack (heatClient, canonicalName);
             LOGGER.debug (heatStack.getStackStatus ());
         }
-
         return new StackInfoMapper(heatStack).map();
     }
 
+       private Stack pollStackForCompletion(String cloudSiteId, String tenantId, String stackName, int timeoutMinutes,
+                       boolean backout, Heat heatClient, Stack heatStack, String canonicalName)
+                       throws MsoException, MsoOpenstackException {
+               int createPollInterval = Integer.parseInt(this.environment.getProperty(createPollIntervalProp, createPollIntervalDefault));
+               int pollTimeout = (timeoutMinutes * 60) + createPollInterval;
+               int deletePollInterval = createPollInterval;
+               int deletePollTimeout = pollTimeout;
+               boolean createTimedOut = false;
+               StringBuilder stackErrorStatusReason = new StringBuilder("");
+               LOGGER.debug("createPollInterval=" + createPollInterval + ", pollTimeout=" + pollTimeout);
+
+               while (true) {
+                   try {
+                       heatStack = queryHeatStack (heatClient, canonicalName);
+                       LOGGER.debug (heatStack.getStackStatus () + " (" + canonicalName + ")");
+                       try {
+                           LOGGER.debug("Current stack " + this.getOutputsAsStringBuilder(heatStack).toString());
+                       } catch (Exception e) {
+                           LOGGER.debug("an error occurred trying to print out the current outputs of the stack", e);
+                       }
+
+                       if ("CREATE_IN_PROGRESS".equals (heatStack.getStackStatus ())) {                       
+                           if (pollTimeout <= 0) {
+                               LOGGER.error (MessageEnum.RA_CREATE_STACK_TIMEOUT, cloudSiteId, tenantId, stackName, heatStack.getStackStatus (), "", "", MsoLogger.ErrorCode.AvailabilityError, "Create stack timeout");
+                               createTimedOut = true;
+                               break;
+                           }
+                           sleep(createPollInterval * 1000L);
+                           pollTimeout -= createPollInterval;
+                               LOGGER.debug("pollTimeout remaining: " + pollTimeout);
+                       } else {                    
+                               stackErrorStatusReason.append("Stack error (" + heatStack.getStackStatus() + "): " + heatStack.getStackStatusReason());
+                           break;
+                       }
+                   } catch (MsoException me) {
+                       // Cannot query the stack status. Something is wrong.
+                       // Try to roll back the stack
+                       if (!backout)
+                       {
+                               LOGGER.warn(MessageEnum.RA_CREATE_STACK_ERR, "Create Stack errored, stack deletion suppressed", "", "", MsoLogger.ErrorCode.BusinessProcesssError, "Exception in Create Stack, stack deletion suppressed");
+                       }
+                       else
+                       {
+                               try {
+                                       LOGGER.debug("Create Stack error - unable to query for stack status - attempting to delete stack: " + canonicalName + " - This will likely fail and/or we won't be able to query to see if delete worked");
+                                       OpenStackRequest <Void> request = heatClient.getStacks ().deleteByName (canonicalName);
+                                       executeAndRecordOpenstackRequest (request);
+                                       boolean deleted = false;
+                                       while (!deleted) {
+                                               try {
+                                                       heatStack = queryHeatStack(heatClient, canonicalName);
+                                                       if (heatStack != null) {
+                                                               LOGGER.debug(heatStack.getStackStatus());
+                                                               if ("DELETE_IN_PROGRESS".equals(heatStack.getStackStatus())) {
+                                                                       if (deletePollTimeout <= 0) {
+                                                                               LOGGER.error (MessageEnum.RA_CREATE_STACK_TIMEOUT, cloudSiteId, tenantId, stackName,
+                                                                                               heatStack.getStackStatus (), "", "", MsoLogger.ErrorCode.AvailabilityError,
+                                                                                               "Rollback: DELETE stack timeout");
+                                                                               break;
+                                                                       } else {
+                                                                               sleep(deletePollInterval * 1000L);
+                                                                               deletePollTimeout -= deletePollInterval;
+                                                                       }
+                                                               } else if ("DELETE_COMPLETE".equals(heatStack.getStackStatus())){
+                                                                       LOGGER.debug("DELETE_COMPLETE for " + canonicalName);
+                                                                       deleted = true;
+                                                                       continue;
+                                                               } else {
+                                                                       //got a status other than DELETE_IN_PROGRESS or DELETE_COMPLETE - so break and evaluate
+                                                                       break;
+                                                               }
+                                                       } else {
+                                                               // assume if we can't find it - it's deleted
+                                                               LOGGER.debug("heatStack returned null - assume the stack " + canonicalName + " has been deleted");
+                                                               deleted = true;
+                                                               continue;
+                                                       }
+
+                                               } catch (Exception e3) {
+                                                       // Just log this one. We will report the original exception.
+                                                       LOGGER.error (MessageEnum.RA_CREATE_STACK_ERR, "Create Stack: Nested exception rolling back stack: " + e3, "", "", MsoLogger.ErrorCode.BusinessProcesssError, "Create Stack: Nested exception rolling back stack on error on query");
+
+                                               }
+                                       }
+                               } catch (Exception e2) {
+                                       // Just log this one. We will report the original exception.
+                                       LOGGER.error (MessageEnum.RA_CREATE_STACK_ERR, "Create Stack: Nested exception rolling back stack: " + e2, "", "", MsoLogger.ErrorCode.BusinessProcesssError, "Create Stack: Nested exception rolling back stack");
+                               }
+                       }
+
+                       // Propagate the original exception from Stack Query.
+                       me.addContext (CREATE_STACK);
+                       throw me;
+                   }
+               }
+
+               if (!"CREATE_COMPLETE".equals (heatStack.getStackStatus ())) {
+                   LOGGER.error (MessageEnum.RA_CREATE_STACK_ERR, "Create Stack error:  Polling complete with non-success status: "
+                                 + heatStack.getStackStatus () + ", " + heatStack.getStackStatusReason (), "", "", MsoLogger.ErrorCode.BusinessProcesssError, "Create Stack error");
+
+                   // Rollback the stack creation, since it is in an indeterminate state.
+                   if (!backout)
+                   {
+                       LOGGER.warn(MessageEnum.RA_CREATE_STACK_ERR, "Create Stack errored, stack deletion suppressed", "", "", MsoLogger.ErrorCode.BusinessProcesssError, "Create Stack error, stack deletion suppressed");
+                   }
+                   else
+                   {
+                       try {
+                               LOGGER.debug("Create Stack errored - attempting to DELETE stack: " + canonicalName);
+                               LOGGER.debug("deletePollInterval=" + deletePollInterval + ", deletePollTimeout=" + deletePollTimeout);
+                               OpenStackRequest <Void> request = heatClient.getStacks ().deleteByName (canonicalName);
+                               executeAndRecordOpenstackRequest (request);
+                               boolean deleted = false;
+                               while (!deleted) {
+                                       try {
+                                               heatStack = queryHeatStack(heatClient, canonicalName);
+                                               if (heatStack != null) {
+                                                       LOGGER.debug(heatStack.getStackStatus() + " (" + canonicalName + ")");
+                                                       if ("DELETE_IN_PROGRESS".equals(heatStack.getStackStatus())) {
+                                                               if (deletePollTimeout <= 0) {
+                                                                       LOGGER.error (MessageEnum.RA_CREATE_STACK_TIMEOUT, cloudSiteId, tenantId, stackName,
+                                                                                       heatStack.getStackStatus (), "", "", MsoLogger.ErrorCode.AvailabilityError,
+                                                                                       "Rollback: DELETE stack timeout");
+                                                                       break;
+                                                               } else {
+                                                                       sleep(deletePollInterval * 1000L);
+                                                                       deletePollTimeout -= deletePollInterval;
+                                                                       LOGGER.debug("deletePollTimeout remaining: " + deletePollTimeout);
+                                                               }
+                                                       } else if ("DELETE_COMPLETE".equals(heatStack.getStackStatus())){
+                                                               LOGGER.debug("DELETE_COMPLETE for " + canonicalName);
+                                                               deleted = true;
+                                                               continue;
+                                                       } else if ("DELETE_FAILED".equals(heatStack.getStackStatus())) {
+                                                               // Warn about this (?) - but still throw the original exception
+                                                               LOGGER.warn(MessageEnum.RA_CREATE_STACK_ERR, "Create Stack errored, stack deletion FAILED", "", "", MsoLogger.ErrorCode.BusinessProcesssError, "Create Stack error, stack deletion FAILED");
+                                                               LOGGER.debug("Stack deletion FAILED on a rollback of a create - " + canonicalName + ", status=" + heatStack.getStackStatus() + ", reason=" + heatStack.getStackStatusReason());
+                                                               break;
+                                                       } else {
+                                                               //got a status other than DELETE_IN_PROGRESS or DELETE_COMPLETE - so break and evaluate
+                                                               break;
+                                                       }
+                                               } else {
+                                                       // assume if we can't find it - it's deleted
+                                                       LOGGER.debug("heatStack returned null - assume the stack " + canonicalName + " has been deleted");
+                                                       deleted = true;
+                                                       continue;
+                                               }
+
+                                       } catch (MsoException me2) {
+                                               // We got an exception on the delete - don't throw this exception - throw the original - just log.
+                                               LOGGER.debug("Exception thrown trying to delete " + canonicalName + " on a create->rollback: " + me2.getContextMessage(), me2);
+                                               LOGGER.warn(MessageEnum.RA_CREATE_STACK_ERR, "Create Stack errored, then stack deletion FAILED - exception thrown", "", "", MsoLogger.ErrorCode.BusinessProcesssError, me2.getContextMessage());
+                                       }
+
+                               } // end while !deleted
+                               StringBuilder errorContextMessage;
+                               if (createTimedOut) {
+                                       errorContextMessage = new StringBuilder("Stack Creation Timeout");
+                               } else {
+                                       errorContextMessage  = stackErrorStatusReason;
+                               }
+                               if (deleted) {
+                                       errorContextMessage.append(" - stack successfully deleted");
+                               } else {
+                                       errorContextMessage.append(" - encountered an error trying to delete the stack");
+                               }
+                       } catch (Exception e2) {
+                               // shouldn't happen - but handle
+                               LOGGER.error (MessageEnum.RA_CREATE_STACK_ERR, "Create Stack: Nested exception rolling back stack: " + e2, "", "", MsoLogger.ErrorCode.BusinessProcesssError, "Exception in Create Stack: rolling back stack");
+                       }
+                   }
+                   MsoOpenstackException me = new MsoOpenstackException(0, "", stackErrorStatusReason.toString());
+                   me.addContext(CREATE_STACK);
+                   throw me;
+               }
+               return heatStack;
+       }
+
     /**
      * Query for a single stack (by Name) in a tenant. This call will always return a
      * StackInfo object. If the stack does not exist, an "empty" StackInfo will be
@@ -708,7 +683,7 @@ public class MsoHeatUtils extends MsoCommonUtils implements VduPlugin{
                     me.addContext (DELETE_STACK);
 
                     // Alarm this condition, stack deletion failed
-                    alarmLogger.sendAlarm (HEAT_ERROR, MsoAlarmLogger.CRITICAL, me.getContextMessage ());
+
 
                     throw me;
                 }
@@ -721,7 +696,7 @@ public class MsoHeatUtils extends MsoCommonUtils implements VduPlugin{
                     me.addContext (DELETE_STACK);
 
                     // Alarm this condition, stack deletion failed
-                    alarmLogger.sendAlarm (HEAT_ERROR, MsoAlarmLogger.CRITICAL, me.getContextMessage ());
+
 
                     throw me;
                 }
@@ -875,19 +850,8 @@ public class MsoHeatUtils extends MsoCommonUtils implements VduPlugin{
      */
     public Heat getHeatClient (CloudSite cloudSite, String tenantId) throws MsoException {
         String cloudId = cloudSite.getId();
-
-        // Check first in the cache of previously authorized clients
-        String cacheKey = cloudId + ":" + tenantId;
-        if (heatClientCache.containsKey (cacheKey)) {
-            if (!heatClientCache.get (cacheKey).isExpired ()) {
-                LOGGER.debug ("Using Cached HEAT Client for " + cacheKey);
-                return heatClientCache.get (cacheKey).getHeatClient ();
-            } else {
-                // Token is expired. Remove it from cache.
-                heatClientCache.remove (cacheKey);
-                LOGGER.debug ("Expired Cached HEAT Client for " + cacheKey);
-            }
-        }
+        // For DCP/LCP, the region should be the cloudId.
+        String region = cloudSite.getRegionId ();
 
         // Obtain an MSO token for the tenant
         CloudIdentity cloudIdentity = cloudSite.getIdentityService();
@@ -895,20 +859,60 @@ public class MsoHeatUtils extends MsoCommonUtils implements VduPlugin{
         MsoTenantUtils tenantUtils = tenantUtilsFactory.getTenantUtilsByServerType(cloudIdentity.getIdentityServerType());
         String keystoneUrl = tenantUtils.getKeystoneUrl(cloudId, cloudIdentity);
         LOGGER.debug("keystoneUrl=" + keystoneUrl);
-        Keystone keystoneTenantClient = new Keystone (keystoneUrl);
-        Access access = null;
-        try {
-               Authentication credentials = authenticationMethodFactory.getAuthenticationFor(cloudIdentity);
-
-               OpenStackRequest <Access> request = keystoneTenantClient.tokens ()
-                       .authenticate (credentials).withTenantId (tenantId);
-
-            access = executeAndRecordOpenstackRequest (request);
-        } catch (OpenStackResponseException e) {
+        String heatUrl = null;
+        String tokenId = null;
+        Calendar expiration = null;
+           try {
+               if (ServerType.KEYSTONE.equals(cloudIdentity.getIdentityServerType())) {
+                       Keystone keystoneTenantClient = new Keystone (keystoneUrl);
+                       Access access = null;
+               
+                       Authentication credentials = authenticationMethodFactory.getAuthenticationFor(cloudIdentity);
+       
+                       OpenStackRequest <Access> request = keystoneTenantClient.tokens ()
+                              .authenticate (credentials).withTenantId (tenantId);
+       
+                   access = executeAndRecordOpenstackRequest (request);
+               
+                       try {
+                               // Isolate trying to printout the region IDs
+                               try {
+                                       LOGGER.debug("access=" + access.toString());
+                                       for (Access.Service service : access.getServiceCatalog()) {
+                                               List<Access.Service.Endpoint> endpoints = service.getEndpoints();
+                                               for (Access.Service.Endpoint endpoint : endpoints) {
+                                                       LOGGER.debug("AIC returned region=" + endpoint.getRegion());
+                                               }
+                                       }
+                               } catch (Exception e) {
+                                       LOGGER.debug("Encountered an error trying to printout Access object returned from AIC. " + e.getMessage());
+                               }
+                           heatUrl = KeystoneUtils.findEndpointURL (access.getServiceCatalog (), "orchestration", region, "public");
+                           LOGGER.debug("heatUrl=" + heatUrl + ", region=" + region);
+                       } catch (RuntimeException e) {
+                           // This comes back for not found (probably an incorrect region ID)
+                           String error = "AIC did not match an orchestration service for: region=" + region + ",cloud=" + cloudIdentity.getIdentityUrl();
+                           throw new MsoAdapterException (error, e);
+                       }
+                       tokenId = access.getToken ().getId ();
+                       expiration = access.getToken ().getExpires ();
+               } else if (ServerType.KEYSTONE_V3.equals(cloudIdentity.getIdentityServerType())) {
+                       try {
+                               KeystoneAuthHolder holder = keystoneV3Authentication.getToken(cloudSite, tenantId, "orchestration");
+                               tokenId = holder.getId();
+                               expiration = holder.getexpiration();
+                               heatUrl = holder.getServiceUrl();
+                       } catch (ServiceEndpointNotFoundException e) {
+                               // This comes back for not found (probably an incorrect region ID)
+                           String error = "cloud did not match an orchestration service for: region=" + region + ",cloud=" + cloudIdentity.getIdentityUrl();
+                           throw new MsoAdapterException (error, e);
+                       }
+               }
+           } catch (OpenStackResponseException e) {
             if (e.getStatus () == 401) {
                 // Authentication error.
                 String error = "Authentication Failure: tenant=" + tenantId + ",cloud=" + cloudIdentity.getId ();
-                alarmLogger.sendAlarm ("MsoAuthenticationError", MsoAlarmLogger.CRITICAL, error);
+
                 throw new MsoAdapterException (error);
             } else {
                 throw keystoneErrorToMsoException (e, TOKEN_AUTH);
@@ -922,63 +926,11 @@ public class MsoHeatUtils extends MsoCommonUtils implements VduPlugin{
             // Catch-all
             throw runtimeExceptionToMsoException (e, TOKEN_AUTH);
         }
-
-        // For DCP/LCP, the region should be the cloudId.
-        String region = cloudSite.getRegionId ();
-        String heatUrl = null;
-        try {
-               // Isolate trying to printout the region IDs
-               try {
-                       LOGGER.debug("access=" + access.toString());
-                       for (Access.Service service : access.getServiceCatalog()) {
-                               List<Access.Service.Endpoint> endpoints = service.getEndpoints();
-                               for (Access.Service.Endpoint endpoint : endpoints) {
-                                       LOGGER.debug("AIC returned region=" + endpoint.getRegion());
-                               }
-                       }
-               } catch (Exception e) {
-                       LOGGER.debug("Encountered an error trying to printout Access object returned from AIC. " + e.getMessage());
-               }
-            heatUrl = KeystoneUtils.findEndpointURL (access.getServiceCatalog (), "orchestration", region, "public");
-            LOGGER.debug("heatUrl=" + heatUrl + ", region=" + region);
-        } catch (RuntimeException e) {
-            // This comes back for not found (probably an incorrect region ID)
-            String error = "AIC did not match an orchestration service for: region=" + region + ",cloud=" + cloudIdentity.getIdentityUrl();
-            alarmLogger.sendAlarm ("MsoConfigurationError", MsoAlarmLogger.CRITICAL, error);
-            throw new MsoAdapterException (error, e);
-        }
-
         Heat heatClient = new Heat (heatUrl);
-        heatClient.token (access.getToken ().getId ());
-
-        heatClientCache.put (cacheKey,
-                             new HeatCacheEntry (heatUrl,
-                                                 access.getToken ().getId (),
-                                                 access.getToken ().getExpires ()));
-        LOGGER.debug ("Caching HEAT Client for " + cacheKey);
-
+        heatClient.token (tokenId);
         return heatClient;
     }
 
-    /**
-     * Forcibly expire a HEAT client from the cache. This call is for use by
-     * the KeystoneClient in case where a tenant is deleted. In that case,
-     * all cached credentials must be purged so that fresh authentication is
-     * done if a similarly named tenant is re-created.
-     * <p>
-     * Note: This is probably only applicable to dev/test environments where
-     * the same Tenant Name is repeatedly used for creation/deletion.
-     * <p>
-     *
-     */
-    public void expireHeatClient (String tenantId, String cloudId) {
-        String cacheKey = cloudId + ":" + tenantId;
-        if (heatClientCache.containsKey (cacheKey)) {
-            heatClientCache.remove (cacheKey);
-            LOGGER.debug ("Deleted Cached HEAT Client for " + cacheKey);
-        }
-    }
-
     /*
      * Query for a Heat Stack. This function is needed in several places, so
      * a common method is useful. This method takes an authenticated Heat Client
@@ -1206,7 +1158,7 @@ public class MsoHeatUtils extends MsoCommonUtils implements VduPlugin{
                sb.append("[END]");
                return sb;
        }
-
 
        public void copyBaseOutputsToInputs(Map<String, Object> inputs,
                        Map<String, Object> otherStackOutputs, List<String> paramNames, Map<String, String> aliases) {
@@ -1702,6 +1654,22 @@ public class MsoHeatUtils extends MsoCommonUtils implements VduPlugin{
 
        return vduStatus;
     }
+    
+       public Resources queryStackResources(String cloudSiteId, String tenantId, String stackName) throws MsoException {
+               CloudSite cloudSite = cloudConfig.getCloudSite(cloudSiteId)
+                               .orElseThrow(() -> new MsoCloudSiteNotFound(cloudSiteId));
+               Heat heatClient = getHeatClient(cloudSite, tenantId);
+               OpenStackRequest<Resources> request = heatClient.getResources().listResources(stackName);
+               return executeAndRecordOpenstackRequest(request);
+       }
+
+       public <R> R executeHeatClientRequest(String url, String cloudSiteId, String tenantId, Class<R> returnType) throws MsoException {
+               CloudSite cloudSite = cloudConfig.getCloudSite(cloudSiteId)
+                               .orElseThrow(() -> new MsoCloudSiteNotFound(cloudSiteId));
+               Heat heatClient = getHeatClient(cloudSite, tenantId);
+               OpenStackRequest<R> request = heatClient.get(url, returnType);
+               return executeAndRecordOpenstackRequest(request);
+       }
 
     protected void sleep(long time) {
        try {