- // Set a time limit on overall polling.
- // Use the resource (template) timeout for Openstack (expressed in minutes)
- // and add one poll interval to give Openstack a chance to fail on its own.s
-
- int createPollInterval = Integer.parseInt(this.environment.getProperty(createPollIntervalProp, createPollIntervalDefault));
- int pollTimeout = (timeoutMinutes * 60) + createPollInterval;
- // New 1610 - poll on delete if we rollback - use same values for now
- int deletePollInterval = createPollInterval;
- int deletePollTimeout = pollTimeout;
- boolean createTimedOut = false;
- StringBuilder stackErrorStatusReason = new StringBuilder("");
- LOGGER.debug("createPollInterval=" + createPollInterval + ", pollTimeout=" + pollTimeout);
-
- while (true) {
- try {
- heatStack = queryHeatStack (heatClient, canonicalName);
- LOGGER.debug (heatStack.getStackStatus () + " (" + canonicalName + ")");
- try {
- LOGGER.debug("Current stack " + this.getOutputsAsStringBuilder(heatStack).toString());
- } catch (Exception e) {
- LOGGER.debug("an error occurred trying to print out the current outputs of the stack", e);
- }
-
- if ("CREATE_IN_PROGRESS".equals (heatStack.getStackStatus ())) {
- // Stack creation is still running.
- // Sleep and try again unless timeout has been reached
- if (pollTimeout <= 0) {
- // Note that this should not occur, since there is a timeout specified
- // in the Openstack call.
- LOGGER.error (MessageEnum.RA_CREATE_STACK_TIMEOUT, cloudSiteId, tenantId, stackName, heatStack.getStackStatus (), "", "", MsoLogger.ErrorCode.AvailabilityError, "Create stack timeout");
- createTimedOut = true;
- break;
- }
-
- sleep(createPollInterval * 1000L);
-
- pollTimeout -= createPollInterval;
- LOGGER.debug("pollTimeout remaining: " + pollTimeout);
- } else {
- //save off the status & reason msg before we attempt delete
- stackErrorStatusReason.append("Stack error (" + heatStack.getStackStatus() + "): " + heatStack.getStackStatusReason());
- break;
- }
- } catch (MsoException me) {
- // Cannot query the stack status. Something is wrong.
- // Try to roll back the stack
- if (!backout)
- {
- LOGGER.warn(MessageEnum.RA_CREATE_STACK_ERR, "Create Stack errored, stack deletion suppressed", "", "", MsoLogger.ErrorCode.BusinessProcesssError, "Exception in Create Stack, stack deletion suppressed");
- }
- else
- {
- try {
- LOGGER.debug("Create Stack error - unable to query for stack status - attempting to delete stack: " + canonicalName + " - This will likely fail and/or we won't be able to query to see if delete worked");
- OpenStackRequest <Void> request = heatClient.getStacks ().deleteByName (canonicalName);
- executeAndRecordOpenstackRequest (request);
- // this may be a waste of time - if we just got an exception trying to query the stack - we'll just
- // get another one, n'est-ce pas?
- boolean deleted = false;
- while (!deleted) {
- try {
- heatStack = queryHeatStack(heatClient, canonicalName);
- if (heatStack != null) {
- LOGGER.debug(heatStack.getStackStatus());
- if ("DELETE_IN_PROGRESS".equals(heatStack.getStackStatus())) {
- if (deletePollTimeout <= 0) {
- LOGGER.error (MessageEnum.RA_CREATE_STACK_TIMEOUT, cloudSiteId, tenantId, stackName,
- heatStack.getStackStatus (), "", "", MsoLogger.ErrorCode.AvailabilityError,
- "Rollback: DELETE stack timeout");
- break;
- } else {
- sleep(deletePollInterval * 1000L);
- deletePollTimeout -= deletePollInterval;
- }
- } else if ("DELETE_COMPLETE".equals(heatStack.getStackStatus())){
- LOGGER.debug("DELETE_COMPLETE for " + canonicalName);
- deleted = true;
- continue;
- } else {
- //got a status other than DELETE_IN_PROGRESS or DELETE_COMPLETE - so break and evaluate
- break;
- }
- } else {
- // assume if we can't find it - it's deleted
- LOGGER.debug("heatStack returned null - assume the stack " + canonicalName + " has been deleted");
- deleted = true;
- continue;
- }
-
- } catch (Exception e3) {
- // Just log this one. We will report the original exception.
- LOGGER.error (MessageEnum.RA_CREATE_STACK_ERR, "Create Stack: Nested exception rolling back stack: " + e3, "", "", MsoLogger.ErrorCode.BusinessProcesssError, "Create Stack: Nested exception rolling back stack on error on query");
-
- }
- }
- } catch (Exception e2) {
- // Just log this one. We will report the original exception.
- LOGGER.error (MessageEnum.RA_CREATE_STACK_ERR, "Create Stack: Nested exception rolling back stack: " + e2, "", "", MsoLogger.ErrorCode.BusinessProcesssError, "Create Stack: Nested exception rolling back stack");
- }
- }
-
- // Propagate the original exception from Stack Query.
- me.addContext (CREATE_STACK);
- throw me;
- }
- }
-
- if (!"CREATE_COMPLETE".equals (heatStack.getStackStatus ())) {
- LOGGER.error (MessageEnum.RA_CREATE_STACK_ERR, "Create Stack error: Polling complete with non-success status: "
- + heatStack.getStackStatus () + ", " + heatStack.getStackStatusReason (), "", "", MsoLogger.ErrorCode.BusinessProcesssError, "Create Stack error");
-
- // Rollback the stack creation, since it is in an indeterminate state.
- if (!backout)
- {
- LOGGER.warn(MessageEnum.RA_CREATE_STACK_ERR, "Create Stack errored, stack deletion suppressed", "", "", MsoLogger.ErrorCode.BusinessProcesssError, "Create Stack error, stack deletion suppressed");
- }
- else
- {
- try {
- LOGGER.debug("Create Stack errored - attempting to DELETE stack: " + canonicalName);
- LOGGER.debug("deletePollInterval=" + deletePollInterval + ", deletePollTimeout=" + deletePollTimeout);
- OpenStackRequest <Void> request = heatClient.getStacks ().deleteByName (canonicalName);
- executeAndRecordOpenstackRequest (request);
- boolean deleted = false;
- while (!deleted) {
- try {
- heatStack = queryHeatStack(heatClient, canonicalName);
- if (heatStack != null) {
- LOGGER.debug(heatStack.getStackStatus() + " (" + canonicalName + ")");
- if ("DELETE_IN_PROGRESS".equals(heatStack.getStackStatus())) {
- if (deletePollTimeout <= 0) {
- LOGGER.error (MessageEnum.RA_CREATE_STACK_TIMEOUT, cloudSiteId, tenantId, stackName,
- heatStack.getStackStatus (), "", "", MsoLogger.ErrorCode.AvailabilityError,
- "Rollback: DELETE stack timeout");
- break;
- } else {
- sleep(deletePollInterval * 1000L);
- deletePollTimeout -= deletePollInterval;
- LOGGER.debug("deletePollTimeout remaining: " + deletePollTimeout);
- }
- } else if ("DELETE_COMPLETE".equals(heatStack.getStackStatus())){
- LOGGER.debug("DELETE_COMPLETE for " + canonicalName);
- deleted = true;
- continue;
- } else if ("DELETE_FAILED".equals(heatStack.getStackStatus())) {
- // Warn about this (?) - but still throw the original exception
- LOGGER.warn(MessageEnum.RA_CREATE_STACK_ERR, "Create Stack errored, stack deletion FAILED", "", "", MsoLogger.ErrorCode.BusinessProcesssError, "Create Stack error, stack deletion FAILED");
- LOGGER.debug("Stack deletion FAILED on a rollback of a create - " + canonicalName + ", status=" + heatStack.getStackStatus() + ", reason=" + heatStack.getStackStatusReason());
- break;
- } else {
- //got a status other than DELETE_IN_PROGRESS or DELETE_COMPLETE - so break and evaluate
- break;
- }
- } else {
- // assume if we can't find it - it's deleted
- LOGGER.debug("heatStack returned null - assume the stack " + canonicalName + " has been deleted");
- deleted = true;
- continue;
- }
-
- } catch (MsoException me2) {
- // We got an exception on the delete - don't throw this exception - throw the original - just log.
- LOGGER.debug("Exception thrown trying to delete " + canonicalName + " on a create->rollback: " + me2.getContextMessage(), me2);
- LOGGER.warn(MessageEnum.RA_CREATE_STACK_ERR, "Create Stack errored, then stack deletion FAILED - exception thrown", "", "", MsoLogger.ErrorCode.BusinessProcesssError, me2.getContextMessage());
- }
-
- } // end while !deleted
- StringBuilder errorContextMessage;
- if (createTimedOut) {
- errorContextMessage = new StringBuilder("Stack Creation Timeout");
- } else {
- errorContextMessage = stackErrorStatusReason;
- }
- if (deleted) {
- errorContextMessage.append(" - stack successfully deleted");
- } else {
- errorContextMessage.append(" - encountered an error trying to delete the stack");
- }
-// MsoOpenstackException me = new MsoOpenstackException(0, "", stackErrorStatusReason.toString());
- // me.addContext(CREATE_STACK);
- // alarmLogger.sendAlarm(HEAT_ERROR, MsoAlarmLogger.CRITICAL, me.getContextMessage());
- // throw me;
- } catch (Exception e2) {
- // shouldn't happen - but handle
- LOGGER.error (MessageEnum.RA_CREATE_STACK_ERR, "Create Stack: Nested exception rolling back stack: " + e2, "", "", MsoLogger.ErrorCode.BusinessProcesssError, "Exception in Create Stack: rolling back stack");
- }
- }
- MsoOpenstackException me = new MsoOpenstackException(0, "", stackErrorStatusReason.toString());
- me.addContext(CREATE_STACK);
- alarmLogger.sendAlarm(HEAT_ERROR, MsoAlarmLogger.CRITICAL, me.getContextMessage());
- throw me;
- }
-