* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Calendar;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
+import java.util.Optional;
import java.util.Set;
import org.onap.so.adapters.vdu.CloudInfo;
import org.onap.so.adapters.vdu.VduStateType;
import org.onap.so.adapters.vdu.VduStatus;
import org.onap.so.cloud.CloudConfig;
+import org.onap.so.cloud.authentication.AuthenticationMethodFactory;
+import org.onap.so.cloud.authentication.KeystoneAuthHolder;
+import org.onap.so.cloud.authentication.KeystoneV3Authentication;
+import org.onap.so.cloud.authentication.ServiceEndpointNotFoundException;
import org.onap.so.db.catalog.beans.CloudIdentity;
import org.onap.so.db.catalog.beans.CloudSite;
-import org.onap.so.cloud.authentication.AuthenticationMethodFactory;
import org.onap.so.db.catalog.beans.HeatTemplate;
import org.onap.so.db.catalog.beans.HeatTemplateParam;
+import org.onap.so.db.catalog.beans.ServerType;
import org.onap.so.logger.MessageEnum;
-import org.onap.so.logger.MsoAlarmLogger;
+
import org.onap.so.logger.MsoLogger;
import org.onap.so.openstack.beans.HeatCacheEntry;
import org.onap.so.openstack.beans.HeatStatus;
import com.woorea.openstack.base.client.OpenStackResponseException;
import com.woorea.openstack.heat.Heat;
import com.woorea.openstack.heat.model.CreateStackParam;
+import com.woorea.openstack.heat.model.Resources;
import com.woorea.openstack.heat.model.Stack;
import com.woorea.openstack.heat.model.Stack.Output;
import com.woorea.openstack.heat.model.Stacks;
private static final String DELETE_STACK = "DeleteStack";
- private static final String HEAT_ERROR = "HeatError";
+ protected static final String HEAT_ERROR = "HeatError";
- private static final String CREATE_STACK = "CreateStack";
-
- // Cache Heat Clients statically. Since there is just one MSO user, there is no
- // benefit to re-authentication on every request (or across different flows). The
- // token will be used until it expires.
- //
- // The cache key is "tenantId:cloudId"
- private static Map <String, HeatCacheEntry> heatClientCache = new HashMap <> ();
+ protected static final String CREATE_STACK = "CreateStack";
// Fetch cloud configuration each time (may be cached in CloudConfig class)
@Autowired
protected CloudConfig cloudConfig;
-
+
@Autowired
private Environment environment;
@Autowired
private AuthenticationMethodFactory authenticationMethodFactory;
-
+
@Autowired
private MsoTenantUtilsFactory tenantUtilsFactory;
- private static final MsoLogger LOGGER = MsoLogger.getMsoLogger (MsoLogger.Catalog.RA, MsoHeatUtils.class);
+ @Autowired
+ private KeystoneV3Authentication keystoneV3Authentication;
+ private static final MsoLogger LOGGER = MsoLogger.getMsoLogger (MsoLogger.Catalog.RA, MsoHeatUtils.class);
+
// Properties names and variables (with default values)
- protected String createPollIntervalProp = "ecomp.mso.adapters.po.pollInterval";
- private String deletePollIntervalProp = "ecomp.mso.adapters.po.pollInterval";
- private String deletePollTimeoutProp = "ecomp.mso.adapters.po.pollTimeout";
+ protected String createPollIntervalProp = "org.onap.so.adapters.po.pollInterval";
+ private String deletePollIntervalProp = "org.onap.so.adapters.po.pollInterval";
+ private String deletePollTimeoutProp = "org.onap.so.adapters.po.pollTimeout";
protected static final String createPollIntervalDefault = "15";
private static final String deletePollIntervalDefault = "15";
-
+
private static final ObjectMapper JSON_MAPPER = new ObjectMapper();
/**
Map <String, Object> files,
Map <String, Object> heatFiles,
boolean backout) throws MsoException {
- // Create local variables checking to see if we have an environment, nested, get_files
- // Could later add some checks to see if it's valid.
- boolean haveEnvtVariable = true;
- if (environment == null || "".equalsIgnoreCase (environment.trim ())) {
- haveEnvtVariable = false;
- LOGGER.debug ("createStack called with no environment variable");
- } else {
- LOGGER.debug ("createStack called with an environment variable: " + environment);
- }
- boolean haveFiles = true;
- if (files == null || files.isEmpty ()) {
- haveFiles = false;
- LOGGER.debug ("createStack called with no files / child template ids");
- } else {
- LOGGER.debug ("createStack called with " + files.size () + " files / child template ids");
+ // Take out the multicloud inputs, if present.
+ for (String key : MsoMulticloudUtils.MULTICLOUD_INPUTS) {
+ if (stackInputs.containsKey(key)) {
+ stackInputs.remove(key);
+ if (stackInputs.isEmpty()) {
+ break;
+ }
+ }
}
- boolean haveHeatFiles = true;
- if (heatFiles == null || heatFiles.isEmpty ()) {
- haveHeatFiles = false;
- LOGGER.debug ("createStack called with no heatFiles");
- } else {
- LOGGER.debug ("createStack called with " + heatFiles.size () + " heatFiles");
- }
+ CreateStackParam stack = createStackParam(stackName, heatTemplate, stackInputs, timeoutMinutes, environment, files, heatFiles);
// Obtain the cloud site information where we will create the stack
CloudSite cloudSite = cloudConfig.getCloudSite(cloudSiteId).orElseThrow(
// This could throw MsoTenantNotFound or MsoOpenstackException (both propagated)
Heat heatClient = getHeatClient (cloudSite, tenantId);
if (heatClient != null) {
- LOGGER.debug("Found: " + heatClient.toString());
+ LOGGER.debug("Found: " + heatClient.toString());
}
LOGGER.debug ("Ready to Create Stack (" + heatTemplate + ") with input params: " + stackInputs);
- //force entire stackInput object to generic Map<String, Object> for openstack compatibility
- ObjectMapper mapper = new ObjectMapper();
- Map<String, Object> normalized = new HashMap<>();
- try {
- normalized = mapper.readValue(mapper.writeValueAsString(stackInputs), new TypeReference<HashMap<String,Object>>() {});
- } catch (IOException e1) {
- LOGGER.debug("could not map json", e1);
- }
-
- // Build up the stack to create
- // Disable auto-rollback, because error reason is lost. Always rollback in the code.
- CreateStackParam stack = new CreateStackParam ();
- stack.setStackName (stackName);
- stack.setTimeoutMinutes (timeoutMinutes);
- stack.setParameters (normalized);
- stack.setTemplate (heatTemplate);
- stack.setDisableRollback (true);
- // TJM New for PO Adapter - add envt variable
- if (haveEnvtVariable) {
- LOGGER.debug ("Found an environment variable - value: " + environment);
- stack.setEnvironment (environment);
- }
- // Now handle nested templates or get_files - have to combine if we have both
- // as they're both treated as "files:" on the stack.
- if (haveFiles && haveHeatFiles) {
- // Let's do this here - not in the bean
- LOGGER.debug ("Found files AND heatFiles - combine and add!");
- Map <String, Object> combinedFiles = new HashMap <> ();
- for (Entry<String, Object> entry : files.entrySet()) {
- combinedFiles.put(entry.getKey(), entry.getValue());
- }
- for (Entry<String, Object> entry : heatFiles.entrySet()) {
- combinedFiles.put(entry.getKey(), entry.getValue());
- }
- stack.setFiles (combinedFiles);
- } else {
- // Handle if we only have one or neither:
- if (haveFiles) {
- LOGGER.debug ("Found files - adding to stack");
- stack.setFiles (files);
- }
- if (haveHeatFiles) {
- LOGGER.debug ("Found heatFiles - adding to stack");
- // the setFiles was modified to handle adding the entries
- stack.setFiles (heatFiles);
- }
- }
-
- // 1802 - attempt to add better formatted printout of request to openstack
- try {
- Map<String, Object> inputs = new HashMap<>();
- for (Entry<String, ?> entry : stackInputs.entrySet()) {
- if (entry.getValue() != null) {
- inputs.put(entry.getKey(), entry.getValue());
- }
- }
- LOGGER.debug(this.printStackRequest(tenantId, heatFiles, files, environment, inputs, stackName, heatTemplate, timeoutMinutes, backout, cloudSiteId));
- } catch (Exception e) {
- // that's okay - this is a nice-to-have
- LOGGER.debug("(had an issue printing nicely formatted request to debuglog) " + e.getMessage());
- }
-
Stack heatStack = null;
try {
- // Execute the actual Openstack command to create the Heat stack
OpenStackRequest <Stack> request = heatClient.getStacks ().create (stack);
- // Begin X-Auth-User
- // Obtain an MSO token for the tenant
CloudIdentity cloudIdentity = cloudSite.getIdentityService();
- // cloudIdentity.getMsoId(), cloudIdentity.getMsoPass()
- //req
request.header ("X-Auth-User", cloudIdentity.getMsoId ());
request.header ("X-Auth-Key", CryptoUtils.decryptCloudConfigPassword(cloudIdentity.getMsoPass ()));
- LOGGER.debug ("headers added, about to executeAndRecordOpenstackRequest");
- //LOGGER.debug(this.requestToStringBuilder(stack).toString());
- // END - try to fix X-Auth-User
heatStack = executeAndRecordOpenstackRequest (request);
} catch (OpenStackResponseException e) {
- // Since this came on the 'Create Stack' command, nothing was changed
- // in the cloud. Return the error as an exception.
- if (e.getStatus () == 409) {
- // Stack already exists. Return a specific error for this case
+ if (e.getStatus () == 409) {
MsoStackAlreadyExists me = new MsoStackAlreadyExists (stackName, tenantId, cloudSiteId);
me.addContext (CREATE_STACK);
throw me;
- } else {
- // Convert the OpenStackResponseException to an MsoOpenstackException
- LOGGER.debug("ERROR STATUS = " + e.getStatus() + ",\n" + e.getMessage() + "\n" + e.getLocalizedMessage());
+ } else {
+ LOGGER.debug("ERROR STATUS = " + e.getStatus() + ",\n" + e.getMessage() + "\n" + e.getLocalizedMessage());
throw heatExceptionToMsoException (e, CREATE_STACK);
}
- } catch (OpenStackConnectException e) {
- // Error connecting to Openstack instance. Convert to an MsoException
+ } catch (OpenStackConnectException e) {
throw heatExceptionToMsoException (e, CREATE_STACK);
- } catch (RuntimeException e) {
- // Catch-all
+ } catch (RuntimeException e) {
throw runtimeExceptionToMsoException (e, CREATE_STACK);
}
// Otherwise, simple query by name returns a 302 redirect.
// NOTE: This is specific to the v1 Orchestration API.
String canonicalName = stackName + "/" + heatStack.getId ();
-
- // If client has requested a final response, poll for stack completion
+
if (pollForCompletion) {
- // Set a time limit on overall polling.
- // Use the resource (template) timeout for Openstack (expressed in minutes)
- // and add one poll interval to give Openstack a chance to fail on its own.s
-
- int createPollInterval = Integer.parseInt(this.environment.getProperty(createPollIntervalProp, createPollIntervalDefault));
- int pollTimeout = (timeoutMinutes * 60) + createPollInterval;
- // New 1610 - poll on delete if we rollback - use same values for now
- int deletePollInterval = createPollInterval;
- int deletePollTimeout = pollTimeout;
- boolean createTimedOut = false;
- StringBuilder stackErrorStatusReason = new StringBuilder("");
- LOGGER.debug("createPollInterval=" + createPollInterval + ", pollTimeout=" + pollTimeout);
-
- while (true) {
- try {
- heatStack = queryHeatStack (heatClient, canonicalName);
- LOGGER.debug (heatStack.getStackStatus () + " (" + canonicalName + ")");
- try {
- LOGGER.debug("Current stack " + this.getOutputsAsStringBuilder(heatStack).toString());
- } catch (Exception e) {
- LOGGER.debug("an error occurred trying to print out the current outputs of the stack", e);
- }
-
- if ("CREATE_IN_PROGRESS".equals (heatStack.getStackStatus ())) {
- // Stack creation is still running.
- // Sleep and try again unless timeout has been reached
- if (pollTimeout <= 0) {
- // Note that this should not occur, since there is a timeout specified
- // in the Openstack call.
- LOGGER.error (MessageEnum.RA_CREATE_STACK_TIMEOUT, cloudSiteId, tenantId, stackName, heatStack.getStackStatus (), "", "", MsoLogger.ErrorCode.AvailabilityError, "Create stack timeout");
- createTimedOut = true;
- break;
- }
-
- sleep(createPollInterval * 1000L);
-
- pollTimeout -= createPollInterval;
- LOGGER.debug("pollTimeout remaining: " + pollTimeout);
- } else {
- //save off the status & reason msg before we attempt delete
- stackErrorStatusReason.append("Stack error (" + heatStack.getStackStatus() + "): " + heatStack.getStackStatusReason());
- break;
- }
- } catch (MsoException me) {
- // Cannot query the stack status. Something is wrong.
- // Try to roll back the stack
- if (!backout)
- {
- LOGGER.warn(MessageEnum.RA_CREATE_STACK_ERR, "Create Stack errored, stack deletion suppressed", "", "", MsoLogger.ErrorCode.BusinessProcesssError, "Exception in Create Stack, stack deletion suppressed");
- }
- else
- {
- try {
- LOGGER.debug("Create Stack error - unable to query for stack status - attempting to delete stack: " + canonicalName + " - This will likely fail and/or we won't be able to query to see if delete worked");
- OpenStackRequest <Void> request = heatClient.getStacks ().deleteByName (canonicalName);
- executeAndRecordOpenstackRequest (request);
- // this may be a waste of time - if we just got an exception trying to query the stack - we'll just
- // get another one, n'est-ce pas?
- boolean deleted = false;
- while (!deleted) {
- try {
- heatStack = queryHeatStack(heatClient, canonicalName);
- if (heatStack != null) {
- LOGGER.debug(heatStack.getStackStatus());
- if ("DELETE_IN_PROGRESS".equals(heatStack.getStackStatus())) {
- if (deletePollTimeout <= 0) {
- LOGGER.error (MessageEnum.RA_CREATE_STACK_TIMEOUT, cloudSiteId, tenantId, stackName,
- heatStack.getStackStatus (), "", "", MsoLogger.ErrorCode.AvailabilityError,
- "Rollback: DELETE stack timeout");
- break;
- } else {
- sleep(deletePollInterval * 1000L);
- deletePollTimeout -= deletePollInterval;
- }
- } else if ("DELETE_COMPLETE".equals(heatStack.getStackStatus())){
- LOGGER.debug("DELETE_COMPLETE for " + canonicalName);
- deleted = true;
- continue;
- } else {
- //got a status other than DELETE_IN_PROGRESS or DELETE_COMPLETE - so break and evaluate
- break;
- }
- } else {
- // assume if we can't find it - it's deleted
- LOGGER.debug("heatStack returned null - assume the stack " + canonicalName + " has been deleted");
- deleted = true;
- continue;
- }
-
- } catch (Exception e3) {
- // Just log this one. We will report the original exception.
- LOGGER.error (MessageEnum.RA_CREATE_STACK_ERR, "Create Stack: Nested exception rolling back stack: " + e3, "", "", MsoLogger.ErrorCode.BusinessProcesssError, "Create Stack: Nested exception rolling back stack on error on query");
-
- }
- }
- } catch (Exception e2) {
- // Just log this one. We will report the original exception.
- LOGGER.error (MessageEnum.RA_CREATE_STACK_ERR, "Create Stack: Nested exception rolling back stack: " + e2, "", "", MsoLogger.ErrorCode.BusinessProcesssError, "Create Stack: Nested exception rolling back stack");
- }
- }
-
- // Propagate the original exception from Stack Query.
- me.addContext (CREATE_STACK);
- throw me;
- }
- }
-
- if (!"CREATE_COMPLETE".equals (heatStack.getStackStatus ())) {
- LOGGER.error (MessageEnum.RA_CREATE_STACK_ERR, "Create Stack error: Polling complete with non-success status: "
- + heatStack.getStackStatus () + ", " + heatStack.getStackStatusReason (), "", "", MsoLogger.ErrorCode.BusinessProcesssError, "Create Stack error");
-
- // Rollback the stack creation, since it is in an indeterminate state.
- if (!backout)
- {
- LOGGER.warn(MessageEnum.RA_CREATE_STACK_ERR, "Create Stack errored, stack deletion suppressed", "", "", MsoLogger.ErrorCode.BusinessProcesssError, "Create Stack error, stack deletion suppressed");
- }
- else
- {
- try {
- LOGGER.debug("Create Stack errored - attempting to DELETE stack: " + canonicalName);
- LOGGER.debug("deletePollInterval=" + deletePollInterval + ", deletePollTimeout=" + deletePollTimeout);
- OpenStackRequest <Void> request = heatClient.getStacks ().deleteByName (canonicalName);
- executeAndRecordOpenstackRequest (request);
- boolean deleted = false;
- while (!deleted) {
- try {
- heatStack = queryHeatStack(heatClient, canonicalName);
- if (heatStack != null) {
- LOGGER.debug(heatStack.getStackStatus() + " (" + canonicalName + ")");
- if ("DELETE_IN_PROGRESS".equals(heatStack.getStackStatus())) {
- if (deletePollTimeout <= 0) {
- LOGGER.error (MessageEnum.RA_CREATE_STACK_TIMEOUT, cloudSiteId, tenantId, stackName,
- heatStack.getStackStatus (), "", "", MsoLogger.ErrorCode.AvailabilityError,
- "Rollback: DELETE stack timeout");
- break;
- } else {
- sleep(deletePollInterval * 1000L);
- deletePollTimeout -= deletePollInterval;
- LOGGER.debug("deletePollTimeout remaining: " + deletePollTimeout);
- }
- } else if ("DELETE_COMPLETE".equals(heatStack.getStackStatus())){
- LOGGER.debug("DELETE_COMPLETE for " + canonicalName);
- deleted = true;
- continue;
- } else if ("DELETE_FAILED".equals(heatStack.getStackStatus())) {
- // Warn about this (?) - but still throw the original exception
- LOGGER.warn(MessageEnum.RA_CREATE_STACK_ERR, "Create Stack errored, stack deletion FAILED", "", "", MsoLogger.ErrorCode.BusinessProcesssError, "Create Stack error, stack deletion FAILED");
- LOGGER.debug("Stack deletion FAILED on a rollback of a create - " + canonicalName + ", status=" + heatStack.getStackStatus() + ", reason=" + heatStack.getStackStatusReason());
- break;
- } else {
- //got a status other than DELETE_IN_PROGRESS or DELETE_COMPLETE - so break and evaluate
- break;
- }
- } else {
- // assume if we can't find it - it's deleted
- LOGGER.debug("heatStack returned null - assume the stack " + canonicalName + " has been deleted");
- deleted = true;
- continue;
- }
-
- } catch (MsoException me2) {
- // We got an exception on the delete - don't throw this exception - throw the original - just log.
- LOGGER.debug("Exception thrown trying to delete " + canonicalName + " on a create->rollback: " + me2.getContextMessage(), me2);
- LOGGER.warn(MessageEnum.RA_CREATE_STACK_ERR, "Create Stack errored, then stack deletion FAILED - exception thrown", "", "", MsoLogger.ErrorCode.BusinessProcesssError, me2.getContextMessage());
- }
-
- } // end while !deleted
- StringBuilder errorContextMessage;
- if (createTimedOut) {
- errorContextMessage = new StringBuilder("Stack Creation Timeout");
- } else {
- errorContextMessage = stackErrorStatusReason;
- }
- if (deleted) {
- errorContextMessage.append(" - stack successfully deleted");
- } else {
- errorContextMessage.append(" - encountered an error trying to delete the stack");
- }
-// MsoOpenstackException me = new MsoOpenstackException(0, "", stackErrorStatusReason.toString());
- // me.addContext(CREATE_STACK);
- // alarmLogger.sendAlarm(HEAT_ERROR, MsoAlarmLogger.CRITICAL, me.getContextMessage());
- // throw me;
- } catch (Exception e2) {
- // shouldn't happen - but handle
- LOGGER.error (MessageEnum.RA_CREATE_STACK_ERR, "Create Stack: Nested exception rolling back stack: " + e2, "", "", MsoLogger.ErrorCode.BusinessProcesssError, "Exception in Create Stack: rolling back stack");
- }
- }
- MsoOpenstackException me = new MsoOpenstackException(0, "", stackErrorStatusReason.toString());
- me.addContext(CREATE_STACK);
- alarmLogger.sendAlarm(HEAT_ERROR, MsoAlarmLogger.CRITICAL, me.getContextMessage());
- throw me;
- }
-
+ heatStack = pollStackForCompletion(cloudSiteId, tenantId, stackName, timeoutMinutes, backout, heatClient,
+ heatStack, canonicalName);
} else {
// Get initial status, since it will have been null after the create.
heatStack = queryHeatStack (heatClient, canonicalName);
LOGGER.debug (heatStack.getStackStatus ());
}
-
return new StackInfoMapper(heatStack).map();
}
+ private Stack pollStackForCompletion(String cloudSiteId, String tenantId, String stackName, int timeoutMinutes,
+ boolean backout, Heat heatClient, Stack heatStack, String canonicalName)
+ throws MsoException, MsoOpenstackException {
+ int createPollInterval = Integer.parseInt(this.environment.getProperty(createPollIntervalProp, createPollIntervalDefault));
+ int pollTimeout = (timeoutMinutes * 60) + createPollInterval;
+ int deletePollInterval = createPollInterval;
+ int deletePollTimeout = pollTimeout;
+ boolean createTimedOut = false;
+ StringBuilder stackErrorStatusReason = new StringBuilder("");
+ LOGGER.debug("createPollInterval=" + createPollInterval + ", pollTimeout=" + pollTimeout);
+
+ while (true) {
+ try {
+ heatStack = queryHeatStack (heatClient, canonicalName);
+ LOGGER.debug (heatStack.getStackStatus () + " (" + canonicalName + ")");
+ try {
+ LOGGER.debug("Current stack " + this.getOutputsAsStringBuilder(heatStack).toString());
+ } catch (Exception e) {
+ LOGGER.debug("an error occurred trying to print out the current outputs of the stack", e);
+ }
+
+ if ("CREATE_IN_PROGRESS".equals (heatStack.getStackStatus ())) {
+ if (pollTimeout <= 0) {
+ LOGGER.error (MessageEnum.RA_CREATE_STACK_TIMEOUT, cloudSiteId, tenantId, stackName, heatStack.getStackStatus (), "", "", MsoLogger.ErrorCode.AvailabilityError, "Create stack timeout");
+ createTimedOut = true;
+ break;
+ }
+ sleep(createPollInterval * 1000L);
+ pollTimeout -= createPollInterval;
+ LOGGER.debug("pollTimeout remaining: " + pollTimeout);
+ } else {
+ stackErrorStatusReason.append("Stack error (" + heatStack.getStackStatus() + "): " + heatStack.getStackStatusReason());
+ break;
+ }
+ } catch (MsoException me) {
+ // Cannot query the stack status. Something is wrong.
+ // Try to roll back the stack
+ if (!backout)
+ {
+ LOGGER.warn(MessageEnum.RA_CREATE_STACK_ERR, "Create Stack errored, stack deletion suppressed", "", "", MsoLogger.ErrorCode.BusinessProcesssError, "Exception in Create Stack, stack deletion suppressed");
+ }
+ else
+ {
+ try {
+ LOGGER.debug("Create Stack error - unable to query for stack status - attempting to delete stack: " + canonicalName + " - This will likely fail and/or we won't be able to query to see if delete worked");
+ OpenStackRequest <Void> request = heatClient.getStacks ().deleteByName (canonicalName);
+ executeAndRecordOpenstackRequest (request);
+ boolean deleted = false;
+ while (!deleted) {
+ try {
+ heatStack = queryHeatStack(heatClient, canonicalName);
+ if (heatStack != null) {
+ LOGGER.debug(heatStack.getStackStatus());
+ if ("DELETE_IN_PROGRESS".equals(heatStack.getStackStatus())) {
+ if (deletePollTimeout <= 0) {
+ LOGGER.error (MessageEnum.RA_CREATE_STACK_TIMEOUT, cloudSiteId, tenantId, stackName,
+ heatStack.getStackStatus (), "", "", MsoLogger.ErrorCode.AvailabilityError,
+ "Rollback: DELETE stack timeout");
+ break;
+ } else {
+ sleep(deletePollInterval * 1000L);
+ deletePollTimeout -= deletePollInterval;
+ }
+ } else if ("DELETE_COMPLETE".equals(heatStack.getStackStatus())){
+ LOGGER.debug("DELETE_COMPLETE for " + canonicalName);
+ deleted = true;
+ continue;
+ } else {
+ //got a status other than DELETE_IN_PROGRESS or DELETE_COMPLETE - so break and evaluate
+ break;
+ }
+ } else {
+ // assume if we can't find it - it's deleted
+ LOGGER.debug("heatStack returned null - assume the stack " + canonicalName + " has been deleted");
+ deleted = true;
+ continue;
+ }
+
+ } catch (Exception e3) {
+ // Just log this one. We will report the original exception.
+ LOGGER.error (MessageEnum.RA_CREATE_STACK_ERR, "Create Stack: Nested exception rolling back stack: " + e3, "", "", MsoLogger.ErrorCode.BusinessProcesssError, "Create Stack: Nested exception rolling back stack on error on query");
+
+ }
+ }
+ } catch (Exception e2) {
+ // Just log this one. We will report the original exception.
+ LOGGER.error (MessageEnum.RA_CREATE_STACK_ERR, "Create Stack: Nested exception rolling back stack: " + e2, "", "", MsoLogger.ErrorCode.BusinessProcesssError, "Create Stack: Nested exception rolling back stack");
+ }
+ }
+
+ // Propagate the original exception from Stack Query.
+ me.addContext (CREATE_STACK);
+ throw me;
+ }
+ }
+
+ if (!"CREATE_COMPLETE".equals (heatStack.getStackStatus ())) {
+ LOGGER.error (MessageEnum.RA_CREATE_STACK_ERR, "Create Stack error: Polling complete with non-success status: "
+ + heatStack.getStackStatus () + ", " + heatStack.getStackStatusReason (), "", "", MsoLogger.ErrorCode.BusinessProcesssError, "Create Stack error");
+
+ // Rollback the stack creation, since it is in an indeterminate state.
+ if (!backout)
+ {
+ LOGGER.warn(MessageEnum.RA_CREATE_STACK_ERR, "Create Stack errored, stack deletion suppressed", "", "", MsoLogger.ErrorCode.BusinessProcesssError, "Create Stack error, stack deletion suppressed");
+ }
+ else
+ {
+ try {
+ LOGGER.debug("Create Stack errored - attempting to DELETE stack: " + canonicalName);
+ LOGGER.debug("deletePollInterval=" + deletePollInterval + ", deletePollTimeout=" + deletePollTimeout);
+ OpenStackRequest <Void> request = heatClient.getStacks ().deleteByName (canonicalName);
+ executeAndRecordOpenstackRequest (request);
+ boolean deleted = false;
+ while (!deleted) {
+ try {
+ heatStack = queryHeatStack(heatClient, canonicalName);
+ if (heatStack != null) {
+ LOGGER.debug(heatStack.getStackStatus() + " (" + canonicalName + ")");
+ if ("DELETE_IN_PROGRESS".equals(heatStack.getStackStatus())) {
+ if (deletePollTimeout <= 0) {
+ LOGGER.error (MessageEnum.RA_CREATE_STACK_TIMEOUT, cloudSiteId, tenantId, stackName,
+ heatStack.getStackStatus (), "", "", MsoLogger.ErrorCode.AvailabilityError,
+ "Rollback: DELETE stack timeout");
+ break;
+ } else {
+ sleep(deletePollInterval * 1000L);
+ deletePollTimeout -= deletePollInterval;
+ LOGGER.debug("deletePollTimeout remaining: " + deletePollTimeout);
+ }
+ } else if ("DELETE_COMPLETE".equals(heatStack.getStackStatus())){
+ LOGGER.debug("DELETE_COMPLETE for " + canonicalName);
+ deleted = true;
+ continue;
+ } else if ("DELETE_FAILED".equals(heatStack.getStackStatus())) {
+ // Warn about this (?) - but still throw the original exception
+ LOGGER.warn(MessageEnum.RA_CREATE_STACK_ERR, "Create Stack errored, stack deletion FAILED", "", "", MsoLogger.ErrorCode.BusinessProcesssError, "Create Stack error, stack deletion FAILED");
+ LOGGER.debug("Stack deletion FAILED on a rollback of a create - " + canonicalName + ", status=" + heatStack.getStackStatus() + ", reason=" + heatStack.getStackStatusReason());
+ break;
+ } else {
+ //got a status other than DELETE_IN_PROGRESS or DELETE_COMPLETE - so break and evaluate
+ break;
+ }
+ } else {
+ // assume if we can't find it - it's deleted
+ LOGGER.debug("heatStack returned null - assume the stack " + canonicalName + " has been deleted");
+ deleted = true;
+ continue;
+ }
+
+ } catch (MsoException me2) {
+ // We got an exception on the delete - don't throw this exception - throw the original - just log.
+ LOGGER.debug("Exception thrown trying to delete " + canonicalName + " on a create->rollback: " + me2.getContextMessage(), me2);
+ LOGGER.warn(MessageEnum.RA_CREATE_STACK_ERR, "Create Stack errored, then stack deletion FAILED - exception thrown", "", "", MsoLogger.ErrorCode.BusinessProcesssError, me2.getContextMessage());
+ }
+
+ } // end while !deleted
+ StringBuilder errorContextMessage;
+ if (createTimedOut) {
+ errorContextMessage = new StringBuilder("Stack Creation Timeout");
+ } else {
+ errorContextMessage = stackErrorStatusReason;
+ }
+ if (deleted) {
+ errorContextMessage.append(" - stack successfully deleted");
+ } else {
+ errorContextMessage.append(" - encountered an error trying to delete the stack");
+ }
+ } catch (Exception e2) {
+ // shouldn't happen - but handle
+ LOGGER.error (MessageEnum.RA_CREATE_STACK_ERR, "Create Stack: Nested exception rolling back stack: " + e2, "", "", MsoLogger.ErrorCode.BusinessProcesssError, "Exception in Create Stack: rolling back stack");
+ }
+ }
+ MsoOpenstackException me = new MsoOpenstackException(0, "", stackErrorStatusReason.toString());
+ me.addContext(CREATE_STACK);
+ throw me;
+ }
+ return heatStack;
+ }
+
/**
* Query for a single stack (by Name) in a tenant. This call will always return a
* StackInfo object. If the stack does not exist, an "empty" StackInfo will be
else {
LOGGER.debug ("Heat Client is NULL" );
}
-
+
executeAndRecordOpenstackRequest (request);
} catch (OpenStackResponseException e) {
if (e.getStatus () == 404) {
if (pollForCompletion) {
// Set a timeout on polling
-
+
int pollInterval = Integer.parseInt(this.environment.getProperty(deletePollIntervalProp, "" + deletePollIntervalDefault));
int pollTimeout = Integer.parseInt(this.environment.getProperty(deletePollTimeoutProp, "" + deletePollIntervalDefault));
me.addContext (DELETE_STACK);
// Alarm this condition, stack deletion failed
- alarmLogger.sendAlarm (HEAT_ERROR, MsoAlarmLogger.CRITICAL, me.getContextMessage ());
+
throw me;
}
me.addContext (DELETE_STACK);
// Alarm this condition, stack deletion failed
- alarmLogger.sendAlarm (HEAT_ERROR, MsoAlarmLogger.CRITICAL, me.getContextMessage ());
+
throw me;
}
// Remove any extraneous parameters (don't throw an error)
Map <String, Object> updatedParams = new HashMap <> ();
List <String> extraParams = new ArrayList <> ();
-
+
for (Entry<String, Object> entry : inputParams.entrySet()) {
if (!paramList.contains(entry.getKey())) {
// This is not a valid parameter for this template
*/
public Heat getHeatClient (CloudSite cloudSite, String tenantId) throws MsoException {
String cloudId = cloudSite.getId();
-
- // Check first in the cache of previously authorized clients
- String cacheKey = cloudId + ":" + tenantId;
- if (heatClientCache.containsKey (cacheKey)) {
- if (!heatClientCache.get (cacheKey).isExpired ()) {
- LOGGER.debug ("Using Cached HEAT Client for " + cacheKey);
- return heatClientCache.get (cacheKey).getHeatClient ();
- } else {
- // Token is expired. Remove it from cache.
- heatClientCache.remove (cacheKey);
- LOGGER.debug ("Expired Cached HEAT Client for " + cacheKey);
- }
- }
+ // For DCP/LCP, the region should be the cloudId.
+ String region = cloudSite.getRegionId ();
// Obtain an MSO token for the tenant
CloudIdentity cloudIdentity = cloudSite.getIdentityService();
MsoTenantUtils tenantUtils = tenantUtilsFactory.getTenantUtilsByServerType(cloudIdentity.getIdentityServerType());
String keystoneUrl = tenantUtils.getKeystoneUrl(cloudId, cloudIdentity);
LOGGER.debug("keystoneUrl=" + keystoneUrl);
- Keystone keystoneTenantClient = new Keystone (keystoneUrl);
- Access access = null;
- try {
- Authentication credentials = authenticationMethodFactory.getAuthenticationFor(cloudIdentity);
-
- OpenStackRequest <Access> request = keystoneTenantClient.tokens ()
- .authenticate (credentials).withTenantId (tenantId);
-
- access = executeAndRecordOpenstackRequest (request);
- } catch (OpenStackResponseException e) {
+ String heatUrl = null;
+ String tokenId = null;
+ Calendar expiration = null;
+ try {
+ if (ServerType.KEYSTONE.equals(cloudIdentity.getIdentityServerType())) {
+ Keystone keystoneTenantClient = new Keystone (keystoneUrl);
+ Access access = null;
+
+ Authentication credentials = authenticationMethodFactory.getAuthenticationFor(cloudIdentity);
+
+ OpenStackRequest <Access> request = keystoneTenantClient.tokens ()
+ .authenticate (credentials).withTenantId (tenantId);
+
+ access = executeAndRecordOpenstackRequest (request);
+
+ try {
+ // Isolate trying to printout the region IDs
+ try {
+ LOGGER.debug("access=" + access.toString());
+ for (Access.Service service : access.getServiceCatalog()) {
+ List<Access.Service.Endpoint> endpoints = service.getEndpoints();
+ for (Access.Service.Endpoint endpoint : endpoints) {
+ LOGGER.debug("AIC returned region=" + endpoint.getRegion());
+ }
+ }
+ } catch (Exception e) {
+ LOGGER.debug("Encountered an error trying to printout Access object returned from AIC. " + e.getMessage());
+ }
+ heatUrl = KeystoneUtils.findEndpointURL (access.getServiceCatalog (), "orchestration", region, "public");
+ LOGGER.debug("heatUrl=" + heatUrl + ", region=" + region);
+ } catch (RuntimeException e) {
+ // This comes back for not found (probably an incorrect region ID)
+ String error = "AIC did not match an orchestration service for: region=" + region + ",cloud=" + cloudIdentity.getIdentityUrl();
+ throw new MsoAdapterException (error, e);
+ }
+ tokenId = access.getToken ().getId ();
+ expiration = access.getToken ().getExpires ();
+ } else if (ServerType.KEYSTONE_V3.equals(cloudIdentity.getIdentityServerType())) {
+ try {
+ KeystoneAuthHolder holder = keystoneV3Authentication.getToken(cloudSite, tenantId, "orchestration");
+ tokenId = holder.getId();
+ expiration = holder.getexpiration();
+ heatUrl = holder.getServiceUrl();
+ } catch (ServiceEndpointNotFoundException e) {
+ // This comes back for not found (probably an incorrect region ID)
+ String error = "cloud did not match an orchestration service for: region=" + region + ",cloud=" + cloudIdentity.getIdentityUrl();
+ throw new MsoAdapterException (error, e);
+ }
+ }
+ } catch (OpenStackResponseException e) {
if (e.getStatus () == 401) {
// Authentication error.
String error = "Authentication Failure: tenant=" + tenantId + ",cloud=" + cloudIdentity.getId ();
- alarmLogger.sendAlarm ("MsoAuthenticationError", MsoAlarmLogger.CRITICAL, error);
+
throw new MsoAdapterException (error);
} else {
throw keystoneErrorToMsoException (e, TOKEN_AUTH);
// Catch-all
throw runtimeExceptionToMsoException (e, TOKEN_AUTH);
}
-
- // For DCP/LCP, the region should be the cloudId.
- String region = cloudSite.getRegionId ();
- String heatUrl = null;
- try {
- // Isolate trying to printout the region IDs
- try {
- LOGGER.debug("access=" + access.toString());
- for (Access.Service service : access.getServiceCatalog()) {
- List<Access.Service.Endpoint> endpoints = service.getEndpoints();
- for (Access.Service.Endpoint endpoint : endpoints) {
- LOGGER.debug("AIC returned region=" + endpoint.getRegion());
- }
- }
- } catch (Exception e) {
- LOGGER.debug("Encountered an error trying to printout Access object returned from AIC. " + e.getMessage());
- }
- heatUrl = KeystoneUtils.findEndpointURL (access.getServiceCatalog (), "orchestration", region, "public");
- LOGGER.debug("heatUrl=" + heatUrl + ", region=" + region);
- } catch (RuntimeException e) {
- // This comes back for not found (probably an incorrect region ID)
- String error = "AIC did not match an orchestration service for: region=" + region + ",cloud=" + cloudIdentity.getIdentityUrl();
- alarmLogger.sendAlarm ("MsoConfigurationError", MsoAlarmLogger.CRITICAL, error);
- throw new MsoAdapterException (error, e);
- }
-
Heat heatClient = new Heat (heatUrl);
- heatClient.token (access.getToken ().getId ());
-
- heatClientCache.put (cacheKey,
- new HeatCacheEntry (heatUrl,
- access.getToken ().getId (),
- access.getToken ().getExpires ()));
- LOGGER.debug ("Caching HEAT Client for " + cacheKey);
-
+ heatClient.token (tokenId);
return heatClient;
}
- /**
- * Forcibly expire a HEAT client from the cache. This call is for use by
- * the KeystoneClient in case where a tenant is deleted. In that case,
- * all cached credentials must be purged so that fresh authentication is
- * done if a similarly named tenant is re-created.
- * <p>
- * Note: This is probably only applicable to dev/test environments where
- * the same Tenant Name is repeatedly used for creation/deletion.
- * <p>
- *
- */
- public void expireHeatClient (String tenantId, String cloudId) {
- String cacheKey = cloudId + ":" + tenantId;
- if (heatClientCache.containsKey (cacheKey)) {
- heatClientCache.remove (cacheKey);
- LOGGER.debug ("Deleted Cached HEAT Client for " + cacheKey);
- }
- }
-
/*
* Query for a Heat Stack. This function is needed in several places, so
* a common method is useful. This method takes an authenticated Heat Client
*/
protected Stack queryHeatStack (Heat heatClient, String stackName) throws MsoException {
if (stackName == null) {
- return null;
+ return null;
}
try {
OpenStackRequest <Stack> request = heatClient.getStacks ().byName (stackName);
}
- private StringBuilder getOutputsAsStringBuilder(Stack heatStack) {
+ protected StringBuilder getOutputsAsStringBuilder(Stack heatStack) {
// This should only be used as a utility to print out the stack outputs
// to the log
StringBuilder sb = new StringBuilder("");
} catch (Exception e) {
LOGGER.debug("Exception :",e);
sb.append("(a LinkedHashMap value that would not convert nicely)");
- }
+ }
} else if (obj instanceof Integer) {
String str = "";
try {
sb.append("[END]");
return sb;
}
-
-
+
+
public void copyBaseOutputsToInputs(Map<String, Object> inputs,
Map<String, Object> otherStackOutputs, List<String> paramNames, Map<String, String> aliases) {
if (inputs == null || otherStackOutputs == null)
}
return;
}
-
+
public List<String> convertCdlToArrayList(String cdl) {
String cdl2 = cdl.trim();
String cdl3;
}
return new ArrayList<>(Arrays.asList(cdl3.split(",")));
}
-
+
/**
* New with 1707 - this method will convert all the String *values* of the inputs
* to their "actual" object type (based on the param type: in the db - which comes from the template):
HashMap<String, Object> newInputs = new HashMap<>();
HashMap<String, HeatTemplateParam> params = new HashMap<>();
HashMap<String, HeatTemplateParam> paramAliases = new HashMap<>();
-
+
if (inputs == null) {
LOGGER.debug("convertInputMap - inputs is null - nothing to do here");
return new HashMap<>();
}
-
+
LOGGER.debug("convertInputMap in MsoHeatUtils called, with " + inputs.size() + " inputs, and template " + template.getArtifactUuid());
try {
LOGGER.debug(template.toString());
} catch (Exception e) {
LOGGER.debug("Exception occurred in convertInputMap:" + e.getMessage(), e);
}
-
+
for (HeatTemplateParam htp : template.getParameters()) {
LOGGER.debug("Adding " + htp.getParamName());
params.put(htp.getParamName(), htp);
if ("string".equalsIgnoreCase(type)) {
// Easiest!
String str = inputs.get(key);
- if (alias)
+ if (alias)
newInputs.put(realName, str);
- else
+ else
newInputs.put(key, str);
} else if ("number".equalsIgnoreCase(type)) {
String integerString = inputs.get(key);
}
return newInputs;
}
-
- /*
- * This helpful method added for Valet
+
+ /*
+ * This helpful method added for Valet
*/
public String getCloudSiteKeystoneUrl(String cloudSiteId) throws MsoCloudSiteNotFound {
String keystone_url = null;
}
return keystone_url;
}
-
+
/*
- * Create a string suitable for being dumped to a debug log that creates a
+ * Create a string suitable for being dumped to a debug log that creates a
* pseudo-JSON request dumping what's being sent to Openstack API in the create or update request
*/
-
- private String printStackRequest(String tenantId,
+
+ private String printStackRequest(String tenantId,
Map<String, Object> heatFiles,
Map<String, Object> nestedTemplates,
String environment,
- Map<String, Object> inputs,
+ Map<String, Object> inputs,
String vfModuleName,
String template,
int timeoutMinutes,
sb.append("{\n");
sb.append(" \"stack_name\": \"" + vfModuleName + "\",\n");
sb.append(" \"disable_rollback\": " + backout + ",\n");
- sb.append(" \"timeout_mins\": " + timeoutMinutes + ",\n");
+ sb.append(" \"timeout_mins\": " + timeoutMinutes + ",\n");
sb.append(" \"template\": {\n");
sb.append(template);
sb.append(" },\n");
sb.append(" \"environment\": {\n");
- if (environment == null)
+ if (environment == null)
sb.append("<none>");
- else
+ else
sb.append(environment);
sb.append(" },\n");
sb.append(" \"files\": {\n");
}
}
sb.append("\n }\n}\n");
-
+
return sb.toString();
}
-
+
/*******************************************************************************
- *
+ *
* Methods (and associated utilities) to implement the VduPlugin interface
- *
+ *
*******************************************************************************/
-
+
/**
* VduPlugin interface for instantiate function.
- *
+ *
* Translate the VduPlugin parameters to the corresponding 'createStack' parameters,
* and then invoke the existing function.
*/
{
String cloudSiteId = cloudInfo.getCloudSiteId();
String tenantId = cloudInfo.getTenantId();
-
+
// Translate the VDU ModelInformation structure to that which is needed for
// creating the Heat stack. Loop through the artifacts, looking specifically
// for MAIN_TEMPLATE and ENVIRONMENT. Any other artifact will
Map<String,Object> nestedTemplates = new HashMap<>();
Map<String,Object> files = new HashMap<>();
String heatEnvironment = null;
-
+
for (VduArtifact vduArtifact: vduModel.getArtifacts()) {
if (vduArtifact.getType() == ArtifactType.MAIN_TEMPLATE) {
heatTemplate = new String(vduArtifact.getContent());
heatEnvironment = new String(vduArtifact.getContent());
}
}
-
+
try {
StackInfo stackInfo = createStack (cloudSiteId,
tenantId,
nestedTemplates,
files,
rollbackOnFailure);
-
+
// Populate a vduInstance from the StackInfo
return stackInfoToVduInstance(stackInfo);
}
throw new VduException ("MsoHeatUtils (instantiateVDU): createStack Exception", e);
}
}
-
-
+
+
/**
* VduPlugin interface for query function.
*/
{
String cloudSiteId = cloudInfo.getCloudSiteId();
String tenantId = cloudInfo.getTenantId();
-
+
try {
// Query the Cloudify Deployment object and populate a VduInstance
StackInfo stackInfo = queryStack (cloudSiteId, tenantId, instanceId);
-
+
return stackInfoToVduInstance(stackInfo);
}
catch (Exception e) {
throw new VduException ("MsoHeatUtile (queryVdu): queryStack Exception ", e);
}
}
-
-
+
+
/**
* VduPlugin interface for delete function.
*/
{
String cloudSiteId = cloudInfo.getCloudSiteId();
String tenantId = cloudInfo.getTenantId();
-
+
try {
// Delete the Heat stack
StackInfo stackInfo = deleteStack (tenantId, cloudSiteId, instanceId, true);
-
+
// Populate a VduInstance based on the deleted Cloudify Deployment object
VduInstance vduInstance = stackInfoToVduInstance(stackInfo);
-
+
// Override return state to DELETED (HeatUtils sets to NOTFOUND)
vduInstance.getStatus().setState(VduStateType.DELETED);
-
+
return vduInstance;
}
catch (Exception e) {
throw new VduException ("Delete VDU Exception", e);
}
}
-
-
+
+
/**
* VduPlugin interface for update function.
- *
+ *
* Update is currently not supported in the MsoHeatUtils implementation of VduPlugin.
* Just return a VduException.
- *
+ *
*/
@Override
public VduInstance updateVdu (
{
throw new VduException ("MsoHeatUtils: updateVdu interface not supported");
}
-
-
+
+
/*
* Convert the local DeploymentInfo object (Cloudify-specific) to a generic VduInstance object
*/
- private VduInstance stackInfoToVduInstance (StackInfo stackInfo)
+ protected VduInstance stackInfoToVduInstance (StackInfo stackInfo)
{
VduInstance vduInstance = new VduInstance();
-
+
// The full canonical name as the instance UUID
vduInstance.setVduInstanceId(stackInfo.getCanonicalName());
vduInstance.setVduInstanceName(stackInfo.getName());
-
+
// Copy inputs and outputs
vduInstance.setInputs(stackInfo.getParameters());
vduInstance.setOutputs(stackInfo.getOutputs());
-
+
// Translate the status elements
vduInstance.setStatus(stackStatusToVduStatus (stackInfo));
-
+
return vduInstance;
}
-
+
private VduStatus stackStatusToVduStatus (StackInfo stackInfo)
{
VduStatus vduStatus = new VduStatus();
-
+
// Map the status fields to more generic VduStatus.
// There are lots of HeatStatus values, so this is a bit long...
HeatStatus heatStatus = stackInfo.getStatus();
String statusMessage = stackInfo.getStatusMessage();
-
+
if (heatStatus == HeatStatus.INIT || heatStatus == HeatStatus.BUILDING) {
vduStatus.setState(VduStateType.INSTANTIATING);
vduStatus.setLastAction((new PluginAction ("create", "in_progress", statusMessage)));
} else {
vduStatus.setState(VduStateType.UNKNOWN);
}
-
+
return vduStatus;
}
- private void sleep(long time) {
+ public Resources queryStackResources(String cloudSiteId, String tenantId, String stackName) throws MsoException {
+ CloudSite cloudSite = cloudConfig.getCloudSite(cloudSiteId)
+ .orElseThrow(() -> new MsoCloudSiteNotFound(cloudSiteId));
+ Heat heatClient = getHeatClient(cloudSite, tenantId);
+ OpenStackRequest<Resources> request = heatClient.getResources().listResources(stackName);
+ return executeAndRecordOpenstackRequest(request);
+ }
+
+ public <R> R executeHeatClientRequest(String url, String cloudSiteId, String tenantId, Class<R> returnType) throws MsoException {
+ CloudSite cloudSite = cloudConfig.getCloudSite(cloudSiteId)
+ .orElseThrow(() -> new MsoCloudSiteNotFound(cloudSiteId));
+ Heat heatClient = getHeatClient(cloudSite, tenantId);
+ OpenStackRequest<R> request = heatClient.get(url, returnType);
+ return executeAndRecordOpenstackRequest(request);
+ }
+
+ protected void sleep(long time) {
try {
Thread.sleep(time);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
}
-
+
}
Code Review / so.git / blobdiff