appc-adapters/appc-iaas-adapter/appc-iaas-adapter-bundle/src/main/java/org/onap/appc/adapter/iaas/provider/operation/impl/RebuildServer.java

   1 /*
   2 * ============LICENSE_START=======================================================
   3 * ONAP : APPC
   4 * ================================================================================
   5 * Copyright (C) 2017-2018 AT&T Intellectual Property. All rights reserved.
   6 * ================================================================================
   7 * Copyright (C) 2017 Amdocs
   8 * =============================================================================
   9 * Modifications Copyright (C) 2019 IBM
  10 * =============================================================================
  11 * Licensed under the Apache License, Version 2.0 (the "License");
  12 * you may not use this file except in compliance with the License.
  13 * You may obtain a copy of the License at
  14 *
  15 *      http://www.apache.org/licenses/LICENSE-2.0
  16 *
  17 * Unless required by applicable law or agreed to in writing, software
  18 * distributed under the License is distributed on an "AS IS" BASIS,
  19 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  20 * See the License for the specific language governing permissions and
  21 * limitations under the License.
  22 *
  23 * ============LICENSE_END=========================================================
  24 */
  25
  26 package org.onap.appc.adapter.iaas.provider.operation.impl;
  27
  28 import com.att.cdp.exceptions.ContextConnectionException;
  29 import com.att.cdp.exceptions.ResourceNotFoundException;
  30 import com.att.cdp.exceptions.ZoneException;
  31 import com.att.cdp.zones.ComputeService;
  32 import com.att.cdp.zones.Context;
  33 import com.att.cdp.zones.ImageService;
  34 import com.att.cdp.zones.Provider;
  35 import com.att.cdp.zones.model.Image;
  36 import com.att.cdp.zones.model.ModelObject;
  37 import com.att.cdp.zones.model.Server;
  38 import com.att.cdp.zones.model.ServerBootSource;
  39 import com.att.eelf.configuration.EELFLogger;
  40 import com.att.eelf.configuration.EELFManager;
  41 import com.att.eelf.i18n.EELFResourceManager;
  42 import org.glassfish.grizzly.http.util.HttpStatus;
  43 import org.onap.appc.Constants;
  44 import com.fasterxml.jackson.databind.JsonNode;
  45 import com.fasterxml.jackson.databind.ObjectMapper;
  46 import org.onap.appc.adapter.iaas.ProviderAdapter;
  47 import org.onap.appc.adapter.iaas.impl.IdentityURL;
  48 import org.onap.appc.adapter.iaas.impl.RequestContext;
  49 import org.onap.appc.adapter.iaas.impl.RequestFailedException;
  50 import org.onap.appc.adapter.iaas.impl.VMURL;
  51 import org.onap.appc.adapter.iaas.provider.operation.common.constants.Property;
  52 import org.onap.appc.adapter.iaas.provider.operation.common.enums.Operation;
  53 import org.onap.appc.adapter.iaas.provider.operation.common.enums.Outcome;
  54 import org.onap.appc.adapter.iaas.provider.operation.impl.base.ProviderServerOperation;
  55 import org.onap.appc.configuration.Configuration;
  56 import org.onap.appc.configuration.ConfigurationFactory;
  57 import org.onap.appc.exceptions.APPCException;
  58 import org.onap.appc.i18n.Msg;
  59 import org.onap.appc.logging.LoggingConstants;
  60 import org.onap.appc.logging.LoggingUtils;
  61 import org.onap.ccsdk.sli.core.sli.SvcLogicContext;
  62 import org.slf4j.MDC;
  63 import java.text.DateFormat;
  64 import java.text.SimpleDateFormat;
  65 import java.util.Date;
  66 import java.util.List;
  67 import java.util.Map;
  68 import java.util.TimeZone;
  69 import static org.onap.appc.adapter.iaas.provider.operation.common.enums.Operation.STOP_SERVICE;
  70 import static org.onap.appc.adapter.utils.Constants.ADAPTER_NAME;
  71 import com.att.cdp.exceptions.StateException;
  72
  73 public class RebuildServer extends ProviderServerOperation {
  74
  75     private static final EELFLogger logger = EELFManager.getInstance().getLogger(RebuildServer.class);
  76     private static EELFLogger metricsLogger = EELFManager.getInstance().getMetricsLogger();
  77     private static final Configuration configuration = ConfigurationFactory.getConfiguration();
  78     // the sleep time used by thread.sleep to give "some time for OpenStack to start
  79     // processing the request"
  80     private long rebuildSleepTime = 10L * 1000L;
  81
  82     /*
  83      * Rebuild the indicated server with the indicated image. This method assumes
  84      * the server has been determined to be in the correct state to do the rebuild.
  85      *
  86      * @param rc
  87      *            The request context that manages the state and recovery of the
  88      *            request for the life of its processing.
  89      * @param server
  90      *            the server to be rebuilt
  91      * @param image
  92      *            The image to be used (or snapshot)
  93      * @throws RequestFailedException
  94      *             if the server does not change state in the allotted time
  95      */
  96     @SuppressWarnings("nls")
  97     private void rebuildServer(RequestContext rc, Server server, String image) throws RequestFailedException {
  98         logger.debug(Msg.REBUILD_SERVER, server.getId());
  99         String msg;
 100         Context context = server.getContext();
 101         Provider provider = context.getProvider();
 102         ComputeService service = context.getComputeService();
 103         /*
 104          * Set Time for Metrics Logger
 105          */
 106         setTimeForMetricsLogger();
 107         try {
 108             while (rc.attempt()) {
 109                 try {
 110                     server.rebuild(image);
 111                     break;
 112                 } catch (ContextConnectionException e) {
 113                     msg = EELFResourceManager.format(Msg.CONNECTION_FAILED_RETRY, provider.getName(), service.getURL(),
 114                             context.getTenant().getName(), context.getTenant().getId(), e.getMessage(),
 115                             Long.toString(rc.getRetryDelay()), Integer.toString(rc.getAttempts()),
 116                             Integer.toString(rc.getRetryLimit()));
 117                     logger.error(msg, e);
 118                     metricsLogger.error(msg, e);
 119                     rc.delay();
 120                 }
 121             }
 122             /*
 123              * We need to provide some time for OpenStack to start processing the request.
 124              */
 125             try {
 126                 Thread.sleep(rebuildSleepTime);
 127             } catch (InterruptedException e) {
 128                 logger.trace("Sleep threw interrupted exception, should never occur");
 129                 metricsLogger.trace("Sleep threw interrupted exception, should never occur");
 130                 Thread.currentThread().interrupt();
 131             }
 132         } catch (ZoneException e) {
 133             msg = EELFResourceManager.format(Msg.REBUILD_SERVER_FAILED, server.getName(), server.getId(),
 134                     e.getMessage());
 135             logger.error(msg);
 136             metricsLogger.error(msg);
 137             throw new RequestFailedException("Rebuild Server", msg, HttpStatus.BAD_GATEWAY_502, server);
 138         }
 139         rc.reset();
 140         /*
 141          * Once we have started the process, now we wait for the final state of stopped.
 142          * This should be the final state (since we started the rebuild with the server
 143          * stopped).
 144          */
 145         waitForStateChange(rc, server, Server.Status.READY);
 146         if (rc.isFailed()) {
 147             msg = EELFResourceManager.format(Msg.CONNECTION_FAILED, provider.getName(), service.getURL());
 148             logger.error(msg);
 149             metricsLogger.error(msg);
 150             throw new RequestFailedException("Rebuild Server", msg, HttpStatus.BAD_GATEWAY_502, server);
 151         }
 152         rc.reset();
 153     }
 154
 155     /**
 156      * This method is called to rebuild the provided server.
 157      * <p>
 158      * If the server was booted from a volume, then the request is failed
 159      * immediately and no action is taken. Rebuilding a VM from a bootable volume,
 160      * where the bootable volume itself is not rebuilt, serves no purpose.
 161      * </p>
 162      *
 163      * @param rc
 164      *            The request context that manages the state and recovery of the
 165      *            request for the life of its processing.
 166      * @param server
 167      *            The server to be rebuilt
 168      * @throws ZoneException
 169      *             When error occurs
 170      * @throws RequestFailedException
 171      *             When server status is error
 172      */
 173     @SuppressWarnings("nls")
 174     private void rebuildServer(RequestContext rc, Server server, SvcLogicContext ctx)
 175             throws ZoneException, RequestFailedException {
 176         ServerBootSource builtFrom = server.getBootSource();
 177         /*
 178          * Set Time for Metrics Logger
 179          */
 180         setTimeForMetricsLogger();
 181         String msg;
 182         //Throw error if boot source is unknown
 183         if (ServerBootSource.UNKNOWN.equals(builtFrom)) {
 184             logger.debug("Boot Source Unknown" );
 185             msg = String.format("Error occured when retrieving server boot source [%s]!!!", server.getId());
 186             logger.error(msg);
 187             generateEvent(rc, false,msg);
 188             metricsLogger.error(msg);
 189             throw new RequestFailedException("Rebuild Server", msg, HttpStatus.INTERNAL_SERVER_ERROR_500, server);
 190         }
 191
 192         // Throw exception for non image/snap boot source
 193         if (ServerBootSource.VOLUME.equals(builtFrom)) {
 194             msg = String.format("Rebuilding is currently not supported for servers built from bootable volumes [%s]",
 195                     server.getId());
 196             generateEvent(rc, false, msg);
 197             logger.error(msg);
 198             metricsLogger.error(msg);
 199             throw new RequestFailedException("Rebuild Server", msg, HttpStatus.FORBIDDEN_403, server);
 200         }
 201         /*
 202          * Pending is a bit of a special case. If we find the server is in a pending
 203          * state, then the provider is in the process of changing state of the server.
 204          * So, lets try to wait a little bit and see if the state settles down to one we
 205          * can deal with. If not, then we have to fail the request.
 206          */
 207         Context context = server.getContext();
 208         Provider provider = context.getProvider();
 209         ComputeService service = context.getComputeService();
 210         if (server.getStatus().equals(Server.Status.PENDING)) {
 211             rc.reset();
 212             waitForStateChange(rc, server, Server.Status.READY, Server.Status.RUNNING, Server.Status.ERROR,
 213                     Server.Status.SUSPENDED, Server.Status.PAUSED);
 214         }
 215         // Is the skip Hypervisor check attribute populated?
 216         String skipHypervisorCheck = configuration.getProperty(Property.SKIP_HYPERVISOR_CHECK);
 217         if (skipHypervisorCheck == null && ctx != null) {
 218             skipHypervisorCheck = ctx.getAttribute(ProviderAdapter.SKIP_HYPERVISOR_CHECK);
 219         }
 220         // Always perform Hypervisor Status checks
 221         // unless the skip is set to true
 222         if (skipHypervisorCheck == null || (!skipHypervisorCheck.equalsIgnoreCase("true"))) {
 223             // Check of the Hypervisor for the VM Server is UP and reachable
 224             checkHypervisor(server);
 225         }
 226         /*
 227          * Get the image to use in this priority order: (1) If snapshot-id provided in
 228          * the request, use this (2) If any snapshots exist, then the latest snapshot is
 229          * used (3) Otherwise the image used to construct the VM is used.
 230          */
 231         String imageToUse = "";
 232         try {
 233             ObjectMapper mapper = new ObjectMapper();
 234             String payloadStr = configuration.getProperty(Property.PAYLOAD);
 235             if (payloadStr == null || payloadStr.isEmpty()) {
 236                 payloadStr = ctx.getAttribute(ProviderAdapter.PAYLOAD);
 237             }
 238             JsonNode payloadNode = mapper.readTree(payloadStr);
 239             imageToUse = payloadNode.get(ProviderAdapter.PROPERTY_REQUEST_SNAPSHOT_ID).textValue();
 240             logger.debug("Pulled snapshot-id " + imageToUse + " from the payload");
 241         } catch (Exception e) {
 242             logger.debug("Exception attempting to pull snapshot-id from the payload: " + e.toString());
 243         }
 244         List<Image> snapshots = server.getSnapshots();
 245         if (!imageToUse.isEmpty()) {
 246             logger.debug("Using snapshot-id " + imageToUse + " for the rebuild request");
 247         } else if (snapshots != null && !snapshots.isEmpty()) {
 248             imageToUse = snapshots.get(0).getId();
 249         } else {
 250             imageToUse = server.getImage();
 251             ImageService imageService = server.getContext().getImageService();
 252             rc.reset();
 253             try {
 254                 while (rc.attempt()) {
 255                     try {
 256                         /*
 257                          * We are just trying to make sure that the image exists. We arent interested in
 258                          * the details at this point.
 259                          */
 260                         imageService.getImage(imageToUse);
 261                         break;
 262                     } catch (ContextConnectionException e) {
 263                         msg = EELFResourceManager.format(Msg.CONNECTION_FAILED_RETRY, provider.getName(),
 264                                 imageService.getURL(), context.getTenant().getName(), context.getTenant().getId(),
 265                                 e.getMessage(), Long.toString(rc.getRetryDelay()), Integer.toString(rc.getAttempts()),
 266                                 Integer.toString(rc.getRetryLimit()));
 267                         logger.error(msg, e);
 268                         metricsLogger.error(msg);
 269                         rc.delay();
 270                     }
 271                 }
 272             } catch (ZoneException e) {
 273                 msg = EELFResourceManager.format(Msg.IMAGE_NOT_FOUND, imageToUse, "rebuild");
 274                 generateEvent(rc, false, msg);
 275                 logger.error(msg);
 276                 metricsLogger.error(msg);
 277                 throw new RequestFailedException("Rebuild Server", msg, HttpStatus.METHOD_NOT_ALLOWED_405, server);
 278             }
 279         }
 280         if (rc.isFailed()) {
 281             msg = EELFResourceManager.format(Msg.CONNECTION_FAILED, provider.getName(), service.getURL());
 282             logger.error(msg);
 283             metricsLogger.error(msg);
 284             throw new RequestFailedException("Rebuild Server", msg, HttpStatus.BAD_GATEWAY_502, server);
 285         }
 286         rc.reset();
 287         /*
 288          * We determine what to do based on the current state of the server
 289          */
 290         switch (server.getStatus()) {
 291         case DELETED:
 292             // Nothing to do, the server is gone
 293             msg = EELFResourceManager.format(Msg.SERVER_DELETED, server.getName(), server.getId(), server.getTenantId(),
 294                     "rebuilt");
 295             generateEvent(rc, false, msg);
 296             logger.error(msg);
 297             metricsLogger.error(msg);
 298             throw new RequestFailedException("Rebuild Server", msg, HttpStatus.METHOD_NOT_ALLOWED_405, server);
 299         case RUNNING:
 300             // Attempt to stop the server, then rebuild it
 301             stopServer(rc, server);
 302             rc.reset();
 303             rebuildServer(rc, server, imageToUse);
 304             rc.reset();
 305             startServer(rc, server);
 306             generateEvent(rc, true, Outcome.SUCCESS.toString());
 307             metricsLogger.info("Server status: RUNNING");
 308             break;
 309         case ERROR:
 310             msg = EELFResourceManager.format(Msg.SERVER_ERROR_STATE, server.getName(), server.getId(),
 311                     server.getTenantId(), "rebuild");
 312             generateEvent(rc, false, msg);
 313             logger.error(msg);
 314             metricsLogger.error(msg);
 315             throw new RequestFailedException("Rebuild Server", msg, HttpStatus.METHOD_NOT_ALLOWED_405, server);
 316         case READY:
 317             // Attempt to rebuild the server
 318             rebuildServer(rc, server, imageToUse);
 319             rc.reset();
 320             startServer(rc, server);
 321             generateEvent(rc, true, Outcome.SUCCESS.toString());
 322             metricsLogger.info("Server status: READY");
 323             break;
 324         case PAUSED:
 325             // if paused, un-pause it, stop it, and rebuild it
 326             unpauseServer(rc, server);
 327             rc.reset();
 328             stopServer(rc, server);
 329             rc.reset();
 330             rebuildServer(rc, server, imageToUse);
 331             rc.reset();
 332             startServer(rc, server);
 333             generateEvent(rc, true, Outcome.SUCCESS.toString());
 334             metricsLogger.info("Server status: PAUSED");
 335             break;
 336         case SUSPENDED:
 337             // Attempt to resume the suspended server, stop it, and rebuild it
 338             resumeServer(rc, server);
 339             rc.reset();
 340             stopServer(rc, server);
 341             rc.reset();
 342             rebuildServer(rc, server, imageToUse);
 343             rc.reset();
 344             startServer(rc, server);
 345             generateEvent(rc, true, Outcome.SUCCESS.toString());
 346             metricsLogger.info("Server status: SUSPENDED");
 347             break;
 348         default:
 349             // Hmmm, unknown status, should never occur
 350             msg = EELFResourceManager.format(Msg.UNKNOWN_SERVER_STATE, server.getName(), server.getId(),
 351                     server.getTenantId(), server.getStatus().name());
 352             generateEvent(rc, false, msg);
 353             logger.error(msg);
 354             metricsLogger.error(msg);
 355             throw new RequestFailedException("Rebuild Server", msg, HttpStatus.METHOD_NOT_ALLOWED_405, server);
 356         }
 357     }
 358
 359     /**
 360      * @see org.onap.appc.adapter.iaas.ProviderAdapter#rebuildServer(java.util.Map,
 361      *      org.onap.ccsdk.sli.core.sli.SvcLogicContext)
 362      */
 363     @SuppressWarnings("nls")
 364     public Server rebuildServer(Map<String, String> params, SvcLogicContext ctx) throws APPCException {
 365         Server server = null;
 366         RequestContext rc = new RequestContext(ctx);
 367         rc.isAlive();
 368         setTimeForMetricsLogger();
 369         String msg;
 370         try {
 371             validateParametersExist(params, ProviderAdapter.PROPERTY_INSTANCE_URL,
 372                     ProviderAdapter.PROPERTY_PROVIDER_NAME);
 373
 374             String appName = configuration.getProperty(Constants.PROPERTY_APPLICATION_NAME);
 375             String vm_url = params.get(ProviderAdapter.PROPERTY_INSTANCE_URL);
 376             VMURL vm = VMURL.parseURL(vm_url);
 377             if (validateVM(rc, appName, vm_url, vm))
 378                 return null;
 379             IdentityURL ident = IdentityURL.parseURL(params.get(ProviderAdapter.PROPERTY_IDENTITY_URL));
 380             String identStr = (ident == null) ? null : ident.toString();
 381             ctx.setAttribute("REBUILD_STATUS", "ERROR");
 382             Context context = null;
 383             String tenantName = "Unknown";// to be used also in case of exception
 384             try {
 385                 context = getContext(rc, vm_url, identStr);
 386                 if (context != null) {
 387                     tenantName = context.getTenantName();// this varaible also is used in case of exception
 388                     rc.reset();
 389                     server = lookupServer(rc, context, vm.getServerId());
 390                     logger.debug(Msg.SERVER_FOUND, vm_url, tenantName, server.getStatus().toString());
 391                     // Manually checking image service until new PAL release
 392                     if (hasImageAccess(rc, context)) {
 393                         rebuildServer(rc, server, ctx);
 394                         doSuccess(rc);
 395                         ctx.setAttribute("REBUILD_STATUS", "SUCCESS");
 396                     } else {
 397                         msg = EELFResourceManager.format(Msg.REBUILD_SERVER_FAILED, server.getName(), server.getId(),
 398                                 "Accessing Image Service Failed");
 399                         logger.error(msg);
 400                         metricsLogger.error(msg);
 401                         doFailure(rc, HttpStatus.FORBIDDEN_403, msg);
 402                     }
 403                     context.close();
 404                 } else {
 405                     ctx.setAttribute("REBUILD_STATUS", "CONTEXT_NOT_FOUND");
 406                 }
 407             } catch (StateException ex) {
 408                 logger.error(ex.getMessage());
 409                 ctx.setAttribute("REBUILD_STATUS", "ERROR");
 410                 doFailure(rc, HttpStatus.CONFLICT_409, ex.getMessage());
 411             }
 412             catch (RequestFailedException e) {
 413                 doFailure(rc, e.getStatus(), e.getMessage());
 414                 ctx.setAttribute("REBUILD_STATUS", "ERROR");
 415             } catch (ResourceNotFoundException e) {
 416                 msg = EELFResourceManager.format(Msg.SERVER_NOT_FOUND, e, vm_url);
 417                 ctx.setAttribute("REBUILD_STATUS", "ERROR");
 418                 logger.error(msg);
 419                 metricsLogger.error(msg);
 420                 doFailure(rc, HttpStatus.NOT_FOUND_404, msg);
 421             } catch (Exception e1) {
 422                 msg = EELFResourceManager.format(Msg.SERVER_OPERATION_EXCEPTION, e1, e1.getClass().getSimpleName(),
 423                         STOP_SERVICE.toString(), vm_url, tenantName);
 424                 ctx.setAttribute("REBUILD_STATUS", "ERROR");
 425                 logger.error(msg, e1);
 426                 metricsLogger.error(msg);
 427                 doFailure(rc, HttpStatus.INTERNAL_SERVER_ERROR_500, msg);
 428             }
 429         } catch (RequestFailedException e) {
 430
 431             ctx.setAttribute("REBUILD_STATUS", "ERROR");
 432             doFailure(rc, e.getStatus(), e.getMessage());
 433         }
 434         return server;
 435     }
 436
 437     @Override
 438     protected ModelObject executeProviderOperation(Map<String, String> params, SvcLogicContext context)
 439             throws APPCException {
 440         setMDC(Operation.REBUILD_SERVICE.toString(), "App-C IaaS Adapter:Rebuild", ADAPTER_NAME);
 441         logOperation(Msg.REBUILDING_SERVER, params, context);
 442         setTimeForMetricsLogger();
 443         metricsLogger.info("Executing Provider Operation: Rebuild");
 444         return rebuildServer(params, context);
 445     }
 446
 447     private void setTimeForMetricsLogger() {
 448         String timestamp = LoggingUtils.generateTimestampStr(((Date) new Date()).toInstant());
 449         MDC.put(LoggingConstants.MDCKeys.BEGIN_TIMESTAMP, timestamp);
 450         MDC.put(LoggingConstants.MDCKeys.END_TIMESTAMP, timestamp);
 451         MDC.put(LoggingConstants.MDCKeys.ELAPSED_TIME, "0");
 452         MDC.put(LoggingConstants.MDCKeys.STATUS_CODE, LoggingConstants.StatusCodes.COMPLETE);
 453         MDC.put(LoggingConstants.MDCKeys.TARGET_ENTITY, "cdp");
 454         MDC.put(LoggingConstants.MDCKeys.TARGET_SERVICE_NAME, "rebuild server");
 455         MDC.put(LoggingConstants.MDCKeys.CLASS_NAME,
 456                 "org.onap.appc.adapter.iaas.provider.operation.impl.RebuildServer");
 457
 458     }
 459
 460     /**
 461      * Sets the sleep time used by thread.sleep to give "some time for OpenStack to
 462      * start processing the request".
 463      *
 464      * @param millis
 465      *            Time to sleep in milliseconds
 466      */
 467     public void setRebuildSleepTime(long millis) {
 468         this.rebuildSleepTime = millis;
 469     }
 470 }