appc-adapters/appc-iaas-adapter/appc-iaas-adapter-bundle/src/main/java/org/openecomp/appc/adapter/iaas/provider/operation/impl/EvacuateServer.java

   1 /*-
   2  * ============LICENSE_START=======================================================
   3  * openECOMP : APP-C
   4  * ================================================================================
   5  * Copyright (C) 2017 AT&T Intellectual Property. All rights
   6  *                                              reserved.
   7  * ================================================================================
   8  * Licensed under the Apache License, Version 2.0 (the "License");
   9  * you may not use this file except in compliance with the License.
  10  * You may obtain a copy of the License at
  11  *
  12  *      http://www.apache.org/licenses/LICENSE-2.0
  13  *
  14  * Unless required by applicable law or agreed to in writing, software
  15  * distributed under the License is distributed on an "AS IS" BASIS,
  16  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  17  * See the License for the specific language governing permissions and
  18  * limitations under the License.
  19  * ============LICENSE_END=========================================================
  20  */
  21
  22 package org.openecomp.appc.adapter.iaas.provider.operation.impl;
  23
  24 import org.openecomp.appc.Constants;
  25 import org.openecomp.appc.adapter.iaas.ProviderAdapter;
  26 import org.openecomp.appc.adapter.iaas.impl.IdentityURL;
  27 import org.openecomp.appc.adapter.iaas.impl.ProviderAdapterImpl;
  28 import org.openecomp.appc.adapter.iaas.impl.RequestContext;
  29 import org.openecomp.appc.adapter.iaas.impl.RequestFailedException;
  30 import org.openecomp.appc.adapter.iaas.impl.VMURL;
  31 import org.openecomp.appc.adapter.iaas.provider.operation.common.enums.Operation;
  32 import org.openecomp.appc.adapter.iaas.provider.operation.impl.base.ProviderServerOperation;
  33 import org.openecomp.appc.configuration.Configuration;
  34 import org.openecomp.appc.configuration.ConfigurationFactory;
  35 import org.openecomp.appc.exceptions.APPCException;
  36 import org.openecomp.appc.i18n.Msg;
  37 import com.att.cdp.exceptions.ContextConnectionException;
  38 import com.att.cdp.exceptions.ResourceNotFoundException;
  39 import com.att.cdp.exceptions.ZoneException;
  40 import com.att.cdp.zones.ComputeService;
  41 import com.att.cdp.zones.Context;
  42 import com.att.cdp.zones.Provider;
  43 import com.att.cdp.zones.model.Hypervisor;
  44 import com.att.cdp.zones.model.Hypervisor.Status;
  45 import com.att.cdp.zones.model.Hypervisor.State;
  46 import com.att.cdp.zones.model.Image;
  47 import com.att.cdp.zones.model.ModelObject;
  48 import com.att.cdp.zones.model.Server;
  49 import com.att.eelf.configuration.EELFLogger;
  50 import com.att.eelf.configuration.EELFManager;
  51 import com.att.eelf.i18n.EELFResourceManager;
  52 import org.openecomp.sdnc.sli.SvcLogicContext;
  53 import org.glassfish.grizzly.http.util.HttpStatus;
  54 import org.slf4j.MDC;
  55
  56 import java.text.DateFormat;
  57 import java.text.SimpleDateFormat;
  58 import java.util.Arrays;
  59 import java.util.Date;
  60 import java.util.List;
  61 import java.util.Map;
  62 import java.util.TimeZone;
  63
  64 import static org.openecomp.appc.adapter.utils.Constants.ADAPTER_NAME;
  65
  66 public class EvacuateServer extends ProviderServerOperation {
  67
  68     private static final EELFLogger logger = EELFManager.getInstance().getLogger(EvacuateServer.class);
  69     private static EELFLogger metricsLogger = EELFManager.getInstance().getMetricsLogger();
  70     private static final Configuration configuration = ConfigurationFactory.getConfiguration();
  71     private ProviderAdapterImpl paImpl = null;
  72
  73
  74     private void evacuateServer(RequestContext rc, @SuppressWarnings("unused") Server server, String target_host) throws ZoneException, RequestFailedException {
  75
  76         String msg;
  77         Context ctx = server.getContext();
  78         Provider provider = ctx.getProvider();
  79         ComputeService service = ctx.getComputeService();
  80
  81         /*
  82          * Pending is a bit of a special case. If we find the server is in a pending state, then the provider is in the
  83          * process of changing state of the server. So, lets try to wait a little bit and see if the state settles down
  84          * to one we can deal with. If not, then we have to fail the request.
  85          */
  86         try {
  87                 if (server.getStatus().equals(Server.Status.PENDING)) {
  88                         waitForStateChange(rc, server, Server.Status.READY, Server.Status.RUNNING, Server.Status.ERROR, Server.Status.SUSPENDED, Server.Status.PAUSED);
  89                 }
  90         } catch (RequestFailedException e) {
  91                 // evacuate is a special case. If the server is still in a Pending state, we want to continue with evacuate
  92                 logger.info("Evacuate server - ignore RequestFailedException from waitForStateChange() ...");
  93         }
  94
  95         /*
  96          * Set Time for Metrics Logger
  97          */
  98         long startTime = System.currentTimeMillis();
  99         TimeZone tz = TimeZone.getTimeZone("UTC");
 100         DateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZ");
 101         df.setTimeZone(tz);
 102         String startTimeStr = df.format(new Date());
 103         long endTime = System.currentTimeMillis();
 104         long duration = endTime - startTime;
 105         String endTimeStr = String.valueOf(endTime);
 106         String durationStr = String.valueOf(duration);
 107         String endTimeStrUTC = df.format(new Date());
 108         MDC.put("EndTimestamp", endTimeStrUTC);
 109         MDC.put("ElapsedTime", durationStr);
 110         MDC.put("TargetEntity", "cdp");
 111         MDC.put("TargetServiceName", "evacuate server");
 112         MDC.put("ClassName", "org.openecomp.appc.adapter.iaas.provider.operation.impl.EvacuteServer");
 113
 114
 115         try {
 116             while (rc.attempt()) {
 117                 try {
 118                         logger.debug("Calling CDP moveServer - server id = " + server.getId());
 119                     service.moveServer(server.getId(), target_host);
 120                     // Wait for completion, expecting the server to go to a non pending state
 121                     waitForStateChange(rc, server, Server.Status.READY, Server.Status.RUNNING, Server.Status.ERROR, Server.Status.SUSPENDED, Server.Status.PAUSED);
 122                     break;
 123                 } catch (ContextConnectionException e) {
 124                     msg = EELFResourceManager.format(Msg.CONNECTION_FAILED_RETRY, provider.getName(), service.getURL(),
 125                             ctx.getTenant().getName(), ctx.getTenant().getId(), e.getMessage(),
 126                             Long.toString(rc.getRetryDelay()), Integer.toString(rc.getAttempts()),
 127                             Integer.toString(rc.getRetryLimit()));
 128                     logger.error(msg, e);
 129                     metricsLogger.error(msg,e);
 130                     rc.delay();
 131                 }
 132             }
 133
 134         } catch (ZoneException e) {
 135             msg =
 136                     EELFResourceManager.format(Msg.EVACUATE_SERVER_FAILED, server.getName(), server.getId(), e.getMessage());
 137             logger.error(msg);
 138             metricsLogger.error(msg);
 139             throw new RequestFailedException("Evacute Server", msg, HttpStatus.BAD_GATEWAY_502, server);
 140         }
 141
 142         if (rc.isFailed()) {
 143             msg = EELFResourceManager.format(Msg.CONNECTION_FAILED, provider.getName(), service.getURL());
 144             logger.error(msg);
 145             metricsLogger.error(msg);
 146             throw new RequestFailedException("Evacuate Server", msg, HttpStatus.BAD_GATEWAY_502, server);
 147         }
 148         rc.reset();
 149     }
 150
 151
 152     /**
 153      * @see org.openecomp.appc.adapter.iaas.ProviderAdapter#evacuateServer(java.util.Map, org.openecomp.sdnc.sli.SvcLogicContext)
 154      */
 155     private Server evacuateServer(Map<String, String> params, SvcLogicContext ctx) throws APPCException {
 156         Server server = null;
 157         RequestContext rc = new RequestContext(ctx);
 158         rc.isAlive();
 159
 160         String appName = configuration.getProperty(Constants.PROPERTY_APPLICATION_NAME);
 161         String msg;
 162
 163         /*
 164          * Set Time for Metrics Logger
 165          */
 166         long startTime = System.currentTimeMillis();
 167         TimeZone tz = TimeZone.getTimeZone("UTC");
 168         DateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZ");
 169         df.setTimeZone(tz);
 170         String startTimeStr = df.format(new Date());
 171         long endTime = System.currentTimeMillis();
 172         long duration = endTime - startTime;
 173         String endTimeStr = String.valueOf(endTime);
 174         String durationStr = String.valueOf(duration);
 175         String endTimeStrUTC = df.format(new Date());
 176         MDC.put("EndTimestamp", endTimeStrUTC);
 177         MDC.put("ElapsedTime", durationStr);
 178         MDC.put("TargetEntity", "cdp");
 179         MDC.put("TargetServiceName", "evacuate server");
 180         MDC.put("ClassName", "org.openecomp.appc.adapter.iaas.provider.operation.impl.EvacuateServer");
 181
 182         ctx.setAttribute("EVACUATE_STATUS", "ERROR");
 183         try {
 184             validateParametersExist(params, ProviderAdapter.PROPERTY_INSTANCE_URL,
 185                     ProviderAdapter.PROPERTY_PROVIDER_NAME);
 186             String vm_url = params.get(ProviderAdapter.PROPERTY_INSTANCE_URL);
 187
 188             VMURL vm = VMURL.parseURL(vm_url);
 189             if (validateVM(rc, appName, vm_url, vm)) return null;
 190
 191             IdentityURL ident = IdentityURL.parseURL(params.get(ProviderAdapter.PROPERTY_IDENTITY_URL));
 192             String identStr = (ident == null) ? null : ident.toString();
 193
 194             // retrieve the optional parameters
 195             String rebuild_vm = params.get(ProviderAdapter.PROPERTY_REBUILD_VM);
 196             String targethost_id = params.get(ProviderAdapter.PROPERTY_TARGETHOST_ID);
 197
 198             Context context = null;
 199             try {
 200                 context = getContext(rc, vm_url, identStr);
 201                 if (context != null) {
 202
 203                     server = lookupServer(rc, context, vm.getServerId());
 204                     logger.debug(Msg.SERVER_FOUND, vm_url, context.getTenantName(), server.getStatus().toString());
 205
 206                     // check target host status
 207                         if (isComputeNodeDown(context, targethost_id)) {
 208                                 msg = EELFResourceManager.format(Msg.EVACUATE_SERVER_FAILED, server.getName(), server.getId(),
 209                                                 "Target host " + targethost_id +" status is not UP/ENABLED");
 210                         logger.error(msg);
 211                         metricsLogger.error(msg);
 212                         throw new RequestFailedException("Evacuate Server", msg, HttpStatus.BAD_REQUEST_400, server);
 213                         }
 214
 215                     // save hypervisor name before evacuate
 216                     String hypervisor = server.getHypervisor().getHostName();
 217
 218                     evacuateServer(rc, server, targethost_id);
 219
 220                     server.refreshAll();
 221                     String hypervisor_after_evacuate = server.getHypervisor().getHostName();
 222                     logger.debug("Hostname before evacuate: " + hypervisor + ", After evacuate: " + hypervisor_after_evacuate);
 223
 224                     // check hypervisor host name after evacuate. If it is unchanged, the evacuate failed.
 225                     if ((hypervisor != null) && (hypervisor.equals(hypervisor_after_evacuate))) {
 226                         msg = EELFResourceManager.format(Msg.EVACUATE_SERVER_FAILED, server.getName(), server.getId(),
 227                                         "Hypervisor host " + hypervisor + " after evacuate is the same as before evacuate. Provider (ex. Openstack) recovery actions may be needed.");
 228                         logger.error(msg);
 229                         metricsLogger.error(msg);
 230                         throw new RequestFailedException("Evacuate Server", msg, HttpStatus.INTERNAL_SERVER_ERROR_500, server);
 231
 232                     }
 233
 234                     // check VM status after evacuate
 235                     if (server.getStatus() == Server.Status.ERROR) {
 236                         msg = EELFResourceManager.format(Msg.EVACUATE_SERVER_FAILED, server.getName(), server.getId(),
 237                                         "VM is in ERROR state after evacuate. Provider (ex. Openstack) recovery actions may be needed.");
 238                         logger.error(msg);
 239                         metricsLogger.error(msg);
 240                         throw new RequestFailedException("Evacuate Server", msg, HttpStatus.INTERNAL_SERVER_ERROR_500, server);
 241                     }
 242
 243                     context.close();
 244                     doSuccess(rc);
 245                     ctx.setAttribute("EVACUATE_STATUS", "SUCCESS");
 246
 247                     // If a snapshot exists, do a rebuild to apply the latest snapshot to the evacuated server.
 248                     // This is the default behavior unless the optional parameter is set to FALSE.
 249                     if ((rebuild_vm == null) || !(rebuild_vm.equalsIgnoreCase("false"))) {
 250                         List<Image> snapshots = server.getSnapshots();
 251                         if (snapshots == null || snapshots.isEmpty()) {
 252                                 logger.debug("No snapshots available - skipping rebuild after evacuate");
 253                         } else if (paImpl != null) {
 254                                 logger.debug("Executing a rebuild after evacuate");
 255                                 paImpl.rebuildServer(params, ctx);
 256                                 // Check error code for rebuild errors. Evacuate had set it to 200 after
 257                                 // a successful evacuate. Rebuild updates the error code.
 258                                 String rebuildErrorCode = ctx.getAttribute(org.openecomp.appc.Constants.ATTRIBUTE_ERROR_CODE);
 259                             if (rebuildErrorCode != null) {
 260                                 try {
 261                                         int error_code = Integer.parseInt(rebuildErrorCode);
 262                                         if (error_code != HttpStatus.OK_200.getStatusCode()) {
 263                                                 logger.debug("Rebuild after evacuate failed - error code=" + error_code
 264                                                                 + ", message=" + ctx.getAttribute(org.openecomp.appc.Constants.ATTRIBUTE_ERROR_MESSAGE));
 265                                                 msg = EELFResourceManager.format(Msg.EVACUATE_SERVER_REBUILD_FAILED, server.getName(), hypervisor,
 266                                                         hypervisor_after_evacuate, ctx.getAttribute(org.openecomp.appc.Constants.ATTRIBUTE_ERROR_MESSAGE));
 267                                         logger.error(msg);
 268                                         metricsLogger.error(msg);
 269                                                 ctx.setAttribute("EVACUATE_STATUS", "ERROR");
 270                                                 // update error message while keeping the error code the same as before
 271                                                 doFailure(rc, HttpStatus.getHttpStatus(error_code), msg);
 272                                         }
 273                                 } catch (NumberFormatException e) {
 274                                         // ignore
 275                                 }
 276                                         }
 277                         }
 278                     }
 279
 280                 }
 281             } catch (ResourceNotFoundException e) {
 282                 msg = EELFResourceManager.format(Msg.SERVER_NOT_FOUND, e, vm_url);
 283                 logger.error(msg);
 284                 metricsLogger.error(msg);
 285                 doFailure(rc, HttpStatus.NOT_FOUND_404, msg);
 286             } catch (RequestFailedException e) {
 287                 doFailure(rc, e.getStatus(), e.getMessage());
 288             } catch (Throwable t) {
 289                 msg = EELFResourceManager.format(Msg.SERVER_OPERATION_EXCEPTION, t, t.getClass().getSimpleName(),
 290                         Operation.EVACUATE_SERVICE.toString(), vm_url, context == null ? "Unknown" : context.getTenantName());
 291                 logger.error(msg, t);
 292                 metricsLogger.error(msg, t);
 293                 doFailure(rc, HttpStatus.INTERNAL_SERVER_ERROR_500, msg);
 294             }
 295         } catch (RequestFailedException e) {
 296                 msg = EELFResourceManager.format(Msg.EVACUATE_SERVER_FAILED, "n/a", "n/a", e.getMessage());
 297             logger.error(msg);
 298             metricsLogger.error(msg);
 299             doFailure(rc, e.getStatus(), e.getMessage());
 300         }
 301
 302         return server;
 303     }
 304
 305     /*
 306      * Check if a Compute node is down.
 307      *
 308      * This method attempts to find a given host in the list of hypervisors for a given
 309      * context. The only case where a node is considered down is if a matching hypervisor
 310      * is found and it's state and status are not UP/ENABLED.
 311      *
 312      * @param context
 313      *                  The current context
 314      * @param host
 315      *                  The host name (short or fully qualified) of a compute node
 316      * @return true if the node is determined as down, false for all other cases
 317      */
 318     private boolean isComputeNodeDown(Context context, String host) throws ZoneException {
 319         ComputeService service = context.getComputeService();
 320         boolean node_down = false;
 321
 322         // Check host status. A node is considered down only if a matching target host is
 323         //  found and it's state/status is not UP/ENABLED.
 324         if ((host != null) && !(host.isEmpty())) {
 325                 List<Hypervisor> hypervisors = service.getHypervisors();
 326                 logger.debug("List of Hypervisors retrieved: " + Arrays.toString(hypervisors.toArray()));
 327             for (Hypervisor h : hypervisors) {
 328                 if (h.getHostName().startsWith(host)) {
 329                         // host matches one of the hypervisors
 330                     State hstate = h.getState();
 331                     Status hstatus = h.getStatus();
 332                     logger.debug("Host matching hypervisor: " + h.getHostName() + ", State/Status: "
 333                                                 + hstate.toString() + "/" + hstatus.toString());
 334                     if ((hstate != null) && (hstatus != null)) {
 335                         if (!(hstate.equals(State.UP)) || !(hstatus.equals(Status.ENABLED))) {
 336                                 node_down = true;
 337                         }
 338                     }
 339                 }
 340             }
 341         }
 342         return node_down;
 343     }
 344
 345     @Override
 346     protected ModelObject executeProviderOperation(Map<String, String> params, SvcLogicContext context) throws APPCException {
 347
 348         setMDC(Operation.EVACUATE_SERVICE.toString(), "App-C IaaS Adapter:Evacuate", ADAPTER_NAME);
 349         logOperation(Msg.EVACUATING_SERVER, params, context);
 350
 351         /*
 352          * Set Time for Metrics Logger
 353          */
 354         long startTime = System.currentTimeMillis();
 355         TimeZone tz = TimeZone.getTimeZone("UTC");
 356         DateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZ");
 357         df.setTimeZone(tz);
 358         String startTimeStr = df.format(new Date());
 359         long endTime = System.currentTimeMillis();
 360         long duration = endTime - startTime;
 361         String endTimeStr = String.valueOf(endTime);
 362         String durationStr = String.valueOf(duration);
 363         String endTimeStrUTC = df.format(new Date());
 364         MDC.put("EndTimestamp", endTimeStrUTC);
 365         MDC.put("ElapsedTime", durationStr);
 366         MDC.put("TargetEntity", "cdp");
 367         MDC.put("TargetServiceName", "evacuate server");
 368         MDC.put("ClassName", "org.openecomp.appc.adapter.iaas.provider.operation.impl.EvacuateServer");
 369
 370
 371         metricsLogger.info("Executing Provider Operation: Evacuate");
 372         return evacuateServer(params, context);
 373     }
 374
 375     public void setProvideAdapterRef(ProviderAdapterImpl pai) {
 376         paImpl = pai;
 377     }
 378 }