appc-adapters/appc-iaas-adapter/appc-iaas-adapter-bundle/src/main/java/org/openecomp/appc/adapter/iaas/provider/operation/impl/EvacuateServer.java

   1 /*-
   2  * ============LICENSE_START=======================================================
   3  * ONAP : APPC
   4  * ================================================================================
   5  * Copyright (C) 2017 AT&T Intellectual Property. All rights reserved.
   6  * ================================================================================
   7  * Copyright (C) 2017 Amdocs
   8  * =============================================================================
   9  * Licensed under the Apache License, Version 2.0 (the "License");
  10  * you may not use this file except in compliance with the License.
  11  * You may obtain a copy of the License at
  12  *
  13  *      http://www.apache.org/licenses/LICENSE-2.0
  14  *
  15  * Unless required by applicable law or agreed to in writing, software
  16  * distributed under the License is distributed on an "AS IS" BASIS,
  17  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  18  * See the License for the specific language governing permissions and
  19  * limitations under the License.
  20  *
  21  * ECOMP is a trademark and service mark of AT&T Intellectual Property.
  22  * ============LICENSE_END=========================================================
  23  */
  24
  25 package org.openecomp.appc.adapter.iaas.provider.operation.impl;
  26
  27 import org.openecomp.appc.Constants;
  28 import org.openecomp.appc.adapter.iaas.ProviderAdapter;
  29 import org.openecomp.appc.adapter.iaas.impl.IdentityURL;
  30 import org.openecomp.appc.adapter.iaas.impl.ProviderAdapterImpl;
  31 import org.openecomp.appc.adapter.iaas.impl.RequestContext;
  32 import org.openecomp.appc.adapter.iaas.impl.RequestFailedException;
  33 import org.openecomp.appc.adapter.iaas.impl.VMURL;
  34 import org.openecomp.appc.adapter.iaas.provider.operation.common.enums.Operation;
  35 import org.openecomp.appc.adapter.iaas.provider.operation.impl.base.ProviderServerOperation;
  36 import org.openecomp.appc.configuration.Configuration;
  37 import org.openecomp.appc.configuration.ConfigurationFactory;
  38 import org.openecomp.appc.exceptions.APPCException;
  39 import org.openecomp.appc.i18n.Msg;
  40 import com.att.cdp.exceptions.ContextConnectionException;
  41 import com.att.cdp.exceptions.ResourceNotFoundException;
  42 import com.att.cdp.exceptions.ZoneException;
  43 import com.att.cdp.zones.ComputeService;
  44 import com.att.cdp.zones.Context;
  45 import com.att.cdp.zones.Provider;
  46 import com.att.cdp.zones.model.Hypervisor;
  47 import com.att.cdp.zones.model.Hypervisor.Status;
  48 import com.att.cdp.zones.model.Hypervisor.State;
  49 import com.att.cdp.zones.model.Image;
  50 import com.att.cdp.zones.model.ModelObject;
  51 import com.att.cdp.zones.model.Server;
  52 import com.att.eelf.configuration.EELFLogger;
  53 import com.att.eelf.configuration.EELFManager;
  54 import com.att.eelf.i18n.EELFResourceManager;
  55 import org.openecomp.sdnc.sli.SvcLogicContext;
  56 import org.glassfish.grizzly.http.util.HttpStatus;
  57 import org.slf4j.MDC;
  58
  59 import java.text.DateFormat;
  60 import java.text.SimpleDateFormat;
  61 import java.util.Arrays;
  62 import java.util.Date;
  63 import java.util.List;
  64 import java.util.Map;
  65 import java.util.TimeZone;
  66
  67 import static org.openecomp.appc.adapter.utils.Constants.ADAPTER_NAME;
  68
  69 public class EvacuateServer extends ProviderServerOperation {
  70
  71     private static final EELFLogger logger = EELFManager.getInstance().getLogger(EvacuateServer.class);
  72     private static EELFLogger metricsLogger = EELFManager.getInstance().getMetricsLogger();
  73     private static final Configuration configuration = ConfigurationFactory.getConfiguration();
  74     private ProviderAdapterImpl paImpl = null;
  75
  76
  77     private void evacuateServer(RequestContext rc, @SuppressWarnings("unused") Server server, String target_host)
  78             throws ZoneException, RequestFailedException {
  79
  80         Context ctx = server.getContext();
  81         Provider provider = ctx.getProvider();
  82         ComputeService service = ctx.getComputeService();
  83
  84         /*
  85          * Pending is a bit of a special case. If we find the server is in a pending state, then the provider is in the
  86          * process of changing state of the server. So, lets try to wait a little bit and see if the state settles down
  87          * to one we can deal with. If not, then we have to fail the request.
  88          */
  89         try {
  90             if (server.getStatus().equals(Server.Status.PENDING)) {
  91                 waitForStateChange(rc, server, Server.Status.READY, Server.Status.RUNNING, Server.Status.ERROR,
  92                         Server.Status.SUSPENDED, Server.Status.PAUSED);
  93             }
  94         } catch (RequestFailedException e) {
  95             // evacuate is a special case. If the server is still in a Pending state, we want to continue with evacuate
  96             logger.info("Evacuate server - ignore RequestFailedException from waitForStateChange() ...");
  97         }
  98
  99         SetTimeForMetricsLogger();
 100
 101         String msg;
 102         try {
 103             while (rc.attempt()) {
 104                 try {
 105                     logger.debug("Calling CDP moveServer - server id = " + server.getId());
 106                     service.moveServer(server.getId(), target_host);
 107                     // Wait for completion, expecting the server to go to a non pending state
 108                     waitForStateChange(rc, server, Server.Status.READY, Server.Status.RUNNING,
 109                             Server.Status.ERROR, Server.Status.SUSPENDED, Server.Status.PAUSED);
 110                     break;
 111                 } catch (ContextConnectionException e) {
 112                     msg = EELFResourceManager.format(Msg.CONNECTION_FAILED_RETRY, provider.getName(), service.getURL(),
 113                             ctx.getTenant().getName(), ctx.getTenant().getId(), e.getMessage(),
 114                             Long.toString(rc.getRetryDelay()), Integer.toString(rc.getAttempts()),
 115                             Integer.toString(rc.getRetryLimit()));
 116                     logger.error(msg, e);
 117                     metricsLogger.error(msg, e);
 118                     rc.delay();
 119                 }
 120             }
 121
 122         } catch (ZoneException e) {
 123             msg = EELFResourceManager.format(Msg.EVACUATE_SERVER_FAILED,
 124                     server.getName(), server.getId(), e.getMessage());
 125             logger.error(msg);
 126             metricsLogger.error(msg);
 127             throw new RequestFailedException("Evacute Server", msg, HttpStatus.BAD_GATEWAY_502, server);
 128         }
 129
 130         if (rc.isFailed()) {
 131             msg = EELFResourceManager.format(Msg.CONNECTION_FAILED, provider.getName(), service.getURL());
 132             logger.error(msg);
 133             metricsLogger.error(msg);
 134             throw new RequestFailedException("Evacuate Server", msg, HttpStatus.BAD_GATEWAY_502, server);
 135         }
 136         rc.reset();
 137     }
 138
 139
 140     /**
 141      * @see org.openecomp.appc.adapter.iaas.ProviderAdapter#evacuateServer(java.util.Map, org.openecomp.sdnc.sli.SvcLogicContext)
 142      */
 143     private Server evacuateServer(Map<String, String> params, SvcLogicContext ctx) throws APPCException {
 144         Server server = null;
 145         RequestContext rc = new RequestContext(ctx);
 146         rc.isAlive();
 147
 148         SetTimeForMetricsLogger();
 149
 150         String msg;
 151         ctx.setAttribute("EVACUATE_STATUS", "ERROR");
 152         try {
 153             validateParametersExist(params, ProviderAdapter.PROPERTY_INSTANCE_URL,
 154                     ProviderAdapter.PROPERTY_PROVIDER_NAME);
 155
 156             String appName = configuration.getProperty(Constants.PROPERTY_APPLICATION_NAME);
 157             String vm_url = params.get(ProviderAdapter.PROPERTY_INSTANCE_URL);
 158             VMURL vm = VMURL.parseURL(vm_url);
 159             if (validateVM(rc, appName, vm_url, vm)) return null;
 160
 161             IdentityURL ident = IdentityURL.parseURL(params.get(ProviderAdapter.PROPERTY_IDENTITY_URL));
 162             String identStr = (ident == null) ? null : ident.toString();
 163
 164             // retrieve the optional parameters
 165             String rebuild_vm = params.get(ProviderAdapter.PROPERTY_REBUILD_VM);
 166             String targethost_id = params.get(ProviderAdapter.PROPERTY_TARGETHOST_ID);
 167
 168             Context context = null;
 169             try {
 170                 context = getContext(rc, vm_url, identStr);
 171                 if (context != null) {
 172
 173                     server = lookupServer(rc, context, vm.getServerId());
 174                     logger.debug(Msg.SERVER_FOUND, vm_url, context.getTenantName(), server.getStatus().toString());
 175
 176                     // check target host status
 177                     if (isComputeNodeDown(context, targethost_id)) {
 178                         msg = EELFResourceManager.format(Msg.EVACUATE_SERVER_FAILED, server.getName(),
 179                                 server.getId(), "Target host " + targethost_id + " status is not UP/ENABLED");
 180                         logger.error(msg);
 181                         metricsLogger.error(msg);
 182                         throw new RequestFailedException("Evacuate Server", msg, HttpStatus.BAD_REQUEST_400, server);
 183                     }
 184
 185                     // save hypervisor name before evacuate
 186                     String hypervisor = server.getHypervisor().getHostName();
 187
 188                     evacuateServer(rc, server, targethost_id);
 189
 190                     server.refreshAll();
 191                     String hypervisor_after_evacuate = server.getHypervisor().getHostName();
 192                     logger.debug("Hostname before evacuate: " + hypervisor + ", After evacuate: "
 193                             + hypervisor_after_evacuate);
 194
 195                     // check hypervisor host name after evacuate. If it is unchanged, the evacuate failed.
 196                     if ((hypervisor != null) && (hypervisor.equals(hypervisor_after_evacuate))) {
 197                         msg = EELFResourceManager.format(Msg.EVACUATE_SERVER_FAILED, server.getName(),
 198                                 server.getId(), "Hypervisor host " + hypervisor
 199                                         + " after evacuate is the same as before evacuate. Provider (ex. Openstack) recovery actions may be needed.");
 200                         logger.error(msg);
 201                         metricsLogger.error(msg);
 202                         throw new RequestFailedException("Evacuate Server", msg,
 203                                 HttpStatus.INTERNAL_SERVER_ERROR_500, server);
 204
 205                     }
 206
 207                     // check VM status after evacuate
 208                     if (server.getStatus() == Server.Status.ERROR) {
 209                         msg = EELFResourceManager.format(Msg.EVACUATE_SERVER_FAILED, server.getName(),
 210                                 server.getId(),
 211                                 "VM is in ERROR state after evacuate. Provider (ex. Openstack) recovery actions may be needed.");
 212                         logger.error(msg);
 213                         metricsLogger.error(msg);
 214                         throw new RequestFailedException("Evacuate Server", msg,
 215                                 HttpStatus.INTERNAL_SERVER_ERROR_500, server);
 216                     }
 217
 218                     context.close();
 219                     doSuccess(rc);
 220                     ctx.setAttribute("EVACUATE_STATUS", "SUCCESS");
 221
 222                     // If a snapshot exists, do a rebuild to apply the latest snapshot to the evacuated server.
 223                     // This is the default behavior unless the optional parameter is set to FALSE.
 224                     if ((rebuild_vm == null) || !(rebuild_vm.equalsIgnoreCase("false"))) {
 225                         List<Image> snapshots = server.getSnapshots();
 226                         if (snapshots == null || snapshots.isEmpty()) {
 227                             logger.debug("No snapshots available - skipping rebuild after evacuate");
 228                         } else if (paImpl != null) {
 229                             logger.debug("Executing a rebuild after evacuate");
 230                             paImpl.rebuildServer(params, ctx);
 231                             // Check error code for rebuild errors. Evacuate had set it to 200 after
 232                             // a successful evacuate. Rebuild updates the error code.
 233                             String rebuildErrorCode =
 234                                     ctx.getAttribute(org.openecomp.appc.Constants.ATTRIBUTE_ERROR_CODE);
 235                             if (rebuildErrorCode != null) {
 236                                 try {
 237                                     int error_code = Integer.parseInt(rebuildErrorCode);
 238                                     if (error_code != HttpStatus.OK_200.getStatusCode()) {
 239                                         logger.debug("Rebuild after evacuate failed - error code=" + error_code
 240                                                 + ", message=" + ctx.getAttribute(
 241                                                 org.openecomp.appc.Constants.ATTRIBUTE_ERROR_MESSAGE));
 242                                         msg = EELFResourceManager.format(Msg.EVACUATE_SERVER_REBUILD_FAILED,
 243                                                 server.getName(), hypervisor, hypervisor_after_evacuate,
 244                                                 ctx.getAttribute(org.openecomp.appc.Constants.ATTRIBUTE_ERROR_MESSAGE));
 245                                         logger.error(msg);
 246                                         metricsLogger.error(msg);
 247                                         ctx.setAttribute("EVACUATE_STATUS", "ERROR");
 248                                         // update error message while keeping the error code the same as before
 249                                         doFailure(rc, HttpStatus.getHttpStatus(error_code), msg);
 250                                     }
 251                                 } catch (NumberFormatException e) {
 252                                     // ignore
 253                                 }
 254                             }
 255                         }
 256                     }
 257
 258                 }
 259             } catch (ResourceNotFoundException e) {
 260                 msg = EELFResourceManager.format(Msg.SERVER_NOT_FOUND, e, vm_url);
 261                 logger.error(msg);
 262                 metricsLogger.error(msg);
 263                 doFailure(rc, HttpStatus.NOT_FOUND_404, msg);
 264             } catch (RequestFailedException e) {
 265                 doFailure(rc, e.getStatus(), e.getMessage());
 266             } catch (Exception e1) {
 267                 msg = EELFResourceManager.format(Msg.SERVER_OPERATION_EXCEPTION, e1, e1.getClass().getSimpleName(),
 268                         Operation.EVACUATE_SERVICE.toString(),
 269                         vm_url, context == null ? "Unknown" : context.getTenantName());
 270                 logger.error(msg, e1);
 271                 metricsLogger.error(msg, e1);
 272                 doFailure(rc, HttpStatus.INTERNAL_SERVER_ERROR_500, msg);
 273             }
 274         } catch (RequestFailedException e) {
 275             msg = EELFResourceManager.format(Msg.EVACUATE_SERVER_FAILED, "n/a", "n/a", e.getMessage());
 276             logger.error(msg);
 277             metricsLogger.error(msg);
 278             doFailure(rc, e.getStatus(), e.getMessage());
 279         }
 280
 281         return server;
 282     }
 283
 284     /*
 285      * Check if a Compute node is down.
 286      *
 287      * This method attempts to find a given host in the list of hypervisors for a given
 288      * context. The only case where a node is considered down is if a matching hypervisor
 289      * is found and it's state and status are not UP/ENABLED.
 290      *
 291      * @param context
 292      *             The current context
 293      * @param host
 294      *             The host name (short or fully qualified) of a compute node
 295      * @return true if the node is determined as down, false for all other cases
 296      */
 297     private boolean isComputeNodeDown(Context context, String host) throws ZoneException {
 298         ComputeService service = context.getComputeService();
 299         boolean node_down = false;
 300
 301         // Check host status. A node is considered down only if a matching target host is
 302         //  found and it's state/status is not UP/ENABLED.
 303         if ((host != null) && !(host.isEmpty())) {
 304             List<Hypervisor> hypervisors = service.getHypervisors();
 305             logger.debug("List of Hypervisors retrieved: " + Arrays.toString(hypervisors.toArray()));
 306             for (Hypervisor h : hypervisors) {
 307                 if (h.getHostName().startsWith(host)) {
 308                     // host matches one of the hypervisors
 309                     State hstate = h.getState();
 310                     Status hstatus = h.getStatus();
 311                     logger.debug("Host matching hypervisor: " + h.getHostName() + ", State/Status: "
 312                             + hstate.toString() + "/" + hstatus.toString());
 313                     if (hstate != State.UP || hstatus != Status.ENABLED) {
 314                         node_down = true;
 315                     }
 316                 }
 317             }
 318         }
 319         return node_down;
 320     }
 321
 322     @Override
 323     protected ModelObject executeProviderOperation(Map<String, String> params, SvcLogicContext context)
 324             throws APPCException {
 325         setMDC(Operation.EVACUATE_SERVICE.toString(), "App-C IaaS Adapter:Evacuate", ADAPTER_NAME);
 326         logOperation(Msg.EVACUATING_SERVER, params, context);
 327
 328         SetTimeForMetricsLogger();
 329
 330         metricsLogger.info("Executing Provider Operation: Evacuate");
 331         return evacuateServer(params, context);
 332     }
 333
 334     private void SetTimeForMetricsLogger() {
 335         long startTime = System.currentTimeMillis();
 336         TimeZone tz = TimeZone.getTimeZone("UTC");
 337         DateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZ");
 338         df.setTimeZone(tz);
 339         long endTime = System.currentTimeMillis();
 340         long duration = endTime - startTime;
 341         String durationStr = String.valueOf(duration);
 342         String endTimeStrUTC = df.format(new Date());
 343         MDC.put("EndTimestamp", endTimeStrUTC);
 344         MDC.put("ElapsedTime", durationStr);
 345         MDC.put("TargetEntity", "cdp");
 346         MDC.put("TargetServiceName", "evacuate server");
 347         MDC.put("ClassName", "org.openecomp.appc.adapter.iaas.provider.operation.impl.EvacuateServer");
 348     }
 349
 350     public void setProvideAdapterRef(ProviderAdapterImpl pai) {
 351         paImpl = pai;
 352     }
 353 }