Fix LCM evacuate issue
[appc.git] / appc-adapters / appc-iaas-adapter / appc-iaas-adapter-bundle / src / main / java / org / onap / appc / adapter / iaas / provider / operation / impl / EvacuateServer.java
index 5d60ae2..2ce8ab4 100644 (file)
@@ -2,9 +2,13 @@
  * ============LICENSE_START=======================================================
  * ONAP : APPC
  * ================================================================================
- * Copyright (C) 2017 AT&T Intellectual Property. All rights reserved.
+ * Copyright (C) 2017-2018 AT&T Intellectual Property. All rights reserved.
  * ================================================================================
  * Copyright (C) 2017 Amdocs
+ * ================================================================================
+ * Modifications Copyright (C) 2019 Ericsson
+ * =============================================================================
+ * Modification Copyright (C) 2019 IBM
  * =============================================================================
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -18,7 +22,6 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  * 
- * ECOMP is a trademark and service mark of AT&T Intellectual Property.
  * ============LICENSE_END=========================================================
  */
 
@@ -37,6 +40,8 @@ import org.onap.appc.configuration.Configuration;
 import org.onap.appc.configuration.ConfigurationFactory;
 import org.onap.appc.exceptions.APPCException;
 import org.onap.appc.i18n.Msg;
+import org.onap.appc.logging.LoggingConstants;
+import org.onap.appc.logging.LoggingUtils;
 import com.att.cdp.exceptions.ContextConnectionException;
 import com.att.cdp.exceptions.ResourceNotFoundException;
 import com.att.cdp.exceptions.ZoneException;
@@ -55,34 +60,33 @@ import com.att.eelf.i18n.EELFResourceManager;
 import org.onap.ccsdk.sli.core.sli.SvcLogicContext;
 import org.glassfish.grizzly.http.util.HttpStatus;
 import org.slf4j.MDC;
-import java.text.DateFormat;
-import java.text.SimpleDateFormat;
+import java.io.IOException;
 import java.util.Arrays;
 import java.util.Date;
 import java.util.List;
 import java.util.Map;
-import java.util.TimeZone;
 import static org.onap.appc.adapter.utils.Constants.ADAPTER_NAME;
 
 public class EvacuateServer extends ProviderServerOperation {
 
+    private static final String EVACUATE_STATUS = "EVACUATE_STATUS";
+    private static final String EVACUATE_SERVER = "Evacuate Server";
+
     private static final EELFLogger logger = EELFManager.getInstance().getLogger(EvacuateServer.class);
     private static EELFLogger metricsLogger = EELFManager.getInstance().getMetricsLogger();
     private static final Configuration configuration = ConfigurationFactory.getConfiguration();
     private ProviderAdapterImpl paImpl = null;
 
-
-    private void evacuateServer(RequestContext rc, @SuppressWarnings("unused") Server server, String target_host)
+    private void evacuateServer(RequestContext rc, @SuppressWarnings("unused") Server server, String targetHost)
             throws ZoneException, RequestFailedException {
-
         Context ctx = server.getContext();
         Provider provider = ctx.getProvider();
         ComputeService service = ctx.getComputeService();
-
         /*
-         * Pending is a bit of a special case. If we find the server is in a pending state, then the provider is in the
-         * process of changing state of the server. So, lets try to wait a little bit and see if the state settles down
-         * to one we can deal with. If not, then we have to fail the request.
+         * Pending is a bit of a special case. If we find the server is in a pending
+         * state, then the provider is in the process of changing state of the server.
+         * So, lets try to wait a little bit and see if the state settles down to one we
+         * can deal with. If not, then we have to fail the request.
          */
         try {
             if (server.getStatus().equals(Server.Status.PENDING)) {
@@ -90,51 +94,55 @@ public class EvacuateServer extends ProviderServerOperation {
                         Server.Status.SUSPENDED, Server.Status.PAUSED);
             }
         } catch (RequestFailedException e) {
-            // evacuate is a special case. If the server is still in a Pending state, we want to
+            // evacuate is a special case. If the server is still in a Pending state, we
+            // want to
             // continue with evacuate
-            logger.info("Evacuate server - ignore RequestFailedException from waitForStateChange() ...");
+            logger.info("Evacuate server - ignore RequestFailedException from waitForStateChange() ...", e);
         }
-
-        SetTimeForMetricsLogger();
-
+        setTimeForMetricsLogger();
         String msg;
         try {
-            while (rc.attempt()) {
-                try {
-                    logger.debug("Calling CDP moveServer - server id = " + server.getId());
-                    service.moveServer(server.getId(), target_host);
-                    // Wait for completion, expecting the server to go to a non pending state
-                    waitForStateChange(rc, server, Server.Status.READY, Server.Status.RUNNING, Server.Status.ERROR,
-                            Server.Status.SUSPENDED, Server.Status.PAUSED);
-                    break;
-                } catch (ContextConnectionException e) {
-                    msg = EELFResourceManager.format(Msg.CONNECTION_FAILED_RETRY, provider.getName(), service.getURL(),
-                            ctx.getTenant().getName(), ctx.getTenant().getId(), e.getMessage(),
-                            Long.toString(rc.getRetryDelay()), Integer.toString(rc.getAttempts()),
-                            Integer.toString(rc.getRetryLimit()));
-                    logger.error(msg, e);
-                    metricsLogger.error(msg, e);
-                    rc.delay();
-                }
-            }
-
+            evacuateServerNested(rc, service, server, provider, targetHost);
         } catch (ZoneException e) {
             msg = EELFResourceManager.format(Msg.EVACUATE_SERVER_FAILED, server.getName(), server.getId(),
                     e.getMessage());
-            logger.error(msg);
+            logger.error(msg, e);
             metricsLogger.error(msg);
-            throw new RequestFailedException("Evacute Server", msg, HttpStatus.BAD_GATEWAY_502, server);
+            throw new RequestFailedException(EVACUATE_SERVER, msg, HttpStatus.BAD_GATEWAY_502, server);
         }
-
         if (rc.isFailed()) {
             msg = EELFResourceManager.format(Msg.CONNECTION_FAILED, provider.getName(), service.getURL());
             logger.error(msg);
             metricsLogger.error(msg);
-            throw new RequestFailedException("Evacuate Server", msg, HttpStatus.BAD_GATEWAY_502, server);
+            throw new RequestFailedException(EVACUATE_SERVER, msg, HttpStatus.BAD_GATEWAY_502, server);
         }
         rc.reset();
     }
 
+    private void evacuateServerNested(RequestContext rcCtx, ComputeService svc, Server server, Provider provider,
+            String targetHost) throws ZoneException, RequestFailedException {
+        String msg;
+        Context ctx = server.getContext();
+        rcCtx.reset();
+        while (rcCtx.attempt()) {
+            try {
+                logger.debug("Calling CDP moveServer - server id = " + server.getId());
+                svc.moveServer(server.getId(), targetHost);
+                // Wait for completion, expecting the server to go to a non pending state
+                waitForStateChange(rcCtx, server, Server.Status.READY, Server.Status.RUNNING, Server.Status.ERROR,
+                        Server.Status.SUSPENDED, Server.Status.PAUSED);
+                break;
+            } catch (ContextConnectionException e) {
+                msg = EELFResourceManager.format(Msg.CONNECTION_FAILED_RETRY, provider.getName(), svc.getURL(),
+                        ctx.getTenant().getName(), ctx.getTenant().getId(), e.getMessage(),
+                        Long.toString(rcCtx.getRetryDelay()), Integer.toString(rcCtx.getAttempts()),
+                        Integer.toString(rcCtx.getRetryLimit()));
+                logger.error(msg, e);
+                metricsLogger.error(msg, e);
+                rcCtx.delay();
+            }
+        }
+    }
 
     /**
      * @see org.onap.appc.adapter.iaas.ProviderAdapter#evacuateServer(java.util.Map,
@@ -144,181 +152,203 @@ public class EvacuateServer extends ProviderServerOperation {
         Server server = null;
         RequestContext rc = new RequestContext(ctx);
         rc.isAlive();
-
-        SetTimeForMetricsLogger();
-
+        setTimeForMetricsLogger();
         String msg;
-        ctx.setAttribute("EVACUATE_STATUS", "ERROR");
+        ctx.setAttribute(EVACUATE_STATUS, "ERROR");
         try {
             validateParametersExist(params, ProviderAdapter.PROPERTY_INSTANCE_URL,
                     ProviderAdapter.PROPERTY_PROVIDER_NAME);
 
             String appName = configuration.getProperty(Constants.PROPERTY_APPLICATION_NAME);
-            String vm_url = params.get(ProviderAdapter.PROPERTY_INSTANCE_URL);
-            VMURL vm = VMURL.parseURL(vm_url);
-            if (validateVM(rc, appName, vm_url, vm))
+            String vmUrl = params.get(ProviderAdapter.PROPERTY_INSTANCE_URL);
+            VMURL vm = VMURL.parseURL(vmUrl);
+            if (validateVM(rc, appName, vmUrl, vm)) {
                 return null;
+            }
+            server = evacuateServerMapNestedFirst(params, server, rc, ctx, vm, vmUrl);
+        } catch (RequestFailedException e) {
+            msg = EELFResourceManager.format(Msg.EVACUATE_SERVER_FAILED, "n/a", "n/a", e.getMessage());
+            logger.error(msg, e);
+            metricsLogger.error(msg);
+            doFailure(rc, e.getStatus(), e.getMessage());
+        }
+        return server;
+    }
 
-            IdentityURL ident = IdentityURL.parseURL(params.get(ProviderAdapter.PROPERTY_IDENTITY_URL));
-            String identStr = (ident == null) ? null : ident.toString();
-
-            // retrieve the optional parameters
-            String rebuild_vm = params.get(ProviderAdapter.PROPERTY_REBUILD_VM);
-            String targethost_id = params.get(ProviderAdapter.PROPERTY_TARGETHOST_ID);
-
-            Context context = null;
-            try {
-                context = getContext(rc, vm_url, identStr);
-                if (context != null) {
-
-                    server = lookupServer(rc, context, vm.getServerId());
-                    logger.debug(Msg.SERVER_FOUND, vm_url, context.getTenantName(), server.getStatus().toString());
-
-                    // check target host status
-                    if (isComputeNodeDown(context, targethost_id)) {
-                        msg = EELFResourceManager.format(Msg.EVACUATE_SERVER_FAILED, server.getName(), server.getId(),
-                                "Target host " + targethost_id + " status is not UP/ENABLED");
-                        logger.error(msg);
-                        metricsLogger.error(msg);
-                        throw new RequestFailedException("Evacuate Server", msg, HttpStatus.BAD_REQUEST_400, server);
-                    }
-
-                    // save hypervisor name before evacuate
-                    String hypervisor = server.getHypervisor().getHostName();
-
-                    evacuateServer(rc, server, targethost_id);
-
-                    server.refreshAll();
-                    String hypervisor_after_evacuate = server.getHypervisor().getHostName();
-                    logger.debug("Hostname before evacuate: " + hypervisor + ", After evacuate: "
-                            + hypervisor_after_evacuate);
-
-                    // check hypervisor host name after evacuate. If it is unchanged, the evacuate
-                    // failed.
-                    if ((hypervisor != null) && (hypervisor.equals(hypervisor_after_evacuate))) {
-                        msg = EELFResourceManager.format(Msg.EVACUATE_SERVER_FAILED, server.getName(), server.getId(),
-                                "Hypervisor host " + hypervisor
-                                        + " after evacuate is the same as before evacuate. Provider (ex. Openstack) recovery actions may be needed.");
-                        logger.error(msg);
-                        metricsLogger.error(msg);
-                        throw new RequestFailedException("Evacuate Server", msg, HttpStatus.INTERNAL_SERVER_ERROR_500,
-                                server);
-
-                    }
-
-                    // check VM status after evacuate
-                    if (server.getStatus() == Server.Status.ERROR) {
-                        msg = EELFResourceManager.format(Msg.EVACUATE_SERVER_FAILED, server.getName(), server.getId(),
-                                "VM is in ERROR state after evacuate. Provider (ex. Openstack) recovery actions may be needed.");
-                        logger.error(msg);
-                        metricsLogger.error(msg);
-                        throw new RequestFailedException("Evacuate Server", msg, HttpStatus.INTERNAL_SERVER_ERROR_500,
-                                server);
-                    }
-
-                    context.close();
-                    doSuccess(rc);
-                    ctx.setAttribute("EVACUATE_STATUS", "SUCCESS");
-
-                    // If a snapshot exists, do a rebuild to apply the latest snapshot to the
-                    // evacuated server.
-                    // This is the default behavior unless the optional parameter is set to FALSE.
-                    if ((rebuild_vm == null) || !(rebuild_vm.equalsIgnoreCase("false"))) {
-                        List<Image> snapshots = server.getSnapshots();
-                        if (snapshots == null || snapshots.isEmpty()) {
-                            logger.debug("No snapshots available - skipping rebuild after evacuate");
-                        } else if (paImpl != null) {
-                            logger.debug("Executing a rebuild after evacuate");
-                            paImpl.rebuildServer(params, ctx);
-                            // Check error code for rebuild errors. Evacuate had set it to 200 after
-                            // a successful evacuate. Rebuild updates the error code.
-                            String rebuildErrorCode =
-                                    ctx.getAttribute(org.onap.appc.Constants.ATTRIBUTE_ERROR_CODE);
-                            if (rebuildErrorCode != null) {
-                                try {
-                                    int error_code = Integer.parseInt(rebuildErrorCode);
-                                    if (error_code != HttpStatus.OK_200.getStatusCode()) {
-                                        logger.debug("Rebuild after evacuate failed - error code=" + error_code
-                                                + ", message=" + ctx.getAttribute(
-                                                        org.onap.appc.Constants.ATTRIBUTE_ERROR_MESSAGE));
-                                        msg = EELFResourceManager.format(Msg.EVACUATE_SERVER_REBUILD_FAILED,
-                                                server.getName(), hypervisor, hypervisor_after_evacuate,
-                                                ctx.getAttribute(org.onap.appc.Constants.ATTRIBUTE_ERROR_MESSAGE));
-                                        logger.error(msg);
-                                        metricsLogger.error(msg);
-                                        ctx.setAttribute("EVACUATE_STATUS", "ERROR");
-                                        // update error message while keeping the error code the
-                                        // same as before
-                                        doFailure(rc, HttpStatus.getHttpStatus(error_code), msg);
-                                    }
-                                } catch (NumberFormatException e) {
-                                    // ignore
-                                }
-                            }
-                        }
+    private Server evacuateServerMapNestedFirst(Map<String, String> params, Server server, RequestContext rqstCtx,
+            SvcLogicContext ctx, VMURL vm, String vmUrl) throws APPCException {
+        String msg;
+        Context context;
+        IdentityURL ident = IdentityURL.parseURL(params.get(ProviderAdapter.PROPERTY_IDENTITY_URL));
+        String identStr = (ident == null) ? null : ident.toString();
+        // retrieve the optional parameters
+        String rebuildVm = params.get(ProviderAdapter.PROPERTY_REBUILD_VM);
+        String targetHostId = params.get(ProviderAdapter.PROPERTY_TARGETHOST_ID);
+        String tenantName = "Unknown";// to be used also in case of exception
+        try {
+            context = getContext(rqstCtx, vmUrl, identStr);
+            if (context != null) {
+                tenantName = context.getTenantName();// this variable also is used in case of exception
+                server = lookupServer(rqstCtx, context, vm.getServerId());
+                logger.debug(Msg.SERVER_FOUND, vmUrl, tenantName, server.getStatus().toString());
+                // check target host status
+                checkHostStatus(server, targetHostId, context);
+                // save hypervisor name before evacuate
+                String hypervisor = server.getHypervisor().getHostName();
+                evacuateServer(rqstCtx, server, targetHostId);
+                server.refreshAll();
+                String hypervisorAfterEvacuate = server.getHypervisor().getHostName();
+                logger.debug(
+                        "Hostname before evacuate: " + hypervisor + ", After evacuate: " + hypervisorAfterEvacuate);
+                // check hypervisor host name after evacuate. If it is unchanged, the evacuate
+                // failed.
+                checkHypervisor(server, hypervisor, hypervisorAfterEvacuate);
+                // check VM status after evacuate
+                checkStatus(server);
+                context.close();
+                doSuccess(rqstCtx);
+                ctx.setAttribute(EVACUATE_STATUS, "SUCCESS");
+                // If a snapshot exists, do a rebuild to apply the latest snapshot to the
+                // evacuated server.
+                // This is the default behavior unless the optional parameter is set to FALSE.
+                if (rebuildVm == null || !"false".equalsIgnoreCase(rebuildVm)) {
+                    List<Image> snapshots = server.getSnapshots();
+                    if (snapshots == null || snapshots.isEmpty()) {
+                        logger.debug("No snapshots available - skipping rebuild after evacuate");
+                    } else if (paImpl != null) {
+                        logger.debug("Executing a rebuild after evacuate");
+                        paImpl.rebuildServer(params, ctx);
+                        // Check error code for rebuild errors. Evacuate had set it to 200 after
+                        // a successful evacuate. Rebuild updates the error code.
+                        evacuateServerMapNestedSecond(server, rqstCtx, ctx, hypervisor, hypervisorAfterEvacuate);
                     }
-
                 }
-            } catch (ResourceNotFoundException e) {
-                msg = EELFResourceManager.format(Msg.SERVER_NOT_FOUND, e, vm_url);
-                logger.error(msg);
-                metricsLogger.error(msg);
-                doFailure(rc, HttpStatus.NOT_FOUND_404, msg);
-            } catch (RequestFailedException e) {
-                doFailure(rc, e.getStatus(), e.getMessage());
-            } catch (Exception e1) {
-                msg = EELFResourceManager.format(Msg.SERVER_OPERATION_EXCEPTION, e1, e1.getClass().getSimpleName(),
-                        Operation.EVACUATE_SERVICE.toString(), vm_url,
-                        context == null ? "Unknown" : context.getTenantName());
-                logger.error(msg, e1);
-                metricsLogger.error(msg, e1);
-                doFailure(rc, HttpStatus.INTERNAL_SERVER_ERROR_500, msg);
             }
+        } catch (ResourceNotFoundException e) {
+            msg = EELFResourceManager.format(Msg.SERVER_NOT_FOUND, e, vmUrl);
+            logger.error(msg);
+            metricsLogger.error(msg);
+            doFailure(rqstCtx, HttpStatus.NOT_FOUND_404, msg);
         } catch (RequestFailedException e) {
-            msg = EELFResourceManager.format(Msg.EVACUATE_SERVER_FAILED, "n/a", "n/a", e.getMessage());
+            logger.error("Request failed", e);
+            doFailure(rqstCtx, e.getStatus(), e.getMessage());
+        } catch (IOException | ZoneException e1) {
+            msg = EELFResourceManager.format(Msg.SERVER_OPERATION_EXCEPTION, e1, e1.getClass().getSimpleName(),
+                    Operation.EVACUATE_SERVICE.toString(), vmUrl, tenantName);
+            logger.error(msg, e1);
+            metricsLogger.error(msg, e1);
+            doFailure(rqstCtx, HttpStatus.INTERNAL_SERVER_ERROR_500, e1.getMessage());
+        } catch (Exception e1) {
+            msg = EELFResourceManager.format(Msg.SERVER_OPERATION_EXCEPTION, e1, e1.getClass().getSimpleName(),
+                    Operation.EVACUATE_SERVICE.toString(), vmUrl, tenantName);
+            logger.error(msg, e1);
+            metricsLogger.error(msg);
+            doFailure(rqstCtx, HttpStatus.INTERNAL_SERVER_ERROR_500, e1.getMessage());
+        }
+        return server;
+    }
+
+    private void evacuateServerMapNestedSecond(Server server, RequestContext rc, SvcLogicContext ctx, String hypervisor,
+            String hypervisorAfterEvacuate) {
+        String msg;
+        String rebuildErrorCode = ctx.getAttribute(org.onap.appc.Constants.ATTRIBUTE_ERROR_CODE);
+        if (rebuildErrorCode != null) {
+            try {
+                int errorCode = Integer.parseInt(rebuildErrorCode);
+                if (errorCode != HttpStatus.OK_200.getStatusCode()) {
+                    logger.debug("Rebuild after evacuate failed - error code=" + errorCode + ", message="
+                            + ctx.getAttribute(org.onap.appc.Constants.ATTRIBUTE_ERROR_MESSAGE));
+                    msg = EELFResourceManager.format(Msg.EVACUATE_SERVER_REBUILD_FAILED, server.getName(), hypervisor,
+                            hypervisorAfterEvacuate, ctx.getAttribute(org.onap.appc.Constants.ATTRIBUTE_ERROR_MESSAGE));
+                    logger.error(msg);
+                    metricsLogger.error(msg);
+                    ctx.setAttribute(EVACUATE_STATUS, "ERROR");
+                    // update error message while keeping the error code the
+                    // same as before
+                    doFailure(rc, HttpStatus.getHttpStatus(errorCode), msg);
+                }
+            } catch (NumberFormatException e) {
+                // ignore
+            }
+        }
+    }
+
+    private void checkHostStatus(Server server, String targetHostId, Context context)
+            throws ZoneException, RequestFailedException {
+        if (isComputeNodeDown(context, targetHostId)) {
+            String msg = EELFResourceManager.format(Msg.EVACUATE_SERVER_FAILED, server.getName(), server.getId(),
+                    "Target host " + targetHostId + " status is not UP/ENABLED");
             logger.error(msg);
             metricsLogger.error(msg);
-            doFailure(rc, e.getStatus(), e.getMessage());
+            throw new RequestFailedException(EVACUATE_SERVER, msg, HttpStatus.BAD_REQUEST_400, server);
         }
+    }
 
-        return server;
+    private void checkHypervisor(Server server, String hypervisor, String hypervisorAfterEvacuate)
+            throws RequestFailedException {
+        if (hypervisor != null && hypervisor.equals(hypervisorAfterEvacuate)) {
+            String msg = EELFResourceManager.format(Msg.EVACUATE_SERVER_FAILED, server.getName(), server.getId(),
+                    "Hypervisor host " + hypervisor
+                            + " after evacuate is the same as before evacuate. Provider (ex. Openstack) recovery actions may be needed.");
+            logger.error(msg);
+            metricsLogger.error(msg);
+            throw new RequestFailedException(EVACUATE_SERVER, msg, HttpStatus.INTERNAL_SERVER_ERROR_500, server);
+        }
     }
 
-    /*
+    private void checkStatus(Server server) throws RequestFailedException {
+        if (server.getStatus() == Server.Status.ERROR) {
+            String msg = EELFResourceManager.format(Msg.EVACUATE_SERVER_FAILED, server.getName(), server.getId(),
+                    "VM is in ERROR state after evacuate. Provider (ex. Openstack) recovery actions may be needed.");
+            logger.error(msg);
+            metricsLogger.error(msg);
+            throw new RequestFailedException(EVACUATE_SERVER, msg, HttpStatus.INTERNAL_SERVER_ERROR_500, server);
+        }
+    }
+
+    /**
      * Check if a Compute node is down.
      * 
-     * This method attempts to find a given host in the list of hypervisors for a given context. The only case where a
-     * node is considered down is if a matching hypervisor is found and it's state and status are not UP/ENABLED.
+     * This method attempts to find a given host in the list of hypervisors for a
+     * given context. The only case where a node is considered down is if a matching
+     * hypervisor is found and it's state and status are not UP/ENABLED.
      * 
-     * @param context The current context
+     * @param context
+     *            The current context
      * 
-     * @param host The host name (short or fully qualified) of a compute node
+     * @param host
+     *            The host name (short or fully qualified) of a compute node
      * 
      * @return true if the node is determined as down, false for all other cases
      */
     private boolean isComputeNodeDown(Context context, String host) throws ZoneException {
         ComputeService service = context.getComputeService();
-        boolean node_down = false;
-
-        // Check host status. A node is considered down only if a matching target host is
-        // found and it's state/status is not UP/ENABLED.
-        if ((host != null) && !(host.isEmpty())) {
+        boolean nodeDown = false;
+        if (host != null && !host.isEmpty()) {
             List<Hypervisor> hypervisors = service.getHypervisors();
             logger.debug("List of Hypervisors retrieved: " + Arrays.toString(hypervisors.toArray()));
-            for (Hypervisor h : hypervisors) {
-                if (h.getHostName().startsWith(host)) {
-                    // host matches one of the hypervisors
-                    State hstate = h.getState();
-                    Status hstatus = h.getStatus();
-                    logger.debug("Host matching hypervisor: " + h.getHostName() + ", State/Status: " + hstate.toString()
-                            + "/" + hstatus.toString());
-                    if (hstate != State.UP || hstatus != Status.ENABLED) {
-                        node_down = true;
-                    }
-                }
+            for (Hypervisor hv : hypervisors) {
+                nodeDown = isNodeDown(host, nodeDown, hv);
             }
         }
-        return node_down;
+        return nodeDown;
+    }
+
+    private boolean isNodeDown(String host, boolean nodeDown, Hypervisor hv) {
+        if (isHostMatchesHypervisor(host, hv)) {
+            State hstate = hv.getState();
+            Status hstatus = hv.getStatus();
+            logger.debug("Host matching hypervisor: " + hv.getHostName() + ", State/Status: " + hstate.toString() + "/"
+                    + hstatus.toString());
+            if (hstate != State.UP || hstatus != Status.ENABLED) {
+                return true;
+            }
+        }
+        return nodeDown;
+    }
+
+    private boolean isHostMatchesHypervisor(String host, Hypervisor hypervisor) {
+        return hypervisor.getHostName().startsWith(host);
     }
 
     @Override
@@ -326,27 +356,21 @@ public class EvacuateServer extends ProviderServerOperation {
             throws APPCException {
         setMDC(Operation.EVACUATE_SERVICE.toString(), "App-C IaaS Adapter:Evacuate", ADAPTER_NAME);
         logOperation(Msg.EVACUATING_SERVER, params, context);
-
-        SetTimeForMetricsLogger();
-
+        setTimeForMetricsLogger();
         metricsLogger.info("Executing Provider Operation: Evacuate");
         return evacuateServer(params, context);
     }
 
-    private void SetTimeForMetricsLogger() {
-        long startTime = System.currentTimeMillis();
-        TimeZone tz = TimeZone.getTimeZone("UTC");
-        DateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZ");
-        df.setTimeZone(tz);
-        long endTime = System.currentTimeMillis();
-        long duration = endTime - startTime;
-        String durationStr = String.valueOf(duration);
-        String endTimeStrUTC = df.format(new Date());
-        MDC.put("EndTimestamp", endTimeStrUTC);
-        MDC.put("ElapsedTime", durationStr);
-        MDC.put("TargetEntity", "cdp");
-        MDC.put("TargetServiceName", "evacuate server");
-        MDC.put("ClassName", "org.onap.appc.adapter.iaas.provider.operation.impl.EvacuateServer");
+    private void setTimeForMetricsLogger() {
+        String timestamp = LoggingUtils.generateTimestampStr((new Date()).toInstant());
+        MDC.put(LoggingConstants.MDCKeys.BEGIN_TIMESTAMP, timestamp);
+        MDC.put(LoggingConstants.MDCKeys.END_TIMESTAMP, timestamp);
+        MDC.put(LoggingConstants.MDCKeys.ELAPSED_TIME, "0");
+        MDC.put(LoggingConstants.MDCKeys.STATUS_CODE, LoggingConstants.StatusCodes.COMPLETE);
+        MDC.put(LoggingConstants.MDCKeys.TARGET_ENTITY, "cdp");
+        MDC.put(LoggingConstants.MDCKeys.TARGET_SERVICE_NAME, "evacuate server");
+        MDC.put(LoggingConstants.MDCKeys.CLASS_NAME,
+                "org.onap.appc.adapter.iaas.provider.operation.impl.EvacuateServer");
     }
 
     public void setProvideAdapterRef(ProviderAdapterImpl pai) {