grToolkit/provider/src/main/java/org/onap/ccsdk/sli/plugins/grtoolkit/GrToolkitProvider.java

   1 /*-
   2  * ============LICENSE_START=======================================================
   3  * openECOMP : SDN-C
   4  * ================================================================================
   5  * Copyright (C) 2018 AT&T Intellectual Property. All rights
   6  *                      reserved.
   7  * ================================================================================
   8  * Licensed under the Apache License, Version 2.0 (the "License");
   9  * you may not use this file except in compliance with the License.
  10  * You may obtain a copy of the License at
  11  *
  12  *      http://www.apache.org/licenses/LICENSE-2.0
  13  *
  14  * Unless required by applicable law or agreed to in writing, software
  15  * distributed under the License is distributed on an "AS IS" BASIS,
  16  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  17  * See the License for the specific language governing permissions and
  18  * limitations under the License.
  19  * ============LICENSE_END=========================================================
  20  */
  21
  22 package org.onap.ccsdk.sli.plugins.grtoolkit;
  23
  24 import java.io.BufferedReader;
  25 import java.io.File;
  26 import java.io.FileInputStream;
  27 import java.io.FileReader;
  28 import java.io.IOException;
  29 import java.io.InputStreamReader;
  30 import java.io.OutputStream;
  31 import java.net.HttpURLConnection;
  32 import java.net.URL;
  33 import java.nio.charset.StandardCharsets;
  34 import java.sql.Connection;
  35 import java.sql.SQLException;
  36 import java.util.ArrayList;
  37 import java.util.Collection;
  38 import java.util.HashMap;
  39 import java.util.Map;
  40 import java.util.Properties;
  41 import java.util.List;
  42 import java.util.concurrent.ExecutorService;
  43 import java.util.concurrent.Executors;
  44 import java.util.regex.Matcher;
  45 import java.util.regex.Pattern;
  46 import javax.annotation.Nonnull;
  47
  48 import com.google.common.util.concurrent.Futures;
  49 import com.google.common.util.concurrent.ListenableFuture;
  50
  51 import org.onap.ccsdk.sli.core.dblib.DbLibService;
  52 import org.onap.ccsdk.sli.plugins.grtoolkit.data.ClusterActor;
  53 import org.onap.ccsdk.sli.plugins.grtoolkit.data.MemberBuilder;
  54
  55 import org.json.JSONArray;
  56 import org.json.JSONException;
  57 import org.json.JSONObject;
  58
  59 import org.opendaylight.controller.cluster.datastore.DistributedDataStoreInterface;
  60 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  61 import org.opendaylight.controller.md.sal.binding.api.DataTreeChangeListener;
  62 import org.opendaylight.controller.md.sal.binding.api.NotificationPublishService;
  63 import org.opendaylight.controller.sal.binding.api.BindingAwareBroker;
  64 import org.opendaylight.controller.sal.binding.api.RpcProviderRegistry;
  65 import org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.AdminHealthInput;
  66 import org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.AdminHealthOutput;
  67 import org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.AdminHealthOutputBuilder;
  68 import org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.ClusterHealthInput;
  69 import org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.ClusterHealthOutput;
  70 import org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.ClusterHealthOutputBuilder;
  71 import org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.DatabaseHealthInput;
  72 import org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.DatabaseHealthOutput;
  73 import org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.DatabaseHealthOutputBuilder;
  74 import org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.FailoverInput;
  75 import org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.FailoverOutput;
  76 import org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.FailoverOutputBuilder;
  77 import org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.GrToolkitService;
  78 import org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.HaltAkkaTrafficInput;
  79 import org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.HaltAkkaTrafficOutput;
  80 import org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.HaltAkkaTrafficOutputBuilder;
  81 import org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.Member;
  82 import org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.ResumeAkkaTrafficInput;
  83 import org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.ResumeAkkaTrafficOutput;
  84 import org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.ResumeAkkaTrafficOutputBuilder;
  85 import org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.Site;
  86 import org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.SiteHealthInput;
  87 import org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.SiteHealthOutput;
  88 import org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.SiteHealthOutputBuilder;
  89 import org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.SiteIdentifierInput;
  90 import org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.SiteIdentifierOutput;
  91 import org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.SiteIdentifierOutputBuilder;
  92 import org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.site.health.output.SitesBuilder;
  93 import org.opendaylight.yangtools.yang.common.RpcResult;
  94 import org.opendaylight.yangtools.yang.common.RpcResultBuilder;
  95
  96 import org.slf4j.Logger;
  97 import org.slf4j.LoggerFactory;
  98
  99 public class GrToolkitProvider implements AutoCloseable, GrToolkitService, DataTreeChangeListener {
 100     private static final String APP_NAME = "gr-toolkit";
 101     private static final String PROPERTIES_FILE = System.getenv("SDNC_CONFIG_DIR") + "/gr-toolkit.properties";
 102     private static final String HEALTHY = "HEALTHY";
 103     private static final String FAULTY = "FAULTY";
 104     private static final String VALUE = "value";
 105     private static final String OUTPUT = "output";
 106     private String akkaConfig;
 107     private String jolokiaClusterPath;
 108     private String shardManagerPath;
 109     private String shardPathTemplate;
 110     private String credentials;
 111     private String httpProtocol;
 112     private String siteIdentifier = System.getenv("SITE_NAME");
 113     private final Logger log = LoggerFactory.getLogger(GrToolkitProvider.class);
 114     private final ExecutorService executor;
 115     protected DataBroker dataBroker;
 116     protected NotificationPublishService notificationService;
 117     protected RpcProviderRegistry rpcRegistry;
 118     protected BindingAwareBroker.RpcRegistration<GrToolkitService> rpcRegistration;
 119     protected DbLibService dbLib;
 120     private String member;
 121     private ClusterActor self;
 122     private HashMap<String, ClusterActor> memberMap;
 123     private SiteConfiguration siteConfiguration;
 124     private Properties properties;
 125     private DistributedDataStoreInterface configDatastore;
 126     public GrToolkitProvider(DataBroker dataBroker,
 127                              NotificationPublishService notificationProviderService,
 128                              RpcProviderRegistry rpcProviderRegistry,
 129                              DistributedDataStoreInterface configDatastore,
 130                              DbLibService dbLibService) {
 131         this.log.info("Creating provider for {}", APP_NAME);
 132         this.executor = Executors.newFixedThreadPool(1);
 133         this.dataBroker = dataBroker;
 134         this.notificationService = notificationProviderService;
 135         this.rpcRegistry = rpcProviderRegistry;
 136         this.configDatastore = configDatastore;
 137         this.dbLib = dbLibService;
 138         initialize();
 139     }
 140
 141     private void initialize() {
 142         log.info("Initializing provider for {}", APP_NAME);
 143         // Create the top level containers
 144         createContainers();
 145         setProperties();
 146         defineMembers();
 147
 148         rpcRegistration = rpcRegistry.addRpcImplementation(GrToolkitService.class, this);
 149         log.info("Initialization complete for {}", APP_NAME);
 150     }
 151
 152     private void setProperties() {
 153         log.info("Loading properties from {}", PROPERTIES_FILE);
 154         properties = new Properties();
 155         File propertiesFile = new File(PROPERTIES_FILE);
 156         if(!propertiesFile.exists()) {
 157             log.warn("Properties file not found.");
 158             return;
 159         }
 160         try(FileInputStream fileInputStream = new FileInputStream(propertiesFile)) {
 161             properties.load(fileInputStream);
 162             if(!properties.containsKey(PropertyKeys.SITE_IDENTIFIER)) {
 163                 properties.put(PropertyKeys.SITE_IDENTIFIER, "Unknown Site");
 164             }
 165             String port = "true".equals(properties.getProperty(PropertyKeys.CONTROLLER_USE_SSL).trim()) ? properties.getProperty(PropertyKeys.CONTROLLER_PORT_SSL).trim() : properties.getProperty(PropertyKeys.CONTROLLER_PORT_HTTP).trim();
 166             httpProtocol = "true".equals(properties.getProperty(PropertyKeys.CONTROLLER_USE_SSL).trim()) ? "https://" : "http://";
 167             akkaConfig = properties.getProperty(PropertyKeys.AKKA_CONF_LOCATION).trim();
 168             jolokiaClusterPath = ":" + port + properties.getProperty(PropertyKeys.MBEAN_CLUSTER).trim();
 169             shardManagerPath = ":" + port + properties.getProperty(PropertyKeys.MBEAN_SHARD_MANAGER).trim();
 170             shardPathTemplate = ":" + port + properties.getProperty(PropertyKeys.MBEAN_SHARD_CONFIG).trim();
 171             if(siteIdentifier == null || siteIdentifier.isEmpty()) {
 172                 siteIdentifier = properties.getProperty(PropertyKeys.SITE_IDENTIFIER).trim();
 173             }
 174             credentials = properties.getProperty(PropertyKeys.CONTROLLER_CREDENTIALS).trim();
 175             log.info("Loaded properties.");
 176         } catch(IOException e) {
 177             log.error("Error loading properties.", e);
 178         }
 179     }
 180
 181     private void defineMembers() {
 182         member = configDatastore.getActorContext().getCurrentMemberName().getName();
 183         log.info("Cluster member: {}", member);
 184
 185         log.info("Parsing akka.conf for cluster memberMap...");
 186         try {
 187             File akkaConfigFile = new File(this.akkaConfig);
 188             try(FileReader fileReader = new FileReader(akkaConfigFile);
 189                 BufferedReader bufferedReader = new BufferedReader(fileReader)) {
 190                 String line;
 191                 while((line = bufferedReader.readLine()) != null) {
 192                     if(line.contains("seed-nodes =")) {
 193                         parseSeedNodes(line);
 194                         break;
 195                     }
 196                 }
 197             }
 198         } catch(IOException e) {
 199             log.error("Couldn't load akka", e);
 200         } catch(NullPointerException e) {
 201             log.error("akkaConfig is null. Check properties file and restart {} bundle.", APP_NAME);
 202         }
 203         log.info("self:\n{}", self);
 204     }
 205
 206     private void createContainers() {
 207         // Replace with MD-SAL write for FailoverStatus
 208     }
 209
 210     protected void initializeChild() {
 211         // Override if you have custom initialization intelligence
 212     }
 213
 214     @Override
 215     public void close() throws Exception {
 216         log.info("Closing provider for {}", APP_NAME);
 217         executor.shutdown();
 218         rpcRegistration.close();
 219         log.info("Successfully closed provider for {}", APP_NAME);
 220     }
 221
 222     @Override
 223     public void onDataTreeChanged(@Nonnull Collection changes) {
 224         log.info("onDataTreeChanged() called. but there is no change here");
 225     }
 226
 227     @Override
 228     public ListenableFuture<RpcResult<ClusterHealthOutput>> clusterHealth(ClusterHealthInput input) {
 229         log.info("{}:cluster-health invoked.", APP_NAME);
 230         getControllerHealth();
 231         return buildClusterHealthOutput("200");
 232     }
 233
 234     @Override
 235     public ListenableFuture<RpcResult<SiteHealthOutput>> siteHealth(SiteHealthInput input) {
 236         log.info("{}:site-health invoked.", APP_NAME);
 237         getControllerHealth();
 238         return buildSiteHealthOutput("200", getAdminHealth(), getDatabaseHealth());
 239     }
 240
 241     @Override
 242     public ListenableFuture<RpcResult<DatabaseHealthOutput>> databaseHealth(DatabaseHealthInput input) {
 243         log.info("{}:database-health invoked.", APP_NAME);
 244         DatabaseHealthOutputBuilder outputBuilder = new DatabaseHealthOutputBuilder();
 245         outputBuilder.setStatus("200");
 246         outputBuilder.setHealth(getDatabaseHealth());
 247         outputBuilder.setServedBy(member);
 248
 249         return Futures.immediateFuture(RpcResultBuilder.<DatabaseHealthOutput>status(true).withResult(outputBuilder.build()).build());
 250     }
 251
 252     @Override
 253     public ListenableFuture<RpcResult<AdminHealthOutput>> adminHealth(AdminHealthInput input) {
 254         log.info("{}:admin-health invoked.", APP_NAME);
 255         AdminHealthOutputBuilder outputBuilder = new AdminHealthOutputBuilder();
 256         outputBuilder.setStatus("200");
 257         outputBuilder.setHealth(getAdminHealth());
 258         outputBuilder.setServedBy(member);
 259
 260         return Futures.immediateFuture(RpcResultBuilder.<AdminHealthOutput>status(true).withResult(outputBuilder.build()).build());
 261     }
 262
 263     @Override
 264     public ListenableFuture<RpcResult<HaltAkkaTrafficOutput>> haltAkkaTraffic(HaltAkkaTrafficInput input) {
 265         log.info("{}:halt-akka-traffic invoked.", APP_NAME);
 266         HaltAkkaTrafficOutputBuilder outputBuilder = new HaltAkkaTrafficOutputBuilder();
 267         outputBuilder.setStatus("200");
 268         modifyIpTables(IpTables.ADD, input.getNodeInfo().toArray());
 269         outputBuilder.setServedBy(member);
 270
 271         return Futures.immediateFuture(RpcResultBuilder.<HaltAkkaTrafficOutput>status(true).withResult(outputBuilder.build()).build());
 272     }
 273
 274     @Override
 275     public ListenableFuture<RpcResult<ResumeAkkaTrafficOutput>> resumeAkkaTraffic(ResumeAkkaTrafficInput input) {
 276         log.info("{}:resume-akka-traffic invoked.", APP_NAME);
 277         ResumeAkkaTrafficOutputBuilder outputBuilder = new ResumeAkkaTrafficOutputBuilder();
 278         outputBuilder.setStatus("200");
 279         modifyIpTables(IpTables.DELETE, input.getNodeInfo().toArray());
 280         outputBuilder.setServedBy(member);
 281
 282         return Futures.immediateFuture(RpcResultBuilder.<ResumeAkkaTrafficOutput>status(true).withResult(outputBuilder.build()).build());
 283     }
 284
 285     @Override
 286     public ListenableFuture<RpcResult<SiteIdentifierOutput>> siteIdentifier(SiteIdentifierInput input) {
 287         log.info("{}:site-identifier invoked.", APP_NAME);
 288         SiteIdentifierOutputBuilder outputBuilder = new SiteIdentifierOutputBuilder();
 289         outputBuilder.setStatus("200");
 290         outputBuilder.setId(siteIdentifier);
 291         outputBuilder.setServedBy(member);
 292
 293         return Futures.immediateFuture(RpcResultBuilder.<SiteIdentifierOutput>status(true).withResult(outputBuilder.build()).build());
 294     }
 295
 296     @Override
 297     public ListenableFuture<RpcResult<FailoverOutput>> failover(FailoverInput input) {
 298         log.info("{}:failover invoked.", APP_NAME);
 299         FailoverOutputBuilder outputBuilder = new FailoverOutputBuilder();
 300         outputBuilder.setServedBy(member);
 301         if(siteConfiguration != SiteConfiguration.GEO) {
 302             log.info("Cannot failover non-GEO site.");
 303             outputBuilder.setMessage("Failover aborted. This is not a GEO configuration.");
 304             outputBuilder.setStatus("400");
 305             return Futures.immediateFuture(RpcResultBuilder.<FailoverOutput>status(true).withResult(outputBuilder.build()).build());
 306         }
 307         ArrayList<ClusterActor> activeSite = new ArrayList<>();
 308         ArrayList<ClusterActor> standbySite = new ArrayList<>();
 309
 310         log.info("Performing preliminary cluster health check...");
 311         // Necessary to populate all member info. Health is not used for judgement calls.
 312         getControllerHealth();
 313
 314         log.info("Determining active site...");
 315         for(Map.Entry<String, ClusterActor> entry : memberMap.entrySet()) {
 316             String key = entry.getKey();
 317             ClusterActor clusterActor = entry.getValue();
 318             if(clusterActor.isVoting()) {
 319                 activeSite.add(clusterActor);
 320                 log.debug("Active Site member: {}", key);
 321             }
 322             else {
 323                 standbySite.add(clusterActor);
 324                 log.debug("Standby Site member: {}", key);
 325             }
 326         }
 327
 328         String port = "true".equals(properties.getProperty(PropertyKeys.CONTROLLER_USE_SSL)) ? properties.getProperty(PropertyKeys.CONTROLLER_PORT_SSL) : properties.getProperty(PropertyKeys.CONTROLLER_PORT_HTTP);
 329
 330         if(Boolean.parseBoolean(input.getBackupData())) {
 331             backupMdSal(activeSite, port);
 332         }
 333
 334         if(!changeClusterVoting(outputBuilder, activeSite, standbySite, port))
 335             return Futures.immediateFuture(RpcResultBuilder.<FailoverOutput>status(true).withResult(outputBuilder.build()).build());
 336
 337         if(Boolean.parseBoolean(input.getIsolate())) {
 338             isolateSiteFromCluster(activeSite, standbySite, port);
 339
 340             if(Boolean.parseBoolean(input.getDownUnreachable())) {
 341                 downUnreachableNodes(activeSite, standbySite, port);
 342             }
 343         }
 344
 345         log.info("{}:failover complete.", APP_NAME);
 346
 347         outputBuilder.setMessage("Failover complete.");
 348         outputBuilder.setStatus("200");
 349         return Futures.immediateFuture(RpcResultBuilder.<FailoverOutput>status(true).withResult(outputBuilder.build()).build());
 350     }
 351
 352     private void isolateSiteFromCluster(ArrayList<ClusterActor> activeSite, ArrayList<ClusterActor> standbySite, String port) {
 353         log.info("Halting Akka traffic...");
 354         for(ClusterActor actor : standbySite) {
 355             try {
 356                 log.info("Halting Akka traffic for: {}", actor.getNode());
 357                 // Build JSON with activeSite actor Node and actor  AkkaPort
 358                 JSONObject akkaInput = new JSONObject();
 359                 JSONObject inputBlock = new JSONObject();
 360                 JSONArray votingStateArray = new JSONArray();
 361                 JSONObject nodeInfo;
 362                 for(ClusterActor node : activeSite) {
 363                     nodeInfo = new JSONObject();
 364                     nodeInfo.put("node", node.getNode());
 365                     nodeInfo.put("port", node.getAkkaPort());
 366                     votingStateArray.put(nodeInfo);
 367                 }
 368                 inputBlock.put("node-info", votingStateArray);
 369                 akkaInput.put("input", inputBlock);
 370                 getRequestContent(httpProtocol + actor.getNode() + ":" + port + "/restconf/operations/gr-toolkit:halt-akka-traffic", HttpMethod.POST, akkaInput.toString());
 371             } catch(IOException e) {
 372                 log.error("Could not halt Akka traffic for: " + actor.getNode(), e);
 373             }
 374         }
 375     }
 376
 377     private void downUnreachableNodes(ArrayList<ClusterActor> activeSite, ArrayList<ClusterActor> standbySite, String port) {
 378         log.info("Setting site unreachable...");
 379         JSONObject jolokiaInput = new JSONObject();
 380         jolokiaInput.put("type", "EXEC");
 381         jolokiaInput.put("mbean", "akka:type=Cluster");
 382         jolokiaInput.put("operation", "down");
 383         JSONArray arguments = new JSONArray();
 384         for(ClusterActor actor : activeSite) {
 385             // Build Jolokia input
 386             // May need to change from akka port to actor.getAkkaPort()
 387             arguments.put("akka.tcp://opendaylight-cluster-data@" + actor.getNode() + ":" + properties.getProperty(PropertyKeys.CONTROLLER_PORT_AKKA));
 388         }
 389         jolokiaInput.put("arguments", arguments);
 390         log.debug("{}", jolokiaInput);
 391         try {
 392             log.info("Setting nodes unreachable");
 393             getRequestContent(httpProtocol + standbySite.get(0).getNode() + ":" + port + "/jolokia", HttpMethod.POST, jolokiaInput.toString());
 394         } catch(IOException e) {
 395             log.error("Error setting nodes unreachable", e);
 396         }
 397     }
 398
 399     private boolean changeClusterVoting(FailoverOutputBuilder outputBuilder, ArrayList<ClusterActor> activeSite, ArrayList<ClusterActor> standbySite, String port) {
 400         log.info("Changing voting for all shards to standby site...");
 401         try {
 402             JSONObject votingInput = new JSONObject();
 403             JSONObject inputBlock = new JSONObject();
 404             JSONArray votingStateArray = new JSONArray();
 405             JSONObject memberVotingState;
 406             for(ClusterActor actor : activeSite) {
 407                 memberVotingState = new JSONObject();
 408                 memberVotingState.put("member-name", actor.getMember());
 409                 memberVotingState.put("voting", false);
 410                 votingStateArray.put(memberVotingState);
 411             }
 412             for(ClusterActor actor : standbySite) {
 413                 memberVotingState = new JSONObject();
 414                 memberVotingState.put("member-name", actor.getMember());
 415                 memberVotingState.put("voting", true);
 416                 votingStateArray.put(memberVotingState);
 417             }
 418             inputBlock.put("member-voting-state", votingStateArray);
 419             votingInput.put("input", inputBlock);
 420             log.debug("{}", votingInput);
 421             // Change voting all shards
 422             getRequestContent(httpProtocol + self.getNode() + ":" + port + "/restconf/operations/cluster-admin:change-member-voting-states-for-all-shards", HttpMethod.POST, votingInput.toString());
 423         } catch(IOException e) {
 424             log.error("Changing voting", e);
 425             outputBuilder.setMessage("Failover aborted. Failed to change voting.");
 426             outputBuilder.setStatus("500");
 427             return false;
 428         }
 429         return true;
 430     }
 431
 432     private void backupMdSal(ArrayList<ClusterActor> activeSite, String port) {
 433         log.info("Backing up data...");
 434         try {
 435             log.info("Scheduling backup for: {}", activeSite.get(0).getNode());
 436             getRequestContent(httpProtocol + activeSite.get(0).getNode() + ":" + port + "/restconf/operations/data-export-import:schedule-export", HttpMethod.POST, "{ \"input\": { \"run-at\": \"30\" } }");
 437         } catch(IOException e) {
 438             log.error("Error backing up MD-SAL", e);
 439         }
 440         for(ClusterActor actor : activeSite) {
 441             try {
 442                 // Move data offsite
 443                 log.info("Backing up data for: {}", actor.getNode());
 444                 getRequestContent(httpProtocol + actor.getNode() + ":" + port + "/restconf/operations/daexim-offsite-backup:backup-data", HttpMethod.POST);
 445             } catch(IOException e) {
 446                 log.error("Error backing up data.", e);
 447             }
 448         }
 449     }
 450
 451     private ListenableFuture<RpcResult<ClusterHealthOutput>> buildClusterHealthOutput(String statusCode) {
 452         ClusterHealthOutputBuilder outputBuilder = new ClusterHealthOutputBuilder();
 453         outputBuilder.setStatus(statusCode);
 454         outputBuilder.setMembers((List) new ArrayList<Member>());
 455         int site1Health = 0;
 456         int site2Health = 0;
 457
 458         for(Map.Entry<String, ClusterActor> entry : memberMap.entrySet()) {
 459             ClusterActor clusterActor = entry.getValue();
 460             if(clusterActor.isUp() && !clusterActor.isUnreachable()) {
 461                 if(ClusterActor.SITE_1.equals(clusterActor.getSite()))
 462                     site1Health++;
 463                 else if(ClusterActor.SITE_2.equals(clusterActor.getSite()))
 464                     site2Health++;
 465             }
 466             outputBuilder.getMembers().add(new MemberBuilder(clusterActor).build());
 467         }
 468         if(siteConfiguration == SiteConfiguration.SOLO) {
 469             outputBuilder.setSite1Health(HEALTHY);
 470         }
 471         else if(site1Health > 1) {
 472             outputBuilder.setSite1Health(HEALTHY);
 473         }
 474         else {
 475             outputBuilder.setSite1Health(FAULTY);
 476         }
 477
 478         if(siteConfiguration == SiteConfiguration.GEO && site2Health > 1) {
 479             outputBuilder.setSite2Health(HEALTHY);
 480         }
 481         else if(siteConfiguration == SiteConfiguration.GEO) {
 482             outputBuilder.setSite2Health(FAULTY);
 483         }
 484
 485         outputBuilder.setServedBy(member);
 486         RpcResult<ClusterHealthOutput> rpcResult = RpcResultBuilder.<ClusterHealthOutput>status(true).withResult(outputBuilder.build()).build();
 487         return Futures.immediateFuture(rpcResult);
 488     }
 489
 490     private ListenableFuture<RpcResult<SiteHealthOutput>> buildSiteHealthOutput(String statusCode, String adminHealth, String databaseHealth) {
 491         SiteHealthOutputBuilder outputBuilder = new SiteHealthOutputBuilder();
 492         outputBuilder.setStatus(statusCode);
 493         outputBuilder.setSites((List) new ArrayList<Site>());
 494
 495         if(siteConfiguration != SiteConfiguration.GEO) {
 496             int healthyODLs = 0;
 497             SitesBuilder builder = new SitesBuilder();
 498             for(Map.Entry<String, ClusterActor> entry : memberMap.entrySet()) {
 499                 ClusterActor clusterActor = entry.getValue();
 500                 if(clusterActor.isUp() && !clusterActor.isUnreachable()) {
 501                     healthyODLs++;
 502                 }
 503             }
 504             if(siteConfiguration != SiteConfiguration.SOLO) {
 505                 builder.setHealth(HEALTHY);
 506                 builder.setRole("ACTIVE");
 507                 builder.setId(siteIdentifier);
 508             }
 509             else {
 510                 builder = getSitesBuilder(healthyODLs, true, HEALTHY.equals(adminHealth), HEALTHY.equals(databaseHealth), siteIdentifier);
 511             }
 512             outputBuilder.getSites().add(builder.build());
 513         }
 514         else {
 515             int site1HealthyODLs = 0;
 516             int site2HealthyODLs = 0;
 517             boolean site1Voting = false;
 518             boolean site2Voting = false;
 519             boolean performedCrossSiteHealthCheck = false;
 520             boolean crossSiteAdminHealthy = false;
 521             boolean crossSiteDbHealthy = false;
 522             String crossSiteIdentifier = "UNKNOWN_SITE";
 523             String port = "true".equals(properties.getProperty(PropertyKeys.CONTROLLER_USE_SSL)) ? properties.getProperty(PropertyKeys.CONTROLLER_PORT_SSL) : properties.getProperty(PropertyKeys.CONTROLLER_PORT_HTTP);
 524             if(isSite1()) {
 525                 // Make calls over to site 2 healthchecks
 526                 for(Map.Entry<String, ClusterActor> entry : memberMap.entrySet()) {
 527                     ClusterActor clusterActor = entry.getValue();
 528                     if(clusterActor.isUp() && !clusterActor.isUnreachable()) {
 529                         if(ClusterActor.SITE_1.equals(clusterActor.getSite())) {
 530                             site1HealthyODLs++;
 531                             if(clusterActor.isVoting()) {
 532                                 site1Voting = true;
 533                             }
 534                         }
 535                         else {
 536                             site2HealthyODLs++;
 537                             if(clusterActor.isVoting()) {
 538                                 site2Voting = true;
 539                             }
 540                             if(!performedCrossSiteHealthCheck) {
 541                                 try {
 542                                     String content = getRequestContent(httpProtocol + clusterActor.getNode() + ":" + port + "/restconf/operations/gr-toolkit:site-identifier", HttpMethod.POST);
 543                                     crossSiteIdentifier = new JSONObject(content).getJSONObject(OUTPUT).getString("id");
 544                                     crossSiteDbHealthy = crossSiteHealthRequest(httpProtocol + clusterActor.getNode() + ":" + port + "/restconf/operations/gr-toolkit:database-health");
 545                                     crossSiteAdminHealthy = crossSiteHealthRequest(httpProtocol + clusterActor.getNode() + ":" + port + "/restconf/operations/gr-toolkit:admin-health");
 546                                     performedCrossSiteHealthCheck = true;
 547                                 } catch(Exception e) {
 548                                     log.info("Cannot get cross site health from {}", clusterActor.getNode());
 549                                     log.info("siteIdentifier: {} | dbHealth: {} | adminHealth: {}", crossSiteIdentifier, crossSiteDbHealthy, crossSiteAdminHealthy);
 550                                     log.error("Site Health Error", e);
 551                                 }
 552                             }
 553                         }
 554                     }
 555                 }
 556                 SitesBuilder builder = getSitesBuilder(site1HealthyODLs, site1Voting, HEALTHY.equals(adminHealth), HEALTHY.equals(databaseHealth), siteIdentifier);
 557                 outputBuilder.getSites().add(builder.build());
 558                 builder = getSitesBuilder(site2HealthyODLs, site2Voting, crossSiteAdminHealthy, crossSiteDbHealthy, crossSiteIdentifier);
 559                 outputBuilder.getSites().add(builder.build());
 560             }
 561             else {
 562                 // Make calls over to site 1 healthchecks
 563                 for(Map.Entry<String, ClusterActor> entry : memberMap.entrySet()) {
 564                     ClusterActor clusterActor = entry.getValue();
 565                     if(clusterActor.isUp() && !clusterActor.isUnreachable()) {
 566                         if(ClusterActor.SITE_1.equals(clusterActor.getSite())) {
 567                             site1HealthyODLs++;
 568                             if(clusterActor.isVoting()) {
 569                                 site1Voting = true;
 570                             }
 571                             if(!performedCrossSiteHealthCheck) {
 572                                 try {
 573                                     String content = getRequestContent(httpProtocol + clusterActor.getNode() + ":" + port + "/restconf/operations/gr-toolkit:site-identifier", HttpMethod.POST);
 574                                     crossSiteIdentifier = new JSONObject(content).getJSONObject(OUTPUT).getString("id");
 575                                     crossSiteDbHealthy = crossSiteHealthRequest(httpProtocol + clusterActor.getNode() + ":" + port + "/restconf/operations/gr-toolkit:database-health");
 576                                     crossSiteAdminHealthy = crossSiteHealthRequest(httpProtocol + clusterActor.getNode() + ":" + port + "/restconf/operations/gr-toolkit:admin-health");
 577                                     performedCrossSiteHealthCheck = true;
 578                                 } catch(Exception e) {
 579                                     log.info("Cannot get cross site health from {}", clusterActor.getNode());
 580                                     log.info("siteIdentifier: {} | dbHealth: {} | adminHealth: {}", crossSiteIdentifier, crossSiteDbHealthy, crossSiteAdminHealthy);
 581                                     log.error("Site Health Error", e);
 582                                 }
 583                             }
 584                         }
 585                         else {
 586                             site2HealthyODLs++;
 587                             if(clusterActor.isVoting()) {
 588                                 site2Voting = true;
 589                             }
 590                         }
 591                     }
 592                 }
 593                 // Build Output
 594                 SitesBuilder builder = getSitesBuilder(site1HealthyODLs, site1Voting, crossSiteAdminHealthy, crossSiteDbHealthy, crossSiteIdentifier);
 595                 outputBuilder.getSites().add(builder.build());
 596                 builder = getSitesBuilder(site2HealthyODLs, site2Voting, HEALTHY.equals(adminHealth), HEALTHY.equals(databaseHealth), siteIdentifier);
 597                 outputBuilder.getSites().add(builder.build());
 598             }
 599         }
 600
 601         outputBuilder.setServedBy(member);
 602         RpcResult<SiteHealthOutput> rpcResult = RpcResultBuilder.<SiteHealthOutput>status(true).withResult(outputBuilder.build()).build();
 603         return Futures.immediateFuture(rpcResult);
 604     }
 605
 606     private SitesBuilder getSitesBuilder(int siteHealthyODLs, boolean siteVoting, boolean adminHealthy, boolean dbHealthy, String siteIdentifier) {
 607         SitesBuilder builder = new SitesBuilder();
 608         if(siteHealthyODLs > 1) {
 609             builder.setHealth(HEALTHY);
 610         }
 611         else {
 612             log.warn("{} Healthy ODLs: {}", siteIdentifier, siteHealthyODLs);
 613             builder.setHealth(FAULTY);
 614         }
 615         if(!adminHealthy) {
 616             log.warn("{} Admin Health: {}", siteIdentifier, FAULTY);
 617             builder.setHealth(FAULTY);
 618         }
 619         if(!dbHealthy) {
 620             log.warn("{} Database Health: {}", siteIdentifier, FAULTY);
 621             builder.setHealth(FAULTY);
 622         }
 623         if(siteVoting) {
 624             builder.setRole("ACTIVE");
 625         }
 626         else {
 627             builder.setRole("STANDBY");
 628         }
 629         builder.setId(siteIdentifier);
 630         return builder;
 631     }
 632
 633     private boolean isSite1() {
 634         int memberNumber = Integer.parseInt(member.split("-")[1]);
 635         boolean isSite1 = memberNumber < 4;
 636         log.info("isSite1(): {}", isSite1);
 637         return isSite1;
 638     }
 639
 640     private void parseSeedNodes(String line) {
 641         memberMap = new HashMap<>();
 642         line = line.substring(line.indexOf("[\""), line.indexOf(']'));
 643         String[] splits = line.split(",");
 644
 645         for(int ndx = 0; ndx < splits.length; ndx++) {
 646             String nodeName = splits[ndx];
 647             int delimLocation = nodeName.indexOf('@');
 648             String port = nodeName.substring(splits[ndx].indexOf(':', delimLocation) + 1, splits[ndx].indexOf('"', splits[ndx].indexOf(':')));
 649             splits[ndx] = nodeName.substring(delimLocation + 1, splits[ndx].indexOf(':', delimLocation));
 650             log.info("Adding node: {}:{}", splits[ndx], port);
 651             ClusterActor clusterActor = new ClusterActor();
 652             clusterActor.setNode(splits[ndx]);
 653             clusterActor.setAkkaPort(port);
 654             clusterActor.setMember("member-" + (ndx + 1));
 655             if(ndx < 3) {
 656                 clusterActor.setSite(ClusterActor.SITE_1);
 657             }
 658             else {
 659                 clusterActor.setSite(ClusterActor.SITE_2);
 660             }
 661
 662             if(member.equals(clusterActor.getMember())) {
 663                 self = clusterActor;
 664             }
 665             memberMap.put(clusterActor.getNode(), clusterActor);
 666             log.info("{}", clusterActor);
 667         }
 668
 669         if(memberMap.size() == 1) {
 670             log.info("1 member found. This is a solo environment.");
 671             siteConfiguration = SiteConfiguration.SOLO;
 672         }
 673         else if(memberMap.size() == 3) {
 674             log.info("This is a single site.");
 675             siteConfiguration = SiteConfiguration.SINGLE;
 676         }
 677         else if(memberMap.size() == 6) {
 678             log.info("This is a georedundant site.");
 679             siteConfiguration = SiteConfiguration.GEO;
 680         }
 681     }
 682
 683     private void getMemberStatus(ClusterActor clusterActor) throws IOException {
 684         log.info("Getting member status for {}", clusterActor.getNode());
 685         String content = getRequestContent(httpProtocol + clusterActor.getNode() + jolokiaClusterPath, HttpMethod.GET);
 686         try {
 687             JSONObject responseJson = new JSONObject(content);
 688             JSONObject responseValue = responseJson.getJSONObject(VALUE);
 689             clusterActor.setUp("Up".equals(responseValue.getString("MemberStatus")));
 690             clusterActor.setUnreachable(false);
 691         } catch(JSONException e) {
 692             log.error("Error parsing response from {}", clusterActor.getNode(), e);
 693             clusterActor.setUp(false);
 694             clusterActor.setUnreachable(true);
 695         }
 696     }
 697
 698     private void getShardStatus(ClusterActor clusterActor) throws IOException {
 699         log.info("Getting shard status for {}", clusterActor.getNode());
 700         String content = getRequestContent(httpProtocol + clusterActor.getNode() + shardManagerPath, HttpMethod.GET);
 701         try {
 702             JSONObject responseValue = new JSONObject(content).getJSONObject(VALUE);
 703             JSONArray shardList = responseValue.getJSONArray("LocalShards");
 704
 705             String pattern = "-config$";
 706             Pattern r = Pattern.compile(pattern);
 707             Matcher m;
 708             for(int ndx = 0; ndx < shardList.length(); ndx++) {
 709                 String configShardName = shardList.getString(ndx);
 710                 m = r.matcher(configShardName);
 711                 String operationalShardName = m.replaceFirst("-operational");
 712                 String shardConfigPath = String.format(shardPathTemplate, configShardName);
 713                 String shardOperationalPath = String.format(shardPathTemplate, operationalShardName).replace("Config", "Operational");
 714                 extractShardInfo(clusterActor, configShardName, shardConfigPath);
 715                 extractShardInfo(clusterActor, operationalShardName, shardOperationalPath);
 716             }
 717         } catch(JSONException e) {
 718             log.error("Error parsing response from " + clusterActor.getNode(), e);
 719         }
 720     }
 721
 722     private void extractShardInfo(ClusterActor clusterActor, String shardName, String shardPath) throws IOException {
 723         log.info("Extracting shard info for {}", shardName);
 724         log.debug("Pulling config info for {} from: {}", shardName, shardPath);
 725         String content = getRequestContent(httpProtocol + clusterActor.getNode() + shardPath, HttpMethod.GET);
 726         log.debug("Response: {}", content);
 727
 728         try {
 729             JSONObject shardValue = new JSONObject(content).getJSONObject(VALUE);
 730             clusterActor.setVoting(shardValue.getBoolean("Voting"));
 731             if(shardValue.getString("PeerAddresses").length() > 0) {
 732                 clusterActor.getReplicaShards().add(shardName);
 733                 if(shardValue.getString("Leader").startsWith(clusterActor.getMember())) {
 734                     clusterActor.getShardLeader().add(shardName);
 735                 }
 736             }
 737             else {
 738                 clusterActor.getNonReplicaShards().add(shardName);
 739             }
 740             JSONArray followerInfo = shardValue.getJSONArray("FollowerInfo");
 741             for(int followerNdx = 0; followerNdx < followerInfo.length(); followerNdx++) {
 742                 int commitIndex = shardValue.getInt("CommitIndex");
 743                 int matchIndex = followerInfo.getJSONObject(followerNdx).getInt("matchIndex");
 744                 if(commitIndex != -1 && matchIndex != -1) {
 745                     int commitsBehind = commitIndex - matchIndex;
 746                     clusterActor.getCommits().put(followerInfo.getJSONObject(followerNdx).getString("id"), commitsBehind);
 747                 }
 748             }
 749         } catch(JSONException e) {
 750             log.error("Error parsing response from " + clusterActor.getNode(), e);
 751         }
 752     }
 753
 754     private void getControllerHealth() {
 755         for(Map.Entry<String, ClusterActor> entry : memberMap.entrySet()) {
 756             ClusterActor clusterActor = entry.getValue();
 757             String key = entry.getKey();
 758             try {
 759                 // First flush out the old values
 760                 clusterActor.flush();
 761                 log.info("Gathering info for {}", clusterActor.getNode());
 762                 getMemberStatus(clusterActor);
 763                 getShardStatus(clusterActor);
 764                 log.info("MemberInfo:\n{}", clusterActor);
 765             } catch(IOException e) {
 766                 log.error("Connection Error", e);
 767                 memberMap.get(key).setUnreachable(true);
 768                 memberMap.get(key).setUp(false);
 769                 log.info("MemberInfo:\n{}", memberMap.get(key));
 770             }
 771         }
 772     }
 773
 774     private void modifyIpTables(IpTables task, Object[] nodeInfo) {
 775         log.info("Modifying IPTables rules...");
 776         if(task == IpTables.ADD) {
 777             for(Object node : nodeInfo) {
 778                 org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.halt.akka.traffic.input.NodeInfo n =
 779                         (org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.halt.akka.traffic.input.NodeInfo) node;
 780                 log.info("Isolating {}", n.getNode());
 781                 executeCommand(String.format("sudo /sbin/iptables -A INPUT -p tcp --destination-port %s -j DROP -s %s", properties.get(PropertyKeys.CONTROLLER_PORT_AKKA), n.getNode()));
 782                 executeCommand(String.format("sudo /sbin/iptables -A OUTPUT -p tcp --destination-port %s -j DROP -d %s", n.getPort(), n.getNode()));
 783             }
 784
 785         } else if(task == IpTables.DELETE) {
 786             for(Object node : nodeInfo) {
 787                 org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.resume.akka.traffic.input.NodeInfo n =
 788                         (org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.resume.akka.traffic.input.NodeInfo) node;
 789                 log.info("De-isolating {}", n.getNode());
 790                 executeCommand(String.format("sudo /sbin/iptables -D INPUT -p tcp --destination-port %s -j DROP -s %s", properties.get(PropertyKeys.CONTROLLER_PORT_AKKA), n.getNode()));
 791                 executeCommand(String.format("sudo /sbin/iptables -D OUTPUT -p tcp --destination-port %s -j DROP -d %s", n.getPort(), n.getNode()));
 792             }
 793
 794         }
 795         executeCommand("sudo /sbin/iptables -L");
 796     }
 797
 798     private void executeCommand(String command) {
 799         log.info("Executing command: {}", command);
 800         String[] cmd = command.split(" ");
 801         try {
 802             Process p = Runtime.getRuntime().exec(cmd);
 803             BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(p.getInputStream()));
 804             String inputLine;
 805             StringBuilder content = new StringBuilder();
 806             while((inputLine = bufferedReader.readLine()) != null) {
 807                 content.append(inputLine);
 808             }
 809             bufferedReader.close();
 810             log.info("{}", content);
 811         } catch(IOException e) {
 812             log.error("Error executing command", e);
 813         }
 814     }
 815
 816     private boolean crossSiteHealthRequest(String path) throws IOException {
 817         String content = getRequestContent(path, HttpMethod.POST);
 818         try {
 819             JSONObject responseJson = new JSONObject(content);
 820             JSONObject responseValue = responseJson.getJSONObject(OUTPUT);
 821             return HEALTHY.equals(responseValue.getString("health"));
 822         } catch(JSONException e) {
 823             log.error("Error parsing JSON", e);
 824             throw new IOException();
 825         }
 826     }
 827
 828     private String getAdminHealth() {
 829         String protocol = "true".equals(properties.getProperty(PropertyKeys.ADM_USE_SSL)) ? "https://" : "http://";
 830         String port = "true".equals(properties.getProperty(PropertyKeys.ADM_USE_SSL)) ? properties.getProperty(PropertyKeys.ADM_PORT_SSL) : properties.getProperty(PropertyKeys.ADM_PORT_HTTP);
 831         String path = protocol + properties.getProperty(PropertyKeys.ADM_FQDN) + ":" + port + properties.getProperty(PropertyKeys.ADM_HEALTHCHECK);
 832         log.info("Requesting healthcheck from {}", path);
 833         try {
 834             int response = getRequestStatus(path, HttpMethod.GET);
 835             log.info("Response: {}", response);
 836             if(response == 200)
 837                 return HEALTHY;
 838             return FAULTY;
 839         } catch(IOException e) {
 840             log.error("Problem getting ADM health.", e);
 841             return FAULTY;
 842         }
 843     }
 844
 845     private String getDatabaseHealth() {
 846         log.info("Determining database health...");
 847         try {
 848             Connection connection = dbLib.getConnection();
 849             log.debug("DBLib isActive(): {}", dbLib.isActive());
 850             log.debug("DBLib isReadOnly(): {}", connection.isReadOnly());
 851             log.debug("DBLib isClosed(): {}", connection.isClosed());
 852             if(!dbLib.isActive() || connection.isClosed() || connection.isReadOnly()) {
 853                 log.warn("Database is FAULTY");
 854                 connection.close();
 855                 return FAULTY;
 856             }
 857             connection.close();
 858             log.info("Database is HEALTHY");
 859         } catch(SQLException e) {
 860             log.error("Database is FAULTY");
 861             log.error("Error", e);
 862             return FAULTY;
 863         }
 864
 865         return HEALTHY;
 866     }
 867
 868     private String getRequestContent(String path, HttpMethod method) throws IOException {
 869         return getRequestContent(path, method, null);
 870     }
 871
 872     private String getRequestContent(String path, HttpMethod method, String input) throws IOException {
 873         HttpURLConnection connection = getConnection(path);
 874         connection.setRequestMethod(method.getMethod());
 875         connection.setDoInput(true);
 876
 877         if(input != null) {
 878             sendPayload(input, connection);
 879         }
 880
 881         BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(connection.getInputStream()));
 882         String inputLine;
 883         StringBuilder content = new StringBuilder();
 884         while((inputLine = bufferedReader.readLine()) != null) {
 885             content.append(inputLine);
 886         }
 887         bufferedReader.close();
 888         connection.disconnect();
 889
 890         String response = content.toString();
 891         log.debug("getRequestContent(): Response:\n{}", response);
 892         return response;
 893     }
 894
 895     private int getRequestStatus(String path, HttpMethod method) throws IOException {
 896         return getRequestStatus(path, method, null);
 897     }
 898
 899     private int getRequestStatus(String path, HttpMethod method, String input) throws IOException {
 900         HttpURLConnection connection = getConnection(path);
 901         connection.setRequestMethod(method.getMethod());
 902         connection.setDoInput(true);
 903
 904         if(input != null) {
 905             sendPayload(input, connection);
 906         }
 907         int response = connection.getResponseCode();
 908         log.info("Received {} response code from {}", response, path);
 909         connection.disconnect();
 910         return response;
 911     }
 912
 913     private void sendPayload(String input, HttpURLConnection connection) throws IOException {
 914         byte[] out = input.getBytes(StandardCharsets.UTF_8);
 915         int length = out.length;
 916
 917         connection.setFixedLengthStreamingMode(length);
 918         connection.setRequestProperty("Content-Type", "application/json");
 919         connection.setDoOutput(true);
 920         connection.connect();
 921         try(OutputStream os = connection.getOutputStream()) {
 922             os.write(out);
 923         }
 924     }
 925
 926     private HttpURLConnection getConnection(String host) throws IOException {
 927         log.info("Getting connection to: {}", host);
 928         URL url = new URL(host);
 929         String auth = "Basic " + javax.xml.bind.DatatypeConverter.printBase64Binary(credentials.getBytes());
 930         HttpURLConnection connection = (HttpURLConnection) url.openConnection();
 931         connection.addRequestProperty("Authorization", auth);
 932         connection.setRequestProperty("Connection", "keep-alive");
 933         connection.setRequestProperty("Proxy-Connection", "keep-alive");
 934         return connection;
 935     }
 936
 937     enum IpTables {
 938         ADD,
 939         DELETE
 940     }
 941
 942     enum SiteConfiguration {
 943         SOLO,
 944         SINGLE,
 945         GEO
 946     }
 947
 948     enum HttpMethod {
 949         GET("GET"),
 950         POST("POST");
 951
 952         private String method;
 953         HttpMethod(String method) {
 954             this.method = method;
 955         }
 956         public String getMethod() {
 957             return method;
 958         }
 959     }
 960
 961     class PropertyKeys {
 962         static final String SITE_IDENTIFIER = "site.identifier";
 963         static final String CONTROLLER_USE_SSL = "controller.useSsl";
 964         static final String CONTROLLER_PORT_SSL = "controller.port.ssl";
 965         static final String CONTROLLER_PORT_HTTP = "controller.port.http";
 966         static final String CONTROLLER_PORT_AKKA = "controller.port.akka";
 967         static final String CONTROLLER_CREDENTIALS = "controller.credentials";
 968         static final String AKKA_CONF_LOCATION = "akka.conf.location";
 969         static final String MBEAN_CLUSTER = "mbean.cluster";
 970         static final String MBEAN_SHARD_MANAGER  = "mbean.shardManager";
 971         static final String MBEAN_SHARD_CONFIG = "mbean.shard.config";
 972         static final String ADM_USE_SSL = "adm.useSsl";
 973         static final String ADM_PORT_SSL = "adm.port.ssl";
 974         static final String ADM_PORT_HTTP = "adm.port.http";
 975         static final String ADM_FQDN = "adm.fqdn";
 976         static final String ADM_HEALTHCHECK= "adm.healthcheck";
 977     }
 978 }