grToolkit/provider/src/main/java/org/onap/ccsdk/sli/plugins/grtoolkit/GrToolkitProvider.java

   1 /*-
   2  * ============LICENSE_START=======================================================
   3  * openECOMP : SDN-C
   4  * ================================================================================
   5  * Copyright (C) 2018 AT&T Intellectual Property. All rights
   6  *                      reserved.
   7  * ================================================================================
   8  * Licensed under the Apache License, Version 2.0 (the "License");
   9  * you may not use this file except in compliance with the License.
  10  * You may obtain a copy of the License at
  11  *
  12  *      http://www.apache.org/licenses/LICENSE-2.0
  13  *
  14  * Unless required by applicable law or agreed to in writing, software
  15  * distributed under the License is distributed on an "AS IS" BASIS,
  16  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  17  * See the License for the specific language governing permissions and
  18  * limitations under the License.
  19  * ============LICENSE_END=========================================================
  20  */
  21
  22 package org.onap.ccsdk.sli.plugins.grtoolkit;
  23
  24 import java.io.BufferedReader;
  25 import java.io.File;
  26 import java.io.FileInputStream;
  27 import java.io.FileReader;
  28 import java.io.IOException;
  29 import java.io.InputStreamReader;
  30 import java.io.OutputStream;
  31 import java.net.HttpURLConnection;
  32 import java.net.URL;
  33 import java.nio.charset.StandardCharsets;
  34 import java.sql.Connection;
  35 import java.sql.SQLException;
  36 import java.util.ArrayList;
  37 import java.util.Collection;
  38 import java.util.HashMap;
  39 import java.util.Map;
  40 import java.util.Properties;
  41 import java.util.List;
  42 import java.util.concurrent.ExecutorService;
  43 import java.util.concurrent.Executors;
  44 import java.util.regex.Matcher;
  45 import java.util.regex.Pattern;
  46 import javax.annotation.Nonnull;
  47
  48 import com.google.common.util.concurrent.Futures;
  49 import com.google.common.util.concurrent.ListenableFuture;
  50
  51 import org.onap.ccsdk.sli.core.dblib.DBLibConnection;
  52 import org.onap.ccsdk.sli.core.dblib.DbLibService;
  53 import org.onap.ccsdk.sli.plugins.grtoolkit.data.ClusterActor;
  54 import org.onap.ccsdk.sli.plugins.grtoolkit.data.MemberBuilder;
  55
  56 import org.json.JSONArray;
  57 import org.json.JSONException;
  58 import org.json.JSONObject;
  59
  60 import org.opendaylight.controller.cluster.datastore.DistributedDataStoreInterface;
  61 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  62 import org.opendaylight.controller.md.sal.binding.api.DataTreeChangeListener;
  63 import org.opendaylight.controller.md.sal.binding.api.NotificationPublishService;
  64 import org.opendaylight.controller.sal.binding.api.BindingAwareBroker;
  65 import org.opendaylight.controller.sal.binding.api.RpcProviderRegistry;
  66 import org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.AdminHealthInput;
  67 import org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.AdminHealthOutput;
  68 import org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.AdminHealthOutputBuilder;
  69 import org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.ClusterHealthInput;
  70 import org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.ClusterHealthOutput;
  71 import org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.ClusterHealthOutputBuilder;
  72 import org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.DatabaseHealthInput;
  73 import org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.DatabaseHealthOutput;
  74 import org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.DatabaseHealthOutputBuilder;
  75 import org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.FailoverInput;
  76 import org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.FailoverOutput;
  77 import org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.FailoverOutputBuilder;
  78 import org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.GrToolkitService;
  79 import org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.HaltAkkaTrafficInput;
  80 import org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.HaltAkkaTrafficOutput;
  81 import org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.HaltAkkaTrafficOutputBuilder;
  82 import org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.Member;
  83 import org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.ResumeAkkaTrafficInput;
  84 import org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.ResumeAkkaTrafficOutput;
  85 import org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.ResumeAkkaTrafficOutputBuilder;
  86 import org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.Site;
  87 import org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.SiteHealthInput;
  88 import org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.SiteHealthOutput;
  89 import org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.SiteHealthOutputBuilder;
  90 import org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.SiteIdentifierInput;
  91 import org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.SiteIdentifierOutput;
  92 import org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.SiteIdentifierOutputBuilder;
  93 import org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.site.health.output.SitesBuilder;
  94 import org.opendaylight.yangtools.yang.common.RpcResult;
  95 import org.opendaylight.yangtools.yang.common.RpcResultBuilder;
  96
  97 import org.slf4j.Logger;
  98 import org.slf4j.LoggerFactory;
  99
 100 public class GrToolkitProvider implements AutoCloseable, GrToolkitService, DataTreeChangeListener {
 101     private static final String APP_NAME = "gr-toolkit";
 102     private static final String PROPERTIES_FILE = System.getenv("SDNC_CONFIG_DIR") + "/gr-toolkit.properties";
 103     private static final String HEALTHY = "HEALTHY";
 104     private static final String FAULTY = "FAULTY";
 105     private static final String VALUE = "value";
 106     private String akkaConfig;
 107     private String jolokiaClusterPath;
 108     private String shardManagerPath;
 109     private String shardPathTemplate;
 110     private String credentials;
 111     private String httpProtocol;
 112     private String siteIdentifier = System.getenv("SITE_NAME");
 113     private final Logger log = LoggerFactory.getLogger(GrToolkitProvider.class);
 114     private final ExecutorService executor;
 115     protected DataBroker dataBroker;
 116     protected NotificationPublishService notificationService;
 117     protected RpcProviderRegistry rpcRegistry;
 118     protected BindingAwareBroker.RpcRegistration<GrToolkitService> rpcRegistration;
 119     protected DbLibService dbLib;
 120     private String member;
 121     private ClusterActor self;
 122     private HashMap<String, ClusterActor> memberMap;
 123     private SiteConfiguration siteConfiguration;
 124     private Properties properties;
 125     private DistributedDataStoreInterface configDatastore;
 126     public GrToolkitProvider(DataBroker dataBroker,
 127                              NotificationPublishService notificationProviderService,
 128                              RpcProviderRegistry rpcProviderRegistry,
 129                              DistributedDataStoreInterface configDatastore,
 130                              DbLibService dbLibService) {
 131         this.log.info("Creating provider for {}", APP_NAME);
 132         this.executor = Executors.newFixedThreadPool(1);
 133         this.dataBroker = dataBroker;
 134         this.notificationService = notificationProviderService;
 135         this.rpcRegistry = rpcProviderRegistry;
 136         this.configDatastore = configDatastore;
 137         this.dbLib = dbLibService;
 138         initialize();
 139     }
 140
 141     private void initialize() {
 142         log.info("Initializing provider for {}", APP_NAME);
 143         // Create the top level containers
 144         createContainers();
 145         setProperties();
 146         defineMembers();
 147
 148         rpcRegistration = rpcRegistry.addRpcImplementation(GrToolkitService.class, this);
 149         log.info("Initialization complete for {}", APP_NAME);
 150     }
 151
 152     private void setProperties() {
 153         log.info("Loading properties from {}", PROPERTIES_FILE);
 154         properties = new Properties();
 155         File propertiesFile = new File(PROPERTIES_FILE);
 156         if(!propertiesFile.exists()) {
 157             log.warn("Properties file not found.");
 158             return;
 159         }
 160         try(FileInputStream fileInputStream = new FileInputStream(propertiesFile)) {
 161             properties.load(fileInputStream);
 162             if(!properties.containsKey(PropertyKeys.SITE_IDENTIFIER)) {
 163                 properties.put(PropertyKeys.SITE_IDENTIFIER, "Unknown Site");
 164             }
 165             String port = "true".equals(properties.getProperty(PropertyKeys.CONTROLLER_USE_SSL).trim()) ? properties.getProperty(PropertyKeys.CONTROLLER_PORT_SSL).trim() : properties.getProperty(PropertyKeys.CONTROLLER_PORT_HTTP).trim();
 166             httpProtocol = "true".equals(properties.getProperty(PropertyKeys.CONTROLLER_USE_SSL).trim()) ? "https://" : "http://";
 167             akkaConfig = properties.getProperty(PropertyKeys.AKKA_CONF_LOCATION).trim();
 168             jolokiaClusterPath = ":" + port + properties.getProperty(PropertyKeys.MBEAN_CLUSTER).trim();
 169             shardManagerPath = ":" + port + properties.getProperty(PropertyKeys.MBEAN_SHARD_MANAGER).trim();
 170             shardPathTemplate = ":" + port + properties.getProperty(PropertyKeys.MBEAN_SHARD_CONFIG).trim();
 171             if(siteIdentifier == null || siteIdentifier.isEmpty()) {
 172                 siteIdentifier = properties.getProperty(PropertyKeys.SITE_IDENTIFIER).trim();
 173             }
 174             credentials = properties.getProperty(PropertyKeys.CONTROLLER_CREDENTIALS).trim();
 175             log.info("Loaded properties.");
 176         } catch(IOException e) {
 177             log.error("Error loading properties.", e);
 178         }
 179     }
 180
 181     private void defineMembers() {
 182         member = configDatastore.getActorContext().getCurrentMemberName().getName();
 183         log.info("Cluster member: {}", member);
 184
 185         log.info("Parsing akka.conf for cluster memberMap...");
 186         try {
 187             File akkaConfigFile = new File(this.akkaConfig);
 188             try(FileReader fileReader = new FileReader(akkaConfigFile);
 189                 BufferedReader bufferedReader = new BufferedReader(fileReader)) {
 190                 String line;
 191                 while((line = bufferedReader.readLine()) != null) {
 192                     if(line.contains("seed-nodes =")) {
 193                         parseSeedNodes(line);
 194                         break;
 195                     }
 196                 }
 197             }
 198         } catch(IOException e) {
 199             log.error("Couldn't load akka", e);
 200         } catch(NullPointerException e) {
 201             log.error("akkaConfig is null. Check properties file and restart {} bundle.", APP_NAME);
 202         }
 203         log.info("self:\n{}", self);
 204     }
 205
 206     private void createContainers() {
 207         // Replace with MD-SAL write for FailoverStatus
 208     }
 209
 210     protected void initializeChild() {
 211         // Override if you have custom initialization intelligence
 212     }
 213
 214     @Override
 215     public void close() throws Exception {
 216         log.info("Closing provider for {}", APP_NAME);
 217         executor.shutdown();
 218         rpcRegistration.close();
 219         log.info("Successfully closed provider for {}", APP_NAME);
 220     }
 221
 222     @Override
 223     public void onDataTreeChanged(@Nonnull Collection changes) {
 224         log.info("onDataTreeChanged() called. but there is no change here");
 225     }
 226
 227     @Override
 228     public ListenableFuture<RpcResult<ClusterHealthOutput>> clusterHealth(ClusterHealthInput input) {
 229         log.info("{}:cluster-health invoked.", APP_NAME);
 230         getControllerHealth();
 231         return buildClusterHealthOutput("200");
 232     }
 233
 234     @Override
 235     public ListenableFuture<RpcResult<SiteHealthOutput>> siteHealth(SiteHealthInput input) {
 236         log.info("{}:site-health invoked.", APP_NAME);
 237         getControllerHealth();
 238         return buildSiteHealthOutput("200", getAdminHealth(), getDatabaseHealth());
 239     }
 240
 241     @Override
 242     public ListenableFuture<RpcResult<DatabaseHealthOutput>> databaseHealth(DatabaseHealthInput input) {
 243         log.info("{}:database-health invoked.", APP_NAME);
 244         DatabaseHealthOutputBuilder outputBuilder = new DatabaseHealthOutputBuilder();
 245         outputBuilder.setStatus("200");
 246         outputBuilder.setHealth(getDatabaseHealth());
 247
 248         return Futures.immediateFuture(RpcResultBuilder.<DatabaseHealthOutput>status(true).withResult(outputBuilder.build()).build());
 249     }
 250
 251     @Override
 252     public ListenableFuture<RpcResult<AdminHealthOutput>> adminHealth(AdminHealthInput input) {
 253         log.info("{}:admin-health invoked.", APP_NAME);
 254         AdminHealthOutputBuilder outputBuilder = new AdminHealthOutputBuilder();
 255         outputBuilder.setStatus("200");
 256         outputBuilder.setHealth(getAdminHealth());
 257
 258         return Futures.immediateFuture(RpcResultBuilder.<AdminHealthOutput>status(true).withResult(outputBuilder.build()).build());
 259     }
 260
 261     @Override
 262     public ListenableFuture<RpcResult<HaltAkkaTrafficOutput>> haltAkkaTraffic(HaltAkkaTrafficInput input) {
 263         log.info("{}:halt-akka-traffic invoked.", APP_NAME);
 264         HaltAkkaTrafficOutputBuilder outputBuilder = new HaltAkkaTrafficOutputBuilder();
 265         outputBuilder.setStatus("200");
 266         modifyIpTables(IpTables.ADD, input.getNodeInfo().toArray());
 267
 268         return Futures.immediateFuture(RpcResultBuilder.<HaltAkkaTrafficOutput>status(true).withResult(outputBuilder.build()).build());
 269     }
 270
 271     @Override
 272     public ListenableFuture<RpcResult<ResumeAkkaTrafficOutput>> resumeAkkaTraffic(ResumeAkkaTrafficInput input) {
 273         log.info("{}:resume-akka-traffic invoked.", APP_NAME);
 274         ResumeAkkaTrafficOutputBuilder outputBuilder = new ResumeAkkaTrafficOutputBuilder();
 275         outputBuilder.setStatus("200");
 276         modifyIpTables(IpTables.DELETE, input.getNodeInfo().toArray());
 277
 278         return Futures.immediateFuture(RpcResultBuilder.<ResumeAkkaTrafficOutput>status(true).withResult(outputBuilder.build()).build());
 279     }
 280
 281     @Override
 282     public ListenableFuture<RpcResult<SiteIdentifierOutput>> siteIdentifier(SiteIdentifierInput input) {
 283         log.info("{}:site-identifier invoked.", APP_NAME);
 284         SiteIdentifierOutputBuilder outputBuilder = new SiteIdentifierOutputBuilder();
 285         outputBuilder.setStatus("200");
 286         outputBuilder.setId(siteIdentifier);
 287
 288         return Futures.immediateFuture(RpcResultBuilder.<SiteIdentifierOutput>status(true).withResult(outputBuilder.build()).build());
 289     }
 290
 291     @Override
 292     public ListenableFuture<RpcResult<FailoverOutput>> failover(FailoverInput input) {
 293         log.info("{}:failover invoked.", APP_NAME);
 294         FailoverOutputBuilder outputBuilder = new FailoverOutputBuilder();
 295         if(siteConfiguration != SiteConfiguration.GEO) {
 296             log.info("Cannot failover non-GEO site.");
 297             outputBuilder.setMessage("Failover aborted. This is not a GEO configuration.");
 298             outputBuilder.setStatus("400");
 299             return Futures.immediateFuture(RpcResultBuilder.<FailoverOutput>status(true).withResult(outputBuilder.build()).build());
 300         }
 301         ArrayList<ClusterActor> activeSite = new ArrayList<>();
 302         ArrayList<ClusterActor> standbySite = new ArrayList<>();
 303
 304         log.info("Performing preliminary cluster health check...");
 305         // Necessary to populate all member info. Health is not used for judgement calls.
 306         getControllerHealth();
 307
 308         log.info("Determining active site...");
 309         for(Map.Entry<String, ClusterActor> entry : memberMap.entrySet()) {
 310             String key = entry.getKey();
 311             ClusterActor clusterActor = entry.getValue();
 312             if(clusterActor.isVoting()) {
 313                 activeSite.add(clusterActor);
 314                 log.debug("Active Site member: {}", key);
 315             }
 316             else {
 317                 standbySite.add(clusterActor);
 318                 log.debug("Standby Site member: {}", key);
 319             }
 320         }
 321
 322         String port = "true".equals(properties.getProperty(PropertyKeys.CONTROLLER_USE_SSL)) ? properties.getProperty(PropertyKeys.CONTROLLER_PORT_SSL) : properties.getProperty(PropertyKeys.CONTROLLER_PORT_HTTP);
 323
 324         if(Boolean.parseBoolean(input.getBackupData())) {
 325             backupMdSal(activeSite, port);
 326         }
 327
 328         if(!changeClusterVoting(outputBuilder, activeSite, standbySite, port))
 329             return Futures.immediateFuture(RpcResultBuilder.<FailoverOutput>status(true).withResult(outputBuilder.build()).build());
 330
 331         if(Boolean.parseBoolean(input.getIsolate())) {
 332             isolateSiteFromCluster(activeSite, standbySite, port);
 333
 334             if(Boolean.parseBoolean(input.getDownUnreachable())) {
 335                 downUnreachableNodes(activeSite, standbySite, port);
 336             }
 337         }
 338
 339         log.info("{}:failover complete.", APP_NAME);
 340
 341         outputBuilder.setMessage("Failover complete.");
 342         outputBuilder.setStatus("200");
 343         return Futures.immediateFuture(RpcResultBuilder.<FailoverOutput>status(true).withResult(outputBuilder.build()).build());
 344     }
 345
 346     private void isolateSiteFromCluster(ArrayList<ClusterActor> activeSite, ArrayList<ClusterActor> standbySite, String port) {
 347         log.info("Halting Akka traffic...");
 348         for(ClusterActor actor : standbySite) {
 349             try {
 350                 log.info("Halting Akka traffic for: {}", actor.getNode());
 351                 // Build JSON with activeSite actor Node and actor  AkkaPort
 352                 JSONObject akkaInput = new JSONObject();
 353                 JSONObject inputBlock = new JSONObject();
 354                 JSONArray votingStateArray = new JSONArray();
 355                 JSONObject nodeInfo;
 356                 for(ClusterActor node : activeSite) {
 357                     nodeInfo = new JSONObject();
 358                     nodeInfo.put("node", node.getNode());
 359                     nodeInfo.put("port", node.getAkkaPort());
 360                     votingStateArray.put(nodeInfo);
 361                 }
 362                 inputBlock.put("node-info", votingStateArray);
 363                 akkaInput.put("input", inputBlock);
 364                 getRequestContent(httpProtocol + actor.getNode() + ":" + port + "/restconf/operations/gr-toolkit:halt-akka-traffic", HttpMethod.POST, akkaInput.toString());
 365             } catch(IOException e) {
 366                 log.error("Could not halt Akka traffic for: " + actor.getNode(), e);
 367             }
 368         }
 369     }
 370
 371     private void downUnreachableNodes(ArrayList<ClusterActor> activeSite, ArrayList<ClusterActor> standbySite, String port) {
 372         log.info("Setting site unreachable...");
 373         JSONObject jolokiaInput = new JSONObject();
 374         jolokiaInput.put("type", "EXEC");
 375         jolokiaInput.put("mbean", "akka:type=Cluster");
 376         jolokiaInput.put("operation", "down");
 377         JSONArray arguments = new JSONArray();
 378         for(ClusterActor actor : activeSite) {
 379             // Build Jolokia input
 380             // May need to change from akka port to actor.getAkkaPort()
 381             arguments.put("akka.tcp://opendaylight-cluster-data@" + actor.getNode() + ":" + properties.getProperty(PropertyKeys.CONTROLLER_PORT_AKKA));
 382         }
 383         jolokiaInput.put("arguments", arguments);
 384         log.debug("{}", jolokiaInput);
 385         try {
 386             log.info("Setting nodes unreachable");
 387             getRequestContent(httpProtocol + standbySite.get(0).getNode() + ":" + port + "/jolokia", HttpMethod.POST, jolokiaInput.toString());
 388         } catch(IOException e) {
 389             log.error("Error setting nodes unreachable", e);
 390         }
 391     }
 392
 393     private boolean changeClusterVoting(FailoverOutputBuilder outputBuilder, ArrayList<ClusterActor> activeSite, ArrayList<ClusterActor> standbySite, String port) {
 394         log.info("Changing voting for all shards to standby site...");
 395         try {
 396             JSONObject votingInput = new JSONObject();
 397             JSONObject inputBlock = new JSONObject();
 398             JSONArray votingStateArray = new JSONArray();
 399             JSONObject memberVotingState;
 400             for(ClusterActor actor : activeSite) {
 401                 memberVotingState = new JSONObject();
 402                 memberVotingState.put("member-name", actor.getMember());
 403                 memberVotingState.put("voting", false);
 404                 votingStateArray.put(memberVotingState);
 405             }
 406             for(ClusterActor actor : standbySite) {
 407                 memberVotingState = new JSONObject();
 408                 memberVotingState.put("member-name", actor.getMember());
 409                 memberVotingState.put("voting", true);
 410                 votingStateArray.put(memberVotingState);
 411             }
 412             inputBlock.put("member-voting-state", votingStateArray);
 413             votingInput.put("input", inputBlock);
 414             log.debug("{}", votingInput);
 415             // Change voting all shards
 416             getRequestContent(httpProtocol + self.getNode() + ":" + port + "/restconf/operations/cluster-admin:change-member-voting-states-for-all-shards", HttpMethod.POST, votingInput.toString());
 417         } catch(IOException e) {
 418             log.error("Changing voting", e);
 419             outputBuilder.setMessage("Failover aborted. Failed to change voting.");
 420             outputBuilder.setStatus("500");
 421             return false;
 422         }
 423         return true;
 424     }
 425
 426     private void backupMdSal(ArrayList<ClusterActor> activeSite, String port) {
 427         log.info("Backing up data...");
 428         try {
 429             log.info("Scheduling backup for: {}", activeSite.get(0).getNode());
 430             getRequestContent(httpProtocol + activeSite.get(0).getNode() + ":" + port + "/restconf/operations/data-export-import:schedule-export", HttpMethod.POST, "{ \"input\": { \"run-at\": \"30\" } }");
 431         } catch(IOException e) {
 432             log.error("Error backing up MD-SAL", e);
 433         }
 434         for(ClusterActor actor : activeSite) {
 435             try {
 436                 // Move data offsite
 437                 log.info("Backing up data for: {}", actor.getNode());
 438                 getRequestContent(httpProtocol + actor.getNode() + ":" + port + "/restconf/operations/daexim-offsite-backup:backup-data", HttpMethod.POST);
 439             } catch(IOException e) {
 440                 log.error("Error backing up data.", e);
 441             }
 442         }
 443     }
 444
 445     private ListenableFuture<RpcResult<ClusterHealthOutput>> buildClusterHealthOutput(String statusCode) {
 446         ClusterHealthOutputBuilder outputBuilder = new ClusterHealthOutputBuilder();
 447         outputBuilder.setStatus(statusCode);
 448         outputBuilder.setMembers((List) new ArrayList<Member>());
 449         int site1Health = 0;
 450         int site2Health = 0;
 451
 452         for(Map.Entry<String, ClusterActor> entry : memberMap.entrySet()) {
 453             ClusterActor clusterActor = entry.getValue();
 454             if(clusterActor.isUp() && !clusterActor.isUnreachable()) {
 455                 if(ClusterActor.SITE_1.equals(clusterActor.getSite()))
 456                     site1Health++;
 457                 else if(ClusterActor.SITE_2.equals(clusterActor.getSite()))
 458                     site2Health++;
 459             }
 460             outputBuilder.getMembers().add(new MemberBuilder(clusterActor).build());
 461         }
 462         if(siteConfiguration == SiteConfiguration.SOLO) {
 463             outputBuilder.setSite1Health(HEALTHY);
 464         }
 465         else if(site1Health > 1) {
 466             outputBuilder.setSite1Health(HEALTHY);
 467         }
 468         else {
 469             outputBuilder.setSite1Health(FAULTY);
 470         }
 471
 472         if(siteConfiguration == SiteConfiguration.GEO && site2Health > 1) {
 473             outputBuilder.setSite2Health(HEALTHY);
 474         }
 475         else if(siteConfiguration == SiteConfiguration.GEO) {
 476             outputBuilder.setSite2Health(FAULTY);
 477         }
 478
 479         RpcResult<ClusterHealthOutput> rpcResult = RpcResultBuilder.<ClusterHealthOutput>status(true).withResult(outputBuilder.build()).build();
 480         return Futures.immediateFuture(rpcResult);
 481     }
 482
 483     private ListenableFuture<RpcResult<SiteHealthOutput>> buildSiteHealthOutput(String statusCode, String adminHealth, String databaseHealth) {
 484         SiteHealthOutputBuilder outputBuilder = new SiteHealthOutputBuilder();
 485         outputBuilder.setStatus(statusCode);
 486         outputBuilder.setSites((List) new ArrayList<Site>());
 487
 488         if(siteConfiguration != SiteConfiguration.GEO) {
 489             int healthyODLs = 0;
 490             SitesBuilder builder = new SitesBuilder();
 491             for(Map.Entry<String, ClusterActor> entry : memberMap.entrySet()) {
 492                 ClusterActor clusterActor = entry.getValue();
 493                 if(clusterActor.isUp() && !clusterActor.isUnreachable()) {
 494                     healthyODLs++;
 495                 }
 496             }
 497             if(siteConfiguration != SiteConfiguration.SOLO) {
 498                 builder.setHealth(HEALTHY);
 499                 builder.setRole("ACTIVE");
 500                 builder.setId(siteIdentifier);
 501             }
 502             else {
 503                 builder = getSitesBuilder(healthyODLs, true, HEALTHY.equals(adminHealth), HEALTHY.equals(databaseHealth), siteIdentifier);
 504             }
 505             outputBuilder.getSites().add(builder.build());
 506         }
 507         else {
 508             int site1HealthyODLs = 0;
 509             int site2HealthyODLs = 0;
 510             boolean site1Voting = false;
 511             boolean site2Voting = false;
 512             boolean performedCrossSiteHealthCheck = false;
 513             boolean crossSiteAdminHealthy = false;
 514             boolean crossSiteDbHealthy = false;
 515             String crossSiteIdentifier = "UNKNOWN_SITE";
 516             String port = "true".equals(properties.getProperty(PropertyKeys.CONTROLLER_USE_SSL)) ? properties.getProperty(PropertyKeys.CONTROLLER_PORT_SSL) : properties.getProperty(PropertyKeys.CONTROLLER_PORT_HTTP);
 517             if(isSite1()) {
 518                 // Make calls over to site 2 healthchecks
 519                 for(Map.Entry<String, ClusterActor> entry : memberMap.entrySet()) {
 520                     ClusterActor clusterActor = entry.getValue();
 521                     if(clusterActor.isUp() && !clusterActor.isUnreachable()) {
 522                         if(ClusterActor.SITE_1.equals(clusterActor.getSite())) {
 523                             site1HealthyODLs++;
 524                             if(clusterActor.isVoting()) {
 525                                 site1Voting = true;
 526                             }
 527                         }
 528                         else {
 529                             site2HealthyODLs++;
 530                             if(clusterActor.isVoting()) {
 531                                 site2Voting = true;
 532                             }
 533                             if(!performedCrossSiteHealthCheck) {
 534                                 try {
 535                                     String content = getRequestContent(httpProtocol + clusterActor.getNode() + ":" + port + "/restconf/operations/gr-toolkit:site-identifier", HttpMethod.POST);
 536                                     crossSiteIdentifier = new JSONObject(content).getJSONObject("output").getString("id");
 537                                     crossSiteDbHealthy = crossSiteHealthRequest(httpProtocol + clusterActor.getNode() + ":" + port + "/restconf/operations/gr-toolkit:database-health");
 538                                     crossSiteAdminHealthy = crossSiteHealthRequest(httpProtocol + clusterActor.getNode() + ":" + port + "/restconf/operations/gr-toolkit:admin-health");
 539                                     performedCrossSiteHealthCheck = true;
 540                                 } catch(Exception e) {
 541                                     log.info("Cannot get site identifier from {}", clusterActor.getNode());
 542                                     log.error("Site Health Error", e);
 543                                 }
 544                             }
 545                         }
 546                     }
 547                 }
 548                 SitesBuilder builder = getSitesBuilder(site1HealthyODLs, site1Voting, HEALTHY.equals(adminHealth), HEALTHY.equals(databaseHealth), siteIdentifier);
 549                 outputBuilder.getSites().add(builder.build());
 550                 builder = getSitesBuilder(site2HealthyODLs, site2Voting, crossSiteAdminHealthy, crossSiteDbHealthy, crossSiteIdentifier);
 551                 outputBuilder.getSites().add(builder.build());
 552             }
 553             else {
 554                 // Make calls over to site 1 healthchecks
 555                 for(Map.Entry<String, ClusterActor> entry : memberMap.entrySet()) {
 556                     ClusterActor clusterActor = entry.getValue();
 557                     if(clusterActor.isUp() && !clusterActor.isUnreachable()) {
 558                         if(ClusterActor.SITE_1.equals(clusterActor.getSite())) {
 559                             site1HealthyODLs++;
 560                             if(clusterActor.isVoting()) {
 561                                 site1Voting = true;
 562                             }
 563                             if(!performedCrossSiteHealthCheck) {
 564                                 try {
 565                                     String content = getRequestContent(httpProtocol + clusterActor.getNode() + ":" + port + "/restconf/operations/gr-toolkit:site-identifier", HttpMethod.POST);
 566                                     crossSiteIdentifier = new JSONObject(content).getJSONObject("output").getString("id");
 567                                     crossSiteDbHealthy = crossSiteHealthRequest(httpProtocol + clusterActor.getNode() + ":" + port + "/restconf/operations/gr-toolkit:database-health");
 568                                     crossSiteAdminHealthy = crossSiteHealthRequest(httpProtocol + clusterActor.getNode() + ":" + port + "/restconf/operations/gr-toolkit:admin-health");
 569                                     performedCrossSiteHealthCheck = true;
 570                                 } catch(Exception e) {
 571                                     log.info("Cannot get site identifier from {}", clusterActor.getNode());
 572                                     log.error("Site Health Error", e);
 573                                 }
 574                             }
 575                         }
 576                         else {
 577                             site2HealthyODLs++;
 578                             if(clusterActor.isVoting()) {
 579                                 site2Voting = true;
 580                             }
 581                         }
 582                     }
 583                 }
 584                 // Build Output
 585                 SitesBuilder builder = getSitesBuilder(site1HealthyODLs, site1Voting, crossSiteAdminHealthy, crossSiteDbHealthy, crossSiteIdentifier);
 586                 outputBuilder.getSites().add(builder.build());
 587                 builder = getSitesBuilder(site2HealthyODLs, site2Voting, HEALTHY.equals(adminHealth), HEALTHY.equals(databaseHealth), siteIdentifier);
 588                 outputBuilder.getSites().add(builder.build());
 589             }
 590         }
 591
 592         RpcResult<SiteHealthOutput> rpcResult = RpcResultBuilder.<SiteHealthOutput>status(true).withResult(outputBuilder.build()).build();
 593         return Futures.immediateFuture(rpcResult);
 594     }
 595
 596     private SitesBuilder getSitesBuilder(int siteHealthyODLs, boolean siteVoting, boolean adminHealthy, boolean dbHealthy, String siteIdentifier) {
 597         SitesBuilder builder = new SitesBuilder();
 598         if(siteHealthyODLs > 1) {
 599             builder.setHealth(HEALTHY);
 600         }
 601         else {
 602             log.warn("{} Healthy ODLs: {}", siteIdentifier, siteHealthyODLs);
 603             builder.setHealth(FAULTY);
 604         }
 605         if(!adminHealthy) {
 606             log.warn("{} Admin Health: {}", siteIdentifier, FAULTY);
 607             builder.setHealth(FAULTY);
 608         }
 609         if(!dbHealthy) {
 610             log.warn("{} Database Health: {}", siteIdentifier, FAULTY);
 611             builder.setHealth(FAULTY);
 612         }
 613         if(siteVoting) {
 614             builder.setRole("ACTIVE");
 615         }
 616         else {
 617             builder.setRole("STANDBY");
 618         }
 619         builder.setId(siteIdentifier);
 620         return builder;
 621     }
 622
 623     private boolean isSite1() {
 624         int memberNumber = Integer.parseInt(member.split("-")[1]);
 625         boolean isSite1 = memberNumber < 4;
 626         log.info("isSite1(): {}", isSite1);
 627         return isSite1;
 628     }
 629
 630     private void parseSeedNodes(String line) {
 631         memberMap = new HashMap<>();
 632         line = line.substring(line.indexOf("[\""), line.indexOf(']'));
 633         String[] splits = line.split(",");
 634
 635         for(int ndx = 0; ndx < splits.length; ndx++) {
 636             String nodeName = splits[ndx];
 637             int delimLocation = nodeName.indexOf('@');
 638             String port = nodeName.substring(splits[ndx].indexOf(':', delimLocation) + 1, splits[ndx].indexOf('"', splits[ndx].indexOf(':')));
 639             splits[ndx] = nodeName.substring(delimLocation + 1, splits[ndx].indexOf(':', delimLocation));
 640             log.info("Adding node: {}:{}", splits[ndx], port);
 641             ClusterActor clusterActor = new ClusterActor();
 642             clusterActor.setNode(splits[ndx]);
 643             clusterActor.setAkkaPort(port);
 644             clusterActor.setMember("member-" + (ndx + 1));
 645             if(ndx < 3) {
 646                 clusterActor.setSite(ClusterActor.SITE_1);
 647             }
 648             else {
 649                 clusterActor.setSite(ClusterActor.SITE_2);
 650             }
 651
 652             if(member.equals(clusterActor.getMember())) {
 653                 self = clusterActor;
 654             }
 655             memberMap.put(clusterActor.getNode(), clusterActor);
 656             log.info("{}", clusterActor);
 657         }
 658
 659         if(memberMap.size() == 1) {
 660             log.info("1 member found. This is a solo environment.");
 661             siteConfiguration = SiteConfiguration.SOLO;
 662         }
 663         else if(memberMap.size() == 3) {
 664             log.info("This is a single site.");
 665             siteConfiguration = SiteConfiguration.SINGLE;
 666         }
 667         else if(memberMap.size() == 6) {
 668             log.info("This is a georedundant site.");
 669             siteConfiguration = SiteConfiguration.GEO;
 670         }
 671     }
 672
 673     private void getMemberStatus(ClusterActor clusterActor) throws IOException {
 674         log.info("Getting member status for {}", clusterActor.getNode());
 675         String content = getRequestContent(httpProtocol + clusterActor.getNode() + jolokiaClusterPath, HttpMethod.GET);
 676         try {
 677             JSONObject responseJson = new JSONObject(content);
 678             JSONObject responseValue = responseJson.getJSONObject(VALUE);
 679             clusterActor.setUp("Up".equals(responseValue.getString("MemberStatus")));
 680             clusterActor.setUnreachable(false);
 681         } catch(JSONException e) {
 682             log.error("Error parsing response from {}", clusterActor.getNode(), e);
 683             clusterActor.setUp(false);
 684             clusterActor.setUnreachable(true);
 685         }
 686     }
 687
 688     private void getShardStatus(ClusterActor clusterActor) throws IOException {
 689         log.info("Getting shard status for {}", clusterActor.getNode());
 690         String content = getRequestContent(httpProtocol + clusterActor.getNode() + shardManagerPath, HttpMethod.GET);
 691         try {
 692             JSONObject responseValue = new JSONObject(content).getJSONObject(VALUE);
 693             JSONArray shardList = responseValue.getJSONArray("LocalShards");
 694
 695             String pattern = "-config$";
 696             Pattern r = Pattern.compile(pattern);
 697             Matcher m;
 698             for(int ndx = 0; ndx < shardList.length(); ndx++) {
 699                 String configShardName = shardList.getString(ndx);
 700                 m = r.matcher(configShardName);
 701                 String operationalShardName = m.replaceFirst("-operational");
 702                 String shardConfigPath = String.format(shardPathTemplate, configShardName);
 703                 String shardOperationalPath = String.format(shardPathTemplate, operationalShardName).replace("Config", "Operational");
 704                 extractShardInfo(clusterActor, configShardName, shardConfigPath);
 705                 extractShardInfo(clusterActor, operationalShardName, shardOperationalPath);
 706             }
 707         } catch(JSONException e) {
 708             log.error("Error parsing response from " + clusterActor.getNode(), e);
 709         }
 710     }
 711
 712     private void extractShardInfo(ClusterActor clusterActor, String shardName, String shardPath) throws IOException {
 713         log.info("Extracting shard info for {}", shardName);
 714         log.debug("Pulling config info for {} from: {}", shardName, shardPath);
 715         String content = getRequestContent(httpProtocol + clusterActor.getNode() + shardPath, HttpMethod.GET);
 716         log.debug("Response: {}", content);
 717
 718         try {
 719             JSONObject shardValue = new JSONObject(content).getJSONObject(VALUE);
 720             clusterActor.setVoting(shardValue.getBoolean("Voting"));
 721             if(shardValue.getString("PeerAddresses").length() > 0) {
 722                 clusterActor.getReplicaShards().add(shardName);
 723                 if(shardValue.getString("Leader").startsWith(clusterActor.getMember())) {
 724                     clusterActor.getShardLeader().add(shardName);
 725                 }
 726             }
 727             else {
 728                 clusterActor.getNonReplicaShards().add(shardName);
 729             }
 730             JSONArray followerInfo = shardValue.getJSONArray("FollowerInfo");
 731             for(int followerNdx = 0; followerNdx < followerInfo.length(); followerNdx++) {
 732                 int commitIndex = shardValue.getInt("CommitIndex");
 733                 int matchIndex = followerInfo.getJSONObject(followerNdx).getInt("matchIndex");
 734                 if(commitIndex != -1 && matchIndex != -1) {
 735                     int commitsBehind = commitIndex - matchIndex;
 736                     clusterActor.getCommits().put(followerInfo.getJSONObject(followerNdx).getString("id"), commitsBehind);
 737                 }
 738             }
 739         } catch(JSONException e) {
 740             log.error("Error parsing response from " + clusterActor.getNode(), e);
 741         }
 742     }
 743
 744     private void getControllerHealth() {
 745         for(Map.Entry<String, ClusterActor> entry : memberMap.entrySet()) {
 746             ClusterActor clusterActor = entry.getValue();
 747             String key = entry.getKey();
 748             try {
 749                 // First flush out the old values
 750                 clusterActor.flush();
 751                 log.info("Gathering info for {}", clusterActor.getNode());
 752                 getMemberStatus(clusterActor);
 753                 getShardStatus(clusterActor);
 754                 log.info("MemberInfo:\n{}", clusterActor);
 755             } catch(IOException e) {
 756                 log.error("Connection Error", e);
 757                 memberMap.get(key).setUnreachable(true);
 758                 memberMap.get(key).setUp(false);
 759                 log.info("MemberInfo:\n{}", memberMap.get(key));
 760             }
 761         }
 762     }
 763
 764     private void modifyIpTables(IpTables task, Object[] nodeInfo) {
 765         log.info("Modifying IPTables rules...");
 766         if(task == IpTables.ADD) {
 767             for(Object node : nodeInfo) {
 768                 org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.halt.akka.traffic.input.NodeInfo n =
 769                         (org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.halt.akka.traffic.input.NodeInfo) node;
 770                 log.info("Isolating {}", n.getNode());
 771                 executeCommand(String.format("sudo /sbin/iptables -A INPUT -p tcp --destination-port %s -j DROP -s %s", properties.get(PropertyKeys.CONTROLLER_PORT_AKKA), n.getNode()));
 772                 executeCommand(String.format("sudo /sbin/iptables -A OUTPUT -p tcp --destination-port %s -j DROP -s %s", n.getPort(), n.getNode()));
 773             }
 774
 775         } else if(task == IpTables.DELETE) {
 776             for(Object node : nodeInfo) {
 777                 org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.resume.akka.traffic.input.NodeInfo n =
 778                         (org.opendaylight.yang.gen.v1.org.onap.ccsdk.sli.plugins.gr.toolkit.rev180926.resume.akka.traffic.input.NodeInfo) node;
 779                 log.info("De-isolating {}", n.getNode());
 780                 executeCommand(String.format("sudo /sbin/iptables -D INPUT -p tcp --destination-port %s -j DROP -s %s", properties.get(PropertyKeys.CONTROLLER_PORT_AKKA), n.getNode()));
 781                 executeCommand(String.format("sudo /sbin/iptables -D OUTPUT -p tcp --destination-port %s -j DROP -s %s", n.getPort(), n.getNode()));
 782             }
 783
 784         }
 785         executeCommand("sudo /sbin/iptables -L");
 786     }
 787
 788     private void executeCommand(String command) {
 789         log.info("Executing command: {}", command);
 790         String[] cmd = command.split(" ");
 791         try {
 792             Process p = Runtime.getRuntime().exec(cmd);
 793             BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(p.getInputStream()));
 794             String inputLine;
 795             StringBuilder content = new StringBuilder();
 796             while((inputLine = bufferedReader.readLine()) != null) {
 797                 content.append(inputLine);
 798             }
 799             bufferedReader.close();
 800             log.info("{}", content);
 801         } catch(IOException e) {
 802             log.error("Error executing command", e);
 803         }
 804     }
 805
 806     private boolean crossSiteHealthRequest(String path) throws IOException {
 807         String content = getRequestContent(path, HttpMethod.POST);
 808         try {
 809             JSONObject responseJson = new JSONObject(content);
 810             JSONObject responseValue = responseJson.getJSONObject(VALUE);
 811             return HEALTHY.equals(responseValue.getString("health"));
 812         } catch(JSONException e) {
 813             log.error("Error parsing JSON", e);
 814             throw new IOException();
 815         }
 816     }
 817
 818     private String getAdminHealth() {
 819         String protocol = "true".equals(properties.getProperty(PropertyKeys.ADM_USE_SSL)) ? "https://" : "http://";
 820         String port = "true".equals(properties.getProperty(PropertyKeys.ADM_USE_SSL)) ? properties.getProperty(PropertyKeys.ADM_PORT_SSL) : properties.getProperty(PropertyKeys.ADM_PORT_HTTP);
 821         String path = protocol + properties.getProperty(PropertyKeys.ADM_FQDN) + ":" + port + properties.getProperty(PropertyKeys.ADM_HEALTHCHECK);
 822         log.info("Requesting healthcheck from {}", path);
 823         try {
 824             int response = getRequestStatus(path, HttpMethod.GET);
 825             log.info("Response: {}", response);
 826             if(response == 200)
 827                 return HEALTHY;
 828             return FAULTY;
 829         } catch(IOException e) {
 830             log.error("Problem getting ADM health.", e);
 831             return FAULTY;
 832         }
 833     }
 834
 835     private String getDatabaseHealth() {
 836         log.info("Determining database health...");
 837         try {
 838             Connection connection = dbLib.getConnection();
 839             log.info("DBLib isActive(): {}", dbLib.isActive());
 840             log.info("DBLib isReadOnly(): {}", connection.isReadOnly());
 841             log.info("DBLib isClosed(): {}", connection.isClosed());
 842             if(!dbLib.isActive() || connection.isClosed() || connection.isReadOnly()) {
 843                 log.warn("Database is FAULTY");
 844                 connection.close();
 845                 return FAULTY;
 846             }
 847             connection.close();
 848             log.info("Database is HEALTHY");
 849         } catch(SQLException e) {
 850             log.error("Database is FAULTY");
 851             log.error("Error", e);
 852             return FAULTY;
 853         }
 854
 855         return HEALTHY;
 856     }
 857
 858     private String getRequestContent(String path, HttpMethod method) throws IOException {
 859         return getRequestContent(path, method, null);
 860     }
 861
 862     private String getRequestContent(String path, HttpMethod method, String input) throws IOException {
 863         HttpURLConnection connection = getConnection(path);
 864         connection.setRequestMethod(method.getMethod());
 865         connection.setDoInput(true);
 866
 867         if(input != null) {
 868             sendPayload(input, connection);
 869         }
 870
 871         BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(connection.getInputStream()));
 872         String inputLine;
 873         StringBuilder content = new StringBuilder();
 874         while((inputLine = bufferedReader.readLine()) != null) {
 875             content.append(inputLine);
 876         }
 877         bufferedReader.close();
 878         connection.disconnect();
 879         return content.toString();
 880     }
 881
 882     private int getRequestStatus(String path, HttpMethod method) throws IOException {
 883         return getRequestStatus(path, method, null);
 884     }
 885
 886     private int getRequestStatus(String path, HttpMethod method, String input) throws IOException {
 887         HttpURLConnection connection = getConnection(path);
 888         connection.setRequestMethod(method.getMethod());
 889         connection.setDoInput(true);
 890
 891         if(input != null) {
 892             sendPayload(input, connection);
 893         }
 894         int response = connection.getResponseCode();
 895         log.info("Received {} response code from {}", response, path);
 896         connection.disconnect();
 897         return response;
 898     }
 899
 900     private void sendPayload(String input, HttpURLConnection connection) throws IOException {
 901         byte[] out = input.getBytes(StandardCharsets.UTF_8);
 902         int length = out.length;
 903
 904         connection.setFixedLengthStreamingMode(length);
 905         connection.setRequestProperty("Content-Type", "application/json");
 906         connection.setDoOutput(true);
 907         connection.connect();
 908         try(OutputStream os = connection.getOutputStream()) {
 909             os.write(out);
 910         }
 911     }
 912
 913     private HttpURLConnection getConnection(String host) throws IOException {
 914         log.info("Getting connection to: {}", host);
 915         URL url = new URL(host);
 916         String auth = "Basic " + javax.xml.bind.DatatypeConverter.printBase64Binary(credentials.getBytes());
 917         HttpURLConnection connection = (HttpURLConnection) url.openConnection();
 918         connection.addRequestProperty("Authorization", auth);
 919         connection.setRequestProperty("Connection", "keep-alive");
 920         connection.setRequestProperty("Proxy-Connection", "keep-alive");
 921         return connection;
 922     }
 923
 924     enum IpTables {
 925         ADD,
 926         DELETE
 927     }
 928
 929     enum SiteConfiguration {
 930         SOLO,
 931         SINGLE,
 932         GEO
 933     }
 934
 935     enum HttpMethod {
 936         GET("GET"),
 937         POST("POST");
 938
 939         private String method;
 940         HttpMethod(String method) {
 941             this.method = method;
 942         }
 943         public String getMethod() {
 944             return method;
 945         }
 946     }
 947
 948     class PropertyKeys {
 949         static final String SITE_IDENTIFIER = "site.identifier";
 950         static final String CONTROLLER_USE_SSL = "controller.useSsl";
 951         static final String CONTROLLER_PORT_SSL = "controller.port.ssl";
 952         static final String CONTROLLER_PORT_HTTP = "controller.port.http";
 953         static final String CONTROLLER_PORT_AKKA = "controller.port.akka";
 954         static final String CONTROLLER_CREDENTIALS = "controller.credentials";
 955         static final String AKKA_CONF_LOCATION = "akka.conf.location";
 956         static final String MBEAN_CLUSTER = "mbean.cluster";
 957         static final String MBEAN_SHARD_MANAGER  = "mbean.shardManager";
 958         static final String MBEAN_SHARD_CONFIG = "mbean.shard.config";
 959         static final String ADM_USE_SSL = "adm.useSsl";
 960         static final String ADM_PORT_SSL = "adm.port.ssl";
 961         static final String ADM_PORT_HTTP = "adm.port.http";
 962         static final String ADM_FQDN = "adm.fqdn";
 963         static final String ADM_HEALTHCHECK= "adm.healthcheck";
 964     }
 965 }