1 /*******************************************************************************
2 * ============LICENSE_START==================================================
4 * * ===========================================================================
5 * * Copyright © 2017 AT&T Intellectual Property. All rights reserved.
6 * * ===========================================================================
7 * * Licensed under the Apache License, Version 2.0 (the "License");
8 * * you may not use this file except in compliance with the License.
9 * * You may obtain a copy of the License at
11 * * http://www.apache.org/licenses/LICENSE-2.0
13 * * Unless required by applicable law or agreed to in writing, software
14 * * distributed under the License is distributed on an "AS IS" BASIS,
15 * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 * * See the License for the specific language governing permissions and
17 * * limitations under the License.
18 * * ============LICENSE_END====================================================
20 * * ECOMP is a trademark and service mark of AT&T Intellectual Property.
22 ******************************************************************************/
25 package org.onap.dmaap.datarouter.node;
27 import static java.lang.System.exit;
29 import com.att.eelf.configuration.EELFLogger;
30 import com.att.eelf.configuration.EELFManager;
32 import java.io.FileInputStream;
33 import java.io.IOException;
34 import java.io.InputStreamReader;
35 import java.io.Reader;
37 import java.nio.file.Files;
38 import java.util.Objects;
39 import java.util.Properties;
40 import java.util.Timer;
41 import org.onap.dmaap.datarouter.node.eelf.EelfMsgs;
45 * Maintain the configuration of a Data Router node
47 * <p>The NodeConfigManager is the single point of contact for servlet, delivery, event logging, and log retention
48 * subsystems to access configuration information.
50 * <p>There are two basic sets of configuration data. The static local configuration data, stored in a local
51 * configuration file (created as part of installation by SWM), and the dynamic global configuration data fetched from
52 * the data router provisioning server.
54 public class NodeConfigManager implements DeliveryQueueHelper {
56 private static final String CHANGE_ME = "changeme";
57 private static final String NODE_CONFIG_MANAGER = "NodeConfigManager";
58 private static EELFLogger eelfLogger = EELFManager.getInstance().getLogger(NodeConfigManager.class);
59 private static NodeConfigManager base = new NodeConfigManager();
61 private Timer timer = new Timer("Node Configuration Timer", true);
62 private long maxfailuretimer;
63 private long initfailuretimer;
64 private long waitForFileProcessFailureTimer;
65 private long expirationtimer;
66 private double failurebackoff;
67 private long fairtimelimit;
68 private int fairfilelimit;
69 private double fdpstart;
70 private double fdpstop;
71 private int deliverythreads;
72 private String provurl;
73 private String provhost;
74 private IsFrom provcheck;
78 private String spooldir;
79 private String logdir;
80 private long logretention;
81 private String redirfile;
82 private String kstype;
83 private String ksfile;
84 private String kspass;
86 private String tstype;
87 private String tsfile;
88 private String tspass;
89 private String myname;
90 private RedirManager rdmgr;
91 private RateLimitedOperation pfetcher;
92 private NodeConfig config;
94 private PublishId pid;
96 private TaskList configtasks = new TaskList();
97 private String eventlogurl;
98 private String eventlogprefix;
99 private String eventlogsuffix;
100 private String eventloginterval;
101 private boolean followredirects;
102 private String[] enabledprotocols;
103 private String aafType;
104 private String aafInstance;
105 private String aafAction;
106 private String aafURL;
107 private boolean cadiEnabled;
108 private NodeAafPropsUtils nodeAafPropsUtils;
112 * Initialize the configuration of a Data Router node.
114 private NodeConfigManager() {
116 Properties drNodeProperties = new Properties();
118 eelfLogger.debug("NODE0301 Loading local config file node.properties");
119 drNodeProperties.load(new FileInputStream(System
120 .getProperty("org.onap.dmaap.datarouter.node.properties", "/opt/app/datartr/etc/node.properties")));
121 } catch (Exception e) {
122 NodeUtils.setIpAndFqdnForEelf(NODE_CONFIG_MANAGER);
123 eelfLogger.error(EelfMsgs.MESSAGE_PROPERTIES_LOAD_ERROR, e,
124 System.getProperty("org.onap.dmaap.datarouter.node.properties",
125 "/opt/app/datartr/etc/node.properties"));
127 provurl = drNodeProperties.getProperty("ProvisioningURL", "https://dmaap-dr-prov:8443/internal/prov");
128 String aafPropsFilePath = drNodeProperties
129 .getProperty("AAFPropsFilePath", "/opt/app/osaaf/local/org.onap.dmaap-dr.props");
131 nodeAafPropsUtils = new NodeAafPropsUtils(new File(aafPropsFilePath));
132 } catch (IOException e) {
133 eelfLogger.error("NODE0314 Failed to load AAF props. Exiting", e);
137 * START - AAF changes: TDP EPIC US# 307413
138 * Pull AAF settings from node.properties
140 aafType = drNodeProperties.getProperty("AAFType", "org.onap.dmaap-dr.feed");
141 aafInstance = drNodeProperties.getProperty("AAFInstance", "legacy");
142 aafAction = drNodeProperties.getProperty("AAFAction", "publish");
143 cadiEnabled = Boolean.parseBoolean(drNodeProperties.getProperty("CadiEnabled", "false"));
144 aafURL = nodeAafPropsUtils.getPropAccess().getProperty("aaf_locate_url", "https://aaf-locate:8095");
146 * END - AAF changes: TDP EPIC US# 307413
147 * Pull AAF settings from node.properties
149 //Disable and enable protocols*/
150 enabledprotocols = ((drNodeProperties.getProperty("NodeHttpsProtocols")).trim()).split("\\|");
152 provhost = (new URL(provurl)).getHost();
153 } catch (Exception e) {
154 NodeUtils.setIpAndFqdnForEelf(NODE_CONFIG_MANAGER);
155 eelfLogger.error(EelfMsgs.MESSAGE_BAD_PROV_URL, e, provurl);
158 eelfLogger.debug("NODE0303 Provisioning server is " + provhost);
159 eventlogurl = drNodeProperties.getProperty("LogUploadURL", "https://feeds-drtr.web.att.com/internal/logs");
160 provcheck = new IsFrom(provhost);
161 gfport = Integer.parseInt(drNodeProperties.getProperty("IntHttpPort", "8080"));
162 svcport = Integer.parseInt(drNodeProperties.getProperty("IntHttpsPort", "8443"));
163 port = Integer.parseInt(drNodeProperties.getProperty("ExtHttpsPort", "443"));
164 spooldir = drNodeProperties.getProperty("SpoolDir", "spool");
165 File fdir = new File(spooldir + "/f");
167 for (File junk : Objects.requireNonNull(fdir.listFiles())) {
169 Files.deleteIfExists(junk.toPath());
170 } catch (IOException e) {
171 eelfLogger.error("NODE0313 Failed to clear junk files from " + fdir.getPath(), e);
174 logdir = drNodeProperties.getProperty("LogDir", "logs");
175 (new File(logdir)).mkdirs();
176 logretention = Long.parseLong(drNodeProperties.getProperty("LogRetention", "30")) * 86400000L;
177 eventlogprefix = logdir + "/events";
178 eventlogsuffix = ".log";
179 redirfile = drNodeProperties.getProperty("RedirectionFile", "etc/redirections.dat");
180 kstype = drNodeProperties.getProperty("KeyStoreType", "PKCS12");
181 ksfile = nodeAafPropsUtils.getPropAccess().getProperty("cadi_keystore");
182 kspass = nodeAafPropsUtils.getDecryptedPass("cadi_keystore_password");
183 kpass = nodeAafPropsUtils.getDecryptedPass("cadi_keystore_password");
184 tstype = drNodeProperties.getProperty("TrustStoreType", "jks");
185 tsfile = nodeAafPropsUtils.getPropAccess().getProperty("cadi_truststore");
186 tspass = nodeAafPropsUtils.getDecryptedPass("cadi_truststore_password");
187 if (tsfile != null && tsfile.length() > 0) {
188 System.setProperty("javax.net.ssl.trustStoreType", tstype);
189 System.setProperty("javax.net.ssl.trustStore", tsfile);
190 System.setProperty("javax.net.ssl.trustStorePassword", tspass);
192 nak = drNodeProperties.getProperty("NodeAuthKey", "Node123!");
193 quiesce = new File(drNodeProperties.getProperty("QuiesceFile", "etc/SHUTDOWN"));
194 myname = NodeUtils.getCanonicalName(kstype, ksfile, kspass);
195 if (myname == null) {
196 NodeUtils.setIpAndFqdnForEelf(NODE_CONFIG_MANAGER);
197 eelfLogger.error(EelfMsgs.MESSAGE_KEYSTORE_FETCH_ERROR, ksfile);
198 eelfLogger.error("NODE0309 Unable to fetch canonical name from keystore file " + ksfile);
201 eelfLogger.debug("NODE0304 My certificate says my name is " + myname);
202 pid = new PublishId(myname);
203 long minrsinterval = Long.parseLong(drNodeProperties.getProperty("MinRedirSaveInterval", "10000"));
204 long minpfinterval = Long.parseLong(drNodeProperties.getProperty("MinProvFetchInterval", "10000"));
205 rdmgr = new RedirManager(redirfile, minrsinterval, timer);
206 pfetcher = new RateLimitedOperation(minpfinterval, timer) {
211 eelfLogger.debug("NODE0305 Attempting to fetch configuration at " + provurl);
216 * Get the default node configuration manager.
218 public static NodeConfigManager getInstance() {
222 private void localconfig() {
223 followredirects = Boolean.parseBoolean(getProvParam("FOLLOW_REDIRECTS", "false"));
224 eventloginterval = getProvParam("LOGROLL_INTERVAL", "30s");
225 initfailuretimer = 10000;
226 waitForFileProcessFailureTimer = 600000;
227 maxfailuretimer = 3600000;
228 expirationtimer = 86400000;
229 failurebackoff = 2.0;
230 deliverythreads = 40;
232 fairtimelimit = 60000;
236 initfailuretimer = (long) (Double.parseDouble(getProvParam("DELIVERY_INIT_RETRY_INTERVAL")) * 1000);
237 } catch (Exception e) {
238 eelfLogger.trace("Error parsing DELIVERY_INIT_RETRY_INTERVAL", e);
241 waitForFileProcessFailureTimer = (long) (Double.parseDouble(getProvParam("DELIVERY_FILE_PROCESS_INTERVAL"))
243 } catch (Exception e) {
244 eelfLogger.trace("Error parsing DELIVERY_FILE_PROCESS_INTERVAL", e);
247 maxfailuretimer = (long) (Double.parseDouble(getProvParam("DELIVERY_MAX_RETRY_INTERVAL")) * 1000);
248 } catch (Exception e) {
249 eelfLogger.trace("Error parsing DELIVERY_MAX_RETRY_INTERVAL", e);
252 expirationtimer = (long) (Double.parseDouble(getProvParam("DELIVERY_MAX_AGE")) * 1000);
253 } catch (Exception e) {
254 eelfLogger.trace("Error parsing DELIVERY_MAX_AGE", e);
257 failurebackoff = Double.parseDouble(getProvParam("DELIVERY_RETRY_RATIO"));
258 } catch (Exception e) {
259 eelfLogger.trace("Error parsing DELIVERY_RETRY_RATIO", e);
262 deliverythreads = Integer.parseInt(getProvParam("DELIVERY_THREADS"));
263 } catch (Exception e) {
264 eelfLogger.trace("Error parsing DELIVERY_THREADS", e);
267 fairfilelimit = Integer.parseInt(getProvParam("FAIR_FILE_LIMIT"));
268 } catch (Exception e) {
269 eelfLogger.trace("Error parsing FAIR_FILE_LIMIT", e);
272 fairtimelimit = (long) (Double.parseDouble(getProvParam("FAIR_TIME_LIMIT")) * 1000);
273 } catch (Exception e) {
274 eelfLogger.trace("Error parsing FAIR_TIME_LIMIT", e);
277 fdpstart = Double.parseDouble(getProvParam("FREE_DISK_RED_PERCENT")) / 100.0;
278 } catch (Exception e) {
279 eelfLogger.trace("Error parsing FREE_DISK_RED_PERCENT", e);
282 fdpstop = Double.parseDouble(getProvParam("FREE_DISK_YELLOW_PERCENT")) / 100.0;
283 } catch (Exception e) {
284 eelfLogger.trace("Error parsing FREE_DISK_YELLOW_PERCENT", e);
286 if (fdpstart < 0.01) {
289 if (fdpstart > 0.5) {
292 if (fdpstop < fdpstart) {
300 private void fetchconfig() {
302 eelfLogger.debug("NodeConfigMan.fetchConfig: provurl:: " + provurl);
303 Reader reader = new InputStreamReader((new URL(provurl)).openStream());
304 config = new NodeConfig(new ProvData(reader), myname, spooldir, port, nak);
306 configtasks.startRun();
308 } catch (Exception e) {
309 NodeUtils.setIpAndFqdnForEelf("fetchconfigs");
310 eelfLogger.error(EelfMsgs.MESSAGE_CONF_FAILED, e.toString());
311 eelfLogger.error("NODE0306 Configuration failed " + e.toString() + " - try again later", e);
316 private void runTasks() {
318 while ((rr = configtasks.next()) != null) {
321 } catch (Exception e) {
322 eelfLogger.error("NODE0518 Exception fetchconfig: " + e);
328 * Process a gofetch request from a particular IP address. If the IP address is not an IP address we would go to to
329 * fetch the provisioning data, ignore the request. If the data has been fetched very recently (default 10
330 * seconds), wait a while before fetching again.
332 public synchronized void gofetch(String remoteAddr) {
333 if (provcheck.isReachable(remoteAddr)) {
334 eelfLogger.debug("NODE0307 Received configuration fetch request from provisioning server " + remoteAddr);
337 eelfLogger.debug("NODE0308 Received configuration fetch request from unexpected server " + remoteAddr);
344 public boolean isConfigured() {
345 return (config != null);
351 public boolean isShutdown() {
352 return (quiesce.exists());
356 * Given a routing string, get the targets.
358 * @param routing Target string
359 * @return array of targets
361 public Target[] parseRouting(String routing) {
362 return (config.parseRouting(routing));
366 * Given a set of credentials and an IP address, is this request from another node.
368 * @param credentials Credentials offered by the supposed node
369 * @param ip IP address the request came from
370 * @return If the credentials and IP address are recognized, true, otherwise false.
372 public boolean isAnotherNode(String credentials, String ip) {
373 return (config.isAnotherNode(credentials, ip));
377 * Check whether publication is allowed.
379 * @param feedid The ID of the feed being requested
380 * @param credentials The offered credentials
381 * @param ip The requesting IP address
382 * @return True if the IP and credentials are valid for the specified feed.
384 public String isPublishPermitted(String feedid, String credentials, String ip) {
385 return (config.isPublishPermitted(feedid, credentials, ip));
389 * Check whether publication is allowed for AAF Feed.
391 * @param feedid The ID of the feed being requested
392 * @param ip The requesting IP address
393 * @return True if the IP and credentials are valid for the specified feed.
395 public String isPublishPermitted(String feedid, String ip) {
396 return (config.isPublishPermitted(feedid, ip));
400 * Check whether delete file is allowed.
402 * @param subId The ID of the subscription being requested
403 * @return True if the delete file is permitted for the subscriber.
405 public boolean isDeletePermitted(String subId) {
406 return (config.isDeletePermitted(subId));
410 * Check who the user is given the feed ID and the offered credentials.
412 * @param feedid The ID of the feed specified
413 * @param credentials The offered credentials
414 * @return Null if the credentials are invalid or the user if they are valid.
416 public String getAuthUser(String feedid, String credentials) {
417 return (config.getAuthUser(feedid, credentials));
421 * AAF changes: TDP EPIC US# 307413 Check AAF_instance for feed ID in NodeConfig.
423 * @param feedid The ID of the feed specified
425 public String getAafInstance(String feedid) {
426 return (config.getAafInstance(feedid));
429 public String getAafInstance() {
434 * Check if the publish request should be sent to another node based on the feedid, user, and source IP address.
436 * @param feedid The ID of the feed specified
437 * @param user The publishing user
438 * @param ip The IP address of the publish endpoint
439 * @return Null if the request should be accepted or the correct hostname if it should be sent to another node.
441 public String getIngressNode(String feedid, String user, String ip) {
442 return (config.getIngressNode(feedid, user, ip));
446 * Get a provisioned configuration parameter (from the provisioning server configuration).
448 * @param name The name of the parameter
449 * @return The value of the parameter or null if it is not defined.
451 public String getProvParam(String name) {
452 return (config.getProvParam(name));
456 * Get a provisioned configuration parameter (from the provisioning server configuration).
458 * @param name The name of the parameter
459 * @param defaultValue The value to use if the parameter is not defined
460 * @return The value of the parameter or deflt if it is not defined.
462 public String getProvParam(String name, String defaultValue) {
463 name = config.getProvParam(name);
471 * Generate a publish ID.
473 public String getPublishId() {
478 * Get all the outbound spooling destinations. This will include both subscriptions and nodes.
480 public DestInfo[] getAllDests() {
481 return (config.getAllDests());
485 * Register a task to run whenever the configuration changes.
487 public void registerConfigTask(Runnable task) {
488 configtasks.addTask(task);
492 * Deregister a task to run whenever the configuration changes.
494 public void deregisterConfigTask(Runnable task) {
495 configtasks.removeTask(task);
499 * Get the URL to deliver a message to.
501 * @param destinationInfo The destination information
502 * @param fileid The file ID
503 * @return The URL to deliver to
505 public String getDestURL(DestInfo destinationInfo, String fileid) {
506 String subid = destinationInfo.getSubId();
507 String purl = destinationInfo.getURL();
508 if (followredirects && subid != null) {
509 purl = rdmgr.lookup(subid, purl);
511 return (purl + "/" + fileid);
515 * Set up redirection on receipt of a 3XX from a target URL.
517 public boolean handleRedirection(DestInfo destinationInfo, String redirto, String fileid) {
518 fileid = "/" + fileid;
519 String subid = destinationInfo.getSubId();
520 String purl = destinationInfo.getURL();
521 if (followredirects && subid != null && redirto.endsWith(fileid)) {
522 redirto = redirto.substring(0, redirto.length() - fileid.length());
523 if (!redirto.equals(purl)) {
524 rdmgr.redirect(subid, purl, redirto);
532 * Handle unreachable target URL.
534 public void handleUnreachable(DestInfo destinationInfo) {
535 String subid = destinationInfo.getSubId();
536 if (followredirects && subid != null) {
542 * Get the timeout before retrying after an initial delivery failure.
544 public long getInitFailureTimer() {
545 return (initfailuretimer);
549 * Get the timeout before retrying after delivery and wait for file processing.
551 public long getWaitForFileProcessFailureTimer() {
552 return (waitForFileProcessFailureTimer);
556 * Get the maximum timeout between delivery attempts.
558 public long getMaxFailureTimer() {
559 return (maxfailuretimer);
563 * Get the ratio between consecutive delivery attempts.
565 public double getFailureBackoff() {
566 return (failurebackoff);
570 * Get the expiration timer for deliveries.
572 public long getExpirationTimer() {
573 return (expirationtimer);
577 * Get the maximum number of file delivery attempts before checking if another queue has work to be performed.
579 public int getFairFileLimit() {
580 return (fairfilelimit);
584 * Get the maximum amount of time spent delivering files before checking if another queue has work to be performed.
586 public long getFairTimeLimit() {
587 return (fairtimelimit);
591 * Get the targets for a feed.
593 * @param feedid The feed ID
594 * @return The targets this feed should be delivered to
596 public Target[] getTargets(String feedid) {
597 return (config.getTargets(feedid));
601 * Get the spool directory for temporary files.
603 public String getSpoolDir() {
604 return (spooldir + "/f");
608 * Get the spool directory for a subscription.
610 public String getSpoolDir(String subid, String remoteaddr) {
611 if (provcheck.isFrom(remoteaddr)) {
612 String sdir = config.getSpoolDir(subid);
614 eelfLogger.debug("NODE0310 Received subscription reset request for subscription " + subid
615 + " from provisioning server " + remoteaddr);
617 eelfLogger.debug("NODE0311 Received subscription reset request for unknown subscription " + subid
618 + " from provisioning server " + remoteaddr);
622 eelfLogger.debug("NODE0312 Received subscription reset request from unexpected server " + remoteaddr);
628 * Get the base directory for spool directories.
630 public String getSpoolBase() {
635 * Get the key store type.
637 public String getKSType() {
642 * Get the key store file.
644 public String getKSFile() {
649 * Get the key store password.
651 public String getKSPass() {
656 * Get the key password.
658 public String getKPass() {
665 public int getHttpPort() {
670 * Get the https port.
672 public int getHttpsPort() {
677 * Get the externally visible https port.
679 public int getExtHttpsPort() {
684 * Get the external name of this machine.
686 public String getMyName() {
691 * Get the number of threads to use for delivery.
693 public int getDeliveryThreads() {
694 return (deliverythreads);
698 * Get the URL for uploading the event log data.
700 public String getEventLogUrl() {
701 return (eventlogurl);
705 * Get the prefix for the names of event log files.
707 public String getEventLogPrefix() {
708 return (eventlogprefix);
712 * Get the suffix for the names of the event log files.
714 public String getEventLogSuffix() {
715 return (eventlogsuffix);
719 * Get the interval between event log file rollovers.
721 public String getEventLogInterval() {
722 return (eventloginterval);
726 * Should I follow redirects from subscribers.
728 public boolean isFollowRedirects() {
729 return (followredirects);
733 * Get the directory where the event and node log files live.
735 public String getLogDir() {
740 * How long do I keep log files (in milliseconds).
742 public long getLogRetention() {
743 return (logretention);
749 public Timer getTimer() {
754 * Get the feed ID for a subscription.
756 * @param subid The subscription ID
757 * @return The feed ID
759 public String getFeedId(String subid) {
760 return (config.getFeedId(subid));
764 * Get the authorization string this node uses.
766 * @return The Authorization string for this node
768 public String getMyAuth() {
769 return (config.getMyAuth());
773 * Get the fraction of free spool disk space where we start throwing away undelivered files. This is
774 * FREE_DISK_RED_PERCENT / 100.0. Default is 0.05. Limited by 0.01 <= FreeDiskStart <= 0.5.
776 public double getFreeDiskStart() {
781 * Get the fraction of free spool disk space where we stop throwing away undelivered files. This is
782 * FREE_DISK_YELLOW_PERCENT / 100.0. Default is 0.2. Limited by FreeDiskStart <= FreeDiskStop <= 0.5.
784 public double getFreeDiskStop() {
789 * Disable and enable protocols.
791 public String[] getEnabledprotocols() {
792 return enabledprotocols;
795 public String getAafType() {
799 public String getAafAction() {
804 * Get aafURL from SWM variable
806 public String getAafURL() {
810 public boolean getCadiEnabled() {
814 public NodeAafPropsUtils getNodeAafPropsUtils() {
815 return nodeAafPropsUtils;
819 * Builds the permissions string to be verified.
821 * @param aafInstance The aaf instance
822 * @return The permissions
824 protected String getPermission(String aafInstance) {
826 String type = getAafType();
827 String action = getAafAction();
828 if ("".equals(aafInstance)) {
829 aafInstance = getAafInstance();
831 return type + "|" + aafInstance + "|" + action;
832 } catch (Exception e) {
833 eelfLogger.error("NODE0543 NodeConfigManager.getPermission: ", e);