1 /*******************************************************************************
2 * ============LICENSE_START==================================================
4 * * ===========================================================================
5 * * Copyright © 2017 AT&T Intellectual Property. All rights reserved.
6 * * ===========================================================================
7 * * Licensed under the Apache License, Version 2.0 (the "License");
8 * * you may not use this file except in compliance with the License.
9 * * You may obtain a copy of the License at
11 * * http://www.apache.org/licenses/LICENSE-2.0
13 * * Unless required by applicable law or agreed to in writing, software
14 * * distributed under the License is distributed on an "AS IS" BASIS,
15 * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 * * See the License for the specific language governing permissions and
17 * * limitations under the License.
18 * * ============LICENSE_END====================================================
20 * * ECOMP is a trademark and service mark of AT&T Intellectual Property.
22 ******************************************************************************/
25 package org.onap.dmaap.datarouter.node;
27 import static java.lang.System.exit;
28 import static java.lang.System.getProperty;
30 import com.att.eelf.configuration.EELFLogger;
31 import com.att.eelf.configuration.EELFManager;
33 import java.io.FileInputStream;
34 import java.io.IOException;
35 import java.io.InputStreamReader;
36 import java.io.Reader;
38 import java.nio.file.Files;
39 import java.util.Objects;
40 import java.util.Properties;
41 import java.util.Timer;
42 import org.onap.dmaap.datarouter.node.eelf.EelfMsgs;
46 * Maintain the configuration of a Data Router node
48 * <p>The NodeConfigManager is the single point of contact for servlet, delivery, event logging, and log retention
49 * subsystems to access configuration information.
51 * <p>There are two basic sets of configuration data. The static local configuration data, stored in a local
52 * configuration file (created as part of installation by SWM), and the dynamic global configuration data fetched from
53 * the data router provisioning server.
55 public class NodeConfigManager implements DeliveryQueueHelper {
57 private static final String NODE_CONFIG_MANAGER = "NodeConfigManager";
58 private static final EELFLogger eelfLogger = EELFManager.getInstance().getLogger(NodeConfigManager.class);
59 private static NodeConfigManager base;
61 private long maxfailuretimer;
62 private long initfailuretimer;
63 private long waitForFileProcessFailureTimer;
64 private long expirationtimer;
65 private double failurebackoff;
66 private long fairtimelimit;
67 private int fairfilelimit;
68 private double fdpstart;
69 private double fdpstop;
70 private int deliverythreads;
71 private final String provurl;
72 private String provhost;
73 private final int intHttpPort;
74 private final int intHttpsPort;
75 private final int extHttpsPort;
76 private String[] enabledprotocols;
77 private final boolean cadiEnabled;
78 private String aafType;
79 private String aafInstance;
80 private String aafAction;
81 private final boolean tlsEnabled;
82 private String kstype;
83 private String ksfile;
84 private String kspass;
86 private String tstype;
87 private String tsfile;
88 private String tspass;
89 private String myname;
90 private final String nak;
91 private final File quiesce;
92 private final String spooldir;
93 private final String logdir;
94 private final long logretention;
95 private final String eventlogurl;
96 private final String eventlogprefix;
97 private final String eventlogsuffix;
98 private String eventloginterval;
99 private boolean followredirects;
100 private final TaskList configtasks = new TaskList();
101 private final PublishId publishId;
102 private final IsFrom provcheck;
103 private final RedirManager rdmgr;
104 private final Timer timer = new Timer("Node Configuration Timer", true);
105 private final RateLimitedOperation pfetcher;
106 private NodeConfig config;
107 private NodeAafPropsUtils nodeAafPropsUtils;
108 private static Properties drNodeProperties;
110 public static Properties getDrNodeProperties() {
111 if (drNodeProperties == null) {
112 try (FileInputStream props = new FileInputStream(getProperty(
113 "org.onap.dmaap.datarouter.node.properties",
114 "/opt/app/datartr/etc/node.properties"))) {
115 drNodeProperties = new Properties();
116 drNodeProperties.load(props);
117 } catch (IOException e) {
118 eelfLogger.error("Failed to load NODE properties: " + e.getMessage(), e);
122 return drNodeProperties;
125 * Initialize the configuration of a Data Router node.
127 private NodeConfigManager() {
128 provurl = getDrNodeProperties().getProperty("ProvisioningURL", "http://dmaap-dr-prov:8080/internal/prov");
130 provhost = (new URL(provurl)).getHost();
131 } catch (Exception e) {
132 NodeUtils.setIpAndFqdnForEelf(NODE_CONFIG_MANAGER);
133 eelfLogger.error(EelfMsgs.MESSAGE_BAD_PROV_URL, e, provurl);
136 eelfLogger.debug("NODE0303 Provisioning server is at: " + provhost);
137 provcheck = new IsFrom(provhost);
139 cadiEnabled = Boolean.parseBoolean(getDrNodeProperties().getProperty("CadiEnabled", "false"));
141 aafType = getDrNodeProperties().getProperty("AAFType", "org.onap.dmaap-dr.feed");
142 aafInstance = getDrNodeProperties().getProperty("AAFInstance", "legacy");
143 aafAction = getDrNodeProperties().getProperty("AAFAction", "publish");
145 tlsEnabled = Boolean.parseBoolean(getDrNodeProperties().getProperty("TlsEnabled", "true"));
146 if (isTlsEnabled()) {
148 kstype = getDrNodeProperties().getProperty("KeyStoreType", "PKCS12");
149 tstype = getDrNodeProperties().getProperty("TrustStoreType", "jks");
150 enabledprotocols = ((getDrNodeProperties().getProperty("NodeHttpsProtocols")).trim()).split("\\|");
151 nodeAafPropsUtils = new NodeAafPropsUtils(new File(getDrNodeProperties()
152 .getProperty("AAFPropsFilePath", "/opt/app/osaaf/local/org.onap.dmaap-dr.props")));
154 if (tsfile != null && tsfile.length() > 0) {
155 System.setProperty("javax.net.ssl.trustStoreType", tstype);
156 System.setProperty("javax.net.ssl.trustStore", tsfile);
157 System.setProperty("javax.net.ssl.trustStorePassword", tspass);
159 myname = NodeUtils.getCanonicalName(kstype, ksfile, kspass);
160 if (myname == null) {
161 NodeUtils.setIpAndFqdnForEelf(NODE_CONFIG_MANAGER);
162 eelfLogger.error(EelfMsgs.MESSAGE_KEYSTORE_FETCH_ERROR, ksfile);
163 eelfLogger.error("NODE0309 Unable to fetch canonical name from keystore file " + ksfile);
166 eelfLogger.debug("NODE0304 My certificate says my name is " + myname);
167 } catch (Exception e) {
168 eelfLogger.error("NODE0314 Failed to load AAF props. Exiting", e);
172 myname = "dmaap-dr-node";
174 eventlogurl = getDrNodeProperties().getProperty("LogUploadURL", "https://feeds-drtr.web.att.com/internal/logs");
175 intHttpPort = Integer.parseInt(getDrNodeProperties().getProperty("IntHttpPort", "80"));
176 intHttpsPort = Integer.parseInt(getDrNodeProperties().getProperty("IntHttpsPort", "443"));
177 extHttpsPort = Integer.parseInt(getDrNodeProperties().getProperty("ExtHttpsPort", "443"));
178 spooldir = getDrNodeProperties().getProperty("SpoolDir", "spool");
180 File fdir = new File(spooldir + "/f");
182 for (File junk : Objects.requireNonNull(fdir.listFiles())) {
184 Files.deleteIfExists(junk.toPath());
185 } catch (IOException e) {
186 eelfLogger.error("NODE0313 Failed to clear junk files from " + fdir.getPath(), e);
189 logdir = getDrNodeProperties().getProperty("LogDir", "logs");
190 (new File(logdir)).mkdirs();
191 logretention = Long.parseLong(getDrNodeProperties().getProperty("LogRetention", "30")) * 86400000L;
192 eventlogprefix = logdir + "/events";
193 eventlogsuffix = ".log";
194 String redirfile = getDrNodeProperties().getProperty("RedirectionFile", "etc/redirections.dat");
195 publishId = new PublishId(myname);
196 nak = getDrNodeProperties().getProperty("NodeAuthKey", "Node123!");
197 quiesce = new File(getDrNodeProperties().getProperty("QuiesceFile", "etc/SHUTDOWN"));
198 rdmgr = new RedirManager(redirfile,
199 Long.parseLong(getDrNodeProperties().getProperty("MinRedirSaveInterval", "10000")), timer);
200 pfetcher = new RateLimitedOperation(
201 Long.parseLong(getDrNodeProperties().getProperty("MinProvFetchInterval", "10000")), timer) {
206 eelfLogger.debug("NODE0305 Attempting to fetch configuration at " + provurl);
210 private void getSslContextData() {
211 ksfile = nodeAafPropsUtils.getPropAccess().getProperty("cadi_keystore");
212 kspass = nodeAafPropsUtils.getDecryptedPass("cadi_keystore_password");
213 kpass = nodeAafPropsUtils.getDecryptedPass("cadi_keystore_password");
214 tsfile = nodeAafPropsUtils.getPropAccess().getProperty("cadi_truststore");
215 tspass = nodeAafPropsUtils.getDecryptedPass("cadi_truststore_password");
219 * Get the default node configuration manager.
221 public static NodeConfigManager getInstance() {
223 base = new NodeConfigManager();
228 private void localconfig() {
229 followredirects = Boolean.parseBoolean(getProvParam("FOLLOW_REDIRECTS", "false"));
230 eventloginterval = getProvParam("LOGROLL_INTERVAL", "30s");
231 initfailuretimer = 10000;
232 waitForFileProcessFailureTimer = 600000;
233 maxfailuretimer = 3600000;
234 expirationtimer = 86400000;
235 failurebackoff = 2.0;
236 deliverythreads = 40;
238 fairtimelimit = 60000;
242 initfailuretimer = (long) (Double.parseDouble(getProvParam("DELIVERY_INIT_RETRY_INTERVAL")) * 1000);
243 } catch (Exception e) {
244 eelfLogger.trace("Error parsing DELIVERY_INIT_RETRY_INTERVAL", e);
247 waitForFileProcessFailureTimer = (long) (Double.parseDouble(getProvParam("DELIVERY_FILE_PROCESS_INTERVAL"))
249 } catch (Exception e) {
250 eelfLogger.trace("Error parsing DELIVERY_FILE_PROCESS_INTERVAL", e);
253 maxfailuretimer = (long) (Double.parseDouble(getProvParam("DELIVERY_MAX_RETRY_INTERVAL")) * 1000);
254 } catch (Exception e) {
255 eelfLogger.trace("Error parsing DELIVERY_MAX_RETRY_INTERVAL", e);
258 expirationtimer = (long) (Double.parseDouble(getProvParam("DELIVERY_MAX_AGE")) * 1000);
259 } catch (Exception e) {
260 eelfLogger.trace("Error parsing DELIVERY_MAX_AGE", e);
263 failurebackoff = Double.parseDouble(getProvParam("DELIVERY_RETRY_RATIO"));
264 } catch (Exception e) {
265 eelfLogger.trace("Error parsing DELIVERY_RETRY_RATIO", e);
268 deliverythreads = Integer.parseInt(getProvParam("DELIVERY_THREADS"));
269 } catch (Exception e) {
270 eelfLogger.trace("Error parsing DELIVERY_THREADS", e);
273 fairfilelimit = Integer.parseInt(getProvParam("FAIR_FILE_LIMIT"));
274 } catch (Exception e) {
275 eelfLogger.trace("Error parsing FAIR_FILE_LIMIT", e);
278 fairtimelimit = (long) (Double.parseDouble(getProvParam("FAIR_TIME_LIMIT")) * 1000);
279 } catch (Exception e) {
280 eelfLogger.trace("Error parsing FAIR_TIME_LIMIT", e);
283 fdpstart = Double.parseDouble(getProvParam("FREE_DISK_RED_PERCENT")) / 100.0;
284 } catch (Exception e) {
285 eelfLogger.trace("Error parsing FREE_DISK_RED_PERCENT", e);
288 fdpstop = Double.parseDouble(getProvParam("FREE_DISK_YELLOW_PERCENT")) / 100.0;
289 } catch (Exception e) {
290 eelfLogger.trace("Error parsing FREE_DISK_YELLOW_PERCENT", e);
292 if (fdpstart < 0.01) {
295 if (fdpstart > 0.5) {
298 if (fdpstop < fdpstart) {
306 private void fetchconfig() {
308 eelfLogger.debug("NodeConfigMan.fetchConfig: provurl:: " + provurl);
309 URL url = new URL(provurl);
310 Reader reader = new InputStreamReader(url.openStream());
311 config = new NodeConfig(new ProvData(reader), myname, spooldir, extHttpsPort, nak);
313 configtasks.startRun();
315 } catch (Exception e) {
316 NodeUtils.setIpAndFqdnForEelf("fetchconfigs");
317 eelfLogger.error(EelfMsgs.MESSAGE_CONF_FAILED, e.toString());
318 eelfLogger.error("NODE0306 Configuration failed " + e + " - try again later", e);
323 private void runTasks() {
325 while ((rr = configtasks.next()) != null) {
328 } catch (Exception e) {
329 eelfLogger.error("NODE0518 Exception fetchconfig: " + e);
335 * Process a gofetch request from a particular IP address. If the IP address is not an IP address we would go to to
336 * fetch the provisioning data, ignore the request. If the data has been fetched very recently (default 10
337 * seconds), wait a while before fetching again.
339 synchronized void gofetch(String remoteAddr) {
340 if (provcheck.isReachable(remoteAddr)) {
341 eelfLogger.debug("NODE0307 Received configuration fetch request from provisioning server " + remoteAddr);
344 eelfLogger.debug("NODE0308 Received configuration fetch request from unexpected server " + remoteAddr);
351 boolean isConfigured() {
352 return config != null;
358 boolean isShutdown() {
359 return quiesce.exists();
363 * Given a routing string, get the targets.
365 * @param routing Target string
366 * @return array of targets
368 Target[] parseRouting(String routing) {
369 return config.parseRouting(routing);
373 * Given a set of credentials and an IP address, is this request from another node.
375 * @param credentials Credentials offered by the supposed node
376 * @param ip IP address the request came from
377 * @return If the credentials and IP address are recognized, true, otherwise false.
379 boolean isAnotherNode(String credentials, String ip) {
380 return config.isAnotherNode(credentials, ip);
384 * Check whether publication is allowed.
386 * @param feedid The ID of the feed being requested
387 * @param credentials The offered credentials
388 * @param ip The requesting IP address
389 * @return True if the IP and credentials are valid for the specified feed.
391 String isPublishPermitted(String feedid, String credentials, String ip) {
392 return config.isPublishPermitted(feedid, credentials, ip);
396 * Check whether publication is allowed for AAF Feed.
398 * @param feedid The ID of the feed being requested
399 * @param ip The requesting IP address
400 * @return True if the IP and credentials are valid for the specified feed.
402 String isPublishPermitted(String feedid, String ip) {
403 return config.isPublishPermitted(feedid, ip);
407 * Check whether delete file is allowed.
409 * @param subId The ID of the subscription being requested
410 * @return True if the delete file is permitted for the subscriber.
412 boolean isDeletePermitted(String subId) {
413 return config.isDeletePermitted(subId);
417 * Check who the user is given the feed ID and the offered credentials.
419 * @param feedid The ID of the feed specified
420 * @param credentials The offered credentials
421 * @return Null if the credentials are invalid or the user if they are valid.
423 String getAuthUser(String feedid, String credentials) {
424 return config.getAuthUser(feedid, credentials);
428 * AAF changes: TDP EPIC US# 307413 Check AAF_instance for feed ID in NodeConfig.
430 * @param feedid The ID of the feed specified
432 String getAafInstance(String feedid) {
433 return config.getAafInstance(feedid);
436 String getAafInstance() {
441 * Check if the publish request should be sent to another node based on the feedid, user, and source IP address.
443 * @param feedid The ID of the feed specified
444 * @param user The publishing user
445 * @param ip The IP address of the publish endpoint
446 * @return Null if the request should be accepted or the correct hostname if it should be sent to another node.
448 String getIngressNode(String feedid, String user, String ip) {
449 return config.getIngressNode(feedid, user, ip);
453 * Get a provisioned configuration parameter (from the provisioning server configuration).
455 * @param name The name of the parameter
456 * @return The value of the parameter or null if it is not defined.
458 private String getProvParam(String name) {
459 return config.getProvParam(name);
463 * Get a provisioned configuration parameter (from the provisioning server configuration).
465 * @param name The name of the parameter
466 * @param defaultValue The value to use if the parameter is not defined
467 * @return The value of the parameter or deflt if it is not defined.
469 private String getProvParam(String name, String defaultValue) {
470 name = config.getProvParam(name);
478 * Generate a publish ID.
480 public String getPublishId() {
481 return publishId.next();
485 * Get all the outbound spooling destinations. This will include both subscriptions and nodes.
487 DestInfo[] getAllDests() {
488 return config.getAllDests();
492 * Register a task to run whenever the configuration changes.
494 void registerConfigTask(Runnable task) {
495 configtasks.addTask(task);
499 * Deregister a task to run whenever the configuration changes.
501 void deregisterConfigTask(Runnable task) {
502 configtasks.removeTask(task);
506 * Get the URL to deliver a message to.
508 * @param destinationInfo The destination information
509 * @param fileid The file ID
510 * @return The URL to deliver to
512 public String getDestURL(DestInfo destinationInfo, String fileid) {
513 String subid = destinationInfo.getSubId();
514 String purl = destinationInfo.getURL();
515 if (followredirects && subid != null) {
516 purl = rdmgr.lookup(subid, purl);
518 return (purl + "/" + fileid);
522 * Set up redirection on receipt of a 3XX from a target URL.
524 public boolean handleRedirection(DestInfo destinationInfo, String redirto, String fileid) {
525 fileid = "/" + fileid;
526 String subid = destinationInfo.getSubId();
527 String purl = destinationInfo.getURL();
528 if (followredirects && subid != null && redirto.endsWith(fileid)) {
529 redirto = redirto.substring(0, redirto.length() - fileid.length());
530 if (!redirto.equals(purl)) {
531 rdmgr.redirect(subid, purl, redirto);
539 * Handle unreachable target URL.
541 public void handleUnreachable(DestInfo destinationInfo) {
542 String subid = destinationInfo.getSubId();
543 if (followredirects && subid != null) {
549 * Get the timeout before retrying after an initial delivery failure.
551 public long getInitFailureTimer() {
552 return initfailuretimer;
556 * Get the timeout before retrying after delivery and wait for file processing.
558 public long getWaitForFileProcessFailureTimer() {
559 return waitForFileProcessFailureTimer;
563 * Get the maximum timeout between delivery attempts.
565 public long getMaxFailureTimer() {
566 return maxfailuretimer;
570 * Get the ratio between consecutive delivery attempts.
572 public double getFailureBackoff() {
573 return failurebackoff;
577 * Get the expiration timer for deliveries.
579 public long getExpirationTimer() {
580 return expirationtimer;
584 * Get the maximum number of file delivery attempts before checking if another queue has work to be performed.
586 public int getFairFileLimit() {
587 return fairfilelimit;
591 * Get the maximum amount of time spent delivering files before checking if another queue has work to be performed.
593 public long getFairTimeLimit() {
594 return fairtimelimit;
598 * Get the targets for a feed.
600 * @param feedid The feed ID
601 * @return The targets this feed should be delivered to
603 Target[] getTargets(String feedid) {
604 return config.getTargets(feedid);
608 * Get the spool directory for temporary files.
610 String getSpoolDir() {
611 return spooldir + "/f";
615 * Get the spool directory for a subscription.
617 String getSpoolDir(String subid, String remoteaddr) {
618 if (provcheck.isFrom(remoteaddr)) {
619 String sdir = config.getSpoolDir(subid);
621 eelfLogger.debug("NODE0310 Received subscription reset request for subscription " + subid
622 + " from provisioning server " + remoteaddr);
624 eelfLogger.debug("NODE0311 Received subscription reset request for unknown subscription " + subid
625 + " from provisioning server " + remoteaddr);
629 eelfLogger.debug("NODE0312 Received subscription reset request from unexpected server " + remoteaddr);
635 * Get the base directory for spool directories.
637 String getSpoolBase() {
642 * Get the key store type.
649 * Get the key store file.
656 * Get the key store password.
663 * Get the key password.
690 * Get the https port.
697 * Get the externally visible https port.
699 int getExtHttpsPort() {
704 * Get the external name of this machine.
711 * Get the number of threads to use for delivery.
713 int getDeliveryThreads() {
714 return deliverythreads;
718 * Get the URL for uploading the event log data.
720 String getEventLogUrl() {
725 * Get the prefix for the names of event log files.
727 String getEventLogPrefix() {
728 return eventlogprefix;
732 * Get the suffix for the names of the event log files.
734 String getEventLogSuffix() {
735 return eventlogsuffix;
739 * Get the interval between event log file rollovers.
741 String getEventLogInterval() {
742 return eventloginterval;
746 * Should I follow redirects from subscribers.
748 public boolean isFollowRedirects() {
749 return followredirects;
753 * Get the directory where the event and node log files live.
760 * How long do I keep log files (in milliseconds).
762 long getLogRetention() {
769 public Timer getTimer() {
774 * Get the feed ID for a subscription.
776 * @param subid The subscription ID
777 * @return The feed ID
779 public String getFeedId(String subid) {
780 return config.getFeedId(subid);
784 * Get the authorization string this node uses.
786 * @return The Authorization string for this node
789 return config.getMyAuth();
793 * Get the fraction of free spool disk space where we start throwing away undelivered files. This is
794 * FREE_DISK_RED_PERCENT / 100.0. Default is 0.05. Limited by 0.01 <= FreeDiskStart <= 0.5.
796 double getFreeDiskStart() {
801 * Get the fraction of free spool disk space where we stop throwing away undelivered files. This is
802 * FREE_DISK_YELLOW_PERCENT / 100.0. Default is 0.2. Limited by FreeDiskStart <= FreeDiskStop <= 0.5.
804 double getFreeDiskStop() {
809 * Disable and enable protocols.
811 String[] getEnabledprotocols() {
812 return enabledprotocols;
815 String getAafType() {
819 String getAafAction() {
823 protected boolean isTlsEnabled() {
827 boolean getCadiEnabled() {
831 NodeAafPropsUtils getNodeAafPropsUtils() {
832 return nodeAafPropsUtils;
836 * Builds the permissions string to be verified.
838 * @param aafInstance The aaf instance
839 * @return The permissions
841 String getPermission(String aafInstance) {
843 String type = getAafType();
844 String action = getAafAction();
845 if ("".equals(aafInstance)) {
846 aafInstance = getAafInstance();
848 return type + "|" + aafInstance + "|" + action;
849 } catch (Exception e) {
850 eelfLogger.error("NODE0543 NodeConfigManager.getPermission: ", e);