1 /*******************************************************************************
2 * ============LICENSE_START==================================================
4 * * ===========================================================================
5 * * Copyright © 2017 AT&T Intellectual Property. All rights reserved.
6 * * ===========================================================================
7 * * Licensed under the Apache License, Version 2.0 (the "License");
8 * * you may not use this file except in compliance with the License.
9 * * You may obtain a copy of the License at
11 * * http://www.apache.org/licenses/LICENSE-2.0
13 * * Unless required by applicable law or agreed to in writing, software
14 * * distributed under the License is distributed on an "AS IS" BASIS,
15 * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 * * See the License for the specific language governing permissions and
17 * * limitations under the License.
18 * * ============LICENSE_END====================================================
20 * * ECOMP is a trademark and service mark of AT&T Intellectual Property.
22 ******************************************************************************/
25 package org.onap.dmaap.datarouter.node;
27 import com.att.eelf.configuration.EELFLogger;
28 import com.att.eelf.configuration.EELFManager;
30 import java.io.FileInputStream;
31 import java.io.InputStreamReader;
32 import java.io.Reader;
34 import java.util.Properties;
35 import java.util.Timer;
36 import org.onap.dmaap.datarouter.node.eelf.EelfMsgs;
40 * Maintain the configuration of a Data Router node
42 * The NodeConfigManager is the single point of contact for servlet, delivery, event logging, and log retention
43 * subsystems to access configuration information.
45 * There are two basic sets of configuration data. The static local configuration data, stored in a local configuration
46 * file (created as part of installation by SWM), and the dynamic global configuration data fetched from the data router
47 * provisioning server.
49 public class NodeConfigManager implements DeliveryQueueHelper {
51 private static final String CHANGE_ME = "changeme";
52 private static final String NODE_CONFIG_MANAGER = "NodeConfigManager";
53 private static EELFLogger eelfLogger = EELFManager.getInstance().getLogger(NodeConfigManager.class);
54 private static NodeConfigManager base = new NodeConfigManager();
56 private Timer timer = new Timer("Node Configuration Timer", true);
57 private long maxfailuretimer;
58 private long initfailuretimer;
59 private long waitForFileProcessFailureTimer;
60 private long expirationtimer;
61 private double failurebackoff;
62 private long fairtimelimit;
63 private int fairfilelimit;
64 private double fdpstart;
65 private double fdpstop;
66 private int deliverythreads;
67 private String provurl;
68 private String provhost;
69 private IsFrom provcheck;
73 private String spooldir;
74 private String logdir;
75 private long logretention;
76 private String redirfile;
77 private String kstype;
78 private String ksfile;
79 private String kspass;
81 private String tstype;
82 private String tsfile;
83 private String tspass;
84 private String myname;
85 private RedirManager rdmgr;
86 private RateLimitedOperation pfetcher;
87 private NodeConfig config;
89 private PublishId pid;
91 private TaskList configtasks = new TaskList();
92 private String eventlogurl;
93 private String eventlogprefix;
94 private String eventlogsuffix;
95 private String eventloginterval;
96 private boolean followredirects;
97 private String[] enabledprotocols;
98 private String aafType;
99 private String aafInstance;
100 private String aafAction;
101 private String aafURL;
102 private boolean cadiEnabled;
106 * Initialize the configuration of a Data Router node
108 private NodeConfigManager() {
110 Properties drNodeProperties = new Properties();
112 eelfLogger.info("NODE0301 Loading local config file node.properties");
113 drNodeProperties.load(new FileInputStream(System
114 .getProperty("org.onap.dmaap.datarouter.node.properties", "/opt/app/datartr/etc/node.properties")));
115 } catch (Exception e) {
116 NodeUtils.setIpAndFqdnForEelf(NODE_CONFIG_MANAGER);
117 eelfLogger.error(EelfMsgs.MESSAGE_PROPERTIES_LOAD_ERROR, e,
118 System.getProperty("org.onap.dmaap.datarouter.node.properties",
119 "/opt/app/datartr/etc/node.properties"));
121 provurl = drNodeProperties.getProperty("ProvisioningURL", "https://dmaap-dr-prov:8443/internal/prov");
123 * START - AAF changes: TDP EPIC US# 307413
124 * Pull AAF settings from node.properties
126 aafType = drNodeProperties.getProperty("AAFType", "org.onap.dmaap-dr.feed");
127 aafInstance = drNodeProperties.getProperty("AAFInstance", "legacy");
128 aafAction = drNodeProperties.getProperty("AAFAction", "publish");
129 aafURL = drNodeProperties.getProperty("AafUrl", "https://aaf-onap-test.osaaf.org:8095");
130 cadiEnabled = Boolean.parseBoolean(drNodeProperties.getProperty("CadiEnabled", "false"));
132 * END - AAF changes: TDP EPIC US# 307413
133 * Pull AAF settings from node.properties
135 //Disable and enable protocols*/
136 enabledprotocols = ((drNodeProperties.getProperty("NodeHttpsProtocols")).trim()).split("\\|");
139 provhost = (new URL(provurl)).getHost();
140 } catch (Exception e) {
141 NodeUtils.setIpAndFqdnForEelf(NODE_CONFIG_MANAGER);
142 eelfLogger.error(EelfMsgs.MESSAGE_BAD_PROV_URL, e, provurl);
145 eelfLogger.info("NODE0303 Provisioning server is " + provhost);
146 eventlogurl = drNodeProperties.getProperty("LogUploadURL", "https://feeds-drtr.web.att.com/internal/logs");
147 provcheck = new IsFrom(provhost);
148 gfport = Integer.parseInt(drNodeProperties.getProperty("IntHttpPort", "8080"));
149 svcport = Integer.parseInt(drNodeProperties.getProperty("IntHttpsPort", "8443"));
150 port = Integer.parseInt(drNodeProperties.getProperty("ExtHttpsPort", "443"));
151 long minpfinterval = Long.parseLong(drNodeProperties.getProperty("MinProvFetchInterval", "10000"));
152 long minrsinterval = Long.parseLong(drNodeProperties.getProperty("MinRedirSaveInterval", "10000"));
153 spooldir = drNodeProperties.getProperty("SpoolDir", "spool");
154 File fdir = new File(spooldir + "/f");
156 for (File junk : fdir.listFiles()) {
161 logdir = drNodeProperties.getProperty("LogDir", "logs");
162 (new File(logdir)).mkdirs();
163 logretention = Long.parseLong(drNodeProperties.getProperty("LogRetention", "30")) * 86400000L;
164 eventlogprefix = logdir + "/events";
165 eventlogsuffix = ".log";
166 redirfile = drNodeProperties.getProperty("RedirectionFile", "etc/redirections.dat");
167 kstype = drNodeProperties.getProperty("KeyStoreType", "jks");
168 ksfile = drNodeProperties.getProperty("KeyStoreFile", "etc/keystore");
169 kspass = drNodeProperties.getProperty("KeyStorePassword", CHANGE_ME);
170 kpass = drNodeProperties.getProperty("KeyPassword", CHANGE_ME);
171 tstype = drNodeProperties.getProperty("TrustStoreType", "jks");
172 tsfile = drNodeProperties.getProperty("TrustStoreFile");
173 tspass = drNodeProperties.getProperty("TrustStorePassword", CHANGE_ME);
174 if (tsfile != null && tsfile.length() > 0) {
175 System.setProperty("javax.net.ssl.trustStoreType", tstype);
176 System.setProperty("javax.net.ssl.trustStore", tsfile);
177 System.setProperty("javax.net.ssl.trustStorePassword", tspass);
179 nak = drNodeProperties.getProperty("NodeAuthKey", "Node123!");
180 quiesce = new File(drNodeProperties.getProperty("QuiesceFile", "etc/SHUTDOWN"));
181 myname = NodeUtils.getCanonicalName(kstype, ksfile, kspass);
182 if (myname == null) {
183 NodeUtils.setIpAndFqdnForEelf(NODE_CONFIG_MANAGER);
184 eelfLogger.error(EelfMsgs.MESSAGE_KEYSTORE_FETCH_ERROR, ksfile);
185 eelfLogger.error("NODE0309 Unable to fetch canonical name from keystore file " + ksfile);
188 eelfLogger.info("NODE0304 My certificate says my name is " + myname);
189 pid = new PublishId(myname);
190 rdmgr = new RedirManager(redirfile, minrsinterval, timer);
191 pfetcher = new RateLimitedOperation(minpfinterval, timer) {
196 eelfLogger.info("NODE0305 Attempting to fetch configuration at " + provurl);
201 * Get the default node configuration manager
203 public static NodeConfigManager getInstance() {
207 private void localconfig() {
208 followredirects = Boolean.parseBoolean(getProvParam("FOLLOW_REDIRECTS", "false"));
209 eventloginterval = getProvParam("LOGROLL_INTERVAL", "30s");
210 initfailuretimer = 10000;
211 waitForFileProcessFailureTimer = 600000;
212 maxfailuretimer = 3600000;
213 expirationtimer = 86400000;
214 failurebackoff = 2.0;
215 deliverythreads = 40;
217 fairtimelimit = 60000;
221 initfailuretimer = (long) (Double.parseDouble(getProvParam("DELIVERY_INIT_RETRY_INTERVAL")) * 1000);
222 } catch (Exception e) {
223 eelfLogger.trace("Error parsing DELIVERY_INIT_RETRY_INTERVAL", e);
226 waitForFileProcessFailureTimer = (long) (Double.parseDouble(getProvParam("DELIVERY_FILE_PROCESS_INTERVAL"))
228 } catch (Exception e) {
229 eelfLogger.trace("Error parsing DELIVERY_FILE_PROCESS_INTERVAL", e);
232 maxfailuretimer = (long) (Double.parseDouble(getProvParam("DELIVERY_MAX_RETRY_INTERVAL")) * 1000);
233 } catch (Exception e) {
234 eelfLogger.trace("Error parsing DELIVERY_MAX_RETRY_INTERVAL", e);
237 expirationtimer = (long) (Double.parseDouble(getProvParam("DELIVERY_MAX_AGE")) * 1000);
238 } catch (Exception e) {
239 eelfLogger.trace("Error parsing DELIVERY_MAX_AGE", e);
242 failurebackoff = Double.parseDouble(getProvParam("DELIVERY_RETRY_RATIO"));
243 } catch (Exception e) {
244 eelfLogger.trace("Error parsing DELIVERY_RETRY_RATIO", e);
247 deliverythreads = Integer.parseInt(getProvParam("DELIVERY_THREADS"));
248 } catch (Exception e) {
249 eelfLogger.trace("Error parsing DELIVERY_THREADS", e);
252 fairfilelimit = Integer.parseInt(getProvParam("FAIR_FILE_LIMIT"));
253 } catch (Exception e) {
254 eelfLogger.trace("Error parsing FAIR_FILE_LIMIT", e);
257 fairtimelimit = (long) (Double.parseDouble(getProvParam("FAIR_TIME_LIMIT")) * 1000);
258 } catch (Exception e) {
259 eelfLogger.trace("Error parsing FAIR_TIME_LIMIT", e);
262 fdpstart = Double.parseDouble(getProvParam("FREE_DISK_RED_PERCENT")) / 100.0;
263 } catch (Exception e) {
264 eelfLogger.trace("Error parsing FREE_DISK_RED_PERCENT", e);
267 fdpstop = Double.parseDouble(getProvParam("FREE_DISK_YELLOW_PERCENT")) / 100.0;
268 } catch (Exception e) {
269 eelfLogger.trace("Error parsing FREE_DISK_YELLOW_PERCENT", e);
271 if (fdpstart < 0.01) {
274 if (fdpstart > 0.5) {
277 if (fdpstop < fdpstart) {
285 private void fetchconfig() {
287 eelfLogger.info("NodeConfigMan.fetchConfig: provurl:: " + provurl);
288 Reader r = new InputStreamReader((new URL(provurl)).openStream());
289 config = new NodeConfig(new ProvData(r), myname, spooldir, port, nak);
291 configtasks.startRun();
293 } catch (Exception e) {
294 NodeUtils.setIpAndFqdnForEelf("fetchconfigs");
295 eelfLogger.error(EelfMsgs.MESSAGE_CONF_FAILED, e.toString());
296 eelfLogger.error("NODE0306 Configuration failed " + e.toString() + " - try again later", e);
301 private void runTasks() {
303 while ((rr = configtasks.next()) != null) {
306 } catch (Exception e) {
307 eelfLogger.error("NODE0518 Exception fetchconfig: " + e);
313 * Process a gofetch request from a particular IP address. If the IP address is not an IP address we would go to to
314 * fetch the provisioning data, ignore the request. If the data has been fetched very recently (default 10
315 * seconds), wait a while before fetching again.
317 public synchronized void gofetch(String remoteAddr) {
318 if (provcheck.isReachable(remoteAddr)) {
319 eelfLogger.info("NODE0307 Received configuration fetch request from provisioning server " + remoteAddr);
322 eelfLogger.info("NODE0308 Received configuration fetch request from unexpected server " + remoteAddr);
329 public boolean isConfigured() {
330 return (config != null);
336 public boolean isShutdown() {
337 return (quiesce.exists());
341 * Given a routing string, get the targets.
343 * @param routing Target string
344 * @return array of targets
346 public Target[] parseRouting(String routing) {
347 return (config.parseRouting(routing));
351 * Given a set of credentials and an IP address, is this request from another node?
353 * @param credentials Credentials offered by the supposed node
354 * @param ip IP address the request came from
355 * @return If the credentials and IP address are recognized, true, otherwise false.
357 public boolean isAnotherNode(String credentials, String ip) {
358 return (config.isAnotherNode(credentials, ip));
362 * Check whether publication is allowed.
364 * @param feedid The ID of the feed being requested
365 * @param credentials The offered credentials
366 * @param ip The requesting IP address
367 * @return True if the IP and credentials are valid for the specified feed.
369 public String isPublishPermitted(String feedid, String credentials, String ip) {
370 return (config.isPublishPermitted(feedid, credentials, ip));
374 * Check whether delete file is allowed.
376 * @param subId The ID of the subscription being requested
377 * @return True if the delete file is permitted for the subscriber.
379 public boolean isDeletePermitted(String subId) {
380 return (config.isDeletePermitted(subId));
384 * Check whether publication is allowed for AAF Feed.
386 * @param feedid The ID of the feed being requested
387 * @param ip The requesting IP address
388 * @return True if the IP and credentials are valid for the specified feed.
390 public String isPublishPermitted(String feedid, String ip) {
391 return (config.isPublishPermitted(feedid, ip));
395 * Check who the user is given the feed ID and the offered credentials.
397 * @param feedid The ID of the feed specified
398 * @param credentials The offered credentials
399 * @return Null if the credentials are invalid or the user if they are valid.
401 public String getAuthUser(String feedid, String credentials) {
402 return (config.getAuthUser(feedid, credentials));
406 * AAF changes: TDP EPIC US# 307413 Check AAF_instance for feed ID in NodeConfig
408 * @param feedid The ID of the feed specified
410 public String getAafInstance(String feedid) {
411 return (config.getAafInstance(feedid));
415 * Check if the publish request should be sent to another node based on the feedid, user, and source IP address.
417 * @param feedid The ID of the feed specified
418 * @param user The publishing user
419 * @param ip The IP address of the publish endpoint
420 * @return Null if the request should be accepted or the correct hostname if it should be sent to another node.
422 public String getIngressNode(String feedid, String user, String ip) {
423 return (config.getIngressNode(feedid, user, ip));
427 * Get a provisioned configuration parameter (from the provisioning server configuration)
429 * @param name The name of the parameter
430 * @return The value of the parameter or null if it is not defined.
432 public String getProvParam(String name) {
433 return (config.getProvParam(name));
437 * Get a provisioned configuration parameter (from the provisioning server configuration)
439 * @param name The name of the parameter
440 * @param defaultValue The value to use if the parameter is not defined
441 * @return The value of the parameter or deflt if it is not defined.
443 public String getProvParam(String name, String defaultValue) {
444 name = config.getProvParam(name);
452 * Generate a publish ID
454 public String getPublishId() {
459 * Get all the outbound spooling destinations. This will include both subscriptions and nodes.
461 public DestInfo[] getAllDests() {
462 return (config.getAllDests());
466 * Register a task to run whenever the configuration changes
468 public void registerConfigTask(Runnable task) {
469 configtasks.addTask(task);
473 * Deregister a task to run whenever the configuration changes
475 public void deregisterConfigTask(Runnable task) {
476 configtasks.removeTask(task);
480 * Get the URL to deliver a message to.
482 * @param destinationInfo The destination information
483 * @param fileid The file ID
484 * @return The URL to deliver to
486 public String getDestURL(DestInfo destinationInfo, String fileid) {
487 String subid = destinationInfo.getSubId();
488 String purl = destinationInfo.getURL();
489 if (followredirects && subid != null) {
490 purl = rdmgr.lookup(subid, purl);
492 return (purl + "/" + fileid);
496 * Is a destination redirected?
498 public boolean isDestRedirected(DestInfo destinfo) {
499 return (followredirects && rdmgr.isRedirected(destinfo.getSubId()));
503 * Set up redirection on receipt of a 3XX from a target URL
505 public boolean handleRedirection(DestInfo destinationInfo, String redirto, String fileid) {
506 fileid = "/" + fileid;
507 String subid = destinationInfo.getSubId();
508 String purl = destinationInfo.getURL();
509 if (followredirects && subid != null && redirto.endsWith(fileid)) {
510 redirto = redirto.substring(0, redirto.length() - fileid.length());
511 if (!redirto.equals(purl)) {
512 rdmgr.redirect(subid, purl, redirto);
520 * Set up redirection on receipt of a 3XX from a target URL
522 public boolean handleRedirectionSubLevel(DeliveryTask task, DestInfo destinfo, String redirto, String fileid) {
523 fileid = "/" + fileid;
524 String subid = destinfo.getSubId();
525 String purl = destinfo.getURL();
526 if (task.getFollowRedirects() && subid != null && redirto.endsWith(fileid)) {
527 redirto = redirto.substring(0, redirto.length() - fileid.length());
528 if (!redirto.equals(purl)) {
529 rdmgr.redirect(subid, purl, redirto);
537 * Handle unreachable target URL
539 public void handleUnreachable(DestInfo destinationInfo) {
540 String subid = destinationInfo.getSubId();
541 if (followredirects && subid != null) {
547 * Get the timeout before retrying after an initial delivery failure
549 public long getInitFailureTimer() {
550 return (initfailuretimer);
554 * Get the timeout before retrying after delivery and wait for file processing
556 public long getWaitForFileProcessFailureTimer() {
557 return (waitForFileProcessFailureTimer);
561 * Get the maximum timeout between delivery attempts
563 public long getMaxFailureTimer() {
564 return (maxfailuretimer);
568 * Get the ratio between consecutive delivery attempts
570 public double getFailureBackoff() {
571 return (failurebackoff);
575 * Get the expiration timer for deliveries
577 public long getExpirationTimer() {
578 return (expirationtimer);
582 * Get the maximum number of file delivery attempts before checking if another queue has work to be performed.
584 public int getFairFileLimit() {
585 return (fairfilelimit);
589 * Get the maximum amount of time spent delivering files before checking if another queue has work to be performed.
591 public long getFairTimeLimit() {
592 return (fairtimelimit);
596 * Get the targets for a feed
598 * @param feedid The feed ID
599 * @return The targets this feed should be delivered to
601 public Target[] getTargets(String feedid) {
602 return (config.getTargets(feedid));
606 * Get the creation date for a feed
608 * @param feedid The feed ID
609 * @return the timestamp of creation date of feed id passed
611 public String getCreatedDate(String feedid) {
612 return (config.getCreatedDate(feedid));
616 * Get the spool directory for temporary files
618 public String getSpoolDir() {
619 return (spooldir + "/f");
623 * Get the base directory for spool directories
625 public String getSpoolBase() {
630 * Get the key store type
632 public String getKSType() {
637 * Get the key store file
639 public String getKSFile() {
644 * Get the key store password
646 public String getKSPass() {
651 * Get the key password
653 public String getKPass() {
660 public int getHttpPort() {
667 public int getHttpsPort() {
672 * Get the externally visible https port
674 public int getExtHttpsPort() {
679 * Get the external name of this machine
681 public String getMyName() {
686 * Get the number of threads to use for delivery
688 public int getDeliveryThreads() {
689 return (deliverythreads);
693 * Get the URL for uploading the event log data
695 public String getEventLogUrl() {
696 return (eventlogurl);
700 * Get the prefix for the names of event log files
702 public String getEventLogPrefix() {
703 return (eventlogprefix);
707 * Get the suffix for the names of the event log files
709 public String getEventLogSuffix() {
710 return (eventlogsuffix);
714 * Get the interval between event log file rollovers
716 public String getEventLogInterval() {
717 return (eventloginterval);
721 * Should I follow redirects from subscribers?
723 public boolean isFollowRedirects() {
724 return (followredirects);
728 * Get the directory where the event and node log files live
730 public String getLogDir() {
735 * How long do I keep log files (in milliseconds)
737 public long getLogRetention() {
738 return (logretention);
744 public Timer getTimer() {
749 * Get the feed ID for a subscription
751 * @param subid The subscription ID
752 * @return The feed ID
754 public String getFeedId(String subid) {
755 return (config.getFeedId(subid));
759 * Get the authorization string this node uses
761 * @return The Authorization string for this node
763 public String getMyAuth() {
764 return (config.getMyAuth());
768 * Get the fraction of free spool disk space where we start throwing away undelivered files. This is
769 * FREE_DISK_RED_PERCENT / 100.0. Default is 0.05. Limited by 0.01 <= FreeDiskStart <= 0.5.
771 public double getFreeDiskStart() {
776 * Get the fraction of free spool disk space where we stop throwing away undelivered files. This is
777 * FREE_DISK_YELLOW_PERCENT / 100.0. Default is 0.2. Limited by FreeDiskStart <= FreeDiskStop <= 0.5.
779 public double getFreeDiskStop() {
784 * Disable and enable protocols
786 public String[] getEnabledprotocols() {
787 return enabledprotocols;
790 public void setEnabledprotocols(String[] enabledprotocols) {
791 this.enabledprotocols = enabledprotocols.clone();
795 * Get the spool directory for a subscription
797 public String getSpoolDir(String subid, String remoteaddr) {
798 if (provcheck.isFrom(remoteaddr)) {
799 String sdir = config.getSpoolDir(subid);
801 eelfLogger.info("NODE0310 Received subscription reset request for subscription " + subid
802 + " from provisioning server " + remoteaddr);
804 eelfLogger.info("NODE0311 Received subscription reset request for unknown subscription " + subid
805 + " from provisioning server " + remoteaddr);
809 eelfLogger.info("NODE0312 Received subscription reset request from unexpected server " + remoteaddr);
814 public String getAafType() {
818 public void setAafType(String aafType) {
819 this.aafType = aafType;
822 public String getAafInstance() {
826 public void setAafInstance(String aafInstance) {
827 this.aafInstance = aafInstance;
830 public String getAafAction() {
834 public void setAafAction(String aafAction) {
835 this.aafAction = aafAction;
839 * Get aafURL from SWM variable
841 public String getAafURL() {
845 public void setAafURL(String aafURL) {
846 this.aafURL = aafURL;
849 public boolean getCadiEnabled() {
853 public void setCadiEnabled(boolean cadiEnabled) {
854 this.cadiEnabled = cadiEnabled;
858 * Builds the permissions string to be verified
860 * @param aafInstance The aaf instance
861 * @return The permissions
863 protected String getPermission(String aafInstance) {
865 String type = getAafType();
866 String action = getAafAction();
867 if ("".equals(aafInstance)) {
868 aafInstance = getAafInstance();
870 return type + "|" + aafInstance + "|" + action;
871 } catch (Exception e) {
872 eelfLogger.error("NODE0543 NodeConfigManager.getPermission: ", e);