1 /*******************************************************************************
2 * ============LICENSE_START==================================================
4 * * ===========================================================================
5 * * Copyright © 2017 AT&T Intellectual Property. All rights reserved.
6 * * ===========================================================================
7 * * Licensed under the Apache License, Version 2.0 (the "License");
8 * * you may not use this file except in compliance with the License.
9 * * You may obtain a copy of the License at
11 * * http://www.apache.org/licenses/LICENSE-2.0
13 * * Unless required by applicable law or agreed to in writing, software
14 * * distributed under the License is distributed on an "AS IS" BASIS,
15 * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 * * See the License for the specific language governing permissions and
17 * * limitations under the License.
18 * * ============LICENSE_END====================================================
20 * * ECOMP is a trademark and service mark of AT&T Intellectual Property.
22 ******************************************************************************/
25 package org.onap.dmaap.datarouter.node;
27 import com.att.eelf.configuration.EELFLogger;
28 import com.att.eelf.configuration.EELFManager;
29 import org.apache.log4j.Logger;
30 import org.onap.aaf.cadi.PropAccess;
31 import org.onap.dmaap.datarouter.node.eelf.EelfMsgs;
34 import java.io.FileInputStream;
35 import java.io.InputStreamReader;
36 import java.io.Reader;
38 import java.util.Properties;
39 import java.util.Timer;
43 * Maintain the configuration of a Data Router node
45 * The NodeConfigManager is the single point of contact for servlet, delivery, event logging, and log retention
46 * subsystems to access configuration information. (Log4J has its own configuration mechanism).
48 * There are two basic sets of configuration data. The static local configuration data, stored in a local configuration
49 * file (created as part of installation by SWM), and the dynamic global configuration data fetched from the data router
50 * provisioning server.
52 public class NodeConfigManager implements DeliveryQueueHelper {
54 private static EELFLogger eelflogger = EELFManager.getInstance()
55 .getLogger(NodeConfigManager.class);
56 private static Logger logger = Logger.getLogger("org.onap.dmaap.datarouter.node.NodeConfigManager");
57 private static NodeConfigManager base = new NodeConfigManager();
59 private Timer timer = new Timer("Node Configuration Timer", true);
60 private long maxfailuretimer;
61 private long initfailuretimer;
62 private long waitForFileProcessFailureTimer;
63 private long expirationtimer;
64 private double failurebackoff;
65 private long fairtimelimit;
66 private int fairfilelimit;
67 private double fdpstart;
68 private double fdpstop;
69 private int deliverythreads;
70 private String provurl;
71 private String provhost;
72 private IsFrom provcheck;
76 private String spooldir;
77 private String logdir;
78 private long logretention;
79 private String redirfile;
80 private String kstype;
81 private String ksfile;
82 private String kspass;
84 private String tstype;
85 private String tsfile;
86 private String tspass;
87 private String myname;
88 private RedirManager rdmgr;
89 private RateLimitedOperation pfetcher;
90 private NodeConfig config;
92 private PublishId pid;
94 private TaskList configtasks = new TaskList();
95 private String eventlogurl;
96 private String eventlogprefix;
97 private String eventlogsuffix;
98 private String eventloginterval;
99 private boolean followredirects;
100 private String [] enabledprotocols;
101 private String aafType;
102 private String aafInstance;
103 private String aafAction;
104 private String aafURL;
105 private boolean cadiEnabled;
109 * Get the default node configuration manager
111 public static NodeConfigManager getInstance() {
116 * Initialize the configuration of a Data Router node
118 private NodeConfigManager() {
120 Properties drNodeProperties = new Properties();
122 logger.info("NODE0301 Loading local config file node.properties");
123 drNodeProperties.load(new FileInputStream(System
124 .getProperty("org.onap.dmaap.datarouter.node.properties", "/opt/app/datartr/etc/node.properties")));
125 } catch (Exception e) {
126 NodeUtils.setIpAndFqdnForEelf("NodeConfigManager");
127 eelflogger.error(EelfMsgs.MESSAGE_PROPERTIES_LOAD_ERROR);
128 logger.error("NODE0301 Unable to load local configuration file " + System
129 .getProperty("org.onap.dmaap.datarouter.node.properties", "/opt/app/datartr/etc/node.properties"), e);
131 provurl = drNodeProperties.getProperty("ProvisioningURL", "https://dmaap-dr-prov:8443/internal/prov");
133 * START - AAF changes: TDP EPIC US# 307413
134 * Pull AAF settings from node.properties
136 aafType = drNodeProperties.getProperty("AAFType", "org.onap.dmaap-dr.feed");
137 aafInstance = drNodeProperties.getProperty("AAFInstance", "legacy");
138 aafAction = drNodeProperties.getProperty("AAFAction", "publish");
139 aafURL = drNodeProperties.getProperty("AafUrl", "https://aaf-onap-test.osaaf.org:8095");
140 cadiEnabled = Boolean.parseBoolean(drNodeProperties.getProperty("CadiEnabled", "false"));
142 * END - AAF changes: TDP EPIC US# 307413
143 * Pull AAF settings from node.properties
145 //Disable and enable protocols*/
146 enabledprotocols = ((drNodeProperties.getProperty("NodeHttpsProtocols")).trim()).split("\\|");
149 provhost = (new URL(provurl)).getHost();
150 } catch (Exception e) {
151 NodeUtils.setIpAndFqdnForEelf("NodeConfigManager");
152 eelflogger.error(EelfMsgs.MESSAGE_BAD_PROV_URL, provurl);
153 logger.error("NODE0302 Bad provisioning server URL " + provurl);
156 logger.info("NODE0303 Provisioning server is " + provhost);
157 eventlogurl = drNodeProperties.getProperty("LogUploadURL", "https://feeds-drtr.web.att.com/internal/logs");
158 provcheck = new IsFrom(provhost);
159 gfport = Integer.parseInt(drNodeProperties.getProperty("IntHttpPort", "8080"));
160 svcport = Integer.parseInt(drNodeProperties.getProperty("IntHttpsPort", "8443"));
161 port = Integer.parseInt(drNodeProperties.getProperty("ExtHttpsPort", "443"));
162 long minpfinterval = Long.parseLong(drNodeProperties.getProperty("MinProvFetchInterval", "10000"));
163 long minrsinterval = Long.parseLong(drNodeProperties.getProperty("MinRedirSaveInterval", "10000"));
164 spooldir = drNodeProperties.getProperty("SpoolDir", "spool");
165 File fdir = new File(spooldir + "/f");
167 for (File junk : fdir.listFiles()) {
172 logdir = drNodeProperties.getProperty("LogDir", "logs");
173 (new File(logdir)).mkdirs();
174 logretention = Long.parseLong(drNodeProperties.getProperty("LogRetention", "30")) * 86400000L;
175 eventlogprefix = logdir + "/events";
176 eventlogsuffix = ".log";
177 String redirfile = drNodeProperties.getProperty("RedirectionFile", "etc/redirections.dat");
178 kstype = drNodeProperties.getProperty("KeyStoreType", "jks");
179 ksfile = drNodeProperties.getProperty("KeyStoreFile", "etc/keystore");
180 kspass = drNodeProperties.getProperty("KeyStorePassword", "changeme");
181 kpass = drNodeProperties.getProperty("KeyPassword", "changeme");
182 tstype = drNodeProperties.getProperty("TrustStoreType", "jks");
183 tsfile = drNodeProperties.getProperty("TrustStoreFile");
184 tspass = drNodeProperties.getProperty("TrustStorePassword", "changeme");
185 if (tsfile != null && tsfile.length() > 0) {
186 System.setProperty("javax.net.ssl.trustStoreType", tstype);
187 System.setProperty("javax.net.ssl.trustStore", tsfile);
188 System.setProperty("javax.net.ssl.trustStorePassword", tspass);
190 nak = drNodeProperties.getProperty("NodeAuthKey", "Node123!");
191 quiesce = new File(drNodeProperties.getProperty("QuiesceFile", "etc/SHUTDOWN"));
192 myname = NodeUtils.getCanonicalName(kstype, ksfile, kspass);
193 if (myname == null) {
194 NodeUtils.setIpAndFqdnForEelf("NodeConfigManager");
195 eelflogger.error(EelfMsgs.MESSAGE_KEYSTORE_FETCH_ERROR, ksfile);
196 logger.error("NODE0309 Unable to fetch canonical name from keystore file " + ksfile);
199 logger.info("NODE0304 My certificate says my name is " + myname);
200 pid = new PublishId(myname);
201 rdmgr = new RedirManager(redirfile, minrsinterval, timer);
202 pfetcher = new RateLimitedOperation(minpfinterval, timer) {
207 logger.info("NODE0305 Attempting to fetch configuration at " + provurl);
211 private void localconfig() {
212 followredirects = Boolean.parseBoolean(getProvParam("FOLLOW_REDIRECTS", "false"));
213 eventloginterval = getProvParam("LOGROLL_INTERVAL", "30s");
214 initfailuretimer = 10000;
215 waitForFileProcessFailureTimer = 600000;
216 maxfailuretimer = 3600000;
217 expirationtimer = 86400000;
218 failurebackoff = 2.0;
219 deliverythreads = 40;
221 fairtimelimit = 60000;
225 initfailuretimer = (long) (Double.parseDouble(getProvParam("DELIVERY_INIT_RETRY_INTERVAL")) * 1000);
226 } catch (Exception e) {
229 waitForFileProcessFailureTimer = (long) (Double.parseDouble(getProvParam("DELIVERY_FILE_PROCESS_INTERVAL")) * 1000);
230 } catch (Exception e) {
233 maxfailuretimer = (long) (Double.parseDouble(getProvParam("DELIVERY_MAX_RETRY_INTERVAL")) * 1000);
234 } catch (Exception e) {
237 expirationtimer = (long) (Double.parseDouble(getProvParam("DELIVERY_MAX_AGE")) * 1000);
238 } catch (Exception e) {
241 failurebackoff = Double.parseDouble(getProvParam("DELIVERY_RETRY_RATIO"));
242 } catch (Exception e) {
245 deliverythreads = Integer.parseInt(getProvParam("DELIVERY_THREADS"));
246 } catch (Exception e) {
249 fairfilelimit = Integer.parseInt(getProvParam("FAIR_FILE_LIMIT"));
250 } catch (Exception e) {
253 fairtimelimit = (long) (Double.parseDouble(getProvParam("FAIR_TIME_LIMIT")) * 1000);
254 } catch (Exception e) {
257 fdpstart = Double.parseDouble(getProvParam("FREE_DISK_RED_PERCENT")) / 100.0;
258 } catch (Exception e) {
261 fdpstop = Double.parseDouble(getProvParam("FREE_DISK_YELLOW_PERCENT")) / 100.0;
262 } catch (Exception e) {
264 if (fdpstart < 0.01) {
267 if (fdpstart > 0.5) {
270 if (fdpstop < fdpstart) {
278 private void fetchconfig() {
280 logger.info("NodeConfigMan.fetchConfig: provurl:: " + provurl);
281 Reader r = new InputStreamReader((new URL(provurl)).openStream());
282 config = new NodeConfig(new ProvData(r), myname, spooldir, port, nak);
284 configtasks.startRun();
286 while ((rr = configtasks.next()) != null) {
289 } catch (Exception e) {
290 logger.error("NODE0518 Exception fetchconfig: " + e);
293 } catch (Exception e) {
294 NodeUtils.setIpAndFqdnForEelf("fetchconfigs");
295 eelflogger.error(EelfMsgs.MESSAGE_CONF_FAILED, e.toString());
296 logger.error("NODE0306 Configuration failed " + e.toString() + " - try again later", e);
302 * Process a gofetch request from a particular IP address. If the IP address is not an IP address we would go to to
303 * fetch the provisioning data, ignore the request. If the data has been fetched very recently (default 10
304 * seconds), wait a while before fetching again.
306 public synchronized void gofetch(String remoteAddr) {
307 if (provcheck.isReachable(remoteAddr)) {
308 logger.info("NODE0307 Received configuration fetch request from provisioning server " + remoteAddr);
311 logger.info("NODE0308 Received configuration fetch request from unexpected server " + remoteAddr);
318 public boolean isConfigured() {
319 return (config != null);
325 public boolean isShutdown() {
326 return (quiesce.exists());
330 * Given a routing string, get the targets.
332 * @param routing Target string
333 * @return array of targets
335 public Target[] parseRouting(String routing) {
336 return (config.parseRouting(routing));
340 * Given a set of credentials and an IP address, is this request from another node?
342 * @param credentials Credentials offered by the supposed node
343 * @param ip IP address the request came from
344 * @return If the credentials and IP address are recognized, true, otherwise false.
346 public boolean isAnotherNode(String credentials, String ip) {
347 return (config.isAnotherNode(credentials, ip));
351 * Check whether publication is allowed.
353 * @param feedid The ID of the feed being requested
354 * @param credentials The offered credentials
355 * @param ip The requesting IP address
356 * @return True if the IP and credentials are valid for the specified feed.
358 public String isPublishPermitted(String feedid, String credentials, String ip) {
359 return (config.isPublishPermitted(feedid, credentials, ip));
363 * Check whether delete file is allowed.
365 * @param subId The ID of the subscription being requested
366 * @return True if the delete file is permitted for the subscriber.
368 public boolean isDeletePermitted(String subId) {
369 return (config.isDeletePermitted(subId));
373 * Check whether publication is allowed for AAF Feed.
375 * @param feedid The ID of the feed being requested
376 * @param ip The requesting IP address
377 * @return True if the IP and credentials are valid for the specified feed.
379 public String isPublishPermitted(String feedid, String ip) {
380 return(config.isPublishPermitted(feedid, ip));
384 * Check who the user is given the feed ID and the offered credentials.
386 * @param feedid The ID of the feed specified
387 * @param credentials The offered credentials
388 * @return Null if the credentials are invalid or the user if they are valid.
390 public String getAuthUser(String feedid, String credentials) {
391 return (config.getAuthUser(feedid, credentials));
395 * AAF changes: TDP EPIC US# 307413
396 * Check AAF_instance for feed ID in NodeConfig
397 * @param feedid The ID of the feed specified
399 public String getAafInstance(String feedid) {
400 return(config.getAafInstance(feedid));
404 * Check if the publish request should be sent to another node based on the feedid, user, and source IP address.
406 * @param feedid The ID of the feed specified
407 * @param user The publishing user
408 * @param ip The IP address of the publish endpoint
409 * @return Null if the request should be accepted or the correct hostname if it should be sent to another node.
411 public String getIngressNode(String feedid, String user, String ip) {
412 return (config.getIngressNode(feedid, user, ip));
416 * Get a provisioned configuration parameter (from the provisioning server configuration)
418 * @param name The name of the parameter
419 * @return The value of the parameter or null if it is not defined.
421 public String getProvParam(String name) {
422 return (config.getProvParam(name));
426 * Get a provisioned configuration parameter (from the provisioning server configuration)
428 * @param name The name of the parameter
429 * @param defaultValue The value to use if the parameter is not defined
430 * @return The value of the parameter or deflt if it is not defined.
432 public String getProvParam(String name, String defaultValue) {
433 name = config.getProvParam(name);
441 * Generate a publish ID
443 public String getPublishId() {
448 * Get all the outbound spooling destinations. This will include both subscriptions and nodes.
450 public DestInfo[] getAllDests() {
451 return (config.getAllDests());
455 * Register a task to run whenever the configuration changes
457 public void registerConfigTask(Runnable task) {
458 configtasks.addTask(task);
462 * Deregister a task to run whenever the configuration changes
464 public void deregisterConfigTask(Runnable task) {
465 configtasks.removeTask(task);
469 * Get the URL to deliver a message to.
471 * @param destinationInfo The destination information
472 * @param fileid The file ID
473 * @return The URL to deliver to
475 public String getDestURL(DestInfo destinationInfo, String fileid) {
476 String subid = destinationInfo.getSubId();
477 String purl = destinationInfo.getURL();
478 if (followredirects && subid != null) {
479 purl = rdmgr.lookup(subid, purl);
481 return (purl + "/" + fileid);
485 * Is a destination redirected?
487 public boolean isDestRedirected(DestInfo destinfo) {
488 return (followredirects && rdmgr.isRedirected(destinfo.getSubId()));
492 * Set up redirection on receipt of a 3XX from a target URL
494 public boolean handleRedirection(DestInfo destinationInfo, String redirto, String fileid) {
495 fileid = "/" + fileid;
496 String subid = destinationInfo.getSubId();
497 String purl = destinationInfo.getURL();
498 if (followredirects && subid != null && redirto.endsWith(fileid)) {
499 redirto = redirto.substring(0, redirto.length() - fileid.length());
500 if (!redirto.equals(purl)) {
501 rdmgr.redirect(subid, purl, redirto);
509 * Set up redirection on receipt of a 3XX from a target URL
511 public boolean handleRedirectionSubLevel(DeliveryTask task, DestInfo destinfo, String redirto, String fileid) {
512 fileid = "/" + fileid;
513 String subid = destinfo.getSubId();
514 String purl = destinfo.getURL();
515 if (task.getFollowRedirects() && subid != null && redirto.endsWith(fileid)) {
516 redirto = redirto.substring(0, redirto.length() - fileid.length());
517 if (!redirto.equals(purl)) {
518 rdmgr.redirect(subid, purl, redirto);
526 * Handle unreachable target URL
528 public void handleUnreachable(DestInfo destinationInfo) {
529 String subid = destinationInfo.getSubId();
530 if (followredirects && subid != null) {
536 * Get the timeout before retrying after an initial delivery failure
538 public long getInitFailureTimer() {
539 return (initfailuretimer);
543 * Get the timeout before retrying after delivery and wait for file processing
545 public long getWaitForFileProcessFailureTimer() {
546 return (waitForFileProcessFailureTimer);
550 * Get the maximum timeout between delivery attempts
552 public long getMaxFailureTimer() {
553 return (maxfailuretimer);
557 * Get the ratio between consecutive delivery attempts
559 public double getFailureBackoff() {
560 return (failurebackoff);
564 * Get the expiration timer for deliveries
566 public long getExpirationTimer() {
567 return (expirationtimer);
571 * Get the maximum number of file delivery attempts before checking if another queue has work to be performed.
573 public int getFairFileLimit() {
574 return (fairfilelimit);
578 * Get the maximum amount of time spent delivering files before checking if another queue has work to be performed.
580 public long getFairTimeLimit() {
581 return (fairtimelimit);
585 * Get the targets for a feed
587 * @param feedid The feed ID
588 * @return The targets this feed should be delivered to
590 public Target[] getTargets(String feedid) {
591 return (config.getTargets(feedid));
595 * Get the creation date for a feed
596 * @param feedid The feed ID
597 * @return the timestamp of creation date of feed id passed
599 public String getCreatedDate(String feedid) {
600 return(config.getCreatedDate(feedid));
604 * Get the spool directory for temporary files
606 public String getSpoolDir() {
607 return (spooldir + "/f");
611 * Get the base directory for spool directories
613 public String getSpoolBase() {
618 * Get the key store type
620 public String getKSType() {
625 * Get the key store file
627 public String getKSFile() {
632 * Get the key store password
634 public String getKSPass() {
639 * Get the key password
641 public String getKPass() {
648 public int getHttpPort() {
655 public int getHttpsPort() {
660 * Get the externally visible https port
662 public int getExtHttpsPort() {
667 * Get the external name of this machine
669 public String getMyName() {
674 * Get the number of threads to use for delivery
676 public int getDeliveryThreads() {
677 return (deliverythreads);
681 * Get the URL for uploading the event log data
683 public String getEventLogUrl() {
684 return (eventlogurl);
688 * Get the prefix for the names of event log files
690 public String getEventLogPrefix() {
691 return (eventlogprefix);
695 * Get the suffix for the names of the event log files
697 public String getEventLogSuffix() {
698 return (eventlogsuffix);
702 * Get the interval between event log file rollovers
704 public String getEventLogInterval() {
705 return (eventloginterval);
709 * Should I follow redirects from subscribers?
711 public boolean isFollowRedirects() {
712 return (followredirects);
716 * Get the directory where the event and node log files live
718 public String getLogDir() {
723 * How long do I keep log files (in milliseconds)
725 public long getLogRetention() {
726 return (logretention);
732 public Timer getTimer() {
737 * Get the feed ID for a subscription
739 * @param subid The subscription ID
740 * @return The feed ID
742 public String getFeedId(String subid) {
743 return (config.getFeedId(subid));
747 * Get the authorization string this node uses
749 * @return The Authorization string for this node
751 public String getMyAuth() {
752 return (config.getMyAuth());
756 * Get the fraction of free spool disk space where we start throwing away undelivered files. This is
757 * FREE_DISK_RED_PERCENT / 100.0. Default is 0.05. Limited by 0.01 <= FreeDiskStart <= 0.5.
759 public double getFreeDiskStart() {
764 * Get the fraction of free spool disk space where we stop throwing away undelivered files. This is
765 * FREE_DISK_YELLOW_PERCENT / 100.0. Default is 0.2. Limited by FreeDiskStart <= FreeDiskStop <= 0.5.
767 public double getFreeDiskStop() {
772 * Disable and enable protocols
774 public String[] getEnabledprotocols() {
775 return enabledprotocols;
777 public void setEnabledprotocols(String[] enabledprotocols) {
778 this.enabledprotocols = enabledprotocols.clone();
782 * Get the spool directory for a subscription
784 public String getSpoolDir(String subid, String remoteaddr) {
785 if (provcheck.isFrom(remoteaddr)) {
786 String sdir = config.getSpoolDir(subid);
788 logger.info("NODE0310 Received subscription reset request for subscription " + subid
789 + " from provisioning server " + remoteaddr);
791 logger.info("NODE0311 Received subscription reset request for unknown subscription " + subid
792 + " from provisioning server " + remoteaddr);
796 logger.info("NODE0312 Received subscription reset request from unexpected server " + remoteaddr);
801 public String getAafType() {
804 public void setAafType(String aafType) {
805 this.aafType = aafType;
807 public String getAafInstance() {
810 public void setAafInstance(String aafInstance) {
811 this.aafInstance = aafInstance;
813 public String getAafAction() {
816 public void setAafAction(String aafAction) {
817 this.aafAction = aafAction;
820 * Get aafURL from SWM variable
822 public String getAafURL() {
825 public void setAafURL(String aafURL) {
826 this.aafURL = aafURL;
829 public boolean getCadiEnabeld() {
832 public void setCadiEnabled(boolean cadiEnabled) {
833 this.cadiEnabled = cadiEnabled;
837 * Builds the permissions string to be verified
839 * @param aafInstance The aaf instance
840 * @return The permissions
842 protected String getPermission(String aafInstance) {
844 String type = getAafType();
845 String action = getAafAction();
846 if (aafInstance == null || aafInstance.equals("")) {
847 aafInstance = getAafInstance();
849 return type + "|" + aafInstance + "|" + action;
850 } catch (Exception e) {
851 logger.error("NODE0543 NodeConfigManager.getPermission: ", e);