datarouter-node/src/main/java/org/onap/dmaap/datarouter/node/NodeServlet.java

   1 /*******************************************************************************
   2  * ============LICENSE_START==================================================
   3  * * org.onap.dmaap
   4  * * ===========================================================================
   5  * * Copyright © 2017 AT&T Intellectual Property. All rights reserved.
   6  * * ===========================================================================
   7  * * Licensed under the Apache License, Version 2.0 (the "License");
   8  * * you may not use this file except in compliance with the License.
   9  * * You may obtain a copy of the License at
  10  * *
  11  *  *      http://www.apache.org/licenses/LICENSE-2.0
  12  * *
  13  *  * Unless required by applicable law or agreed to in writing, software
  14  * * distributed under the License is distributed on an "AS IS" BASIS,
  15  * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * * See the License for the specific language governing permissions and
  17  * * limitations under the License.
  18  * * ============LICENSE_END====================================================
  19  * *
  20  * * ECOMP is a trademark and service mark of AT&T Intellectual Property.
  21  * *
  22  ******************************************************************************/
  23
  24
  25 package org.onap.dmaap.datarouter.node;
  26
  27 import com.att.eelf.configuration.EELFLogger;
  28 import com.att.eelf.configuration.EELFManager;
  29 import java.io.File;
  30 import java.io.FileOutputStream;
  31 import java.io.FileWriter;
  32 import java.io.IOException;
  33 import java.io.InputStream;
  34 import java.io.OutputStream;
  35 import java.io.Writer;
  36 import java.nio.file.Files;
  37 import java.nio.file.Path;
  38 import java.nio.file.Paths;
  39 import java.util.Enumeration;
  40 import java.util.regex.Pattern;
  41 import javax.servlet.http.HttpServlet;
  42 import javax.servlet.http.HttpServletRequest;
  43 import javax.servlet.http.HttpServletResponse;
  44 import org.apache.log4j.Logger;
  45 import org.jetbrains.annotations.Nullable;
  46 import org.onap.dmaap.datarouter.node.eelf.EelfMsgs;
  47 import org.slf4j.MDC;
  48
  49 import static org.onap.dmaap.datarouter.node.NodeUtils.*;
  50
  51 /**
  52  * Servlet for handling all http and https requests to the data router node
  53  * <p>
  54  * Handled requests are:
  55  * <br>
  56  * GET http://<i>node</i>/internal/fetchProv - fetch the provisioning data
  57  * <br>
  58  * PUT/DELETE https://<i>node</i>/internal/publish/<i>fileid</i> - n2n transfer
  59  * <br>
  60  * PUT/DELETE https://<i>node</i>/publish/<i>feedid</i>/<i>fileid</i> - publsh request
  61  */
  62 public class NodeServlet extends HttpServlet {
  63
  64     private static Logger logger = Logger.getLogger("org.onap.dmaap.datarouter.node.NodeServlet");
  65     private static NodeConfigManager config;
  66     private static Pattern MetaDataPattern;
  67     //Adding EELF Logger Rally:US664892
  68     private static EELFLogger eelflogger = EELFManager.getInstance()
  69             .getLogger(NodeServlet.class);
  70     private final Delivery delivery;
  71
  72     static {
  73         final String ws = "\\s*";
  74         // assume that \\ and \" have been replaced by X
  75         final String string = "\"[^\"]*\"";
  76         //String string = "\"(?:[^\"\\\\]|\\\\.)*\"";
  77         final String number = "[+-]?(?:\\.\\d+|(?:0|[1-9]\\d*)(?:\\.\\d*)?)(?:[eE][+-]?\\d+)?";
  78         final String value = "(?:" + string + "|" + number + "|null|true|false)";
  79         final String item = string + ws + ":" + ws + value + ws;
  80         final String object = ws + "\\{" + ws + "(?:" + item + "(?:" + "," + ws + item + ")*)?\\}" + ws;
  81         MetaDataPattern = Pattern.compile(object, Pattern.DOTALL);
  82     }
  83
  84     NodeServlet(Delivery delivery) {
  85         this.delivery = delivery;
  86     }
  87
  88     /**
  89      * Get the NodeConfigurationManager
  90      */
  91     public void init() {
  92         config = NodeConfigManager.getInstance();
  93         logger.info("NODE0101 Node Servlet Configured");
  94     }
  95
  96     private boolean down(HttpServletResponse resp) throws IOException {
  97         if (config.isShutdown() || !config.isConfigured()) {
  98             sendResponseError(resp, HttpServletResponse.SC_SERVICE_UNAVAILABLE, logger);
  99             logger.info("NODE0102 Rejecting request: Service is being quiesced");
 100             return (true);
 101         }
 102         return (false);
 103     }
 104
 105     /**
 106      * Handle a GET for /internal/fetchProv
 107      */
 108     protected void doGet(HttpServletRequest req, HttpServletResponse resp) {
 109         NodeUtils.setIpAndFqdnForEelf("doGet");
 110         NodeUtils.setRequestIdAndInvocationId(req);
 111         eelflogger.info(EelfMsgs.ENTRY);
 112         try {
 113             eelflogger.info(EelfMsgs.MESSAGE_WITH_BEHALF_AND_FEEDID, req.getHeader("X-DMAAP-DR-ON-BEHALF-OF"),
 114                     getIdFromPath(req) + "");
 115             try {
 116                 if (down(resp)) {
 117                     return;
 118                 }
 119
 120             } catch (IOException ioe) {
 121                 logger.error("IOException" + ioe.getMessage());
 122             }
 123             String path = req.getPathInfo();
 124             String qs = req.getQueryString();
 125             String ip = req.getRemoteAddr();
 126             if (qs != null) {
 127                 path = path + "?" + qs;
 128             }
 129             if ("/internal/fetchProv".equals(path)) {
 130                 config.gofetch(ip);
 131                 resp.setStatus(HttpServletResponse.SC_NO_CONTENT);
 132                 return;
 133             } else if (path.startsWith("/internal/resetSubscription/")) {
 134                 String subid = path.substring(28);
 135                 if (subid.length() != 0 && subid.indexOf('/') == -1) {
 136                     NodeMain.resetQueue(subid, ip);
 137                     resp.setStatus(HttpServletResponse.SC_NO_CONTENT);
 138                     return;
 139                 }
 140             }
 141
 142             logger.info("NODE0103 Rejecting invalid GET of " + path + " from " + ip);
 143             sendResponseError(resp, HttpServletResponse.SC_NOT_FOUND, logger);
 144         } finally {
 145             eelflogger.info(EelfMsgs.EXIT);
 146         }
 147     }
 148
 149     /**
 150      * Handle all PUT requests
 151      */
 152     protected void doPut(HttpServletRequest req, HttpServletResponse resp) {
 153         NodeUtils.setIpAndFqdnForEelf("doPut");
 154         NodeUtils.setRequestIdAndInvocationId(req);
 155         eelflogger.info(EelfMsgs.ENTRY);
 156         eelflogger.info(EelfMsgs.MESSAGE_WITH_BEHALF_AND_FEEDID, req.getHeader("X-DMAAP-DR-ON-BEHALF-OF"),
 157                     getIdFromPath(req) + "");
 158         try {
 159             common(req, resp, true);
 160         } catch (IOException ioe) {
 161             logger.error("IOException" + ioe.getMessage());
 162             eelflogger.info(EelfMsgs.EXIT);
 163         }
 164     }
 165
 166     /**
 167      * Handle all DELETE requests
 168      */
 169     protected void doDelete(HttpServletRequest req, HttpServletResponse resp) {
 170         NodeUtils.setIpAndFqdnForEelf("doDelete");
 171         NodeUtils.setRequestIdAndInvocationId(req);
 172         eelflogger.info(EelfMsgs.ENTRY);
 173         eelflogger.info(EelfMsgs.MESSAGE_WITH_BEHALF_AND_FEEDID, req.getHeader("X-DMAAP-DR-ON-BEHALF-OF"),
 174                 getIdFromPath(req) + "");
 175         try {
 176             common(req, resp, false);
 177         } catch (IOException ioe) {
 178             logger.error("IOException " + ioe.getMessage());
 179             eelflogger.info(EelfMsgs.EXIT);
 180         }
 181     }
 182
 183     private void common(HttpServletRequest req, HttpServletResponse resp, boolean isput) throws IOException {
 184         String fileid = getFileId(req, resp);
 185         if (fileid == null) return;
 186         String feedid = null;
 187         String user = null;
 188         String ip = req.getRemoteAddr();
 189         String lip = req.getLocalAddr();
 190         String pubid = null;
 191         String xpubid = null;
 192         String rcvd = NodeUtils.logts(System.currentTimeMillis()) + ";from=" + ip + ";by=" + lip;
 193         Target[] targets = null;
 194         if (fileid.startsWith("/delete/")) {
 195             deleteFile(req, resp, fileid, pubid);
 196             return;
 197         }
 198         String credentials = req.getHeader("Authorization");
 199         if (credentials == null) {
 200             logger.info("NODE0106 Rejecting unauthenticated PUT or DELETE of " + req.getPathInfo() + " from " + req
 201                     .getRemoteAddr());
 202             resp.sendError(HttpServletResponse.SC_FORBIDDEN, "Authorization header required");
 203             eelflogger.info(EelfMsgs.EXIT);
 204             return;
 205         }
 206         if (fileid.startsWith("/publish/")) {
 207             fileid = fileid.substring(9);
 208             int i = fileid.indexOf('/');
 209             if (i == -1 || i == fileid.length() - 1) {
 210                 logger.info("NODE0105 Rejecting bad URI for PUT or DELETE of " + req.getPathInfo() + " from " + req
 211                         .getRemoteAddr());
 212                 resp.sendError(HttpServletResponse.SC_NOT_FOUND,
 213                         "Invalid request URI.  Expecting <feed-publishing-url>/<fileid>.  Possible missing fileid.");
 214                 eelflogger.info(EelfMsgs.EXIT);
 215                 return;
 216             }
 217             feedid = fileid.substring(0, i);
 218             fileid = fileid.substring(i + 1);
 219             pubid = config.getPublishId();
 220             xpubid = req.getHeader("X-DMAAP-DR-PUBLISH-ID");
 221             targets = config.getTargets(feedid);
 222         } else if (fileid.startsWith("/internal/publish/")) {
 223             if (!config.isAnotherNode(credentials, ip)) {
 224                 logger.info("NODE0107 Rejecting unauthorized node-to-node transfer attempt from " + ip);
 225                 resp.sendError(HttpServletResponse.SC_FORBIDDEN);
 226                 eelflogger.info(EelfMsgs.EXIT);
 227                 return;
 228             }
 229             fileid = fileid.substring(18);
 230             pubid = req.getHeader("X-DMAAP-DR-PUBLISH-ID");
 231             targets = config.parseRouting(req.getHeader("X-DMAAP-DR-ROUTING"));
 232         } else {
 233             logger.info("NODE0105 Rejecting bad URI for PUT or DELETE of " + req.getPathInfo() + " from " + req
 234                     .getRemoteAddr());
 235             resp.sendError(HttpServletResponse.SC_NOT_FOUND,
 236                     "Invalid request URI.  Expecting <feed-publishing-url>/<fileid>.");
 237             eelflogger.info(EelfMsgs.EXIT);
 238             return;
 239         }
 240         if (fileid.indexOf('/') != -1) {
 241             logger.info("NODE0105 Rejecting bad URI for PUT or DELETE of " + req.getPathInfo() + " from " + req
 242                     .getRemoteAddr());
 243             resp.sendError(HttpServletResponse.SC_NOT_FOUND,
 244                     "Invalid request URI.  Expecting <feed-publishing-url>/<fileid>.");
 245             eelflogger.info(EelfMsgs.EXIT);
 246             return;
 247         }
 248         String qs = req.getQueryString();
 249         if (qs != null) {
 250             fileid = fileid + "?" + qs;
 251         }
 252         String hp = config.getMyName();
 253         int xp = config.getExtHttpsPort();
 254         if (xp != 443) {
 255             hp = hp + ":" + xp;
 256         }
 257         String logurl = "https://" + hp + "/internal/publish/" + fileid;
 258         if (feedid != null) {
 259             logurl = "https://" + hp + "/publish/" + feedid + "/" + fileid;
 260             String reason = config.isPublishPermitted(feedid, credentials, ip);
 261             if (reason != null) {
 262                 logger.info(
 263                         "NODE0111 Rejecting unauthorized publish attempt to feed " + feedid + " fileid " + fileid
 264                                 + " from "
 265                                 + ip + " reason " + reason);
 266                 resp.sendError(HttpServletResponse.SC_FORBIDDEN, reason);
 267                 eelflogger.info(EelfMsgs.EXIT);
 268                 return;
 269             }
 270             user = config.getAuthUser(feedid, credentials);
 271             String newnode = config.getIngressNode(feedid, user, ip);
 272             if (newnode != null) {
 273                 String port = "";
 274                 int iport = config.getExtHttpsPort();
 275                 if (iport != 443) {
 276                     port = ":" + iport;
 277                 }
 278                 String redirto = "https://" + newnode + port + "/publish/" + feedid + "/" + fileid;
 279                 logger.info(
 280                         "NODE0108 Redirecting publish attempt for feed " + feedid + " user " + user + " ip " + ip
 281                                 + " to "
 282                                 + redirto);
 283                 resp.sendRedirect(redirto);
 284                 eelflogger.info(EelfMsgs.EXIT);
 285                 return;
 286             }
 287             resp.setHeader("X-DMAAP-DR-PUBLISH-ID", pubid);
 288         }
 289         String fbase = config.getSpoolDir() + "/" + pubid;
 290         File data = new File(fbase);
 291         File meta = new File(fbase + ".M");
 292         OutputStream dos = null;
 293         Writer mw = null;
 294         InputStream is = null;
 295         try {
 296             StringBuffer mx = new StringBuffer();
 297             mx.append(req.getMethod()).append('\t').append(fileid).append('\n');
 298             Enumeration hnames = req.getHeaderNames();
 299             String ctype = null;
 300             boolean hasRequestIdHeader = false;
 301             boolean hasInvocationIdHeader = false;
 302             while (hnames.hasMoreElements()) {
 303                 String hn = (String) hnames.nextElement();
 304                 String hnlc = hn.toLowerCase();
 305                 if ((isput && ("content-type".equals(hnlc) ||
 306                         "content-language".equals(hnlc) ||
 307                         "content-md5".equals(hnlc) ||
 308                         "content-range".equals(hnlc))) ||
 309                         "x-dmaap-dr-meta".equals(hnlc) ||
 310                         (feedid == null && "x-dmaap-dr-received".equals(hnlc)) ||
 311                         (hnlc.startsWith("x-") && !hnlc.startsWith("x-dmaap-dr-"))) {
 312                     Enumeration hvals = req.getHeaders(hn);
 313                     while (hvals.hasMoreElements()) {
 314                         String hv = (String) hvals.nextElement();
 315                         if ("content-type".equals(hnlc)) {
 316                             ctype = hv;
 317                         }
 318                         if ("x-onap-requestid".equals(hnlc)) {
 319                             hasRequestIdHeader = true;
 320                         }
 321                         if ("x-invocationid".equals(hnlc)) {
 322                             hasInvocationIdHeader = true;
 323                         }
 324                         if ("x-dmaap-dr-meta".equals(hnlc)) {
 325                             if (hv.length() > 4096) {
 326                                 logger.info(
 327                                         "NODE0109 Rejecting publish attempt with metadata too long for feed " + feedid
 328                                                 + " user " + user + " ip " + ip);
 329                                 resp.sendError(HttpServletResponse.SC_BAD_REQUEST, "Metadata too long");
 330                                 eelflogger.info(EelfMsgs.EXIT);
 331                                 return;
 332                             }
 333                             if (!MetaDataPattern.matcher(hv.replaceAll("\\\\.", "X")).matches()) {
 334                                 logger.info(
 335                                         "NODE0109 Rejecting publish attempt with malformed metadata for feed " + feedid
 336                                                 + " user " + user + " ip " + ip);
 337                                 resp.sendError(HttpServletResponse.SC_BAD_REQUEST, "Malformed metadata");
 338                                 eelflogger.info(EelfMsgs.EXIT);
 339                                 return;
 340                             }
 341                         }
 342                         mx.append(hn).append('\t').append(hv).append('\n');
 343                     }
 344                 }
 345             }
 346             if(!hasRequestIdHeader){
 347                 mx.append("X-ONAP-RequestID\t").append(MDC.get("RequestId")).append('\n');
 348             }
 349             if(!hasInvocationIdHeader){
 350                 mx.append("X-InvocationID\t").append(MDC.get("InvocationId")).append('\n');
 351             }
 352             mx.append("X-DMAAP-DR-RECEIVED\t").append(rcvd).append('\n');
 353             String metadata = mx.toString();
 354             byte[] buf = new byte[1024 * 1024];
 355             int i;
 356             try {
 357                 is = req.getInputStream();
 358                 dos = new FileOutputStream(data);
 359                 while ((i = is.read(buf)) > 0) {
 360                     dos.write(buf, 0, i);
 361                 }
 362                 is.close();
 363                 is = null;
 364                 dos.close();
 365                 dos = null;
 366             } catch (IOException ioe) {
 367                 long exlen = -1;
 368                 try {
 369                     exlen = Long.parseLong(req.getHeader("Content-Length"));
 370                 } catch (Exception e) {
 371                 }
 372                 StatusLog.logPubFail(pubid, feedid, logurl, req.getMethod(), ctype, exlen, data.length(), ip, user,
 373                         ioe.getMessage());
 374                 eelflogger.info(EelfMsgs.EXIT);
 375                 throw ioe;
 376             }
 377             Path dpath = Paths.get(fbase);
 378             for (Target t : targets) {
 379                 DestInfo di = t.getDestInfo();
 380                 if (di == null) {
 381                     // TODO: unknown destination
 382                     continue;
 383                 }
 384                 String dbase = di.getSpool() + "/" + pubid;
 385                 Files.createLink(Paths.get(dbase), dpath);
 386                 mw = new FileWriter(meta);
 387                 mw.write(metadata);
 388                 if (di.getSubId() == null) {
 389                     mw.write("X-DMAAP-DR-ROUTING\t" + t.getRouting() + "\n");
 390                 }
 391                 mw.close();
 392                 meta.renameTo(new File(dbase + ".M"));
 393
 394             }
 395             resp.setStatus(HttpServletResponse.SC_NO_CONTENT);
 396             resp.getOutputStream().close();
 397             StatusLog.logPub(pubid, feedid, logurl, req.getMethod(), ctype, data.length(), ip, user,
 398                     HttpServletResponse.SC_NO_CONTENT);
 399         } catch (IOException ioe) {
 400             logger.info(
 401                     "NODE0110 IO Exception receiving publish attempt for feed " + feedid + " user " + user + " ip " + ip
 402                             + " " + ioe.toString(), ioe);
 403             eelflogger.info(EelfMsgs.EXIT);
 404             throw ioe;
 405         } finally {
 406             if (is != null) {
 407                 try {
 408                     is.close();
 409                 } catch (Exception e) {
 410                 }
 411             }
 412             if (dos != null) {
 413                 try {
 414                     dos.close();
 415                 } catch (Exception e) {
 416                 }
 417             }
 418             if (mw != null) {
 419                 try {
 420                     mw.close();
 421                 } catch (Exception e) {
 422                 }
 423             }
 424             try {
 425                 data.delete();
 426             } catch (Exception e) {
 427             }
 428             try {
 429                 meta.delete();
 430             } catch (Exception e) {
 431             }
 432         }
 433     }
 434
 435     private void deleteFile(HttpServletRequest req, HttpServletResponse resp, String fileid, String pubid) {
 436         try {
 437             fileid = fileid.substring(8);
 438             int i = fileid.indexOf('/');
 439             if (i == -1 || i == fileid.length() - 1) {
 440                 logger.info("NODE0112 Rejecting bad URI for DELETE of " + req.getPathInfo() + " from " + req
 441                         .getRemoteAddr());
 442                 resp.sendError(HttpServletResponse.SC_NOT_FOUND,
 443                         "Invalid request URI. Expecting <subId>/<pubId>.");
 444                 eelflogger.info(EelfMsgs.EXIT);
 445                 return;
 446             }
 447             String subscriptionId = fileid.substring(0, i);
 448             int subId = Integer.parseInt(subscriptionId);
 449             pubid = fileid.substring(i + 1);
 450             String errorMessage = "Unable to delete files (" + pubid + ", " + pubid + ".M) from DR Node: "
 451                             + config.getMyName() + ".";
 452             int subIdDir = subId - (subId % 100);
 453             if (!isAuthorizedToDelete(resp, subscriptionId, errorMessage)) {
 454                 return;
 455             }
 456             boolean result = delivery.markTaskSuccess(config.getSpoolBase() + "/s/" + subIdDir + "/" + subId, pubid);
 457             if (result) {
 458                 logger.info("NODE0115 Successfully deleted files (" + pubid + ", " + pubid + ".M) from DR Node: "
 459                         + config.getMyName());
 460                 resp.setStatus(HttpServletResponse.SC_OK);
 461                 eelflogger.info(EelfMsgs.EXIT);
 462             } else {
 463                 logger.error("NODE0116 " + errorMessage);
 464                 resp.sendError(HttpServletResponse.SC_NOT_FOUND, "File not found on server.");
 465                 eelflogger.info(EelfMsgs.EXIT);
 466             }
 467         } catch (IOException ioe) {
 468             logger.error("NODE0117 Unable to delete files (" + pubid + ", " + pubid + ".M) from DR Node: "
 469                     + config.getMyName() + ". Error: " + ioe.getMessage());
 470             eelflogger.info(EelfMsgs.EXIT);
 471         }
 472     }
 473
 474     @Nullable
 475     private String getFileId(HttpServletRequest req, HttpServletResponse resp) throws IOException {
 476         if (down(resp)) {
 477             eelflogger.info(EelfMsgs.EXIT);
 478             return null;
 479         }
 480         if (!req.isSecure()) {
 481             logger.info(
 482                     "NODE0104 Rejecting insecure PUT or DELETE of " + req.getPathInfo() + " from " + req
 483                             .getRemoteAddr());
 484             resp.sendError(HttpServletResponse.SC_FORBIDDEN, "https required on publish requests");
 485             eelflogger.info(EelfMsgs.EXIT);
 486             return null;
 487         }
 488         String fileid = req.getPathInfo();
 489         if (fileid == null) {
 490             logger.info("NODE0105 Rejecting bad URI for PUT or DELETE of " + req.getPathInfo() + " from " + req
 491                     .getRemoteAddr());
 492             resp.sendError(HttpServletResponse.SC_NOT_FOUND,
 493                     "Invalid request URI.  Expecting <feed-publishing-url>/<fileid>.");
 494             eelflogger.info(EelfMsgs.EXIT);
 495             return null;
 496         }
 497         return fileid;
 498     }
 499
 500     private boolean isAuthorizedToDelete(HttpServletResponse resp, String subscriptionId, String errorMessage) throws IOException {
 501         try {
 502             boolean deletePermitted = config.isDeletePermitted(subscriptionId);
 503             if (!deletePermitted) {
 504                 logger.error("NODE0113 " + errorMessage + " Error: Subscription "
 505                         + subscriptionId + " is not a privileged subscription");
 506                 resp.sendError(HttpServletResponse.SC_UNAUTHORIZED);
 507                 eelflogger.info(EelfMsgs.EXIT);
 508                 return false;
 509             }
 510         } catch (NullPointerException npe) {
 511             logger.error("NODE0114 " + errorMessage + " Error: Subscription " + subscriptionId + " does not exist");
 512             resp.sendError(HttpServletResponse.SC_NOT_FOUND);
 513             eelflogger.info(EelfMsgs.EXIT);
 514             return false;
 515         }
 516         return true;
 517     }
 518
 519     private int getIdFromPath(HttpServletRequest req) {
 520         String path = req.getPathInfo();
 521         if (path == null || path.length() < 2) {
 522             return -1;
 523         }
 524         try {
 525             return Integer.parseInt(path.substring(1));
 526         } catch (NumberFormatException e) {
 527             return -1;
 528         }
 529     }
 530 }