datarouter-prov/src/main/java/org/onap/dmaap/datarouter/provisioning/InternalServlet.java

   1 /*******************************************************************************
   2  * ============LICENSE_START==================================================
   3  * * org.onap.dmaap
   4  * * ===========================================================================
   5  * * Copyright © 2017 AT&T Intellectual Property. All rights reserved.
   6  * * ===========================================================================
   7  * * Licensed under the Apache License, Version 2.0 (the "License");
   8  * * you may not use this file except in compliance with the License.
   9  * * You may obtain a copy of the License at
  10  * *
  11  *  *      http://www.apache.org/licenses/LICENSE-2.0
  12  * *
  13  *  * Unless required by applicable law or agreed to in writing, software
  14  * * distributed under the License is distributed on an "AS IS" BASIS,
  15  * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * * See the License for the specific language governing permissions and
  17  * * limitations under the License.
  18  * * ============LICENSE_END====================================================
  19  * *
  20  * * ECOMP is a trademark and service mark of AT&T Intellectual Property.
  21  * *
  22  ******************************************************************************/
  23
  24
  25 package org.onap.dmaap.datarouter.provisioning;
  26
  27 import java.io.ByteArrayOutputStream;
  28 import java.io.File;
  29 import java.io.IOException;
  30 import java.io.InputStream;
  31 import java.nio.file.FileStore;
  32 import java.nio.file.FileSystem;
  33 import java.nio.file.Files;
  34 import java.nio.file.Path;
  35 import java.nio.file.Paths;
  36 import java.nio.file.StandardCopyOption;
  37 import java.util.Properties;
  38
  39 import javax.servlet.http.HttpServletRequest;
  40 import javax.servlet.http.HttpServletResponse;
  41
  42 import org.json.JSONArray;
  43 import org.onap.dmaap.datarouter.provisioning.beans.EventLogRecord;
  44 import org.onap.dmaap.datarouter.provisioning.beans.LogRecord;
  45 import org.onap.dmaap.datarouter.provisioning.beans.Parameters;
  46 import org.onap.dmaap.datarouter.provisioning.eelf.EelfMsgs;
  47 import org.onap.dmaap.datarouter.provisioning.utils.DB;
  48 import org.onap.dmaap.datarouter.provisioning.utils.LogfileLoader;
  49 import org.onap.dmaap.datarouter.provisioning.utils.RLEBitSet;
  50
  51 import com.att.eelf.configuration.EELFLogger;
  52 import com.att.eelf.configuration.EELFManager;
  53
  54 /**
  55  * <p>
  56  * This servlet handles requests to URLs under /internal on the provisioning server.
  57  * These include:
  58  * </p>
  59  * <div class="contentContainer">
  60  * <table class="packageSummary" border="0" cellpadding="3" cellspacing="0">
  61  * <caption><span>URL Path Summary</span><span class="tabEnd">&nbsp;</span></caption>
  62  * <tr>
  63  *   <th class="colFirst" width="15%">URL Path</th>
  64  *   <th class="colOne">Method</th>
  65  *   <th class="colLast">Purpose</th>
  66  * </tr>
  67  * <tr class="altColor">
  68  *   <td class="colFirst">/internal/prov</td>
  69  *   <td class="colOne">GET</td>
  70  *   <td class="colLast">used to GET a full JSON copy of the provisioning data.</td>
  71  * </tr>
  72  * <tr class="rowColor">
  73  *   <td class="colFirst">/internal/fetchProv</td>
  74  *   <td class="colOne">GET</td>
  75  *   <td class="colLast">used to signal to a standby POD that the provisioning data should be fetched from the active POD.</td>
  76  * </tr>
  77  * <tr class="altColor">
  78  *   <td class="colFirst" rowspan="2">/internal/logs</td>
  79  *   <td class="colOne">GET</td>
  80  *   <td class="colLast">used to GET an index of log files and individual logs for this provisioning server.</td>
  81  * </tr>
  82  * <tr class="altColor">
  83  *   <td class="colOne">POST</td>
  84  *   <td class="colLast">used to POST log files from the individual nodes to this provisioning server.</td>
  85  * </tr>
  86  * <tr class="rowColor">
  87  *   <td class="colFirst" rowspan="4">/internal/api</td>
  88  *   <td class="colOne">GET</td>
  89  *   <td class="colLast">used to GET an individual parameter value. The parameter name is specified by the path after /api/.</td>
  90  * </tr>
  91  * <tr class="rowColor">
  92  *   <td class="colOne">PUT</td>
  93  *   <td class="colLast">used to set an individual parameter value. The parameter name is specified by the path after /api/.</td>
  94  * </tr>
  95  * <tr class="rowColor">
  96  *   <td class="colOne">DELETE</td>
  97  *   <td class="colLast">used to remove an individual parameter value. The parameter name is specified by the path after /api/.</td>
  98  * </tr>
  99  * <tr class="rowColor">
 100  *   <td class="colOne">POST</td>
 101  *   <td class="colLast">used to create a new individual parameter value. The parameter name is specified by the path after /api/.</td>
 102  * </tr>
 103  * <tr class="altColor">
 104  *   <td class="colFirst">/internal/halt</td>
 105  *   <td class="colOne">GET</td>
 106  *   <td class="colLast">used to halt the server (must be accessed from 127.0.0.1).</td>
 107  * </tr>
 108  * <tr class="rowColor">
 109  *   <td class="colFirst" rowspan="2">/internal/drlogs</td>
 110  *   <td class="colOne">GET</td>
 111  *   <td class="colLast">used to get a list of DR log entries available for retrieval.
 112  *   Note: these are the actual data router log entries sent to the provisioning server
 113  *   by the nodes, not the provisioning server's internal logs (access via /internal/logs above).
 114  *   The range is returned as a list of record sequence numbers.</td>
 115  * </tr>
 116  * <tr class="rowColor">
 117  *   <td class="colOne">POST</td>
 118  *   <td class="colLast">used to retrieve specific log entries.
 119  *   The sequence numbers of the records to fetch are POST-ed; the records matching the sequence numbers are returned.</td>
 120  * </tr>
 121  * <tr class="altColor">
 122  *   <td class="colFirst">/internal/route/*</td>
 123  *   <td class="colOne">*</td>
 124  *   <td class="colLast">URLs under this path are handled via the {@link org.onap.dmaap.datarouter.provisioning.RouteServlet}</td>
 125  * </tr>
 126  * </table>
 127  * </div>
 128  * <p>
 129  * Authorization to use these URLs is a little different than for other URLs on the provisioning server.
 130  * For the most part, the IP address that the request comes from should be either:
 131  * </p>
 132  * <ol>
 133  * <li>an IP address of a provisioning server, or</li>
 134  * <li>the IP address of a node (to allow access to /internal/prov), or</li>
 135  * <li>an IP address from the "<i>special subnet</i>" which is configured with
 136  * the PROV_SPECIAL_SUBNET parameter.
 137  * </ol>
 138  * <p>
 139  * In addition, requests to /internal/halt can ONLY come from localhost (127.0.0.1) on the HTTP port.
 140  * </p>
 141  * <p>
 142  * All DELETE/GET/PUT/POST requests made to /internal/api on this servlet on the standby server are
 143  * proxied to the active server (using the {@link ProxyServlet}) if it is up and reachable.
 144  * </p>
 145  *
 146  * @author Robert Eby
 147  * @version $Id: InternalServlet.java,v 1.23 2014/03/24 18:47:10 eby Exp $
 148  */
 149 @SuppressWarnings("serial")
 150 public class InternalServlet extends ProxyServlet {
 151     private static Integer logseq = new Integer(0); // another piece of info to make log spool file names unique
 152     //Adding EELF Logger Rally:US664892
 153     private static EELFLogger eelflogger = EELFManager.getInstance().getLogger("org.onap.dmaap.datarouter.provisioning.InternalServlet");
 154
 155     /**
 156      * Delete a parameter at the address /internal/api/&lt;parameter&gt;.
 157      * See the <b>Internal API</b> document for details on how this method should be invoked.
 158      */
 159     @Override
 160     public void doDelete(HttpServletRequest req, HttpServletResponse resp) throws IOException {
 161         setIpAndFqdnForEelf("doDelete");
 162         eelflogger.info(EelfMsgs.MESSAGE_WITH_BEHALF_AND_FEEDID, req.getHeader(BEHALF_HEADER),getIdFromPath(req)+"");
 163         EventLogRecord elr = new EventLogRecord(req);
 164         if (!isAuthorizedForInternal(req)) {
 165             elr.setMessage("Unauthorized.");
 166             elr.setResult(HttpServletResponse.SC_FORBIDDEN);
 167             eventlogger.info(elr);
 168             resp.sendError(HttpServletResponse.SC_FORBIDDEN, "Unauthorized.");
 169             return;
 170         }
 171
 172         String path = req.getPathInfo();
 173         if (path.startsWith("/api/")) {
 174             if (isProxyOK(req) && isProxyServer()) {
 175                 super.doDelete(req, resp);
 176                 return;
 177             }
 178             String key = path.substring(5);
 179             if (key.length() > 0) {
 180                 Parameters param = Parameters.getParameter(key);
 181                 if (param != null) {
 182                     if (doDelete(param)) {
 183                         elr.setResult(HttpServletResponse.SC_OK);
 184                         eventlogger.info(elr);
 185                         resp.setStatus(HttpServletResponse.SC_OK);
 186                         provisioningDataChanged();
 187                         provisioningParametersChanged();
 188                     } else {
 189                         // Something went wrong with the DELETE
 190                         elr.setResult(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
 191                         eventlogger.info(elr);
 192                         resp.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, DB_PROBLEM_MSG);
 193                     }
 194                     return;
 195                 }
 196             }
 197         }
 198         resp.sendError(HttpServletResponse.SC_NOT_FOUND, "Bad URL.");
 199     }
 200     /**
 201      * Get some information (such as a parameter) underneath the /internal/ namespace.
 202      * See the <b>Internal API</b> document for details on how this method should be invoked.
 203      */
 204     @Override
 205     public void doGet(HttpServletRequest req, HttpServletResponse resp) throws IOException {
 206         setIpAndFqdnForEelf("doGet");
 207         eelflogger.info(EelfMsgs.MESSAGE_WITH_BEHALF_AND_FEEDID, req.getHeader(BEHALF_HEADER),getIdFromPath(req)+"");
 208         String path = req.getPathInfo();
 209         if (path.equals("/halt") && !req.isSecure()) {
 210             // request to halt the server - can ONLY come from localhost
 211             String remote = req.getRemoteAddr();
 212             if (remote.equals("127.0.0.1")) {
 213                 intlogger.info("PROV0009 Request to HALT received.");
 214                 resp.setStatus(HttpServletResponse.SC_OK);
 215                 Main.shutdown();
 216             } else {
 217                 intlogger.info("PROV0010 Disallowed request to HALT received from "+remote);
 218                 resp.setStatus(HttpServletResponse.SC_FORBIDDEN);
 219             }
 220             return;
 221         }
 222
 223         EventLogRecord elr = new EventLogRecord(req);
 224         if (!isAuthorizedForInternal(req)) {
 225             elr.setMessage("Unauthorized.");
 226             elr.setResult(HttpServletResponse.SC_FORBIDDEN);
 227             eventlogger.info(elr);
 228             resp.sendError(HttpServletResponse.SC_FORBIDDEN, "Unauthorized.");
 229             return;
 230         }
 231         if (path.equals("/fetchProv") && !req.isSecure()) {
 232             // if request came from active_pod or standby_pod and it is not us, reload prov data
 233             SynchronizerTask s = SynchronizerTask.getSynchronizer();
 234             s.doFetch();
 235             resp.setStatus(HttpServletResponse.SC_OK);
 236             return;
 237         }
 238         if (path.equals("/prov")) {
 239             if (isProxyOK(req) && isProxyServer()) {
 240                 if (super.doGetWithFallback(req, resp))
 241                     return;
 242                 // fall back to returning the local data if the remote is unreachable
 243                 intlogger.info("Active server unavailable; falling back to local copy.");
 244             }
 245             Poker p = Poker.getPoker();
 246             resp.setStatus(HttpServletResponse.SC_OK);
 247             resp.setContentType(PROVFULL_CONTENT_TYPE2);
 248             resp.getOutputStream().print(p.getProvisioningString());
 249             return;
 250         }
 251         if (path.equals("/logs") || path.equals("/logs/")) {
 252             resp.setStatus(HttpServletResponse.SC_OK);
 253             resp.setContentType("application/json");
 254             resp.getOutputStream().print(generateLogfileList().toString());
 255             return;
 256         }
 257         if (path.startsWith("/logs/")) {
 258             Properties p = (new DB()).getProperties();
 259             String logdir = p.getProperty("org.onap.dmaap.datarouter.provserver.accesslog.dir");
 260             String logfile = path.substring(6);
 261             if (logdir != null && logfile != null && logfile.indexOf('/') < 0) {
 262                 File log = new File(logdir + "/" + logfile);
 263                 if (log.exists() && log.isFile()) {
 264                     resp.setStatus(HttpServletResponse.SC_OK);
 265                     resp.setContentType("text/plain");
 266                     Path logpath = Paths.get(log.getAbsolutePath());
 267                     Files.copy(logpath, resp.getOutputStream());
 268                     return;
 269                 }
 270             }
 271             resp.sendError(HttpServletResponse.SC_NO_CONTENT, "No file.");
 272             return;
 273         }
 274         if (path.startsWith("/api/")) {
 275             if (isProxyOK(req) && isProxyServer()) {
 276                 super.doGet(req, resp);
 277                 return;
 278             }
 279             String key = path.substring(5);
 280             if (key.length() > 0) {
 281                 Parameters param = Parameters.getParameter(key);
 282                 if (param != null) {
 283                     resp.setStatus(HttpServletResponse.SC_OK);
 284                     resp.setContentType("text/plain");
 285                     resp.getOutputStream().print(param.getValue() + "\n");
 286                     return;
 287                 }
 288             }
 289         }
 290         if (path.equals("/drlogs") || path.equals("/drlogs/")) {
 291             // Special POD <=> POD API to determine what log file records are loaded here
 292             LogfileLoader lfl = LogfileLoader.getLoader();
 293             resp.setStatus(HttpServletResponse.SC_OK);
 294             resp.setContentType("text/plain");
 295             resp.getOutputStream().print(lfl.getBitSet().toString());
 296             return;
 297         }
 298         resp.sendError(HttpServletResponse.SC_NOT_FOUND, "Bad URL.");
 299     }
 300     /**
 301      * Modify a parameter at the address /internal/api/&lt;parameter&gt;.
 302      * See the <b>Internal API</b> document for details on how this method should be invoked.
 303      */
 304     @Override
 305     public void doPut(HttpServletRequest req, HttpServletResponse resp) throws IOException {
 306         setIpAndFqdnForEelf("doPut");
 307         eelflogger.info(EelfMsgs.MESSAGE_WITH_BEHALF_AND_FEEDID, req.getHeader(BEHALF_HEADER),getIdFromPath(req)+"");
 308         EventLogRecord elr = new EventLogRecord(req);
 309         if (!isAuthorizedForInternal(req)) {
 310             elr.setMessage("Unauthorized.");
 311             elr.setResult(HttpServletResponse.SC_FORBIDDEN);
 312             eventlogger.info(elr);
 313             resp.sendError(HttpServletResponse.SC_FORBIDDEN, "Unauthorized.");
 314             return;
 315         }
 316         String path = req.getPathInfo();
 317         if (path.startsWith("/api/")) {
 318             if (isProxyOK(req) && isProxyServer()) {
 319                 super.doPut(req, resp);
 320                 return;
 321             }
 322             String key = path.substring(5);
 323             if (key.length() > 0) {
 324                 Parameters param = Parameters.getParameter(key);
 325                 if (param != null) {
 326                     String t = catValues(req.getParameterValues("val"));
 327                     param.setValue(t);
 328                     if (doUpdate(param)) {
 329                         elr.setResult(HttpServletResponse.SC_OK);
 330                         eventlogger.info(elr);
 331                         resp.setStatus(HttpServletResponse.SC_OK);
 332                         provisioningDataChanged();
 333                         provisioningParametersChanged();
 334                     } else {
 335                         // Something went wrong with the UPDATE
 336                         elr.setResult(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
 337                         eventlogger.info(elr);
 338                         resp.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, DB_PROBLEM_MSG);
 339                     }
 340                     return;
 341                 }
 342             }
 343         }
 344         resp.sendError(HttpServletResponse.SC_NOT_FOUND, "Bad URL.");
 345     }
 346     /**
 347      * Create some new information (such as a parameter or log entries) underneath the /internal/ namespace.
 348      * See the <b>Internal API</b> document for details on how this method should be invoked.
 349      */
 350     @SuppressWarnings("resource")
 351     @Override
 352     public void doPost(HttpServletRequest req, HttpServletResponse resp) throws IOException {
 353         setIpAndFqdnForEelf("doPost");
 354         eelflogger.info(EelfMsgs.MESSAGE_WITH_BEHALF, req.getHeader(BEHALF_HEADER));
 355         EventLogRecord elr = new EventLogRecord(req);
 356         if (!isAuthorizedForInternal(req)) {
 357             elr.setMessage("Unauthorized.");
 358             elr.setResult(HttpServletResponse.SC_FORBIDDEN);
 359             eventlogger.info(elr);
 360             resp.sendError(HttpServletResponse.SC_FORBIDDEN, "Unauthorized.");
 361             return;
 362         }
 363
 364         String path = req.getPathInfo();
 365         if (path.startsWith("/api/")) {
 366             if (isProxyOK(req) && isProxyServer()) {
 367                 super.doPost(req, resp);
 368                 return;
 369             }
 370             String key = path.substring(5);
 371             if (key.length() > 0) {
 372                 Parameters param = Parameters.getParameter(key);
 373                 if (param == null) {
 374                     String t = catValues(req.getParameterValues("val"));
 375                     param = new Parameters(key, t);
 376                     if (doInsert(param)) {
 377                         elr.setResult(HttpServletResponse.SC_OK);
 378                         eventlogger.info(elr);
 379                         resp.setStatus(HttpServletResponse.SC_OK);
 380                         provisioningDataChanged();
 381                         provisioningParametersChanged();
 382                     } else {
 383                         // Something went wrong with the INSERT
 384                         elr.setResult(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
 385                         eventlogger.info(elr);
 386                         resp.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, DB_PROBLEM_MSG);
 387                     }
 388                     return;
 389                 }
 390             }
 391         }
 392
 393         if (path.equals("/logs") || path.equals("/logs/")) {
 394             String ctype = req.getHeader("Content-Type");
 395             if (ctype == null || !ctype.equals("text/plain")) {
 396                 elr.setResult(HttpServletResponse.SC_UNSUPPORTED_MEDIA_TYPE);
 397                 elr.setMessage("Bad media type: "+ctype);
 398                 resp.setStatus(HttpServletResponse.SC_UNSUPPORTED_MEDIA_TYPE);
 399                 eventlogger.info(elr);
 400                 return;
 401             }
 402             String spooldir = (new DB()).getProperties().getProperty("org.onap.dmaap.datarouter.provserver.spooldir");
 403             String spoolname = String.format("%d-%d-", System.currentTimeMillis(), Thread.currentThread().getId());
 404             synchronized (logseq) {
 405                 // perhaps unnecessary, but it helps make the name unique
 406                 spoolname += logseq.toString();
 407                 logseq++;
 408             }
 409             String encoding = req.getHeader("Content-Encoding");
 410             if (encoding != null) {
 411                 if (encoding.trim().equals("gzip")) {
 412                     spoolname += ".gz";
 413                 } else {
 414                     elr.setResult(HttpServletResponse.SC_UNSUPPORTED_MEDIA_TYPE);
 415                     resp.setStatus(HttpServletResponse.SC_UNSUPPORTED_MEDIA_TYPE);
 416                     eventlogger.info(elr);
 417                     return;
 418                 }
 419             }
 420             // Determine space available -- available space must be at least 5%
 421             FileSystem fs = (Paths.get(spooldir)).getFileSystem();
 422             long total = 0;
 423             long avail = 0;
 424             for (FileStore store: fs.getFileStores()) {
 425                 total += store.getTotalSpace();
 426                 avail += store.getUsableSpace();
 427             }
 428             try { fs.close(); } catch (Exception e) { }
 429             if (((avail * 100) / total) < 5) {
 430                 elr.setResult(HttpServletResponse.SC_SERVICE_UNAVAILABLE);
 431                 resp.setStatus(HttpServletResponse.SC_SERVICE_UNAVAILABLE);
 432                 eventlogger.info(elr);
 433                 return;
 434             }
 435             Path tmppath = Paths.get(spooldir, spoolname);
 436             Path donepath = Paths.get(spooldir, "IN."+spoolname);
 437             Files.copy(req.getInputStream(), Paths.get(spooldir, spoolname), StandardCopyOption.REPLACE_EXISTING);
 438             Files.move(tmppath, donepath, StandardCopyOption.REPLACE_EXISTING);
 439             elr.setResult(HttpServletResponse.SC_CREATED);
 440             resp.setStatus(HttpServletResponse.SC_CREATED);
 441             eventlogger.info(elr);
 442             LogfileLoader.getLoader();    // This starts the logfile loader "task"
 443             return;
 444         }
 445
 446         if (path.equals("/drlogs") || path.equals("/drlogs/")) {
 447             // Receive post request and generate log entries
 448             String ctype = req.getHeader("Content-Type");
 449             if (ctype == null || !ctype.equals("text/plain")) {
 450                 elr.setResult(HttpServletResponse.SC_UNSUPPORTED_MEDIA_TYPE);
 451                 elr.setMessage("Bad media type: "+ctype);
 452                 resp.setStatus(HttpServletResponse.SC_UNSUPPORTED_MEDIA_TYPE);
 453                 eventlogger.info(elr);
 454                 return;
 455             }
 456             InputStream is = req.getInputStream();
 457             ByteArrayOutputStream bos = new ByteArrayOutputStream();
 458             int ch = 0;
 459             while ((ch = is.read()) >= 0)
 460                 bos.write(ch);
 461             RLEBitSet bs = new RLEBitSet(bos.toString());    // The set of records to retrieve
 462             elr.setResult(HttpServletResponse.SC_OK);
 463             resp.setStatus(HttpServletResponse.SC_OK);
 464             resp.setContentType("text/plain");
 465             LogRecord.printLogRecords(resp.getOutputStream(), bs);
 466             eventlogger.info(elr);
 467             return;
 468         }
 469
 470         elr.setResult(HttpServletResponse.SC_NOT_FOUND);
 471         resp.sendError(HttpServletResponse.SC_NOT_FOUND, "Bad URL.");
 472         eventlogger.info(elr);
 473     }
 474
 475     private String catValues(String[] v) {
 476         StringBuilder sb = new StringBuilder();
 477         if (v != null) {
 478             String pfx = "";
 479             for (String s : v) {
 480                 sb.append(pfx);
 481                 sb.append(s);
 482                 pfx = "|";
 483             }
 484         }
 485         return sb.toString();
 486     }
 487     private JSONArray generateLogfileList() {
 488         JSONArray ja = new JSONArray();
 489         Properties p = (new DB()).getProperties();
 490         String s = p.getProperty("org.onap.dmaap.datarouter.provserver.accesslog.dir");
 491         if (s != null) {
 492             String[] dirs = s.split(",");
 493             for (String dir : dirs) {
 494                 File f = new File(dir);
 495                 String[] list = f.list();
 496                 if (list != null) {
 497                     for (String s2 : list) {
 498                         if (!s2.startsWith("."))
 499                             ja.put(s2);
 500                     }
 501                 }
 502             }
 503         }
 504         return ja;
 505     }
 506 }