dgbuilder/core_nodes/parsers/70-CSV.js

   1 /**
   2  * Copyright 2014 IBM Corp.
   3  *
   4  * Licensed under the Apache License, Version 2.0 (the "License");
   5  * you may not use this file except in compliance with the License.
   6  * You may obtain a copy of the License at
   7  *
   8  * http://www.apache.org/licenses/LICENSE-2.0
   9  *
  10  * Unless required by applicable law or agreed to in writing, software
  11  * distributed under the License is distributed on an "AS IS" BASIS,
  12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  * See the License for the specific language governing permissions and
  14  * limitations under the License.
  15  **/
  16
  17 module.exports = function(RED) {
  18     "use strict";
  19     function CSVNode(n) {
  20         RED.nodes.createNode(this,n);
  21         this.template = n.temp.split(",");
  22         this.sep = (n.sep || ',').replace("\\t","\t").replace("\\n","\n").replace("\\r","\r");
  23         this.quo = '"';
  24         this.ret = (n.ret || "\n").replace("\\n","\n").replace("\\r","\r");
  25         this.winflag = (this.ret === "\r\n");
  26         this.lineend = "\n";
  27         this.multi = n.multi || "one";
  28         this.hdrin = n.hdrin || false;
  29         this.hdrout = n.hdrout || false;
  30         this.goodtmpl = true;
  31         var node = this;
  32
  33         // pass in an array of column names to be trimed, de-quoted and retrimed
  34         var clean = function(col) {
  35             for (var t = 0; t < col.length; t++) {
  36                 col[t] = col[t].trim(); // remove leading and trailing whitespace
  37                 if (col[t].charAt(0) === '"' && col[t].charAt(col[t].length -1) === '"') {
  38                     // remove leading and trailing quotes (if they exist) - and remove whitepace again.
  39                     col[t] = col[t].substr(1,col[t].length -2).trim();
  40                 }
  41             }
  42             if ((col.length === 1) && (col[0] === "")) { node.goodtmpl = false; }
  43             else { node.goodtmpl = true; }
  44             return col;
  45         }
  46         node.template = clean(node.template);
  47
  48         this.on("input", function(msg) {
  49             if (msg.hasOwnProperty("payload")) {
  50                 if (typeof msg.payload == "object") { // convert object to CSV string
  51                     try {
  52                         var ou = "";
  53                         if (node.hdrout) {
  54                             ou += node.template.join(node.sep) + node.ret;
  55                         }
  56                         if (!Array.isArray(msg.payload)) { msg.payload = [ msg.payload ]; }
  57                         for (var s = 0; s < msg.payload.length; s++) {
  58                             for (var t=0; t < node.template.length; t++) {
  59
  60                                 // aaargh - resorting to eval here - but fairly contained front and back.
  61                                 var p = RED.util.ensureString(eval("msg.payload[s]."+node.template[t]));
  62
  63                                 if (p === "undefined") { p = ""; }
  64                                 if (p.indexOf(node.sep) != -1) { // add quotes if any "commas"
  65                                     ou += node.quo + p + node.quo + node.sep;
  66                                 }
  67                                 else if (p.indexOf(node.quo) != -1) { // add double quotes if any quotes
  68                                     p = p.replace(/"/g, '""');
  69                                     ou += node.quo + p + node.quo + node.sep;
  70                                 }
  71                                 else { ou += p + node.sep; } // otherwise just add
  72                             }
  73                             ou = ou.slice(0,-1) + node.ret; // remove final "comma" and add "newline"
  74                         }
  75                         node.send({payload:ou});
  76                     }
  77                     catch(e) { node.log(e); }
  78                 }
  79                 else if (typeof msg.payload == "string") { // convert CSV string to object
  80                     try {
  81                         var f = true; // flag to indicate if inside or outside a pair of quotes true = outside.
  82                         var j = 0; // pointer into array of template items
  83                         var k = [""]; // array of data for each of the template items
  84                         var o = {}; // output object to build up
  85                         var a = []; // output array is needed for multiline option
  86                         var first = true; // is this the first line
  87                         var tmp = "";
  88
  89                         // For now we are just going to assume that any \r or \n means an end of line...
  90                         //   got to be a weird csv that has singleton \r \n in it for another reason...
  91
  92                         // Now process the whole file/line
  93                         for (var i = 0; i < msg.payload.length; i++) {
  94                             if ((node.hdrin === true) && first) { // if the template is in the first line
  95                                 if ((msg.payload[i] === "\n")||(msg.payload[i] === "\r")) { // look for first line break
  96                                     node.template = clean(tmp.split(node.sep));
  97                                     first = false;
  98                                 }
  99                                 else { tmp += msg.payload[i]; }
 100                             }
 101                             else {
 102                                 if (msg.payload[i] === node.quo) { // if it's a quote toggle inside or outside
 103                                     f = !f;
 104                                     if (msg.payload[i-1] === node.quo) { k[j] += '\"'; } // if it's a quotequote then it's actually a quote
 105                                 }
 106                                 else if ((msg.payload[i] === node.sep) && f) { // if we are outside of quote (ie valid separator
 107                                     if (!node.goodtmpl) { node.template[j] = "col"+(j+1); }
 108                                     if ( node.template[j] && (node.template[j] !== "") && (k[j] !== "" ) ) {
 109                                         if (!isNaN(Number(k[j]))) { k[j] = Number(k[j]); }
 110                                         o[node.template[j]] = k[j];
 111                                     }
 112                                     j += 1;
 113                                     k[j] = "";
 114                                 }
 115                                 else if (f && ((msg.payload[i] === "\n") || (msg.payload[i] === "\r"))) { // handle multiple lines
 116                                     //console.log(j,k,o,k[j]);
 117                                     if ( node.template[j] && (node.template[j] !== "") && (k[j] !== "") ) {
 118                                         if (!isNaN(Number(k[j]))) { k[j] = Number(k[j]); }
 119                                         else { k[j].replace(/\r$/,''); }
 120                                         o[node.template[j]] = k[j];
 121                                     }
 122                                     if (JSON.stringify(o) !== "{}") { // don't send empty objects
 123                                         if (node.multi === "one") { node.send({payload:o}); } // either send
 124                                         else { a.push(o); } // or add to the array
 125                                     }
 126                                     j = 0;
 127                                     k = [""];
 128                                     o = {};
 129                                 }
 130                                 else { // just add to the part of the message
 131                                     k[j] += msg.payload[i];
 132                                 }
 133                             }
 134                         }
 135                         // Finished so finalize and send anything left
 136                         //console.log(j,k,o,k[j]);
 137                         if (!node.goodtmpl) { node.template[j] = "col"+(j+1); }
 138                         if ( node.template[j] && (node.template[j] !== "") && (k[j] !== "") ) {
 139                             if (!isNaN(Number(k[j]))) { k[j] = Number(k[j]); }
 140                             else { k[j].replace(/\r$/,''); }
 141                             o[node.template[j]] = k[j];
 142                         }
 143                         msg.payload = o;
 144                         if (JSON.stringify(o) !== "{}") { // don't send empty objects
 145                             if (node.multi === "one") { node.send({payload:o}); } // either send
 146                             else { a.push(o); } // or add to the aray
 147                         }
 148                         if (node.multi !== "one") { node.send({payload:a}); } // finally send the array
 149                     }
 150                     catch(e) { node.log(e); }
 151                 }
 152                 else { node.log("This node only handles csv strings or js objects."); }
 153             }
 154         });
 155     }
 156     RED.nodes.registerType("csv",CSVNode);
 157 }