test/mocks/datafilecollector-testharness/dr-sim/dmaapDR.js

   1 var http = require('http');
   2 var https = require('https');
   3 var ArgumentParser = require('argparse').ArgumentParser;
   4 var express = require('express');
   5 const stream = require('stream');
   6 var app = express();
   7 var fs = require('fs');
   8 const sleep = (milliseconds) => {
   9   return new Promise(resolve => setTimeout(resolve, milliseconds))
  10 }
  11 var privateKey  = fs.readFileSync('cert/private.key', 'utf8');
  12 var certificate = fs.readFileSync('cert/certificate.crt', 'utf8');
  13 var credentials = {key: privateKey, cert: certificate};
  14
  15 var feeds="1:A";  //Comma separated list of feedId:filePrefix. Default is feedId=1 and file prefix 'A'
  16 var feedNames=[];
  17 var filePrefixes=[];
  18 var feedIndexes=[];
  19
  20
  21 //For execution time calculation
  22 var startTime = Date.now();
  23
  24 //Test case constants
  25 const tc_normal = "normal";
  26 const tc_none_published = "none_published";
  27 const tc_all_published = "all_published"
  28 const tc_10p_no_response = "10p_no_response";
  29 const tc_10first_no_response = "10first_no_response";
  30 const tc_100first_no_response = "100first_no_response";
  31 const tc_all_delay_1s = "all_delay_1s";
  32 const tc_all_delay_10s = "all_delay_10s";
  33 const tc_10p_delay_10s = "10p_delay_10s";
  34 const tc_10p_error_response = "10p_error_response";
  35 const tc_10first_error_response = "10first_error_response";
  36 const tc_100first_error_response = "100first_error_response";
  37
  38 var drr_sim_ip = '127.0.0.1'; //IP for redirect to DR redir sim. Can be changed by env DRR_SIM_IP
  39
  40 //Counters
  41 var ctr_publish_query = [];
  42 var ctr_publish_query_bad_file_prefix = [];
  43 var ctr_publish_query_published = [];
  44 var ctr_publish_query_not_published = [];
  45 var ctr_publish_req = [];
  46 var ctr_publish_req_bad_file_prefix = [];
  47 var ctr_publish_req_redirect = [];
  48 var ctr_publish_req_published = [];
  49 var ctr_double_publish = [];
  50
  51 //db of published files
  52 var published=[];
  53
  54 var parser = new ArgumentParser({
  55           version: '0.0.1',
  56           addHelp:true,
  57           description: 'Datarouter simulator'
  58         });
  59
  60 parser.addArgument('--tc' , { help: 'TC $NoOfTc' } );
  61 parser.addArgument('--printtc' ,
  62                 {
  63                         help: 'Print complete usage help',
  64                         action: 'storeTrue'
  65                 }
  66         );
  67
  68 var args = parser.parseArgs();
  69
  70 if (args.tc==tc_normal) {
  71         console.log("TC: " + args.tc)
  72
  73 } else if (args.tc==tc_none_published) {
  74         console.log("TC: " + args.tc)
  75
  76 } else if (args.tc==tc_all_published) {
  77         console.log("TC: " + args.tc)
  78
  79 } else if (args.tc==tc_10p_no_response) {
  80         console.log("TC: " + args.tc)
  81
  82 } else if (args.tc==tc_10first_no_response) {
  83         console.log("TC: " + args.tc)
  84
  85 } else if (args.tc==tc_100first_no_response) {
  86         console.log("TC: " + args.tc)
  87
  88 } else if (args.tc==tc_all_delay_1s) {
  89         console.log("TC: " + args.tc)
  90
  91 } else if (args.tc==tc_all_delay_10s) {
  92         console.log("TC: " + args.tc)
  93
  94 } else if (args.tc==tc_10p_delay_10s) {
  95         console.log("TC: " + args.tc)
  96
  97 } else if (args.tc==tc_10p_error_response) {
  98         console.log("TC: " + args.tc)
  99
 100 } else if (args.tc==tc_10first_error_response) {
 101         console.log("TC: " + args.tc)
 102
 103 } else if (args.tc==tc_100first_error_response) {
 104         console.log("TC: " + args.tc)
 105 } else {
 106         console.log("No TC specified, use: --tc <tc-id>");
 107         process.exit(0);
 108 }
 109
 110 if (args.printtc) {
 111         console.log("TC " + tc_normal + ": Normal case, query respone based on published files. Publish responde with ok/redirect depending on if file is published or not.");
 112         console.log("TC " + tc_none_published + ": Query responde 'ok'. Publish respond with redirect.");
 113         console.log("TC " + tc_all_published + ": Query respond with filename. Publish respond with 'ok'.");
 114         console.log("TC " + tc_10p_no_response + ": 10% % no response for query and publish. Otherwise normal case.");
 115         console.log("TC " + tc_10first_no_response + ": 10 first queries and requests gives no response for query and publish. Otherwise normal case.");
 116         console.log("TC " + tc_100first_no_response + ": 100 first queries and requests gives no response for query and publish. Otherwise normal case.");
 117         console.log("TC " + tc_all_delay_1s + ": All responses delayed 1s (both query and publish).");
 118         console.log("TC " + tc_all_delay_10s + ": All responses delayed 10s (both query and publish).");
 119         console.log("TC " + tc_10p_delay_10s + ": 10% of responses delayed 10s, (both query and publish).");
 120         console.log("TC " + tc_10p_error_response + ": 10% error response for query and publish. Otherwise normal case.");
 121         console.log("TC " + tc_10first_error_response + ": 10 first queries and requests gives no response for query and publish. Otherwise normal case.");
 122         console.log("TC " + tc_100first_error_response + ": 100 first queries and requests gives no response for query and publish. Otherwise normal case.");
 123
 124         process.exit(0);
 125   }
 126
 127
 128 var bodyParser = require('body-parser')
 129 app.use(bodyParser.urlencoded({ extended: false }))
 130
 131 // parse application/json
 132 app.use(bodyParser.json())
 133
 134 // parse application/vnd.api+json as json
 135 app.use(bodyParser.json({ type: 'application/vnd.api+json' }))
 136
 137 // parse some custom thing into a Buffer (to cater for 60MB files)
 138 //Removed-file data not used in this simulator
 139 //app.use(bodyParser.raw({limit:1024*1024*60, type: 'application/octet-stream' }))
 140
 141 // parse an HTML body into a string
 142 app.use(bodyParser.text({ type: 'text/html' }))
 143
 144
 145
 146 //Is alive function
 147 app.get("/",function(req, res){
 148         res.send("ok");
 149 })
 150
 151 function toCommaList(ctrArray) {
 152         var str="";
 153         for(i=0;i<feedNames.length;i++) {
 154                 if (i!=0) {
 155                         str=str+",";
 156                 }
 157                 str=str+ctrArray[i];
 158         }
 159         return str;
 160 }
 161
 162 function sumList(ctrArray) {
 163         var tmp=0;
 164         for(i=0;i<feedNames.length;i++) {
 165                 tmp=tmp+ctrArray[i];
 166         }
 167         return ""+tmp;
 168 }
 169
 170 function sumListLength(ctrArray) {
 171         var tmp=0;
 172         for(i=0;i<feedNames.length;i++) {
 173                 tmp=tmp+ctrArray[i].length;
 174         }
 175         return ""+tmp;
 176 }
 177
 178 //Counter readout
 179 app.get("/ctr_publish_query",function(req, res){
 180         res.send(""+sumList(ctr_publish_query));
 181 })
 182 app.get("/feeds/ctr_publish_query",function(req, res){
 183         res.send(toCommaList(ctr_publish_query));
 184 })
 185 app.get("/ctr_publish_query/:feedId",function(req, res){
 186         var feedId = req.params.feedId;
 187         res.send(""+ctr_publish_query[feedIndexes[feedId]]);
 188 })
 189
 190 app.get("/ctr_publish_query_bad_file_prefix",function(req, res){
 191         res.send(""+sumList(ctr_publish_query_bad_file_prefix));
 192 })
 193 app.get("/feeds/ctr_publish_query_bad_file_prefix",function(req, res){
 194         res.send(toCommaList(ctr_publish_query_bad_file_prefix));
 195 })
 196 app.get("/ctr_publish_query_bad_file_prefix/:feedId",function(req, res){
 197         var feedId = req.params.feedId;
 198         res.send(""+ctr_publish_query_bad_file_prefix[feedIndexes[feedId]]);
 199 })
 200
 201 app.get("/ctr_publish_query_published",function(req, res){
 202         res.send(""+sumList(ctr_publish_query_published));
 203 })
 204 app.get("/feeds/ctr_publish_query_published",function(req, res){
 205         res.send(toCommaList(ctr_publish_query_published));
 206 })
 207 app.get("/ctr_publish_query_published/:feedId",function(req, res){
 208         var feedId = req.params.feedId;
 209         res.send(""+ctr_publish_query_published[feedIndexes[feedId]]);
 210 })
 211
 212 app.get("/ctr_publish_query_not_published",function(req, res){
 213         res.send(""+sumList(ctr_publish_query_not_published));
 214 })
 215 app.get("/feeds/ctr_publish_query_not_published",function(req, res){
 216         res.send(toCommaList(ctr_publish_query_not_published));
 217 })
 218 app.get("/ctr_publish_query_not_published/:feedId",function(req, res){
 219         var feedId = req.params.feedId;
 220         res.send(""+ctr_publish_query_not_published[feedIndexes[feedId]]);
 221 })
 222
 223 app.get("/ctr_publish_req",function(req, res){
 224         res.send(""+sumList(ctr_publish_req));
 225 })
 226 app.get("/feeds/ctr_publish_req",function(req, res){
 227         res.send(toCommaList(ctr_publish_req));
 228 })
 229 app.get("/ctr_publish_req/:feedId",function(req, res){
 230         var feedId = req.params.feedId;
 231         res.send(""+ctr_publish_req[feedIndexes[feedId]]);
 232 })
 233
 234 app.get("/ctr_publish_req_bad_file_prefix",function(req, res){
 235         res.send(""+sumList(ctr_publish_req_bad_file_prefix));
 236 })
 237 app.get("/feeds/ctr_publish_req_bad_file_prefix",function(req, res){
 238         res.send(toCommaList(ctr_publish_req_bad_file_prefix));
 239 })
 240 app.get("/ctr_publish_req_bad_file_prefix/:feedId",function(req, res){
 241         var feedId = req.params.feedId;
 242         res.send(""+ctr_publish_req_bad_file_prefix[feedIndexes[feedId]]);
 243 })
 244
 245 app.get("/ctr_publish_req_redirect",function(req, res){
 246         res.send(""+sumList(ctr_publish_req_redirect));
 247 })
 248 app.get("/feeds/ctr_publish_req_redirect",function(req, res){
 249         res.send(toCommaList(ctr_publish_req_redirect));
 250 })
 251 app.get("/ctr_publish_req_redirect/:feedId",function(req, res){
 252         var feedId = req.params.feedId;
 253         res.send(""+ctr_publish_req_redirect[feedIndexes[feedId]]);
 254 })
 255
 256 app.get("/ctr_publish_req_published",function(req, res){
 257         res.send(""+sumList(ctr_publish_req_published));
 258 })
 259 app.get("/feeds/ctr_publish_req_published",function(req, res){
 260         res.send(toCommaList(ctr_publish_req_published));
 261 })
 262 app.get("/ctr_publish_req_published/:feedId",function(req, res){
 263         var feedId = req.params.feedId;
 264         res.send(""+ctr_publish_req_published[feedIndexes[feedId]]);
 265 })
 266
 267 app.get("/ctr_published_files",function(req, res){
 268         res.send(""+sumListLength(published));
 269 })
 270 app.get("/feeds/ctr_published_files",function(req, res){
 271         var str="";
 272         for(i=0;i<feedNames.length;i++) {
 273                 if (i!=0) {
 274                         str=str+",";
 275                 }
 276                 str=str+published[i].length;
 277         }
 278         res.send(str);
 279 })
 280 app.get("/ctr_published_files/:feedId",function(req, res){
 281         var feedId = req.params.feedId;
 282         res.send(""+published[feedIndexes[feedId]].length);
 283 })
 284
 285 app.get("/tc_info",function(req, res){
 286         res.send(args.tc);
 287 })
 288 app.get("/ctr_double_publish",function(req, res){
 289         res.send(""+sumList(ctr_double_publish));
 290 })
 291 app.get("/feeds/ctr_double_publish",function(req, res){
 292         var str="";
 293         for(i=0;i<feedNames.length;i++) {
 294                 if (i!=0) {
 295                         str=str+",";
 296                 }
 297                 str=str+ctr_double_publish[i];
 298         }
 299         res.send(str);
 300 })
 301 app.get("/ctr_double_publish/:feedId",function(req, res){
 302         var feedId = req.params.feedId;
 303         res.send(""+ctr_double_publish[feedIndexes[feedId]]);
 304 })
 305
 306 function fmtMSS(s){
 307         return(s-(s%=60))/60+(9<s?':':':0')+s    //Format time diff in mm:ss
 308 }
 309 app.get("/execution_time",function(req, res){
 310         var diff = fmtMSS(Math.floor((Date.now()-startTime)/1000));
 311         res.send(""+diff);
 312 })
 313 app.get("/feeds",function(req, res){
 314         res.send(feeds);
 315 })
 316
 317 function filenameStartsWith(fileName, feedIndex) {
 318         for(i=0;i<filePrefixes[feedIndex].length;i++) {
 319                 var prefix=filePrefixes[feedIndex][i];
 320                 if (fileName.startsWith(prefix)) {
 321                         return true;
 322                 }
 323         }
 324         return false;
 325 }
 326
 327 app.get('/feedlog/:feedId',function(req, res){
 328         console.log("url:"+req.url);
 329         var feedId = req.params.feedId;
 330         ctr_publish_query[feedIndexes[feedId]]++;
 331         var filename = req.query.filename;
 332         if (!filenameStartsWith(filename, feedIndexes[feedId])) {
 333                 ctr_publish_query_bad_file_prefix[feedIndexes[feedId]]++;
 334         }
 335         console.log(filename);
 336         var qtype = req.query.type;
 337         if(typeof(filename) == 'undefined'){
 338                 res.status(400).send({error: 'No filename provided.'});
 339                 return;
 340         } else if(typeof(qtype) == 'undefined'){
 341                 res.status(400).send({error: 'No type provided.'});
 342                 return;
 343         }
 344         var ctr = ctr_publish_query[feedIndexes[feedId]];
 345         //Ugly fix, plus signs replaces with spaces in query params....need to put them back
 346         filename = filename.replace(/ /g,"+");
 347
 348         var sleeptime=0;
 349         if (args.tc==tc_normal) {
 350                 sleeptime=0;
 351         } else if (args.tc==tc_10p_no_response && (ctr%10) == 0) {
 352                 return;
 353         } else if (args.tc==tc_10first_no_response && ctr<11) {
 354                 return;
 355         } else if (args.tc==tc_100first_no_response && ctr<101) {
 356                 return;
 357         } else if (args.tc==tc_all_delay_1s) {
 358                 sleeptime=1000;
 359         } else if (args.tc==tc_all_delay_10s) {
 360                 sleeptime=10000;
 361         } else if (args.tc==tc_10p_delay_10s && (ctr%10) == 0) {
 362                 sleeptime=10000;
 363         } else if (args.tc==tc_10p_error_response && (ctr%10) == 0) {
 364                 res.send(400);
 365                 return;
 366         } else if (args.tc==tc_10first_error_response && ctr<11) {
 367                 res.send(400);
 368                 return;
 369         } else if (args.tc==tc_100first_error_response & ctr<101) {
 370                 res.send(400);
 371                 return;
 372         }
 373
 374         if (published[feedIndexes[feedId]].includes(filename)) {
 375                 ctr_publish_query_published[feedIndexes[feedId]]++;
 376                 strToSend="[" + filename + "]";
 377         } else {
 378                 ctr_publish_query_not_published[feedIndexes[feedId]]++;
 379                 strToSend="[]";
 380         }
 381         if (sleeptime > 0) {
 382                 sleep(sleeptime).then(() => {
 383                         res.send(strToSend);
 384                 });
 385         } else {
 386                 res.send(strToSend);
 387         }
 388 });
 389
 390
 391 app.put('/publish/:feedId/:filename', function (req, res) {
 392         console.log("url:"+req.url);
 393 //      console.log("body (first 25 bytes):"+req.body.slice(0,25));
 394         console.log("headers:"+req.headers);
 395         console.log(JSON.stringify(req.headers));
 396         var feedId = req.params.feedId;
 397         ctr_publish_req[feedIndexes[feedId]]++;
 398
 399         var filename = req.params.filename;
 400         console.log(filename);
 401         if (!filenameStartsWith(filename, feedIndexes[feedId])) {
 402                 ctr_publish_req_bad_file_prefix[feedIndexes[feedId]]++;
 403         }
 404     var ctr = ctr_publish_req[feedIndexes[feedId]];
 405         if (args.tc==tc_normal) {
 406         // Continue
 407         } else if (args.tc==tc_none_published) {
 408                 ctr_publish_req_redirect[feedIndexes[feedId]]++;
 409                 res.redirect(301, 'http://' + drr_sim_ip + ':3908/publish/'+feedId+'/'+filename);
 410                 return;
 411         } else if (args.tc==tc_all_published) {
 412                 ctr_publish_req_published[feedIndexes[feedId]]++;
 413                 res.send("ok");
 414                 return;
 415         }else if (args.tc==tc_10p_no_response && (ctr%10) == 0) {
 416                 return;
 417         } else if (args.tc==tc_10first_no_response && ctr<11) {
 418                 return;
 419         } else if (args.tc==tc_100first_no_response && ctr<101) {
 420                 return;
 421         } else if (args.tc==tc_all_delay_1s) {
 422                 do_publish_delay(res, filename, 1000, feedId);
 423                 return;
 424         } else if (args.tc==tc_all_delay_10s) {
 425                 do_publish_delay(res, filename, 10000, feedId);
 426                 return;
 427         } else if (args.tc==tc_10p_delay_10s && (ctr%10) == 0) {
 428                 do_publish_delay(res, filename, 10000, feedId);
 429                 return;
 430         } else if (args.tc==tc_10p_error_response && (ctr%10) == 0) {
 431                 res.send(400);
 432                 return;
 433         } else if (args.tc==tc_10first_error_response && ctr<11) {
 434                 res.send(400);
 435                 return;
 436         } else if (args.tc==tc_100first_error_response & ctr<101) {
 437                 res.send(400);
 438                 return;
 439         }
 440         if (!published.includes(filename)) {
 441                 ctr_publish_req_redirect[feedIndexes[feedId]]++;
 442                 res.redirect(301, 'http://'+drr_sim_ip+':3908/publish/'+feedId+'/'+filename);
 443         } else {
 444                 ctr_publish_req_published[feedIndexes[feedId]]++;
 445                 res.send("ok");
 446         }
 447         return;
 448 })
 449
 450 function do_publish_delay(res, filename, sleeptime, feedId) {
 451         if (!published.includes(filename)) {
 452                 ctr_publish_req_redirect[feedIndexes[feedId]]++;
 453                 sleep(1000).then(() => {
 454                         res.redirect(301, 'http://'+drr_sim_ip+':3908/publish/'+feedId+'/'+filename);
 455                 });
 456         } else {
 457                 ctr_publish_req_published[feedIndexes[feedId]]++;
 458                 sleep(1000).then(() => {
 459                         res.send("ok");
 460                 });
 461         }
 462 }
 463
 464 //Callback from DR REDIR server, when file is published ok this PUT request update the list of published files.
 465 app.put('/dr_redir_publish/:feedId/:filename', function (req, res) {
 466         console.log("url:"+req.url);
 467         var feedId = req.params.feedId;
 468         var filename = req.params.filename;
 469         console.log(filename);
 470
 471         if (!published[feedIndexes[feedId]].includes(filename)) {
 472                 console.log("File marked as published by callback from DR redir SIM. url: " + req.url);
 473                 published[feedIndexes[feedId]].push(filename);
 474         } else {
 475                 console.log("File already marked as published. Callback from DR redir SIM. url: " + req.url);
 476                 ctr_double_publish[feedIndexes[feedId]]++;
 477         }
 478
 479         res.send("ok");
 480 })
 481
 482 var httpServer = http.createServer(app);
 483 var httpsServer = https.createServer(credentials, app);
 484
 485 var httpPort=3906;
 486 var httpsPort=3907;
 487 httpServer.listen(httpPort);
 488 console.log("DR-simulator listening (http) at "+httpPort);
 489 httpsServer.listen(httpsPort);
 490 console.log("DR-simulator listening (https) at "+httpsPort);
 491
 492 if (process.env.DRR_SIM_IP) {
 493         drr_sim_ip=process.env.DRR_SIM_IP;
 494 }
 495 console.log("Using IP " + drr_sim_ip + " for redirect to DR redir sim");
 496
 497 if (process.env.DR_FEEDS) {
 498         feeds=process.env.DR_FEEDS;
 499 }
 500
 501 console.log("Configured list of feeds mapped to file name prefixes: " + feeds);
 502
 503 feedNames=feeds.split(',');
 504 for(i=0;i<feedNames.length;i++) {
 505         var tmp=feedNames[i].split(':');
 506         feedNames[i]=tmp[0].trim();
 507         feedIndexes[feedNames[i]]=i;
 508         filePrefixes[i]=[]
 509         for(j=1;j<tmp.length;j++) {
 510                 filePrefixes[i][j-1]=tmp[j];
 511         }
 512
 513     ctr_publish_query[i] = 0;
 514     ctr_publish_query_published[i] = 0;
 515     ctr_publish_query_not_published[i] = 0;
 516     ctr_publish_req[i] = 0;
 517     ctr_publish_req_redirect[i] = 0;
 518     ctr_publish_req_published[i] = 0;
 519     ctr_double_publish[i] = 0;
 520     ctr_publish_query_bad_file_prefix[i] = 0;
 521         ctr_publish_req_bad_file_prefix[i] = 0;
 522         published[i] = [];
 523 }
 524
 525 console.log("Parsed mapping between feed id and file name prefix");
 526 for(i=0;i<feedNames.length;i++) {
 527         var fn = feedNames[i];
 528         for (j=0;j<filePrefixes[i].length;j++) {
 529                 console.log("Feed id: " + fn + ", file name prefix: " + filePrefixes[i][j]);
 530         }
 531 }
 532