Merge "Update release notes for Honolulu" into honolulu
[sdnc/oam.git] / admportal / server / netdb_updater.js
1 var log4js = require('log4js');
2 var http = require('http');
3 var async = require('async');
4 var properties = require(process.env.SDNC_CONFIG_DIR + '/netdb-updater.json');
5 var admProperties = require(process.env.SDNC_CONFIG_DIR + '/admportal.json');
6 var csvtojson = require('csvtojson');
7 var mysql = require('mysql');
8 var moment = require('moment');
9 var os = require('os');
10 var fs = require('fs.extra');
11
12 // Check to make sure SDNC_CONFIG_DIR is set
13 var sdnc_config_dir = process.env.SDNC_CONFIG_DIR;
14 if ( typeof sdnc_config_dir == 'undefined' )
15 {
16     console.log('ERROR the SDNC_CONFIG_DIR environmental variable is not set.');
17     return;
18 }
19
20 // SETUP LOGGER
21 log4js.configure(process.env.SDNC_CONFIG_DIR + '/netdb.log4js.json');
22 var logger = log4js.getLogger('netdb');
23 logger.setLevel(properties.netdbLogLevel);
24
25 var yargs = require('yargs')
26   .usage("\nUsage: node netdb_updater -t link_master|router_master")
27   .demand('t')
28   .alias('t', 'table')
29   .example("Example: node netdb_updater -t link_master","Update SDNC LINK_MASTER table from NetDB.")
30   .argv;
31
32 var dbTable = yargs.table;
33 var debug = properties.netdbDebug;
34 var env = properties.netdbEnv; 
35 var retSuccess = false;
36
37 // DB Setup
38 var currentDB = '';
39 var dbConnection = '';
40 var db01 = '';
41 var db02 = '';
42 var count = 0;
43 var errorMsg = [];
44
45 var dbtasks = [];
46 dbtasks.push( function(callback) { checkParams(callback); } );
47 dbtasks.push( function(callback) { dbConnect(callback); } );
48 //dbtasks.push( function(callback) { netdb(callback); } );
49
50 logger.debug('\n\n********** START PROCESSING - Env=' + env + ' Debug=' + debug + ' **********');
51
52 async.series(dbtasks, function(err,result){
53     if(err) {
54                 logger.error(err + ' COUNT: ' + count);
55     }
56     else {
57                 if ( errorMsg.length > 0 ){
58                         logger.error(errorMsg);
59                 }
60         }
61 });
62
63
64 function checkParams(scb){
65         if ( dbTable != 'link_master' && dbTable != 'router_master' ){
66                 scb("Invalid parameter passed in '" + dbTable + " ' exiting.'");
67         }
68         else{
69                 scb(null);
70         }
71 }
72
73
74 async.whilst(
75         function () { return count < properties.netdbRetryInterval },
76         function (callback) {
77                         if ( dbTable == 'link_master' ){
78                                 getLinkMaster(callback);
79                         }
80                         else if (dbTable == 'router_master'){
81                                 getRouterMaster(callback);
82                         }
83                         else{ // should never hit this condition
84                                 logger.debug("Invalid parameter passed in '" + dbTable + " ' exiting.'");
85                         }
86         },
87         function (err) {
88 logger.debug('whilst err function errorMsg = ' + errorMsg);
89                         // report error
90                         if ( errorMsg.length > 0 ){
91                                 logger.debug(errorMsg + ' COUNT: ' + count);
92                                 process.exit(1);
93                         }
94                         else{
95                                 logger.debug('success');
96                                 process.exit(0);
97                         }
98         }
99 );
100
101
102 function returnError(emsg, cb){
103         retSuccess=false;
104         errorMsg.push(emsg);
105         if ( count == properties.netdbRetryInterval ) { logger.error(errorMsg); }
106     setTimeout( function(){
107                 cb(null);
108                 }, properties.netdbWaitTime);
109 }
110
111 function returnSuccess(cb){
112 logger.debug('inside returnSuccess');
113         errorMsg = '';
114         //var cnt = properties.netdbRetryInterval;
115         //logger.debug('b4 inc returnSuccess count=' + count);
116         //count = ++cnt;
117         //logger.debug('after inc returnSuccess count=' + count);
118         //cb(null);
119         retSuccess=true;
120 process.exit(0);
121
122
123 function getRouterMaster(cb){
124
125         logger.info('getRouterMaster debug=' + debug + ' count=' + count);
126
127         // setup connection
128     var netdbEnv = properties.netdbEnv;
129     var auth_param = '';
130     if ( netdbEnv == 'e2e' || netdbEnv == 'prod' ){
131         // conexus network
132         auth_param = '?auth=' + admProperties['ip-addresses']['eth2'] + ';'
133     }else{
134         // app network
135         auth_param = '?auth=' + admProperties['ip-addresses']['eth1:0'] + ';'
136     }
137     var username = properties.netdbUser;;
138     var password = properties.netdbPassword;
139         var date = moment().format('YYYYMMDD');
140     var auth = 'Basic ' + new Buffer(username + ':' + password).toString('base64');
141     var host = properties.netdbHost;
142     var port = properties.netdbPort;
143     var path = '/' + properties.netdbPath
144                 + '/' + properties.netdbNetwork
145                 + '/' + properties.netdbApiName
146                 + auth_param
147                 + 'client=' + properties.netdbClientName + ';'
148                 + 'date=' + date + ';'
149                 + 'reportName=' + dbTable + ';'
150                 + 'type=' + properties.netdbType;
151
152     var header = { 'Content-Type': 'text/csv' };
153     //var header = {'Host': host, 'Authorization': auth, 'Content-Type': 'text/csv' };
154     var options = {
155         method            : "GET",
156         path              : path,
157         host              : host,
158         port              : port,
159         headers           : header
160     };
161
162         logger.debug('options:\n' + JSON.stringify(options,null,2));
163
164     var request = http.request(options, function(response) {
165
166         var response_str = '';
167                 if ( retSuccess == true ){
168                         var cnt = properties.netdbRetryInterval;
169                         count = ++cnt;
170                 }
171                 else{
172                         count++;
173                 }
174
175         logger.debug('STATUS: ' + response.statusCode + ' content-type=' + response.headers['content-type']);
176
177         // Read the response from ODL side
178         response.on('data', function(chunk) {
179             response_str += chunk;
180         });
181
182         response.on('end', function() {
183
184                         logger.debug('HEADERS:' + JSON.stringify(response.headers));
185
186             if(response.statusCode == 200){
187
188                 if(response_str.length > 0){
189
190                                         // save the upload
191                                         try{
192                                                 fs.writeFileSync('/sdncvar/sdnc/files/netdb-updater/' + moment().unix() + ".netdb." + dbTable + '.csv', response_str);
193                                         }
194                                         catch(e){
195                                                 // this is not in reqs, if it fails keep on going.
196                                                 logger.error('Error writing NetDB file:' + e);
197                                         }
198
199                                         if (response.headers['content-type'].indexOf('html') > 0){
200                                                 returnError('Error:Unexpected content-type:' + response.headers['content-type'] + ' returned.\n', cb);
201                                                 return;
202                                         }
203                     // need to parse csv file
204                     var Converter=csvtojson.Converter;
205                     var csvConverter = new Converter({
206                         noheader:true
207                     });
208                     var routerMasterSQL = '';
209
210                     // end_parsed will be emitted once parsing is finished
211                     csvConverter.on("end_parsed", function(respObj){
212
213                         routerMasterSQL = routerMasterSQL.concat("INSERT INTO ROUTER_MASTER (crs_name, loopback_ip)");
214                         for ( var x=0; x < respObj.length; x++ ){
215
216                                 if ( respObj[x].field1.length == 0 ){
217                                 returnError('Required field [crs_name] is null.', cb);
218                                 }
219
220                                                         if (x!=0){
221                                                                 routerMasterSQL = routerMasterSQL.concat(' union ');
222                                                         }
223                                                         routerMasterSQL = routerMasterSQL.concat(" SELECT " 
224                                                                 + "'" + respObj[x].field1 + "',"
225                                 + "'" + respObj[x].field2 + "' FROM DUAL ");
226                         }
227                         //logger.debug('SQL: ' + routerMasterSQL);
228         
229                         if (debug != 'debug' && env != 'dev'){
230
231                                 var tasks = [];
232                                 tasks.push( function(callback) { updateRouterMaster(routerMasterSQL,callback); } );
233                                 async.series(tasks, function(err,result){
234                                 if(err) {
235                                         returnError(err,cb);
236                                         return;
237                                 }
238                                 else {
239                                         logger.info('*** Router Master Table Replaced ***');
240                                         returnSuccess(cb);
241                                         return;
242                                 }
243                                 });
244                         }
245                         else{
246 logger.debug('*** debug ***');
247                                 returnSuccess(cb);
248                                         return;
249                         }
250
251                     });
252                     csvConverter.on("error",function(errMsg,errData){
253                         returnError(errMsg,cb);
254                                                 return;
255                     });
256                     csvConverter.fromString(response_str, function(err,result){
257                         if(err){
258                                                         returnError(err,cb);
259                                                         return;
260                         }
261                     });
262                 }
263                 else{
264                     //logger.debug("no data");
265                                         returnError('no data',cb);
266                                         return;
267                 }
268             }
269             else if(response.statusCode == 404){
270                                 returnError('Router Master Table for ' + date + ' is not Available.',cb);
271                                 return;
272                         }
273             else {
274                                 returnError('Status Code:' + response.statudCode + ' returned for Router Master Table query.',cb);
275                                 return;
276             }
277         });
278     });
279     request.on('error', function(e) {
280             if ( retSuccess == true ){
281                         var cnt = properties.netdbRetryInterval;
282                         count = ++cnt;
283                 }
284                 else{
285                         count++;
286                 }       
287         returnError(e,cb);
288                 return;
289     });
290     request.end();
291 }
292
293 function getLinkMaster(cb){
294
295         logger.info('getLinkMaster debug=' + debug + ' count=' + count);
296
297     // setup connection
298     var netdbEnv = properties.netdbEnv;
299     var auth_param = '';
300     if ( netdbEnv == 'e2e' || netdbEnv == 'prod' ){
301         // conexus network
302         auth_param = '?auth=' + admProperties['ip-addresses']['eth2'] + ';'
303     }else{
304         // app network
305         auth_param = '?auth=' + admProperties['ip-addresses']['eth1:0'] + ';'
306     }
307     var username = properties.netdbUser;;
308     var password = properties.netdbPassword;
309     var auth = 'Basic ' + new Buffer(username + ':' + password).toString('base64');
310     var host = properties.netdbHost;
311     var port = properties.netdbPort;
312         var date = moment().format('YYYYMMDD');
313     var path = '/' + properties.netdbPath
314                 + '/' + properties.netdbNetwork
315                 + '/' + properties.netdbApiName
316                 + auth_param
317                 + 'client=' + properties.netdbClientName + ';'
318                 + 'date=' + date + ';'
319                 + 'reportName=' + dbTable + ';'
320                 + 'type=' + properties.netdbType;
321
322     var header = { 'Content-Type': 'text/csv' };
323     //var header = {'Host': host, 'Authorization': auth, 'Content-Type': 'text/csv' };
324     var options = {
325         method            : "GET",
326         path              : path,
327         host              : host,
328         port              : port,
329         headers           : header
330     };
331
332     logger.debug('options:\n' + JSON.stringify(options,null,2));
333
334         var request = http.request(options, function(response) {
335
336         logger.debug('STATUS: ' + response.statusCode + ' content-type=' + response.headers['content-type']);
337
338                 if ( retSuccess == true ){
339                 var cnt = properties.netdbRetryInterval;
340                 count = ++cnt;
341         }
342                 else{
343                 count++
344                 }
345
346         var response_str = '';
347
348         // Read the response from ODL side
349         response.on('data', function(chunk) {
350             response_str += chunk;
351         });
352
353         response.on('end', function() {
354
355                         logger.debug('HEADERS:' + JSON.stringify(response.headers));
356
357             if(response.statusCode == 200){
358
359                                 if(response_str.length > 0){
360
361                                         //logger.debug('response_str=' + response_str);
362                                         // save the upload
363                     try{
364                                                 fs.writeFileSync('/sdncvar/sdnc/files/netdb-updater/' + moment().unix() + ".netdb." + dbTable + '.csv', response_str);
365                     }
366                     catch(e){
367                         // this is not in reqs, if it fails keep on going.
368                         logger.error('Error writing NetDB file:' + e);
369                     }
370
371                                         if (response.headers['content-type'].indexOf('html') > 0){
372                                                 returnError('Error:Unexpected content-type:' + response.headers['content-type'] + ' returned.\n', cb);
373                                                 return;
374                                         }
375                                         // need to parse csv file
376                                         var Converter=csvtojson.Converter;
377                                         var csvConverter = new Converter({
378                                                 noheader:true
379                                         });
380
381                                         var linkMasterSQL = '';
382
383                                         // end_parsed will be emitted once parsing is finished
384                                         csvConverter.on("end_parsed", function(jsonObj){
385
386                                                 linkMasterSQL = linkMasterSQL.concat("INSERT INTO LINK_MASTER (link_interface_ip, source_crs_name, destination_crs_name, link_speed, default_cost, bundle_name, shutdown)"); 
387                                                 for ( var x=0; x < jsonObj.length; x++ ){
388                                                         if ( jsonObj[x].field1.length == 0 ){
389                                                                 returnError('Required field [link_interface_ip] is null.', cb);
390                                                                 return;
391                                                         }
392                                                         if ( jsonObj[x].field2.length == 0 ){
393                                                                 returnError('Required field [source_crs_name] is null.', cb);
394                                                                 return;
395                                                         }
396                                                         if ( jsonObj[x].field3.length == 0 ){
397                                                                 returnError('Required field [destination_crs_name] is null.', cb);
398                                                                 return;
399                                                         }
400                                                         if (x!=0){
401                                                                 linkMasterSQL = linkMasterSQL.concat(' union ');
402                                                         }
403
404                                                         linkMasterSQL = linkMasterSQL.concat(" SELECT " 
405                                                                 + "'" + jsonObj[x].field1 + "',"
406                                                                 + "'" + jsonObj[x].field2 + "',"
407                                                                 + "'" + jsonObj[x].field3 + "',"
408                                                                 + jsonObj[x].field4 + ","
409                                                                 + jsonObj[x].field5 + ","
410                                                                 + "'" + jsonObj[x].field6 + "',"
411                                                                 + "'" + jsonObj[x].field7 + "' FROM DUAL");
412                                                 }
413                                                 //logger.debug('SQL: ' + linkMasterSQL);
414
415                                                 if (debug != 'debug' && env != 'dev'){
416                                 // update db
417                                                         var tasks = [];
418                                                         tasks.push( function(callback) { updateLinkMaster(linkMasterSQL,callback); } );
419                                                 async.series(tasks, function(err,result){
420                                                                 if(err)
421                                 {
422                                         returnError(err,cb);
423                                                                         return;
424                                 }
425                                 else
426                                 {
427                                                                         logger.info('*** Link Master Table Replaced ***');
428                                         returnSuccess(cb);
429                                                                         return;
430                                 }
431                                                 });
432                                                 }
433                                                 else{
434                                 returnSuccess(cb);
435                                                         return;
436                                                 }
437                                         });
438                                         csvConverter.on("error",function(errMsg,errData){
439                                         returnError(errMsg,cb);
440                                                 return;
441                                         });
442                                         csvConverter.fromString(response_str, function(err,result){
443                                                 if(err){
444                                                 returnError(errMsg,cb);
445                                                         return;
446                                                 }
447                                         });
448                 }
449                 else{
450                                 returnError('no data',cb);
451                                         return;
452                 }
453             }
454             else if(response.statusCode == 404){
455                                 returnError('Link Master Table for ' + date + ' is not Available.',cb);
456                                 return;
457                         }
458             else {
459                                 returnError('Status Code:' + response.statudCode + ' returned for Link Master Table query.',cb);
460                                 return;
461             }
462         });
463         });
464         request.on('error', function(e) {
465                 if ( retSuccess == true ){
466                 var cnt = properties.netdbRetryInterval;
467                 count = ++cnt;
468         }
469                 else{
470                 count++
471                 }
472         returnError(e,cb);
473         return;
474         });
475         request.end();
476 }
477                         
478
479 function dbConnect(callback){
480
481         var l_db01 = admProperties['databases']['0'];
482         var db01Array = l_db01.split("|");
483         db01 = db01Array[0];
484
485         var l_db02 = admProperties['databases']['1'];
486         var db02Array = l_db02.split("|");
487         db02 = db02Array[0];
488
489         if ( admProperties.dbFabric == 'true' )
490         {
491                 logger.debug('connectFabric()');
492
493     // testing 
494     var fabric_connection = mysql.createConnection({
495         host            : admProperties.dbFabricServer,
496         user            : admProperties.dbFabricUser,
497         password        : admProperties.dbFabricPassword,
498         database        : admProperties.dbFabricDB,
499         port            : admProperties.dbFabricPort
500     });
501
502
503     fabric_connection.connect( function(err) {
504
505         if (err) {
506             callback(err);
507             return;
508         }
509         fabric_connection.query('CALL dump.servers()', function(err,rows) {
510
511             var masterDB = '';
512
513             if (err) {
514                 callback(err);
515                 return;
516             }
517             fabric_connection.end();
518             logger.debug('rows: ' + JSON.stringify(rows,null,2));
519
520             // fabric servers
521             for ( var x=0; x<rows.length; x++)
522             {
523                 // database servers
524                 for ( var y=0; y<rows[x].length; y++)
525                 {
526                     var row = rows[x][y];
527                     if (row.group_id == admProperties.dbFabricGroupId)
528                     {
529                         if (row.status == '3' && row.mode == '3'){
530                             masterDB = row.host;
531                         }
532                     }
533                 }
534             }
535             logger.debug('currentDB: ' + currentDB);
536             logger.debug('masterDB: ' + masterDB);
537
538             if (masterDB.length <=0)
539             {
540                 logger.debug('no writable master db');
541                 callback('no writable master db');
542                 return;
543             }
544
545             if ( currentDB != masterDB )
546             {
547                 currentDB = masterDB;
548                 dbConnection = mysql.createConnection({
549                     connectionLimit   : admProperties.dbConnLimit,
550                     host              : currentDB,
551                     user              : admProperties.dbUser,
552                     password          : admProperties.dbPassword,
553                     database          : admProperties.dbName,
554                     multipleStatements: true,
555                     debug             : false
556                 });
557             }
558             logger.debug('new currentDB: ' + currentDB);
559             logger.debug('new masterDB: ' + masterDB);
560             callback(null);
561             return;
562         });
563         fabric_connection.on('error', function(err){
564              logger.debug(err.code);
565              callback(err);
566                      return;
567         });
568     });
569         }
570         else
571         {
572         currentDB = db01;
573
574         var dbConn = mysql.createConnection({
575                 connectionLimit : admProperties.dbConnLimit,
576                 host            : currentDB,
577                 user            : admProperties.dbUser,
578                 password        : admProperties.dbPassword,
579                 database        : admProperties.dbName,
580                 multipleStatements: true,
581                 debug           : false
582         });
583                 logger.debug('initDB currentDB=' + currentDB);
584
585         dbConn.connect(function(err,connection){
586
587                 if(err){
588                 logger.debug( String(err) ); // ALARM
589                 callback(err);
590                 return;
591                 }
592                 var sql = 'select @@read_only';
593                 dbConn.query(sql, function(err,result){
594                 dbConn.end();
595
596                 // @@read_only=0 means db is writable
597                 logger.debug('@@read_only=' + result[0]['@@read_only']);
598                 if ( result[0]['@@read_only'] != '0' )
599                 {
600                         if (currentDB == db01)
601                         {
602                         currentDB = db02;
603                         }
604                         else
605                         {
606                         currentDB = db01;
607                         }
608 logger.debug('initDB reconnect to currentDB '+ currentDB);
609                         var newConnection = mysql.createConnection({
610                         connectionLimit : admProperties.dbConnLimit,
611                         host            : currentDB,
612                         user            : admProperties.dbUser,
613                         password        : admProperties.dbPassword,
614                         database        : admProperties.dbName,
615                         multipleStatements: true,
616                         debug           : false
617                         }); // end create
618                         dbConnection = newConnection;
619                         callback(null);
620                                         return;
621                 }
622                 dbConnection = dbConn;
623                 callback(null);
624                                 return;
625                 });
626         });
627         }
628 }
629
630 function updateLinkMaster(linkMasterSQL,callback){
631
632 logger.debug('updateLinkMaster');
633
634     dbConnection.connect(function(err,connection)
635     {
636         if(err){
637             logger.debug( String(err) ); // ALARM
638             callback(err, 'Unable to get database connection.');
639             return;
640         }
641     }); // end connection
642     dbConnection.beginTransaction(function(err) {
643         if(err){
644             //dbConnection.release();
645             callback(err,String(err));
646             return;
647         }
648         var sql = "DELETE FROM LINK_MASTER";
649         dbConnection.query(sql,function(err,result)
650         {
651             if(err){
652                 //dbConnection.release();
653                 dbConnection.rollback( {rollback: 'NO RELEASE'},function(){
654                     callback(err,String(err));
655                     return;
656                 });
657             }
658             dbConnection.query(linkMasterSQL,function(err,result)
659             {
660                 if(err){
661                     //dbConnection.release();
662                     dbConnection.rollback( {rollback: 'NO RELEASE'},function(){
663                         callback(err,String(err));
664                         return;
665                     });
666                 }
667                 dbConnection.commit(function(err){
668                     if(err){
669                         //dbConnection.release();
670                                 dbConnection.rollback( {rollback: 'NO RELEASE'},function(){
671                             callback(err,String(err));
672                             return;
673                         });
674                     }
675                     //dbConnection.release();
676                     callback(null);
677                 });
678             })
679         });
680     }); // end transaction
681 }
682
683 function updateRouterMaster(routerMasterSQL,callback){
684
685 logger.debug('updateRouterMaster');
686
687     dbConnection.connect(function(err,connection)
688     {
689         if(err){
690             logger.debug( String(err) ); // ALARM
691             callback(err, 'Unable to get database connection.');
692             return;
693         }
694     }); // end connection
695     dbConnection.beginTransaction(function(err) {
696         if(err){
697             //dbConnection.release();
698             callback(err,String(err));
699             return;
700         }
701         var sql = "DELETE FROM ROUTER_MASTER";
702         dbConnection.query(sql,function(err,result)
703         {
704             if(err){
705                 //dbConnection.release();
706                 dbConnection.rollback( function(){
707                     callback(err,String(err));
708                     return;
709                 });
710             }
711             dbConnection.query(routerMasterSQL,function(err,result)
712             {
713                 if(err){
714                     //dbConnection.release();
715                         dbConnection.rollback( function(){
716                         callback(err,String(err));
717                         return;
718                     });
719                 }
720                 dbConnection.commit(function(err){
721                     if(err){
722                         //dbConnection.release();
723                                 dbConnection.rollback( function(){
724                             callback(err,String(err));
725                             return;
726                         });
727                     }
728                     //dbConnection.release();
729                     callback(null);
730                 });
731             })
732         });
733     }); // end transaction
734 }
735
736 /*
737
738         logger.debug('getLinkMaster - count=' + count);
739         if ( true ) {
740                 //fail
741                 setTimeout( function(){
742                         cb(null);}, properties.netdbWaitTime);
743                 return;
744         }
745         // success
746         count = 10;
747         cb(null);
748 }
749 */