vnfs/VES5.0/evel/evel-library/code/evel_library/evel_event_mgr.c

   1 /*************************************************************************//**
   2  *
   3  * Copyright © 2017 AT&T Intellectual Property. All rights reserved.
   4  *
   5  * Unless otherwise specified, all software contained herein is
   6  * Licensed under the Apache License, Version 2.0 (the "License");
   7  * you may not use this file except in compliance with the License.
   8  * You may obtain a copy of the License at
   9  *        http://www.apache.org/licenses/LICENSE-2.0
  10  *
  11  * Unless required by applicable law or agreed to in writing, software
  12  * distributed under the License is distributed on an "AS IS" BASIS,
  13  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14  * See the License for the specific language governing permissions and
  15  * limitations under the License.
  16  * ECOMP is a trademark and service mark of AT&T Intellectual Property.
  17  ****************************************************************************/
  18
  19 /**************************************************************************//**
  20  * @file
  21  * Event Manager
  22  *
  23  * Simple event manager that is responsible for taking events (Heartbeats,
  24  * Faults and Measurements) from the ring-buffer and posting them to the API.
  25  *
  26  ****************************************************************************/
  27
  28 #include <string.h>
  29 #include <assert.h>
  30 #include <stdlib.h>
  31 #include <pthread.h>
  32
  33 #include <curl/curl.h>
  34
  35 #include "evel.h"
  36 #include "evel_internal.h"
  37 #include "ring_buffer.h"
  38 #include "evel_throttle.h"
  39
  40 /**************************************************************************//**
  41  * How long we're prepared to wait for the API service to respond in
  42  * seconds.
  43  *****************************************************************************/
  44 static const int EVEL_API_TIMEOUT = 5;
  45
  46 /**************************************************************************//**
  47  * Wait time if both the collectors are not responding
  48  *****************************************************************************/
  49 static const int EVEL_COLLECTOR_RECONNECTION_WAIT_TIME = 120;
  50
  51 /*****************************************************************************/
  52 /* Prototypes of locally scoped functions.                                   */
  53 /*****************************************************************************/
  54 static size_t read_callback(void *ptr, size_t size, size_t nmemb, void *userp);
  55 static void * event_handler(void *arg);
  56 static bool evel_handle_response_tokens(const MEMORY_CHUNK * const chunk,
  57                                         const jsmntok_t * const json_tokens,
  58                                         const int num_tokens,
  59                                         MEMORY_CHUNK * const post);
  60 static bool evel_tokens_match_command_list(const MEMORY_CHUNK * const chunk,
  61                                            const jsmntok_t * const json_token,
  62                                            const int num_tokens);
  63 static bool evel_token_equals_string(const MEMORY_CHUNK * const chunk,
  64                                      const jsmntok_t * const json_token,
  65                                      const char * check_string);
  66 static EVEL_ERR_CODES evel_setup_curl();
  67
  68 /**************************************************************************//**
  69  * Buffers for error strings from libcurl.
  70  *****************************************************************************/
  71 static char curl_err_string[CURL_ERROR_SIZE] = "<NULL>";
  72
  73 /**************************************************************************//**
  74  * Handle for the API into libcurl.
  75  *****************************************************************************/
  76 static CURL * curl_handle = NULL;
  77 int curr_global_handles = 0;
  78
  79 /**************************************************************************//**
  80  * Special headers that we send.
  81  *****************************************************************************/
  82 static struct curl_slist * hdr_chunk = NULL;
  83
  84 /**************************************************************************//**
  85  * Message queue for sending events to the API.
  86  *****************************************************************************/
  87 static ring_buffer event_buffer;
  88
  89 /**************************************************************************//**
  90  * Single pending priority post, which can be generated as a result of a
  91  * response to an event.  Currently only used to respond to a commandList.
  92  *****************************************************************************/
  93 static MEMORY_CHUNK priority_post;
  94
  95 /**************************************************************************//**
  96  * The thread which is responsible for handling events off of the ring-buffer
  97  * and posting them to the Event Handler API.
  98  *****************************************************************************/
  99 static pthread_t evt_handler_thread;
 100
 101 /**************************************************************************//**
 102  * Variable to convey to the event handler thread what the foreground wants it
 103  * to do.
 104  *****************************************************************************/
 105 static EVT_HANDLER_STATE evt_handler_state = EVT_HANDLER_UNINITIALIZED;
 106
 107 /**************************************************************************//**
 108  * The configured API URL for event and throttling.
 109  *****************************************************************************/
 110 static char * evel_event_api_url;
 111 static char * evel_throt_api_url;
 112 static char * evel_batch_api_url;
 113
 114 static char * evel_bevent_api_url;
 115 static char * evel_bthrot_api_url;
 116 static char * evel_bbatch_api_url;
 117
 118 /**************************************************************************//**
 119  * Storage for other CURL related parameters
 120  *****************************************************************************/
 121 int evel_secure = -1;
 122 int evel_verbosity = -1;
 123
 124 long evel_verify_peer = 0;
 125 long evel_verify_host = 0;
 126
 127 static char * evel_source_ip = NULL;
 128 static char * evel_source_ip_bakup = NULL;
 129 static char * evel_cert_file_path = NULL;
 130 static char * evel_key_file_path = NULL;
 131 static char * evel_ca_info = NULL;
 132 static char * evel_ca_file_path = NULL;
 133 static char * evel_username = NULL;
 134 static char * evel_password = NULL;
 135 static char * evel_username2 = NULL;
 136 static char * evel_password2 = NULL;
 137
 138 static int http_response_code = 0;
 139 static int evel_collector_id = 0;
 140 /**************************************************************************//**
 141  * Initialize the event handler.
 142  *
 143  * Primarily responsible for getting CURL ready for use.
 144  *
 145  * @param[in] event_api_url
 146  *                      The URL where the Vendor Event Listener API is expected
 147  *                      to be.
 148  * @param[in] throt_api_url
 149  *                      The URL where the Throttling API is expected to be.
 150  * @param[in] source_ip  Source IP of VES Agent
 151  * @param[in] ring_buf_size     Initial size of ring buffer
 152  * @param[in] secure     Whether Using http or https
 153  * @param[in] cert_file_path  Path to Client Certificate file
 154  * @param[in] key_file_path   Path to Client key file
 155  * @param[in] ca_info         Path to CA info file
 156  * @param[in] ca_file_path    Path to CA file
 157  * @param[in] verify_peer     Using peer verification or not 0 or 1
 158  * @param[in] verify_host     Using host verification or not 0 or 1
 159  * @param[in] username  The username for the Basic Authentication of requests.
 160  * @param[in] password  The password for the Basic Authentication of requests.
 161  * @param     verbosity 0 for normal operation, positive values for chattier
 162  *                        logs.
 163  *****************************************************************************/
 164 EVEL_ERR_CODES event_handler_initialize(const char * const event_api_url,
 165                                         const char * const bakup_api_url,
 166                                         const char * const throt_api_url,
 167                                         const char * const source_ip,
 168                                         const char * const source_ip_bakup,
 169                                         int ring_buf_size,
 170                                         int secure,
 171                                         const char * const cert_file_path,
 172                                         const char * const key_file_path,
 173                                         const char * const ca_info,
 174                                         const char * const ca_file_path,
 175                                         long verify_peer,
 176                                         long verify_host,
 177                                         const char * const username,
 178                                         const char * const password,
 179                                         const char * const username2,
 180                                         const char * const password2,
 181                                         int verbosity)
 182 {
 183   int rc = EVEL_SUCCESS;
 184   char batch_api_url[EVEL_MAX_URL_LEN + 1] = {0};
 185
 186   EVEL_ENTER();
 187
 188   /***************************************************************************/
 189   /* Check assumptions.                                                      */
 190   /***************************************************************************/
 191   assert(event_api_url != NULL);
 192   assert(throt_api_url != NULL);
 193   assert(username != NULL);
 194   assert(password != NULL);
 195   if( bakup_api_url != NULL )
 196   {
 197     assert(username2 != NULL);
 198     assert(password2 != NULL);
 199   }
 200
 201   /***************************************************************************/
 202   /* Store the API URLs.                                                     */
 203   /***************************************************************************/
 204   evel_event_api_url = strdup(event_api_url);
 205   assert(evel_event_api_url != NULL);
 206   sprintf(batch_api_url,"%s/eventBatch",event_api_url);
 207   evel_batch_api_url = strdup(batch_api_url);
 208   assert(evel_batch_api_url != NULL);
 209   evel_throt_api_url = strdup(throt_api_url);
 210   assert(evel_throt_api_url != NULL);
 211
 212   curr_global_handles = 1;
 213
 214   if( bakup_api_url != NULL )
 215   {
 216     evel_bevent_api_url = strdup(bakup_api_url);
 217     assert(evel_bevent_api_url != NULL);
 218     sprintf(batch_api_url,"%s/eventBatch",bakup_api_url);
 219     evel_bbatch_api_url = strdup(batch_api_url);
 220     assert(evel_bbatch_api_url != NULL);
 221     evel_bthrot_api_url = strdup(throt_api_url);
 222     assert(evel_bthrot_api_url != NULL);
 223     curr_global_handles = 2;
 224   }
 225
 226   /***************************************************************************/
 227   /* Store other parameters
 228   /***************************************************************************/
 229   evel_secure = secure;
 230   evel_verbosity = verbosity;
 231
 232   evel_verify_peer = verify_peer;
 233   evel_verify_host = verify_host;
 234
 235   evel_source_ip = NULL;
 236   if (source_ip != NULL)
 237   {
 238     evel_source_ip = strdup(source_ip);
 239     assert(evel_source_ip != NULL);
 240   }
 241
 242   evel_source_ip_bakup = NULL;
 243   if (source_ip_bakup != NULL)
 244   {
 245     evel_source_ip_bakup = strdup(source_ip_bakup);
 246     assert(evel_source_ip_bakup != NULL);
 247   }
 248
 249   evel_cert_file_path = NULL;
 250   if (cert_file_path != NULL)
 251   {
 252     evel_cert_file_path = strdup(cert_file_path);
 253     assert(evel_cert_file_path != NULL);
 254   }
 255
 256   evel_key_file_path = NULL;
 257   if (key_file_path != NULL)
 258   {
 259     evel_key_file_path = strdup(key_file_path);
 260     assert(evel_key_file_path != NULL);
 261   }
 262
 263   evel_ca_info = NULL;
 264   if (ca_info != NULL)
 265   {
 266     evel_ca_info = strdup(ca_info);
 267     assert(evel_ca_info != NULL);
 268   }
 269
 270   evel_ca_file_path = NULL;
 271   if (ca_file_path != NULL)
 272   {
 273     evel_ca_file_path = strdup(ca_file_path);
 274     assert(evel_ca_file_path != NULL);
 275   }
 276
 277   evel_username = NULL;
 278   if (username != NULL)
 279   {
 280     evel_username = strdup(username);
 281     assert(evel_username != NULL);
 282   }
 283
 284   evel_password = NULL;
 285   if (password != NULL)
 286   {
 287     evel_password = strdup(password);
 288     assert(evel_password != NULL);
 289   }
 290
 291   evel_username2 = NULL;
 292   if (username2 != NULL)
 293   {
 294     evel_username2 = strdup(username2);
 295     assert(evel_username2 != NULL);
 296   }
 297
 298   evel_password2 = NULL;
 299   if (password2 != NULL)
 300   {
 301     evel_password2 = strdup(password2);
 302     assert(evel_password2 != NULL);
 303   }
 304
 305   curl_version_info_data *d = curl_version_info(CURLVERSION_NOW);
 306   /* compare with the 24 bit hex number in 8 bit fields */
 307   if(d->version_num >= 0x072100) {
 308      /* this is libcurl 7.33.0 or later */
 309      EVEL_INFO("7.33 or later Curl version %x.",d->version_num);
 310   }
 311   else {
 312      EVEL_INFO("Old Curl version.");
 313   }
 314
 315   /***************************************************************************/
 316   /* Initialize a message ring-buffer to be used between the foreground and  */
 317   /* the thread which sends the messages.  This can't fail.                  */
 318   /***************************************************************************/
 319   if( ring_buf_size < EVEL_EVENT_BUFFER_DEPTH )
 320   {
 321     log_error_state("Warning: Failed to initialize Ring buffer size to %d. ",
 322                     ring_buf_size);
 323     goto exit_label;
 324   }
 325   ring_buffer_initialize(&event_buffer, EVEL_EVENT_BUFFER_DEPTH);
 326
 327 exit_label:
 328
 329   EVEL_EXIT();
 330
 331   return rc;
 332
 333 }
 334 /**************************************************************************//**
 335  * Setup the curl connection to collector
 336  *
 337  * Primarily responsible for getting CURL ready to send message. Also it would
 338  * be used to swithch over to other collector
 339  *****************************************************************************/
 340 static EVEL_ERR_CODES evel_setup_curl()
 341 {
 342   int rc = EVEL_SUCCESS;
 343   CURLcode curl_rc = CURLE_OK;
 344   char local_address[64];
 345   char * api_url = NULL;
 346   char * username = NULL;
 347   char * password = NULL;
 348   char * source_ip = NULL;
 349
 350   EVEL_ENTER();
 351
 352   if (evel_collector_id > 4)
 353   {
 354     rc = EVEL_CURL_LIBRARY_FAIL;
 355     log_error_state("Wrong evel_collector- value > 4");
 356     goto exit_label;
 357   }
 358
 359   /***************************************************************************/
 360   /* Initialize the local variable with proper global variables that are     */
 361   /* required to setup the connection                                        */
 362   /***************************************************************************/
 363   if (evel_collector_id == 1)
 364   {
 365      api_url = evel_event_api_url;
 366      source_ip = evel_source_ip;
 367      username = evel_username;
 368      password = evel_password;
 369   }
 370   else if (evel_collector_id == 2)
 371   {
 372      api_url = evel_bevent_api_url;
 373      source_ip = evel_source_ip_bakup;
 374      username = evel_username2;
 375      password = evel_password2;
 376   }
 377   else if (evel_collector_id == 3)
 378   {
 379      api_url = evel_batch_api_url;
 380      source_ip = evel_source_ip;
 381      username = evel_username;
 382      password = evel_password;
 383   }
 384   else if (evel_collector_id == 4)
 385   {
 386      api_url = evel_bbatch_api_url;
 387      source_ip = evel_source_ip_bakup;
 388      username = evel_username2;
 389      password = evel_password2;
 390   }
 391
 392   /***************************************************************************/
 393   /* Clean-up the cURL library.                                              */
 394   /***************************************************************************/
 395   if (curl_handle != NULL)
 396   {
 397     curl_easy_cleanup(curl_handle);
 398     curl_handle = NULL;
 399   }
 400   if (hdr_chunk != NULL)
 401   {
 402     curl_slist_free_all(hdr_chunk);
 403     hdr_chunk = NULL;
 404   }
 405
 406   /***************************************************************************/
 407   /* Start the CURL library. Note that this initialization is not threadsafe */
 408   /* which imposes a constraint that the EVEL library is initialized before  */
 409   /* any threads are started.                                                */
 410   /***************************************************************************/
 411   curl_rc = curl_global_init(CURL_GLOBAL_SSL);
 412   if (curl_rc != CURLE_OK)
 413   {
 414     rc = EVEL_CURL_LIBRARY_FAIL;
 415     log_error_state("Failed to initialize libCURL. Error code=%d", curl_rc);
 416     goto exit_label;
 417   }
 418
 419   /***************************************************************************/
 420   /* Get a curl handle which we'll use for all of our output.                */
 421   /***************************************************************************/
 422   curl_handle = curl_easy_init();
 423   if (curl_handle == NULL)
 424   {
 425     rc = EVEL_CURL_LIBRARY_FAIL;
 426     log_error_state("Failed to get libCURL handle");
 427     goto exit_label;
 428   }
 429
 430   /***************************************************************************/
 431   /* Prime the library to give friendly error codes.                         */
 432   /***************************************************************************/
 433   curl_rc = curl_easy_setopt(curl_handle,
 434                              CURLOPT_ERRORBUFFER,
 435                              curl_err_string);
 436   if (curl_rc != CURLE_OK)
 437   {
 438     rc = EVEL_CURL_LIBRARY_FAIL;
 439     log_error_state("Failed to initialize libCURL to provide friendly errors. "
 440                     "Error code=%d", curl_rc);
 441     goto exit_label;
 442   }
 443
 444   /***************************************************************************/
 445   /* If running in verbose mode generate more output.                        */
 446   /***************************************************************************/
 447   if (evel_verbosity > 0)
 448   {
 449     curl_rc = curl_easy_setopt(curl_handle, CURLOPT_VERBOSE, 1L);
 450     if (curl_rc != CURLE_OK)
 451     {
 452       rc = EVEL_CURL_LIBRARY_FAIL;
 453       log_error_state("Failed to initialize libCURL to be verbose. "
 454                       "Error code=%d", curl_rc);
 455       goto exit_label;
 456     }
 457   }
 458
 459   /***************************************************************************/
 460   /* Set the URL for the API.                                                */
 461   /***************************************************************************/
 462   curl_rc = curl_easy_setopt(curl_handle, CURLOPT_URL, api_url);
 463   if (curl_rc != CURLE_OK)
 464   {
 465     rc = EVEL_CURL_LIBRARY_FAIL;
 466     log_error_state("Failed to initialize libCURL with the API URL. "
 467                     "Error code=%d (%s)", curl_rc, curl_err_string);
 468     goto exit_label;
 469   }
 470   EVEL_INFO("Initializing CURL to send events to: %s", api_url);
 471
 472   /***************************************************************************/
 473   /* send all data to this function.                                         */
 474   /***************************************************************************/
 475   curl_rc = curl_easy_setopt(curl_handle,
 476                              CURLOPT_WRITEFUNCTION,
 477                              evel_write_callback);
 478   if (curl_rc != CURLE_OK)
 479   {
 480     rc = EVEL_CURL_LIBRARY_FAIL;
 481     log_error_state("Failed to initialize libCURL with the write callback. "
 482                     "Error code=%d (%s)", curl_rc, curl_err_string);
 483     goto exit_label;
 484   }
 485
 486   /***************************************************************************/
 487   /* configure local ip address if provided */
 488   /* Default ip if NULL */
 489   /***************************************************************************/
 490   if( source_ip != NULL )
 491   {
 492     snprintf(local_address,sizeof(local_address),source_ip);
 493     if( local_address[0] != '\0' )
 494     {
 495       curl_rc = curl_easy_setopt(curl_handle,
 496                              CURLOPT_INTERFACE,
 497                              local_address);
 498       if (curl_rc != CURLE_OK)
 499       {
 500         rc = EVEL_CURL_LIBRARY_FAIL;
 501         log_error_state("Failed to initialize libCURL with the local address. "
 502                     "Error code=%d (%s)", curl_rc, curl_err_string);
 503         goto exit_label;
 504       }
 505     }
 506   }
 507
 508   /***************************************************************************/
 509   /* configure SSL options for HTTPS transfers */
 510   /***************************************************************************/
 511   if( evel_secure )
 512   {
 513     if( evel_cert_file_path != NULL )
 514     {
 515       curl_rc = curl_easy_setopt(curl_handle,
 516                              CURLOPT_SSLCERT,
 517                              evel_cert_file_path);
 518       if (curl_rc != CURLE_OK)
 519       {
 520         rc = EVEL_CURL_LIBRARY_FAIL;
 521         log_error_state("Failed to initialize libCURL with the client cert. "
 522                     "Error code=%d (%s)", curl_rc, curl_err_string);
 523         goto exit_label;
 524       }
 525     }
 526
 527     if( evel_key_file_path != NULL )
 528     {
 529       curl_rc = curl_easy_setopt(curl_handle,
 530                              CURLOPT_SSLKEY,
 531                              evel_key_file_path);
 532       if (curl_rc != CURLE_OK)
 533       {
 534         rc = EVEL_CURL_LIBRARY_FAIL;
 535         log_error_state("Failed to initialize libCURL with the client key. "
 536                     "Error code=%d (%s)", curl_rc, curl_err_string);
 537         goto exit_label;
 538       }
 539     }
 540
 541     if( evel_ca_info != NULL )
 542     {
 543       curl_rc = curl_easy_setopt(curl_handle,
 544                              CURLOPT_CAINFO,
 545                              evel_ca_info);
 546       if (curl_rc != CURLE_OK)
 547       {
 548         rc = EVEL_CURL_LIBRARY_FAIL;
 549         log_error_state("Failed to initialize libCURL with the CA cert file. "
 550                     "Error code=%d (%s)", curl_rc, curl_err_string);
 551         goto exit_label;
 552       }
 553     }
 554
 555     if( evel_ca_file_path != NULL )
 556     {
 557       curl_rc = curl_easy_setopt(curl_handle,
 558                              CURLOPT_CAPATH,
 559                              evel_ca_file_path);
 560       if (curl_rc != CURLE_OK)
 561       {
 562         rc = EVEL_CURL_LIBRARY_FAIL;
 563         log_error_state("Failed to initialize libCURL with the CA cert path. "
 564                     "Error code=%d (%s)", curl_rc, curl_err_string);
 565         goto exit_label;
 566       }
 567     }
 568
 569       curl_rc = curl_easy_setopt(curl_handle,
 570                              CURLOPT_SSL_VERIFYPEER,
 571                              evel_verify_peer);
 572       if (curl_rc != CURLE_OK)
 573       {
 574         rc = EVEL_CURL_LIBRARY_FAIL;
 575         log_error_state("Failed to initialize libCURL with SSL Server verification. "
 576                     "Error code=%d (%s)", curl_rc, curl_err_string);
 577         goto exit_label;
 578       }
 579       curl_rc = curl_easy_setopt(curl_handle,
 580                              CURLOPT_SSL_VERIFYHOST,
 581                              evel_verify_host);
 582       if (curl_rc != CURLE_OK)
 583       {
 584         rc = EVEL_CURL_LIBRARY_FAIL;
 585         log_error_state("Failed to initialize libCURL with Client host verification. "
 586                     "Error code=%d (%s)", curl_rc, curl_err_string);
 587         goto exit_label;
 588       }
 589
 590   }
 591
 592
 593
 594   /***************************************************************************/
 595   /* some servers don't like requests that are made without a user-agent     */
 596   /* field, so we provide one.                                               */
 597   /***************************************************************************/
 598   curl_rc = curl_easy_setopt(curl_handle,
 599                              CURLOPT_USERAGENT,
 600                              "libcurl-agent/1.0");
 601   if (curl_rc != CURLE_OK)
 602   {
 603     rc = EVEL_CURL_LIBRARY_FAIL;
 604     log_error_state("Failed to initialize libCURL to upload. "
 605                     "Error code=%d (%s)", curl_rc, curl_err_string);
 606     goto exit_label;
 607   }
 608
 609   /***************************************************************************/
 610   /* Specify that we are going to POST data.                                 */
 611   /***************************************************************************/
 612   curl_rc = curl_easy_setopt(curl_handle, CURLOPT_POST, 1L);
 613   if (curl_rc != CURLE_OK)
 614   {
 615     rc = EVEL_CURL_LIBRARY_FAIL;
 616     log_error_state("Failed to initialize libCURL to upload. "
 617                     "Error code=%d (%s)", curl_rc, curl_err_string);
 618     goto exit_label;
 619   }
 620
 621   /***************************************************************************/
 622   /* we want to use our own read function.                                   */
 623   /***************************************************************************/
 624   curl_rc = curl_easy_setopt(curl_handle, CURLOPT_READFUNCTION, read_callback);
 625   if (curl_rc != CURLE_OK)
 626   {
 627     rc = EVEL_CURL_LIBRARY_FAIL;
 628     log_error_state("Failed to initialize libCURL to upload using read "
 629                     "function. Error code=%d (%s)", curl_rc, curl_err_string);
 630     goto exit_label;
 631   }
 632
 633   /***************************************************************************/
 634   /* All of our events are JSON encoded.  We also suppress the               */
 635   /* Expect: 100-continue   header that we would otherwise get since it      */
 636   /* confuses some servers.                                                  */
 637   /*                                                                         */
 638   /* @TODO: do AT&T want this behavior?                                      */
 639   /***************************************************************************/
 640   hdr_chunk = curl_slist_append(hdr_chunk, "Content-type: application/json");
 641   hdr_chunk = curl_slist_append(hdr_chunk, "Expect:");
 642
 643   /***************************************************************************/
 644   /* set our custom set of headers.                                         */
 645   /***************************************************************************/
 646   curl_rc = curl_easy_setopt(curl_handle, CURLOPT_HTTPHEADER, hdr_chunk);
 647   if (curl_rc != CURLE_OK)
 648   {
 649     rc = EVEL_CURL_LIBRARY_FAIL;
 650     log_error_state("Failed to initialize libCURL to use custom headers. "
 651                     "Error code=%d (%s)", curl_rc, curl_err_string);
 652     goto exit_label;
 653   }
 654
 655   /***************************************************************************/
 656   /* Set the timeout for the operation.                                      */
 657   /***************************************************************************/
 658   curl_rc = curl_easy_setopt(curl_handle,
 659                              CURLOPT_TIMEOUT,
 660                              EVEL_API_TIMEOUT);
 661   if (curl_rc != CURLE_OK)
 662   {
 663     rc = EVEL_CURL_LIBRARY_FAIL;
 664     log_error_state("Failed to initialize libCURL for API timeout. "
 665                     "Error code=%d (%s)", curl_rc, curl_err_string);
 666     goto exit_label;
 667   }
 668
 669   /***************************************************************************/
 670   /* Set that we want Basic authentication with username:password Base-64    */
 671   /* encoded for the operation.                                              */
 672   /***************************************************************************/
 673   curl_rc = curl_easy_setopt(curl_handle, CURLOPT_HTTPAUTH, CURLAUTH_BASIC);
 674   if (curl_rc != CURLE_OK)
 675   {
 676     rc = EVEL_CURL_LIBRARY_FAIL;
 677     log_error_state("Failed to initialize libCURL for Basic Authentication. "
 678                     "Error code=%d (%s)", curl_rc, curl_err_string);
 679     goto exit_label;
 680   }
 681   curl_rc = curl_easy_setopt(curl_handle, CURLOPT_USERNAME, username);
 682   if (curl_rc != CURLE_OK)
 683   {
 684     rc = EVEL_CURL_LIBRARY_FAIL;
 685     log_error_state("Failed to initialize libCURL with username. "
 686                     "Error code=%d (%s)", curl_rc, curl_err_string);
 687     goto exit_label;
 688   }
 689   curl_rc = curl_easy_setopt(curl_handle, CURLOPT_PASSWORD, password);
 690   if (curl_rc != CURLE_OK)
 691   {
 692     rc = EVEL_CURL_LIBRARY_FAIL;
 693     log_error_state("Failed to initialize libCURL with password. "
 694                     "Error code=%d (%s)", curl_rc, curl_err_string);
 695     goto exit_label;
 696   }
 697
 698   /***************************************************************************/
 699   /* Initialize the priority post buffer to empty.                           */
 700   /***************************************************************************/
 701   priority_post.memory = NULL;
 702
 703 exit_label:
 704   EVEL_EXIT();
 705
 706   return(rc);
 707 }
 708
 709 /**************************************************************************//**
 710  * Run the event handler.
 711  *
 712  * Spawns the thread responsible for handling events and sending them to the
 713  * API.
 714  *
 715  *  @return Status code.
 716  *  @retval ::EVEL_SUCCESS if everything OK.
 717  *  @retval One of ::EVEL_ERR_CODES if there was a problem.
 718  *****************************************************************************/
 719 EVEL_ERR_CODES event_handler_run()
 720 {
 721   EVEL_ERR_CODES rc = EVEL_SUCCESS;
 722   int pthread_rc = 0;
 723
 724   EVEL_ENTER();
 725
 726   /***************************************************************************/
 727   /* Start the event handler thread.                                         */
 728   /***************************************************************************/
 729   evt_handler_state = EVT_HANDLER_INACTIVE;
 730   pthread_rc = pthread_create(&evt_handler_thread, NULL, event_handler, NULL);
 731   if (pthread_rc != 0)
 732   {
 733     rc = EVEL_PTHREAD_LIBRARY_FAIL;
 734     log_error_state("Failed to start event handler thread. "
 735                     "Error code=%d", pthread_rc);
 736   }
 737
 738   EVEL_EXIT()
 739   return rc;
 740 }
 741
 742 /**************************************************************************//**
 743  * Terminate the event handler.
 744  *
 745  * Shuts down the event handler thread in as clean a way as possible. Sets the
 746  * global exit flag and then signals the thread to interrupt it since it's
 747  * most likely waiting on the ring-buffer.
 748  *
 749  * Having achieved an orderly shutdown of the event handler thread, clean up
 750  * the cURL library's resources cleanly.
 751  *
 752  *  @return Status code.
 753  *  @retval ::EVEL_SUCCESS if everything OK.
 754  *  @retval One of ::EVEL_ERR_CODES if there was a problem.
 755  *****************************************************************************/
 756 EVEL_ERR_CODES event_handler_terminate()
 757 {
 758   EVEL_ERR_CODES rc = EVEL_SUCCESS;
 759
 760   EVEL_ENTER();
 761   EVENT_INTERNAL *event = NULL;
 762
 763   /***************************************************************************/
 764   /* Make sure that we were initialized before trying to terminate the       */
 765   /* event handler thread.                                                   */
 766   /***************************************************************************/
 767   if (evt_handler_state != EVT_HANDLER_UNINITIALIZED)
 768   {
 769     /*************************************************************************/
 770     /* Make sure that the event handler knows it's time to die.              */
 771     /*************************************************************************/
 772     event = evel_new_internal_event(EVT_CMD_TERMINATE,"EVELinternal","EVELid");
 773     if (event == NULL)
 774     {
 775       /***********************************************************************/
 776       /* We failed to get an event, but we don't bail out - we will just     */
 777       /* clean up what we can and continue on our way, since we're exiting   */
 778       /* anyway.                                                             */
 779       /***********************************************************************/
 780       EVEL_ERROR("Failed to get internal event - perform dirty exit instead!");
 781     }
 782     else
 783     {
 784       /***********************************************************************/
 785       /* Post the event then wait for the Event Handler to exit.  Set the    */
 786       /* global command, too, in case the ring-buffer is full.               */
 787       /***********************************************************************/
 788       EVEL_DEBUG("Sending event to Event Hander to request it to exit.");
 789       evt_handler_state = EVT_HANDLER_REQUEST_TERMINATE;
 790       evel_post_event((EVENT_HEADER *) event);
 791       pthread_join(evt_handler_thread, NULL);
 792       EVEL_DEBUG("Event Handler thread has exited.");
 793     }
 794   }
 795   else
 796   {
 797     EVEL_DEBUG("Event handler was not initialized, so no need to kill it");
 798   }
 799
 800   /***************************************************************************/
 801   /* Clean-up the cURL library.                                              */
 802   /***************************************************************************/
 803   if (curl_handle != NULL)
 804   {
 805     curl_easy_cleanup(curl_handle);
 806     curl_handle = NULL;
 807   }
 808   if (hdr_chunk != NULL)
 809   {
 810     curl_slist_free_all(hdr_chunk);
 811     hdr_chunk = NULL;
 812   }
 813
 814   /***************************************************************************/
 815   /* Free off the stored API URL strings.                                    */
 816   /***************************************************************************/
 817   if (evel_event_api_url != NULL)
 818   {
 819     free(evel_event_api_url);
 820     evel_event_api_url = NULL;
 821   }
 822   if (evel_batch_api_url != NULL)
 823   {
 824     free(evel_batch_api_url);
 825     evel_batch_api_url = NULL;
 826   }
 827   if (evel_throt_api_url != NULL)
 828   {
 829     free(evel_throt_api_url);
 830     evel_throt_api_url = NULL;
 831   }
 832
 833   EVEL_EXIT();
 834   return rc;
 835 }
 836
 837 /**************************************************************************//**
 838  * Post an event.
 839  *
 840  * @note  So far as the caller is concerned, successfully posting the event
 841  * relinquishes all responsibility for the event - the library will take care
 842  * of freeing the event in due course.
 843
 844  * @param event   The event to be posted.
 845  *
 846  * @returns Status code
 847  * @retval  EVEL_SUCCESS On success
 848  * @retval  "One of ::EVEL_ERR_CODES" On failure.
 849  *****************************************************************************/
 850 EVEL_ERR_CODES evel_post_event(EVENT_HEADER * event)
 851 {
 852   int rc = EVEL_SUCCESS;
 853
 854   EVEL_ENTER();
 855
 856   /***************************************************************************/
 857   /* Check preconditions.                                                    */
 858   /***************************************************************************/
 859   assert(event != NULL);
 860
 861   /***************************************************************************/
 862   /* We need to make sure that we are either initializing or running         */
 863   /* normally before writing the event into the buffer so that we can        */
 864   /* guarantee that the ring-buffer empties  properly on exit.               */
 865   /***************************************************************************/
 866   if ((evt_handler_state == EVT_HANDLER_ACTIVE) ||
 867       (evt_handler_state == EVT_HANDLER_INACTIVE) ||
 868       (evt_handler_state == EVT_HANDLER_REQUEST_TERMINATE))
 869   {
 870     if (ring_buffer_write(&event_buffer, event) == 0)
 871     {
 872       log_error_state("Failed to write event to buffer - event dropped!");
 873       rc = EVEL_EVENT_BUFFER_FULL;
 874       evel_free_event(event);
 875     }
 876   }
 877   else
 878   {
 879     /*************************************************************************/
 880     /* System is not in active operation, so reject the event.               */
 881     /*************************************************************************/
 882     log_error_state("Event Handler system not active - event dropped!");
 883     rc = EVEL_EVENT_HANDLER_INACTIVE;
 884     evel_free_event(event);
 885   }
 886
 887   EVEL_EXIT();
 888   return (rc);
 889 }
 890
 891 /**************************************************************************//**
 892  * Post an event to the Vendor Event Listener API.
 893  *
 894  * @returns Status code
 895  * @retval  EVEL_SUCCESS On success
 896  * @retval  "One of ::EVEL_ERR_CODES" On failure.
 897  *****************************************************************************/
 898 static EVEL_ERR_CODES evel_post_api(char * msg, size_t size)
 899 {
 900   int rc = EVEL_SUCCESS;
 901   CURLcode curl_rc = CURLE_OK;
 902   MEMORY_CHUNK rx_chunk;
 903   MEMORY_CHUNK tx_chunk;
 904
 905   EVEL_ENTER();
 906
 907   /***************************************************************************/
 908   /* Create the memory chunk to be used for the response to the post.  The   */
 909   /* will be realloced.                                                      */
 910   /***************************************************************************/
 911   rx_chunk.memory = malloc(1);
 912   assert(rx_chunk.memory != NULL);
 913   rx_chunk.size = 0;
 914
 915   /***************************************************************************/
 916   /* Create the memory chunk to be sent as the body of the post.             */
 917   /***************************************************************************/
 918   tx_chunk.memory = msg;
 919   tx_chunk.size = size;
 920   EVEL_DEBUG("Sending chunk of size %d", tx_chunk.size);
 921
 922   /***************************************************************************/
 923   /* Point to the data to be received.                                       */
 924   /***************************************************************************/
 925   curl_rc = curl_easy_setopt(curl_handle, CURLOPT_WRITEDATA, &rx_chunk);
 926   if (curl_rc != CURLE_OK)
 927   {
 928     rc = EVEL_CURL_LIBRARY_FAIL;
 929     log_error_state("Failed to initialize libCURL to upload. "
 930                     "Error code=%d (%s)", curl_rc, curl_err_string);
 931     goto exit_label;
 932   }
 933   EVEL_DEBUG("Initialized data to receive");
 934
 935   /***************************************************************************/
 936   /* Pointer to pass to our read function                                    */
 937   /***************************************************************************/
 938   curl_rc = curl_easy_setopt(curl_handle, CURLOPT_READDATA, &tx_chunk);
 939   if (curl_rc != CURLE_OK)
 940   {
 941     rc = EVEL_CURL_LIBRARY_FAIL;
 942     log_error_state("Failed to set upload data for libCURL to upload. "
 943                     "Error code=%d (%s)", curl_rc, curl_err_string);
 944     goto exit_label;
 945   }
 946   EVEL_DEBUG("Initialized data to send");
 947
 948   /***************************************************************************/
 949   /* Size of the data to transmit.                                           */
 950   /***************************************************************************/
 951   curl_rc = curl_easy_setopt(curl_handle,
 952                              CURLOPT_POSTFIELDSIZE,
 953                              tx_chunk.size);
 954   if (curl_rc != CURLE_OK)
 955   {
 956     rc = EVEL_CURL_LIBRARY_FAIL;
 957     log_error_state("Failed to set length of upload data for libCURL to "
 958                     "upload.  Error code=%d (%s)", curl_rc, curl_err_string);
 959     goto exit_label;
 960   }
 961   EVEL_DEBUG("Initialized length of data to send");
 962
 963   /***************************************************************************/
 964   /* Now run off and do what you've been told!                               */
 965   /***************************************************************************/
 966   http_response_code = 0;
 967
 968   curl_rc = curl_easy_perform(curl_handle);
 969   if (curl_rc != CURLE_OK)
 970   {
 971     rc = EVEL_CURL_LIBRARY_FAIL;
 972     log_error_state("Failed to transfer an event to Vendor Event Listener! "
 973                     "Error code=%d (%s)", curl_rc, curl_err_string);
 974     EVEL_ERROR("Dropped event: %s", msg);
 975     goto exit_label;
 976   }
 977
 978   /***************************************************************************/
 979   /* See what response we got - any 2XX response is good.                    */
 980   /***************************************************************************/
 981   curl_easy_getinfo(curl_handle, CURLINFO_RESPONSE_CODE, &http_response_code);
 982   EVEL_DEBUG("HTTP response code: %d", http_response_code);
 983   if ((http_response_code / 100) == 2)
 984   {
 985     /*************************************************************************/
 986     /* If the server responded with data it may be interesting but not a     */
 987     /* problem.                                                              */
 988     /*************************************************************************/
 989     if ((rx_chunk.size > 0) && (rx_chunk.memory != NULL))
 990     {
 991       EVEL_DEBUG("Server returned data = %d (%s)",
 992                  rx_chunk.size,
 993                  rx_chunk.memory);
 994
 995       /***********************************************************************/
 996       /* If this is a response to priority post, then we're not interested.  */
 997       /***********************************************************************/
 998       if (priority_post.memory != NULL)
 999       {
1000         EVEL_ERROR("Ignoring priority post response");
1001       }
1002       else
1003       {
1004         evel_handle_event_response(&rx_chunk, &priority_post);
1005       }
1006     }
1007   }
1008   else
1009   {
1010     EVEL_ERROR("Unexpected HTTP response code: %d with data size %d (%s)",
1011                 http_response_code,
1012                 rx_chunk.size,
1013                 rx_chunk.size > 0 ? rx_chunk.memory : "NONE");
1014     EVEL_ERROR("Potentially dropped event: %s", msg);
1015   }
1016
1017 exit_label:
1018   free(rx_chunk.memory);
1019   EVEL_EXIT();
1020   return(rc);
1021 }
1022
1023 /**************************************************************************//**
1024  * Callback function to provide data to send.
1025  *
1026  * Copy data into the supplied buffer, read_callback::ptr, checking size
1027  * limits.
1028  *
1029  * @returns   Number of bytes placed into read_callback::ptr. 0 for EOF.
1030  *****************************************************************************/
1031 static size_t read_callback(void *ptr, size_t size, size_t nmemb, void *userp)
1032 {
1033   size_t rtn = 0;
1034   size_t bytes_to_write = 0;
1035   MEMORY_CHUNK *tx_chunk = (MEMORY_CHUNK *)userp;
1036
1037   EVEL_ENTER();
1038
1039   bytes_to_write = min(size*nmemb, tx_chunk->size);
1040
1041   if (bytes_to_write > 0)
1042   {
1043     EVEL_DEBUG("Going to try to write %d bytes", bytes_to_write);
1044     strncpy((char *)ptr, tx_chunk->memory, bytes_to_write);
1045     tx_chunk->memory += bytes_to_write;
1046     tx_chunk->size -= bytes_to_write;
1047     rtn = bytes_to_write;
1048   }
1049   else
1050   {
1051     EVEL_DEBUG("Reached EOF");
1052   }
1053
1054   EVEL_EXIT();
1055   return rtn;
1056 }
1057
1058 /**************************************************************************//**
1059  * Callback function to provide returned data.
1060  *
1061  * Copy data into the supplied buffer, write_callback::ptr, checking size
1062  * limits.
1063  *
1064  * @returns   Number of bytes placed into write_callback::ptr. 0 for EOF.
1065  *****************************************************************************/
1066 size_t evel_write_callback(void *contents,
1067                              size_t size,
1068                              size_t nmemb,
1069                              void *userp)
1070 {
1071   size_t realsize = size * nmemb;
1072   MEMORY_CHUNK * rx_chunk = (MEMORY_CHUNK *)userp;
1073
1074   EVEL_ENTER();
1075
1076   EVEL_DEBUG("Called with %d chunks of %d size = %d", nmemb, size, realsize);
1077   EVEL_DEBUG("rx chunk size is %d", rx_chunk->size);
1078
1079   rx_chunk->memory = realloc(rx_chunk->memory, rx_chunk->size + realsize + 1);
1080   if(rx_chunk->memory == NULL) {
1081     /* out of memory! */
1082     printf("not enough memory (realloc returned NULL)\n");
1083     return 0;
1084   }
1085
1086   memcpy(&(rx_chunk->memory[rx_chunk->size]), contents, realsize);
1087   rx_chunk->size += realsize;
1088   rx_chunk->memory[rx_chunk->size] = 0;
1089
1090   EVEL_DEBUG("Rx data: %s", rx_chunk->memory);
1091   EVEL_DEBUG("Returning: %d", realsize);
1092
1093   EVEL_EXIT();
1094   return realsize;
1095 }
1096
1097 /**************************************************************************//**
1098  * Event Handler.
1099  *
1100  * Watch for messages coming on the internal queue and send them to the
1101  * listener.
1102  *
1103  * param[in]  arg  Argument - unused.
1104  *****************************************************************************/
1105 static void * event_handler(void * arg __attribute__ ((unused)))
1106 {
1107   int old_type = 0;
1108   EVENT_HEADER * msg = NULL;
1109   EVENT_INTERNAL * internal_msg = NULL;
1110   int json_size = 0;
1111   char json_body[EVEL_MAX_JSON_BODY];
1112   int rc = EVEL_SUCCESS;
1113   CURLcode curl_rc;
1114   int collector_down_count = 0;
1115   int switch_coll = 0;
1116
1117   EVEL_INFO("Event handler thread started");
1118
1119   /***************************************************************************/
1120   /* Set this thread to be cancellable immediately.                          */
1121   /***************************************************************************/
1122   pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, &old_type);
1123
1124   /***************************************************************************/
1125   /* Set the handler as active, defending against weird situations like      */
1126   /* immediately shutting down after initializing the library so the         */
1127   /* handler never gets started up properly.                                 */
1128   /***************************************************************************/
1129   if (evt_handler_state == EVT_HANDLER_INACTIVE)
1130   {
1131     evt_handler_state = EVT_HANDLER_ACTIVE;
1132   }
1133   else
1134   {
1135     EVEL_ERROR("Event Handler State was not INACTIVE at start-up - "
1136                "Handler will exit immediately!");
1137   }
1138
1139   while (evt_handler_state == EVT_HANDLER_ACTIVE)
1140   {
1141     /*************************************************************************/
1142     /* Wait for a message to be received.                                    */
1143     /*************************************************************************/
1144     EVEL_DEBUG("Event handler getting any messages");
1145     msg = ring_buffer_read(&event_buffer);
1146
1147     /*************************************************************************/
1148     /* Internal events get special treatment while regular events get posted */
1149     /* to the far side.                                                      */
1150     /*************************************************************************/
1151     if (msg->event_domain == EVEL_DOMAIN_BATCH )
1152     {
1153       EVEL_DEBUG("Batch event received");
1154
1155       /***********************************************************************/
1156       /* Encode the event in JSON.                                           */
1157       /***********************************************************************/
1158       json_size = evel_json_encode_batch_event(json_body, EVEL_MAX_JSON_BODY, msg);
1159
1160       /***************************************************************************/
1161       /* Set the connection to collector                                         */
1162       /***************************************************************************/
1163       while (true)
1164       {
1165          evel_collector_id =3;
1166          rc = evel_setup_curl();
1167
1168          if ( rc != EVEL_SUCCESS)
1169          {
1170              EVEL_ERROR("Failed to setup the first collector. Error code=%d", rc);
1171              if (curr_global_handles == 2)
1172              {
1173                 EVEL_DEBUG("Switching to other collector");
1174
1175                 evel_collector_id = 4;
1176
1177                 rc = evel_setup_curl();
1178                 if ( rc != EVEL_SUCCESS)
1179                 {
1180                    EVEL_ERROR("Failed to setup the connection to second collector also, Error code%d", rc);
1181                    sleep(EVEL_COLLECTOR_RECONNECTION_WAIT_TIME);
1182                    collector_down_count = collector_down_count + 1;
1183                    EVEL_ERROR("Collectors setup issue- retry count=%d", collector_down_count);
1184                 }
1185                 else
1186                 {
1187                    break;
1188                    collector_down_count = 0;
1189                 }
1190              }
1191              else
1192              {
1193                 sleep(EVEL_COLLECTOR_RECONNECTION_WAIT_TIME);
1194                 collector_down_count = collector_down_count + 1;
1195                 EVEL_ERROR("Collector setup issue-retry count=%d", collector_down_count);
1196              }
1197          }
1198          else
1199          {
1200             break;
1201             collector_down_count = 0;
1202          }
1203       }
1204
1205       /***************************************************************************/
1206       /* Set the URL for the API.                                                */
1207       /***************************************************************************/
1208       if (evel_collector_id == 3)
1209       {
1210          curl_rc = curl_easy_setopt(curl_handle, CURLOPT_URL, evel_batch_api_url);
1211       }
1212       else
1213       {
1214          curl_rc = curl_easy_setopt(curl_handle, CURLOPT_URL, evel_bbatch_api_url);
1215       }
1216
1217       if (curl_rc != CURLE_OK)
1218       {
1219         rc = EVEL_CURL_LIBRARY_FAIL;
1220         log_error_state("Failed to initialize libCURL with the Batch API URL. "
1221                     "Error code=%d (%s)", curl_rc, curl_err_string);
1222       }
1223
1224       /***********************************************************************/
1225       /* Send the JSON across the API.                                       */
1226       /***********************************************************************/
1227       EVEL_DEBUG("Sending Batch JSON of size %d is: %s", json_size, json_body);
1228       rc = evel_post_api(json_body, json_size);
1229
1230       switch_coll = 0;
1231       if ((rc == EVEL_SUCCESS) && ((http_response_code / 100) != 2))
1232       {
1233          switch_coll = 1;
1234          if (http_response_code == 400) // 400 - Bad JSON related return code
1235             switch_coll = 0;
1236       }
1237
1238       if ((rc != EVEL_SUCCESS)  || (switch_coll == 1))
1239       {
1240         EVEL_ERROR("Failed to transfer the data. Error code=%d", rc);
1241         EVEL_DEBUG("Switching to other collector if any");
1242
1243         while (true)
1244         {
1245            if ((evel_collector_id == 3) && (curr_global_handles == 2))
1246            {
1247              evel_collector_id =4;
1248            }
1249            else if (evel_collector_id == 4)
1250            {
1251              evel_collector_id =3;
1252            }
1253
1254            rc = evel_setup_curl();
1255
1256            if ( rc != EVEL_SUCCESS)
1257            {
1258              sleep(EVEL_COLLECTOR_RECONNECTION_WAIT_TIME);
1259              collector_down_count = collector_down_count + 1;
1260              EVEL_ERROR("Collector setup issue-retry count=%d", collector_down_count);
1261              continue;
1262            }
1263
1264            if (evel_collector_id == 3)
1265            {
1266               curl_rc = curl_easy_setopt(curl_handle, CURLOPT_URL, evel_batch_api_url);
1267            }
1268            else if (evel_collector_id == 4)
1269            {
1270               curl_rc = curl_easy_setopt(curl_handle, CURLOPT_URL, evel_bbatch_api_url);
1271            }
1272
1273            rc = evel_post_api(json_body, json_size);
1274            if ( rc != EVEL_SUCCESS)
1275            {
1276               sleep(EVEL_COLLECTOR_RECONNECTION_WAIT_TIME);
1277               collector_down_count = collector_down_count + 1;
1278               EVEL_ERROR("Collector setup issue-retry count=%d", collector_down_count);
1279               continue;
1280            }
1281            else
1282            {
1283               EVEL_DEBUG("Successfully sent msg after retry=%d", collector_down_count);
1284               break;
1285            }
1286         }
1287       }
1288     }
1289     else if (msg->event_domain != EVEL_DOMAIN_INTERNAL )
1290     {
1291       EVEL_DEBUG("External event received");
1292
1293       /***********************************************************************/
1294       /* Encode the event in JSON.                                           */
1295       /***********************************************************************/
1296       json_size = evel_json_encode_event(json_body, EVEL_MAX_JSON_BODY, msg);
1297
1298       /***************************************************************************/
1299       /* Set the connection to collector                                         */
1300       /***************************************************************************/
1301       while (true)
1302       {
1303          evel_collector_id = 1;
1304          rc = evel_setup_curl();
1305
1306          if ( rc != EVEL_SUCCESS)
1307          {
1308              EVEL_ERROR("Failed to setup the first collector. Error code=%d", rc);
1309              if (curr_global_handles == 2)
1310              {
1311                 EVEL_DEBUG("Switching to other collector");
1312
1313                 evel_collector_id = 2;
1314
1315                 rc = evel_setup_curl();
1316                 if ( rc != EVEL_SUCCESS)
1317                 {
1318                    EVEL_ERROR("Failed to setup the connection to second collector also, Error code%d", rc);
1319                    sleep(EVEL_COLLECTOR_RECONNECTION_WAIT_TIME);
1320                    collector_down_count = collector_down_count + 1;
1321                    EVEL_ERROR("Collectors setup issue- retry count=%d", collector_down_count);
1322                 }
1323                 else
1324                 {
1325                    break;
1326                    collector_down_count = 0;
1327                 }
1328              }
1329              else
1330              {
1331                 sleep(EVEL_COLLECTOR_RECONNECTION_WAIT_TIME);
1332                 collector_down_count = collector_down_count + 1;
1333                 EVEL_ERROR("Collector setup issue-retry count=%d", collector_down_count);
1334              }
1335          }
1336          else
1337          {
1338             break;
1339             collector_down_count = 0;
1340          }
1341       }
1342       /***************************************************************************/
1343       /* Set the URL for the API.                                                */
1344       /***************************************************************************/
1345       if (evel_collector_id == 1)
1346          curl_rc = curl_easy_setopt(curl_handle, CURLOPT_URL, evel_event_api_url);
1347       else
1348          curl_rc = curl_easy_setopt(curl_handle, CURLOPT_URL, evel_bevent_api_url);
1349
1350       if (curl_rc != CURLE_OK)
1351       {
1352         rc = EVEL_CURL_LIBRARY_FAIL;
1353         log_error_state("Failed to initialize libCURL with the API URL. "
1354                     "Error code=%d (%s)", curl_rc, curl_err_string);
1355       }
1356
1357       /***********************************************************************/
1358       /* Send the JSON across the API.                                       */
1359       /***********************************************************************/
1360       EVEL_DEBUG("Sending JSON of size %d is: %s", json_size, json_body);
1361       rc = evel_post_api(json_body, json_size);
1362
1363       switch_coll = 0;
1364       if ((rc == EVEL_SUCCESS) && ((http_response_code / 100) != 2))
1365       {
1366          switch_coll = 1;
1367          if (http_response_code == 400) // 400 - Bad JSON related return code
1368             switch_coll = 0;
1369       }
1370
1371       if ((rc != EVEL_SUCCESS)  || (switch_coll == 1))
1372       {
1373         EVEL_ERROR("Failed to transfer the data. Error code=%d", rc);
1374         EVEL_DEBUG("Switching to other collector if any");
1375
1376         while (true)
1377         {
1378            if ((evel_collector_id == 1) && (curr_global_handles == 2))
1379            {
1380              evel_collector_id =2;
1381            }
1382            else if (evel_collector_id == 2)
1383            {
1384              evel_collector_id =1;
1385            }
1386
1387            rc = evel_setup_curl();
1388
1389            if ( rc != EVEL_SUCCESS)
1390            {
1391              sleep(EVEL_COLLECTOR_RECONNECTION_WAIT_TIME);
1392              collector_down_count = collector_down_count + 1;
1393              EVEL_ERROR("Collector setup issue-retry count=%d", collector_down_count);
1394              continue;
1395            }
1396
1397            if (evel_collector_id == 1)
1398            {
1399               curl_rc = curl_easy_setopt(curl_handle, CURLOPT_URL, evel_event_api_url);
1400            }
1401            else if (evel_collector_id == 2)
1402            {
1403               curl_rc = curl_easy_setopt(curl_handle, CURLOPT_URL, evel_bevent_api_url);
1404            }
1405
1406            rc = evel_post_api(json_body, json_size);
1407            if ( rc != EVEL_SUCCESS)
1408            {
1409               sleep(EVEL_COLLECTOR_RECONNECTION_WAIT_TIME);
1410               collector_down_count = collector_down_count + 1;
1411               EVEL_ERROR("Collector setup issue-retry count=%d", collector_down_count);
1412               continue;
1413            }
1414            else
1415            {
1416               EVEL_DEBUG("Successfully sent msg after retry=%d", collector_down_count);
1417               break;
1418            }
1419         }
1420       }
1421     }
1422     else
1423     {
1424       EVEL_DEBUG("Internal event received");
1425       internal_msg = (EVENT_INTERNAL *) msg;
1426       assert(internal_msg->command == EVT_CMD_TERMINATE);
1427       evt_handler_state = EVT_HANDLER_TERMINATING;
1428     }
1429
1430     /*************************************************************************/
1431     /* We are responsible for freeing the memory.                            */
1432     /*************************************************************************/
1433     evel_free_event(msg);
1434     msg = NULL;
1435
1436     /*************************************************************************/
1437     /* There may be a single priority post to be sent.                       */
1438     /*************************************************************************/
1439     if (priority_post.memory != NULL)
1440     {
1441       EVEL_DEBUG("Priority Post");
1442
1443       /***********************************************************************/
1444       /* Set the URL for the throttling API.                                 */
1445       /***********************************************************************/
1446       curl_rc = curl_easy_setopt(curl_handle, CURLOPT_URL, evel_throt_api_url);
1447       if (curl_rc != CURLE_OK)
1448       {
1449         /*********************************************************************/
1450         /* This is only likely to happen with CURLE_OUT_OF_MEMORY, in which  */
1451         /* case we carry on regardless.                                      */
1452         /*********************************************************************/
1453         EVEL_ERROR("Failed to set throttling URL. Error code=%d", rc);
1454       }
1455       else
1456       {
1457         rc = evel_post_api(priority_post.memory, priority_post.size);
1458         if (rc != EVEL_SUCCESS)
1459         {
1460           EVEL_ERROR("Failed to transfer priority post. Error code=%d", rc);
1461         }
1462       }
1463
1464       /***********************************************************************/
1465       /* Reinstate the URL for the event API.                                */
1466       /***********************************************************************/
1467       curl_rc = curl_easy_setopt(curl_handle, CURLOPT_URL, evel_event_api_url);
1468       if (curl_rc != CURLE_OK)
1469       {
1470         /*********************************************************************/
1471         /* This is only likely to happen with CURLE_OUT_OF_MEMORY, in which  */
1472         /* case we carry on regardless.                                      */
1473         /*********************************************************************/
1474         EVEL_ERROR("Failed to reinstate events URL. Error code=%d", rc);
1475       }
1476
1477       /***********************************************************************/
1478       /* We are responsible for freeing the memory.                          */
1479       /***********************************************************************/
1480       free(priority_post.memory);
1481       priority_post.memory = NULL;
1482     }
1483   }
1484
1485   /***************************************************************************/
1486   /* The event handler is now exiting. The ring-buffer could contain events  */
1487   /* which have not been processed, so deplete those.  Because we've been    */
1488   /* asked to exit we can be confident that the foreground will have stopped */
1489   /* sending events in so we know that this process will conclude!           */
1490   /***************************************************************************/
1491   evt_handler_state = EVT_HANDLER_TERMINATING;
1492   while (!ring_buffer_is_empty(&event_buffer))
1493   {
1494     EVEL_DEBUG("Reading event from buffer");
1495     msg = ring_buffer_read(&event_buffer);
1496     evel_free_event(msg);
1497   }
1498   evt_handler_state = EVT_HANDLER_TERMINATED;
1499   EVEL_INFO("Event handler thread stopped");
1500
1501   return (NULL);
1502 }
1503
1504 /**************************************************************************//**
1505  * Handle a JSON response from the listener, contained in a ::MEMORY_CHUNK.
1506  *
1507  * Tokenize the response, and decode any tokens found.
1508  *
1509  * @param chunk         The memory chunk containing the response.
1510  * @param post          The memory chunk in which to place any resulting POST.
1511  *****************************************************************************/
1512 void evel_handle_event_response(const MEMORY_CHUNK * const chunk,
1513                                 MEMORY_CHUNK * const post)
1514 {
1515   jsmn_parser json_parser;
1516   jsmntok_t json_tokens[EVEL_MAX_RESPONSE_TOKENS];
1517   int num_tokens = 0;
1518
1519   EVEL_ENTER();
1520
1521   /***************************************************************************/
1522   /* Check preconditions.                                                    */
1523   /***************************************************************************/
1524   assert(chunk != NULL);
1525   assert(priority_post.memory == NULL);
1526
1527   EVEL_DEBUG("Response size = %d", chunk->size);
1528   EVEL_DEBUG("Response = %s", chunk->memory);
1529
1530   /***************************************************************************/
1531   /* Initialize the parser and tokenize the response.                        */
1532   /***************************************************************************/
1533   jsmn_init(&json_parser);
1534   num_tokens = jsmn_parse(&json_parser,
1535                           chunk->memory,
1536                           chunk->size,
1537                           json_tokens,
1538                           EVEL_MAX_RESPONSE_TOKENS);
1539
1540   if (num_tokens < 0)
1541   {
1542     EVEL_ERROR("Failed to parse JSON response.  "
1543                "Error code=%d", num_tokens);
1544   }
1545   else if (num_tokens == 0)
1546   {
1547     EVEL_DEBUG("No tokens found in JSON response");
1548   }
1549   else
1550   {
1551     EVEL_DEBUG("Decode JSON response tokens");
1552     if (!evel_handle_response_tokens(chunk, json_tokens, num_tokens, post))
1553     {
1554       EVEL_ERROR("Failed to handle JSON response.");
1555     }
1556   }
1557
1558   EVEL_EXIT();
1559 }
1560
1561 /**************************************************************************//**
1562  * Handle a JSON response from the listener, as a list of tokens from JSMN.
1563  *
1564  * @param chunk         Memory chunk containing the JSON buffer.
1565  * @param json_tokens   Array of tokens to handle.
1566  * @param num_tokens    The number of tokens to handle.
1567  * @param post          The memory chunk in which to place any resulting POST.
1568  * @return true if we handled the response, false otherwise.
1569  *****************************************************************************/
1570 bool evel_handle_response_tokens(const MEMORY_CHUNK * const chunk,
1571                                  const jsmntok_t * const json_tokens,
1572                                  const int num_tokens,
1573                                  MEMORY_CHUNK * const post)
1574 {
1575   bool json_ok = true;
1576
1577   EVEL_ENTER();
1578
1579   /***************************************************************************/
1580   /* Check preconditions.                                                    */
1581   /***************************************************************************/
1582   assert(chunk != NULL);
1583   assert(json_tokens != NULL);
1584   assert(num_tokens < EVEL_MAX_RESPONSE_TOKENS);
1585
1586   /***************************************************************************/
1587   /* Peek at the tokens to decide what the response it, then call the        */
1588   /* appropriate handler to handle it.  There is only one handler at this    */
1589   /* point.                                                                  */
1590   /***************************************************************************/
1591   if (evel_tokens_match_command_list(chunk, json_tokens, num_tokens))
1592   {
1593     json_ok = evel_handle_command_list(chunk, json_tokens, num_tokens, post);
1594   }
1595
1596   EVEL_EXIT();
1597
1598   return json_ok;
1599 }
1600
1601 /**************************************************************************//**
1602  * Determine whether a list of tokens looks like a "commandList" response.
1603  *
1604  * @param chunk         Memory chunk containing the JSON buffer.
1605  * @param json_tokens   Token to check.
1606  * @param num_tokens    The number of tokens to handle.
1607  * @return true if the tokens look like a "commandList" match, or false.
1608  *****************************************************************************/
1609 bool evel_tokens_match_command_list(const MEMORY_CHUNK * const chunk,
1610                                     const jsmntok_t * const json_tokens,
1611                                     const int num_tokens)
1612 {
1613   bool result = false;
1614
1615   EVEL_ENTER();
1616
1617   /***************************************************************************/
1618   /* Make some checks on the basic layout of the commandList.                */
1619   /***************************************************************************/
1620   if ((num_tokens > 3) &&
1621       (json_tokens[0].type == JSMN_OBJECT) &&
1622       (json_tokens[1].type == JSMN_STRING) &&
1623       (json_tokens[2].type == JSMN_ARRAY) &&
1624       (evel_token_equals_string(chunk, &json_tokens[1], "commandList")))
1625   {
1626     result = true;
1627   }
1628
1629   EVEL_EXIT();
1630
1631   return result;
1632 }
1633
1634 /**************************************************************************//**
1635  * Check that a string token matches a given input string.
1636  *
1637  * @param chunk         Memory chunk containing the JSON buffer.
1638  * @param json_token    Token to check.
1639  * @param check_string  String to check it against.
1640  * @return true if the strings match, or false.
1641  *****************************************************************************/
1642 bool evel_token_equals_string(const MEMORY_CHUNK * const chunk,
1643                               const jsmntok_t * json_token,
1644                               const char * check_string)
1645 {
1646   bool result = false;
1647
1648   EVEL_ENTER();
1649
1650   const int token_length = json_token->end - json_token->start;
1651   const char * const token_string = chunk->memory + json_token->start;
1652
1653   if (token_length == (int)strlen(check_string))
1654   {
1655     result = (strncmp(token_string, check_string, token_length) == 0);
1656   }
1657
1658   EVEL_EXIT();
1659
1660   return result;
1661 }