vnfs/VESreporting_vFW5.0_DANOS/evel/evel-library/code/evel_library/evel_event_mgr.c

   1 /*************************************************************************//**
   2  *
   3  * Copyright © 2017 AT&T Intellectual Property. All rights reserved.
   4  *
   5  * Unless otherwise specified, all software contained herein is
   6  * Licensed under the Apache License, Version 2.0 (the "License");
   7  * you may not use this file except in compliance with the License.
   8  * You may obtain a copy of the License at
   9  *        http://www.apache.org/licenses/LICENSE-2.0
  10  *
  11  * Unless required by applicable law or agreed to in writing, software
  12  * distributed under the License is distributed on an "AS IS" BASIS,
  13  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14  * See the License for the specific language governing permissions and
  15  * limitations under the License.
  16  * ECOMP is a trademark and service mark of AT&T Intellectual Property.
  17  ****************************************************************************/
  18
  19 /**************************************************************************//**
  20  * @file
  21  * Event Manager
  22  *
  23  * Simple event manager that is responsible for taking events (Heartbeats,
  24  * Faults and Measurements) from the ring-buffer and posting them to the API.
  25  *
  26  ****************************************************************************/
  27
  28 #include <string.h>
  29 #include <assert.h>
  30 #include <stdlib.h>
  31 #include <pthread.h>
  32
  33 #include <curl/curl.h>
  34
  35 #include "evel.h"
  36 #include "evel_internal.h"
  37 #include "ring_buffer.h"
  38 #include "evel_throttle.h"
  39
  40 /**************************************************************************//**
  41  * How long we're prepared to wait for the API service to respond in
  42  * seconds.
  43  *****************************************************************************/
  44 static const int EVEL_API_TIMEOUT = 5;
  45
  46 /**************************************************************************//**
  47  * Wait time if both the collectors are not responding
  48  *****************************************************************************/
  49 static const int EVEL_COLLECTOR_RECONNECTION_WAIT_TIME = 120;
  50
  51 /*****************************************************************************/
  52 /* Prototypes of locally scoped functions.                                   */
  53 /*****************************************************************************/
  54 static size_t read_callback(void *ptr, size_t size, size_t nmemb, void *userp);
  55 static void * event_handler(void *arg);
  56 static bool evel_handle_response_tokens(const MEMORY_CHUNK * const chunk,
  57                                         const jsmntok_t * const json_tokens,
  58                                         const int num_tokens,
  59                                         MEMORY_CHUNK * const post);
  60 static bool evel_tokens_match_command_list(const MEMORY_CHUNK * const chunk,
  61                                            const jsmntok_t * const json_token,
  62                                            const int num_tokens);
  63 static bool evel_token_equals_string(const MEMORY_CHUNK * const chunk,
  64                                      const jsmntok_t * const json_token,
  65                                      const char * check_string);
  66 static EVEL_ERR_CODES evel_setup_curl();
  67 static EVEL_ERR_CODES evel_send_to_another_collector(const EVEL_EVENT_DOMAINS evel_domain, char * json_body, size_t json_size);
  68
  69 /**************************************************************************//**
  70  * Buffers for error strings from libcurl.
  71  *****************************************************************************/
  72 static char curl_err_string[CURL_ERROR_SIZE] = "<NULL>";
  73
  74 /**************************************************************************//**
  75  * Handle for the API into libcurl.
  76  *****************************************************************************/
  77 static CURL * curl_handle = NULL;
  78 int curr_global_handles = 0;
  79
  80 /**************************************************************************//**
  81  * Special headers that we send.
  82  *****************************************************************************/
  83 static struct curl_slist * hdr_chunk = NULL;
  84
  85 /**************************************************************************//**
  86  * Message queue for sending events to the API.
  87  *****************************************************************************/
  88 static ring_buffer event_buffer;
  89
  90 /**************************************************************************//**
  91  * Single pending priority post, which can be generated as a result of a
  92  * response to an event.  Currently only used to respond to a commandList.
  93  *****************************************************************************/
  94 static MEMORY_CHUNK priority_post;
  95
  96 /**************************************************************************//**
  97  * The thread which is responsible for handling events off of the ring-buffer
  98  * and posting them to the Event Handler API.
  99  *****************************************************************************/
 100 static pthread_t evt_handler_thread;
 101
 102 /**************************************************************************//**
 103  * Variable to convey to the event handler thread what the foreground wants it
 104  * to do.
 105  *****************************************************************************/
 106 static EVT_HANDLER_STATE evt_handler_state = EVT_HANDLER_UNINITIALIZED;
 107
 108 /**************************************************************************//**
 109  * The configured API URL for event and throttling.
 110  *****************************************************************************/
 111 static char * evel_event_api_url;
 112 static char * evel_throt_api_url;
 113 static char * evel_batch_api_url;
 114
 115 static char * evel_bevent_api_url;
 116 static char * evel_bthrot_api_url;
 117 static char * evel_bbatch_api_url;
 118
 119 /**************************************************************************//**
 120  * Storage for other CURL related parameters
 121  *****************************************************************************/
 122 int evel_secure = -1;
 123 int evel_verbosity = -1;
 124
 125 long evel_verify_peer = 0;
 126 long evel_verify_host = 0;
 127
 128 static char * evel_source_ip = NULL;
 129 static char * evel_source_ip_bakup = NULL;
 130 static char * evel_cert_file_path = NULL;
 131 static char * evel_key_file_path = NULL;
 132 static char * evel_ca_info = NULL;
 133 static char * evel_ca_file_path = NULL;
 134 static char * evel_username = NULL;
 135 static char * evel_password = NULL;
 136 static char * evel_username2 = NULL;
 137 static char * evel_password2 = NULL;
 138
 139 static long http_response_code = 0;
 140 static int evel_collector_id = 1;
 141 /**************************************************************************//**
 142  * Initialize the event handler.
 143  *
 144  * Primarily responsible for getting CURL ready for use.
 145  *
 146  * @param[in] event_api_url
 147  *                      The URL where the Vendor Event Listener API is expected
 148  *                      to be.
 149  * @param[in] throt_api_url
 150  *                      The URL where the Throttling API is expected to be.
 151  * @param[in] source_ip  Source IP of VES Agent
 152  * @param[in] ring_buf_size     Initial size of ring buffer
 153  * @param[in] secure     Whether Using http or https
 154  * @param[in] cert_file_path  Path to Client Certificate file
 155  * @param[in] key_file_path   Path to Client key file
 156  * @param[in] ca_info         Path to CA info file
 157  * @param[in] ca_file_path    Path to CA file
 158  * @param[in] verify_peer     Using peer verification or not 0 or 1
 159  * @param[in] verify_host     Using host verification or not 0 or 1
 160  * @param[in] username  The username for the Basic Authentication of requests.
 161  * @param[in] password  The password for the Basic Authentication of requests.
 162  * @param     verbosity 0 for normal operation, positive values for chattier
 163  *                        logs.
 164  *****************************************************************************/
 165 EVEL_ERR_CODES event_handler_initialize(const char * const event_api_url,
 166                                         const char * const bakup_api_url,
 167                                         const char * const throt_api_url,
 168                                         const char * const source_ip,
 169                                         const char * const source_ip_bakup,
 170                                         int ring_buf_size,
 171                                         int secure,
 172                                         const char * const cert_file_path,
 173                                         const char * const key_file_path,
 174                                         const char * const ca_info,
 175                                         const char * const ca_file_path,
 176                                         long verify_peer,
 177                                         long verify_host,
 178                                         const char * const username,
 179                                         const char * const password,
 180                                         const char * const username2,
 181                                         const char * const password2,
 182                                         int verbosity)
 183 {
 184   int rc = EVEL_SUCCESS;
 185   char batch_api_url[EVEL_MAX_URL_LEN + 1] = {0};
 186
 187   EVEL_ENTER();
 188
 189   /***************************************************************************/
 190   /* Check assumptions.                                                      */
 191   /***************************************************************************/
 192   assert(event_api_url != NULL);
 193   assert(throt_api_url != NULL);
 194   assert(username != NULL);
 195   assert(password != NULL);
 196   if( bakup_api_url != NULL )
 197   {
 198     assert(username2 != NULL);
 199     assert(password2 != NULL);
 200   }
 201
 202   /***************************************************************************/
 203   /* Store the API URLs.                                                     */
 204   /***************************************************************************/
 205   evel_event_api_url = strdup(event_api_url);
 206   assert(evel_event_api_url != NULL);
 207   sprintf(batch_api_url,"%s/eventBatch",event_api_url);
 208   evel_batch_api_url = strdup(batch_api_url);
 209   assert(evel_batch_api_url != NULL);
 210   evel_throt_api_url = strdup(throt_api_url);
 211   assert(evel_throt_api_url != NULL);
 212
 213   curr_global_handles = 1;
 214
 215   if( bakup_api_url != NULL )
 216   {
 217     evel_bevent_api_url = strdup(bakup_api_url);
 218     assert(evel_bevent_api_url != NULL);
 219     sprintf(batch_api_url,"%s/eventBatch",bakup_api_url);
 220     evel_bbatch_api_url = strdup(batch_api_url);
 221     assert(evel_bbatch_api_url != NULL);
 222     evel_bthrot_api_url = strdup(throt_api_url);
 223     assert(evel_bthrot_api_url != NULL);
 224     curr_global_handles = 2;
 225   }
 226
 227   /***************************************************************************/
 228   /* Store other parameters                                                  */
 229   /***************************************************************************/
 230   evel_secure = secure;
 231   evel_verbosity = verbosity;
 232
 233   evel_verify_peer = verify_peer;
 234   evel_verify_host = verify_host;
 235
 236   evel_source_ip = NULL;
 237   if (source_ip != NULL)
 238   {
 239     evel_source_ip = strdup(source_ip);
 240     assert(evel_source_ip != NULL);
 241   }
 242
 243   evel_source_ip_bakup = NULL;
 244   if (source_ip_bakup != NULL)
 245   {
 246     evel_source_ip_bakup = strdup(source_ip_bakup);
 247     assert(evel_source_ip_bakup != NULL);
 248   }
 249
 250   evel_cert_file_path = NULL;
 251   if (cert_file_path != NULL)
 252   {
 253     evel_cert_file_path = strdup(cert_file_path);
 254     assert(evel_cert_file_path != NULL);
 255   }
 256
 257   evel_key_file_path = NULL;
 258   if (key_file_path != NULL)
 259   {
 260     evel_key_file_path = strdup(key_file_path);
 261     assert(evel_key_file_path != NULL);
 262   }
 263
 264   evel_ca_info = NULL;
 265   if (ca_info != NULL)
 266   {
 267     evel_ca_info = strdup(ca_info);
 268     assert(evel_ca_info != NULL);
 269   }
 270
 271   evel_ca_file_path = NULL;
 272   if (ca_file_path != NULL)
 273   {
 274     evel_ca_file_path = strdup(ca_file_path);
 275     assert(evel_ca_file_path != NULL);
 276   }
 277
 278   evel_username = NULL;
 279   if (username != NULL)
 280   {
 281     evel_username = strdup(username);
 282     assert(evel_username != NULL);
 283   }
 284
 285   evel_password = NULL;
 286   if (password != NULL)
 287   {
 288     evel_password = strdup(password);
 289     assert(evel_password != NULL);
 290   }
 291
 292   evel_username2 = NULL;
 293   if (username2 != NULL)
 294   {
 295     evel_username2 = strdup(username2);
 296     assert(evel_username2 != NULL);
 297   }
 298
 299   evel_password2 = NULL;
 300   if (password2 != NULL)
 301   {
 302     evel_password2 = strdup(password2);
 303     assert(evel_password2 != NULL);
 304   }
 305
 306   curl_version_info_data *d = curl_version_info(CURLVERSION_NOW);
 307   /* compare with the 24 bit hex number in 8 bit fields */
 308   if(d->version_num >= 0x072100) {
 309      /* this is libcurl 7.33.0 or later */
 310      EVEL_INFO("7.33 or later Curl version %x.",d->version_num);
 311   }
 312   else {
 313      EVEL_INFO("Old Curl version.");
 314   }
 315
 316   /***************************************************************************/
 317   /* Initialize a message ring-buffer to be used between the foreground and  */
 318   /* the thread which sends the messages.  This can't fail.                  */
 319   /***************************************************************************/
 320   if( ring_buf_size < EVEL_EVENT_BUFFER_DEPTH )
 321   {
 322     log_error_state("Warning: Failed to initialize Ring buffer size to %d. ",
 323                     ring_buf_size);
 324     goto exit_label;
 325   }
 326   ring_buffer_initialize(&event_buffer, EVEL_EVENT_BUFFER_DEPTH);
 327
 328 exit_label:
 329
 330   EVEL_EXIT();
 331
 332   return rc;
 333
 334 }
 335 /**************************************************************************//**
 336  * Setup the curl connection to collector
 337  *
 338  * Primarily responsible for getting CURL ready to send message. Also it would
 339  * be used to swithch over to other collector
 340  *****************************************************************************/
 341 static EVEL_ERR_CODES evel_setup_curl()
 342 {
 343   int rc = EVEL_SUCCESS;
 344   CURLcode curl_rc = CURLE_OK;
 345   char local_address[64];
 346   char * api_url = NULL;
 347   char * username = NULL;
 348   char * password = NULL;
 349   char * source_ip = NULL;
 350
 351   EVEL_ENTER();
 352
 353   if (evel_collector_id > 2)
 354   {
 355     rc = EVEL_CURL_LIBRARY_FAIL;
 356     log_error_state("Wrong evel_collector- value > 2");
 357     goto exit_label;
 358   }
 359
 360   /***************************************************************************/
 361   /* Initialize the local variable with proper global variables that are     */
 362   /* required to setup the connection                                        */
 363   /***************************************************************************/
 364   if (evel_collector_id == 1)
 365   {
 366      api_url = evel_event_api_url;
 367      source_ip = evel_source_ip;
 368      username = evel_username;
 369      password = evel_password;
 370   }
 371   else if (evel_collector_id == 2)
 372   {
 373      api_url = evel_bevent_api_url;
 374      source_ip = evel_source_ip_bakup;
 375      username = evel_username2;
 376      password = evel_password2;
 377   }
 378   /***************************************************************************/
 379   /* Clean-up the cURL library.                                              */
 380   /***************************************************************************/
 381   if (curl_handle != NULL)
 382   {
 383     curl_easy_cleanup(curl_handle);
 384     curl_handle = NULL;
 385   }
 386   if (hdr_chunk != NULL)
 387   {
 388     curl_slist_free_all(hdr_chunk);
 389     hdr_chunk = NULL;
 390   }
 391
 392   /***************************************************************************/
 393   /* Start the CURL library. Note that this initialization is not threadsafe */
 394   /* which imposes a constraint that the EVEL library is initialized before  */
 395   /* any threads are started.                                                */
 396   /***************************************************************************/
 397   curl_rc = curl_global_init(CURL_GLOBAL_SSL);
 398   if (curl_rc != CURLE_OK)
 399   {
 400     rc = EVEL_CURL_LIBRARY_FAIL;
 401     log_error_state("Failed to initialize libCURL. Error code=%d", curl_rc);
 402     goto exit_label;
 403   }
 404
 405   /***************************************************************************/
 406   /* Get a curl handle which we'll use for all of our output.                */
 407   /***************************************************************************/
 408   curl_handle = curl_easy_init();
 409   if (curl_handle == NULL)
 410   {
 411     rc = EVEL_CURL_LIBRARY_FAIL;
 412     log_error_state("Failed to get libCURL handle");
 413     goto exit_label;
 414   }
 415
 416   /***************************************************************************/
 417   /* Prime the library to give friendly error codes.                         */
 418   /***************************************************************************/
 419   curl_rc = curl_easy_setopt(curl_handle,
 420                              CURLOPT_ERRORBUFFER,
 421                              curl_err_string);
 422   if (curl_rc != CURLE_OK)
 423   {
 424     rc = EVEL_CURL_LIBRARY_FAIL;
 425     log_error_state("Failed to initialize libCURL to provide friendly errors. "
 426                     "Error code=%d", curl_rc);
 427     goto exit_label;
 428   }
 429
 430   /***************************************************************************/
 431   /* If running in verbose mode generate more output.                        */
 432   /***************************************************************************/
 433   if (evel_verbosity > 0)
 434   {
 435     curl_rc = curl_easy_setopt(curl_handle, CURLOPT_VERBOSE, 1L);
 436     if (curl_rc != CURLE_OK)
 437     {
 438       rc = EVEL_CURL_LIBRARY_FAIL;
 439       log_error_state("Failed to initialize libCURL to be verbose. "
 440                       "Error code=%d", curl_rc);
 441       goto exit_label;
 442     }
 443   }
 444
 445   /***************************************************************************/
 446   /* Set the URL for the API.                                                */
 447   /***************************************************************************/
 448   curl_rc = curl_easy_setopt(curl_handle, CURLOPT_URL, api_url);
 449   if (curl_rc != CURLE_OK)
 450   {
 451     rc = EVEL_CURL_LIBRARY_FAIL;
 452     log_error_state("Failed to initialize libCURL with the API URL. "
 453                     "Error code=%d (%s)", curl_rc, curl_err_string);
 454     goto exit_label;
 455   }
 456   EVEL_INFO("Initializing CURL to send events to: %s", api_url);
 457
 458   /***************************************************************************/
 459   /* send all data to this function.                                         */
 460   /***************************************************************************/
 461   curl_rc = curl_easy_setopt(curl_handle,
 462                              CURLOPT_WRITEFUNCTION,
 463                              evel_write_callback);
 464   if (curl_rc != CURLE_OK)
 465   {
 466     rc = EVEL_CURL_LIBRARY_FAIL;
 467     log_error_state("Failed to initialize libCURL with the write callback. "
 468                     "Error code=%d (%s)", curl_rc, curl_err_string);
 469     goto exit_label;
 470   }
 471
 472   /***************************************************************************/
 473   /* configure local ip address if provided */
 474   /* Default ip if NULL */
 475   /***************************************************************************/
 476   if( source_ip != NULL )
 477   {
 478     snprintf(local_address,sizeof(local_address),source_ip);
 479     if( local_address[0] != '\0' )
 480     {
 481       curl_rc = curl_easy_setopt(curl_handle,
 482                              CURLOPT_INTERFACE,
 483                              local_address);
 484       if (curl_rc != CURLE_OK)
 485       {
 486         rc = EVEL_CURL_LIBRARY_FAIL;
 487         log_error_state("Failed to initialize libCURL with the local address. "
 488                     "Error code=%d (%s)", curl_rc, curl_err_string);
 489         goto exit_label;
 490       }
 491     }
 492   }
 493
 494   /***************************************************************************/
 495   /* configure SSL options for HTTPS transfers */
 496   /***************************************************************************/
 497   if( evel_secure )
 498   {
 499     if( evel_cert_file_path != NULL )
 500     {
 501       curl_rc = curl_easy_setopt(curl_handle,
 502                              CURLOPT_SSLCERT,
 503                              evel_cert_file_path);
 504       if (curl_rc != CURLE_OK)
 505       {
 506         rc = EVEL_CURL_LIBRARY_FAIL;
 507         log_error_state("Failed to initialize libCURL with the client cert. "
 508                     "Error code=%d (%s)", curl_rc, curl_err_string);
 509         goto exit_label;
 510       }
 511     }
 512
 513     if( evel_key_file_path != NULL )
 514     {
 515       curl_rc = curl_easy_setopt(curl_handle,
 516                              CURLOPT_SSLKEY,
 517                              evel_key_file_path);
 518       if (curl_rc != CURLE_OK)
 519       {
 520         rc = EVEL_CURL_LIBRARY_FAIL;
 521         log_error_state("Failed to initialize libCURL with the client key. "
 522                     "Error code=%d (%s)", curl_rc, curl_err_string);
 523         goto exit_label;
 524       }
 525     }
 526
 527     if( evel_ca_info != NULL )
 528     {
 529       curl_rc = curl_easy_setopt(curl_handle,
 530                              CURLOPT_CAINFO,
 531                              evel_ca_info);
 532       if (curl_rc != CURLE_OK)
 533       {
 534         rc = EVEL_CURL_LIBRARY_FAIL;
 535         log_error_state("Failed to initialize libCURL with the CA cert file. "
 536                     "Error code=%d (%s)", curl_rc, curl_err_string);
 537         goto exit_label;
 538       }
 539     }
 540
 541     if( evel_ca_file_path != NULL )
 542     {
 543       curl_rc = curl_easy_setopt(curl_handle,
 544                              CURLOPT_CAPATH,
 545                              evel_ca_file_path);
 546       if (curl_rc != CURLE_OK)
 547       {
 548         rc = EVEL_CURL_LIBRARY_FAIL;
 549         log_error_state("Failed to initialize libCURL with the CA cert path. "
 550                     "Error code=%d (%s)", curl_rc, curl_err_string);
 551         goto exit_label;
 552       }
 553     }
 554
 555       curl_rc = curl_easy_setopt(curl_handle,
 556                              CURLOPT_SSL_VERIFYPEER,
 557                              evel_verify_peer);
 558       if (curl_rc != CURLE_OK)
 559       {
 560         rc = EVEL_CURL_LIBRARY_FAIL;
 561         log_error_state("Failed to initialize libCURL with SSL Server verification. "
 562                     "Error code=%d (%s)", curl_rc, curl_err_string);
 563         goto exit_label;
 564       }
 565       curl_rc = curl_easy_setopt(curl_handle,
 566                              CURLOPT_SSL_VERIFYHOST,
 567                              evel_verify_host);
 568       if (curl_rc != CURLE_OK)
 569       {
 570         rc = EVEL_CURL_LIBRARY_FAIL;
 571         log_error_state("Failed to initialize libCURL with Client host verification. "
 572                     "Error code=%d (%s)", curl_rc, curl_err_string);
 573         goto exit_label;
 574       }
 575
 576   }
 577
 578
 579
 580   /***************************************************************************/
 581   /* some servers don't like requests that are made without a user-agent     */
 582   /* field, so we provide one.                                               */
 583   /***************************************************************************/
 584   curl_rc = curl_easy_setopt(curl_handle,
 585                              CURLOPT_USERAGENT,
 586                              "libcurl-agent/1.0");
 587   if (curl_rc != CURLE_OK)
 588   {
 589     rc = EVEL_CURL_LIBRARY_FAIL;
 590     log_error_state("Failed to initialize libCURL to upload. "
 591                     "Error code=%d (%s)", curl_rc, curl_err_string);
 592     goto exit_label;
 593   }
 594
 595   /***************************************************************************/
 596   /* Specify that we are going to POST data.                                 */
 597   /***************************************************************************/
 598   curl_rc = curl_easy_setopt(curl_handle, CURLOPT_POST, 1L);
 599   if (curl_rc != CURLE_OK)
 600   {
 601     rc = EVEL_CURL_LIBRARY_FAIL;
 602     log_error_state("Failed to initialize libCURL to upload. "
 603                     "Error code=%d (%s)", curl_rc, curl_err_string);
 604     goto exit_label;
 605   }
 606
 607   /***************************************************************************/
 608   /* we want to use our own read function.                                   */
 609   /***************************************************************************/
 610   curl_rc = curl_easy_setopt(curl_handle, CURLOPT_READFUNCTION, read_callback);
 611   if (curl_rc != CURLE_OK)
 612   {
 613     rc = EVEL_CURL_LIBRARY_FAIL;
 614     log_error_state("Failed to initialize libCURL to upload using read "
 615                     "function. Error code=%d (%s)", curl_rc, curl_err_string);
 616     goto exit_label;
 617   }
 618
 619   /***************************************************************************/
 620   /* All of our events are JSON encoded.  We also suppress the               */
 621   /* Expect: 100-continue   header that we would otherwise get since it      */
 622   /* confuses some servers.                                                  */
 623   /*                                                                         */
 624   /* @TODO: do AT&T want this behavior?                                      */
 625   /***************************************************************************/
 626   hdr_chunk = curl_slist_append(hdr_chunk, "Content-type: application/json");
 627   hdr_chunk = curl_slist_append(hdr_chunk, "Expect:");
 628
 629   /***************************************************************************/
 630   /* set our custom set of headers.                                         */
 631   /***************************************************************************/
 632   curl_rc = curl_easy_setopt(curl_handle, CURLOPT_HTTPHEADER, hdr_chunk);
 633   if (curl_rc != CURLE_OK)
 634   {
 635     rc = EVEL_CURL_LIBRARY_FAIL;
 636     log_error_state("Failed to initialize libCURL to use custom headers. "
 637                     "Error code=%d (%s)", curl_rc, curl_err_string);
 638     goto exit_label;
 639   }
 640
 641   /***************************************************************************/
 642   /* Set the timeout for the operation.                                      */
 643   /***************************************************************************/
 644   curl_rc = curl_easy_setopt(curl_handle,
 645                              CURLOPT_TIMEOUT,
 646                              EVEL_API_TIMEOUT);
 647   if (curl_rc != CURLE_OK)
 648   {
 649     rc = EVEL_CURL_LIBRARY_FAIL;
 650     log_error_state("Failed to initialize libCURL for API timeout. "
 651                     "Error code=%d (%s)", curl_rc, curl_err_string);
 652     goto exit_label;
 653   }
 654
 655   /***************************************************************************/
 656   /* Set that we want Basic authentication with username:password Base-64    */
 657   /* encoded for the operation.                                              */
 658   /***************************************************************************/
 659   curl_rc = curl_easy_setopt(curl_handle, CURLOPT_HTTPAUTH, CURLAUTH_BASIC);
 660   if (curl_rc != CURLE_OK)
 661   {
 662     rc = EVEL_CURL_LIBRARY_FAIL;
 663     log_error_state("Failed to initialize libCURL for Basic Authentication. "
 664                     "Error code=%d (%s)", curl_rc, curl_err_string);
 665     goto exit_label;
 666   }
 667   curl_rc = curl_easy_setopt(curl_handle, CURLOPT_USERNAME, username);
 668   if (curl_rc != CURLE_OK)
 669   {
 670     rc = EVEL_CURL_LIBRARY_FAIL;
 671     log_error_state("Failed to initialize libCURL with username. "
 672                     "Error code=%d (%s)", curl_rc, curl_err_string);
 673     goto exit_label;
 674   }
 675   curl_rc = curl_easy_setopt(curl_handle, CURLOPT_PASSWORD, password);
 676   if (curl_rc != CURLE_OK)
 677   {
 678     rc = EVEL_CURL_LIBRARY_FAIL;
 679     log_error_state("Failed to initialize libCURL with password. "
 680                     "Error code=%d (%s)", curl_rc, curl_err_string);
 681     goto exit_label;
 682   }
 683
 684   /***************************************************************************/
 685   /* Initialize the priority post buffer to empty.                           */
 686   /***************************************************************************/
 687   priority_post.memory = NULL;
 688
 689 exit_label:
 690   EVEL_EXIT();
 691
 692   return(rc);
 693 }
 694
 695 /**************************************************************************//**
 696  * Run the event handler.
 697  *
 698  * Spawns the thread responsible for handling events and sending them to the
 699  * API.
 700  *
 701  *  @return Status code.
 702  *  @retval ::EVEL_SUCCESS if everything OK.
 703  *  @retval One of ::EVEL_ERR_CODES if there was a problem.
 704  *****************************************************************************/
 705 EVEL_ERR_CODES event_handler_run()
 706 {
 707   EVEL_ERR_CODES rc = EVEL_SUCCESS;
 708   int pthread_rc = 0;
 709
 710   EVEL_ENTER();
 711
 712   /***************************************************************************/
 713   /* Start the event handler thread.                                         */
 714   /***************************************************************************/
 715   evt_handler_state = EVT_HANDLER_INACTIVE;
 716   pthread_rc = pthread_create(&evt_handler_thread, NULL, event_handler, NULL);
 717   if (pthread_rc != 0)
 718   {
 719     rc = EVEL_PTHREAD_LIBRARY_FAIL;
 720     log_error_state("Failed to start event handler thread. "
 721                     "Error code=%d", pthread_rc);
 722   }
 723
 724   EVEL_EXIT()
 725   return rc;
 726 }
 727
 728 /**************************************************************************//**
 729  * Terminate the event handler.
 730  *
 731  * Shuts down the event handler thread in as clean a way as possible. Sets the
 732  * global exit flag and then signals the thread to interrupt it since it's
 733  * most likely waiting on the ring-buffer.
 734  *
 735  * Having achieved an orderly shutdown of the event handler thread, clean up
 736  * the cURL library's resources cleanly.
 737  *
 738  *  @return Status code.
 739  *  @retval ::EVEL_SUCCESS if everything OK.
 740  *  @retval One of ::EVEL_ERR_CODES if there was a problem.
 741  *****************************************************************************/
 742 EVEL_ERR_CODES event_handler_terminate()
 743 {
 744   EVEL_ERR_CODES rc = EVEL_SUCCESS;
 745
 746   EVEL_ENTER();
 747   EVENT_INTERNAL *event = NULL;
 748
 749   /***************************************************************************/
 750   /* Make sure that we were initialized before trying to terminate the       */
 751   /* event handler thread.                                                   */
 752   /***************************************************************************/
 753   if (evt_handler_state != EVT_HANDLER_UNINITIALIZED)
 754   {
 755     /*************************************************************************/
 756     /* Make sure that the event handler knows it's time to die.              */
 757     /*************************************************************************/
 758     event = evel_new_internal_event(EVT_CMD_TERMINATE,"EVELinternal","EVELid");
 759     if (event == NULL)
 760     {
 761       /***********************************************************************/
 762       /* We failed to get an event, but we don't bail out - we will just     */
 763       /* clean up what we can and continue on our way, since we're exiting   */
 764       /* anyway.                                                             */
 765       /***********************************************************************/
 766       EVEL_ERROR("Failed to get internal event - perform dirty exit instead!");
 767     }
 768     else
 769     {
 770       /***********************************************************************/
 771       /* Post the event then wait for the Event Handler to exit.  Set the    */
 772       /* global command, too, in case the ring-buffer is full.               */
 773       /***********************************************************************/
 774       EVEL_DEBUG("Sending event to Event Hander to request it to exit.");
 775       evt_handler_state = EVT_HANDLER_REQUEST_TERMINATE;
 776       evel_post_event((EVENT_HEADER *) event);
 777       pthread_join(evt_handler_thread, NULL);
 778       EVEL_DEBUG("Event Handler thread has exited.");
 779     }
 780   }
 781   else
 782   {
 783     EVEL_DEBUG("Event handler was not initialized, so no need to kill it");
 784   }
 785
 786   /***************************************************************************/
 787   /* Clean-up the cURL library.                                              */
 788   /***************************************************************************/
 789   if (curl_handle != NULL)
 790   {
 791     curl_easy_cleanup(curl_handle);
 792     curl_handle = NULL;
 793   }
 794   if (hdr_chunk != NULL)
 795   {
 796     curl_slist_free_all(hdr_chunk);
 797     hdr_chunk = NULL;
 798   }
 799
 800   /***************************************************************************/
 801   /* Free off the stored API URL strings.                                    */
 802   /***************************************************************************/
 803   if (evel_event_api_url != NULL)
 804   {
 805     free(evel_event_api_url);
 806     evel_event_api_url = NULL;
 807   }
 808   if (evel_batch_api_url != NULL)
 809   {
 810     free(evel_batch_api_url);
 811     evel_batch_api_url = NULL;
 812   }
 813   if (evel_throt_api_url != NULL)
 814   {
 815     free(evel_throt_api_url);
 816     evel_throt_api_url = NULL;
 817   }
 818
 819   EVEL_EXIT();
 820   return rc;
 821 }
 822
 823 /**************************************************************************//**
 824  * Post an event.
 825  *
 826  * @note  So far as the caller is concerned, successfully posting the event
 827  * relinquishes all responsibility for the event - the library will take care
 828  * of freeing the event in due course.
 829
 830  * @param event   The event to be posted.
 831  *
 832  * @returns Status code
 833  * @retval  EVEL_SUCCESS On success
 834  * @retval  "One of ::EVEL_ERR_CODES" On failure.
 835  *****************************************************************************/
 836 EVEL_ERR_CODES evel_post_event(EVENT_HEADER * event)
 837 {
 838   int rc = EVEL_SUCCESS;
 839
 840   EVEL_ENTER();
 841
 842   /***************************************************************************/
 843   /* Check preconditions.                                                    */
 844   /***************************************************************************/
 845   assert(event != NULL);
 846
 847   /***************************************************************************/
 848   /* We need to make sure that we are either initializing or running         */
 849   /* normally before writing the event into the buffer so that we can        */
 850   /* guarantee that the ring-buffer empties  properly on exit.               */
 851   /***************************************************************************/
 852   if ((evt_handler_state == EVT_HANDLER_ACTIVE) ||
 853       (evt_handler_state == EVT_HANDLER_INACTIVE) ||
 854       (evt_handler_state == EVT_HANDLER_REQUEST_TERMINATE))
 855   {
 856     if (ring_buffer_write(&event_buffer, event) == 0)
 857     {
 858       log_error_state("Failed to write event to buffer - event dropped!");
 859       rc = EVEL_EVENT_BUFFER_FULL;
 860       evel_free_event(event);
 861     }
 862   }
 863   else
 864   {
 865     /*************************************************************************/
 866     /* System is not in active operation, so reject the event.               */
 867     /*************************************************************************/
 868     log_error_state("Event Handler system not active - event dropped!");
 869     rc = EVEL_EVENT_HANDLER_INACTIVE;
 870     evel_free_event(event);
 871   }
 872
 873   EVEL_EXIT();
 874   return (rc);
 875 }
 876
 877 /**************************************************************************//**
 878  * Post an event to the Vendor Event Listener API.
 879  *
 880  * @returns Status code
 881  * @retval  EVEL_SUCCESS On success
 882  * @retval  "One of ::EVEL_ERR_CODES" On failure.
 883  *****************************************************************************/
 884 static EVEL_ERR_CODES evel_post_api(char * msg, size_t size)
 885 {
 886   int rc = EVEL_SUCCESS;
 887   CURLcode curl_rc = CURLE_OK;
 888   MEMORY_CHUNK rx_chunk;
 889   MEMORY_CHUNK tx_chunk;
 890
 891   EVEL_ENTER();
 892
 893   /***************************************************************************/
 894   /* Create the memory chunk to be used for the response to the post.  The   */
 895   /* will be realloced.                                                      */
 896   /***************************************************************************/
 897   rx_chunk.memory = malloc(1);
 898   assert(rx_chunk.memory != NULL);
 899   rx_chunk.size = 0;
 900
 901   /***************************************************************************/
 902   /* Create the memory chunk to be sent as the body of the post.             */
 903   /***************************************************************************/
 904   tx_chunk.memory = msg;
 905   tx_chunk.size = size;
 906   EVEL_DEBUG("Sending chunk of size %d", tx_chunk.size);
 907
 908   /***************************************************************************/
 909   /* Point to the data to be received.                                       */
 910   /***************************************************************************/
 911   curl_rc = curl_easy_setopt(curl_handle, CURLOPT_WRITEDATA, &rx_chunk);
 912   if (curl_rc != CURLE_OK)
 913   {
 914     rc = EVEL_CURL_LIBRARY_FAIL;
 915     log_error_state("Failed to initialize libCURL to upload. "
 916                     "Error code=%d (%s)", curl_rc, curl_err_string);
 917     goto exit_label;
 918   }
 919   EVEL_DEBUG("Initialized data to receive");
 920
 921   /***************************************************************************/
 922   /* Pointer to pass to our read function                                    */
 923   /***************************************************************************/
 924   curl_rc = curl_easy_setopt(curl_handle, CURLOPT_READDATA, &tx_chunk);
 925   if (curl_rc != CURLE_OK)
 926   {
 927     rc = EVEL_CURL_LIBRARY_FAIL;
 928     log_error_state("Failed to set upload data for libCURL to upload. "
 929                     "Error code=%d (%s)", curl_rc, curl_err_string);
 930     goto exit_label;
 931   }
 932   EVEL_DEBUG("Initialized data to send");
 933
 934   /***************************************************************************/
 935   /* Size of the data to transmit.                                           */
 936   /***************************************************************************/
 937   curl_rc = curl_easy_setopt(curl_handle,
 938                              CURLOPT_POSTFIELDSIZE,
 939                              tx_chunk.size);
 940   if (curl_rc != CURLE_OK)
 941   {
 942     rc = EVEL_CURL_LIBRARY_FAIL;
 943     log_error_state("Failed to set length of upload data for libCURL to "
 944                     "upload.  Error code=%d (%s)", curl_rc, curl_err_string);
 945     goto exit_label;
 946   }
 947   EVEL_DEBUG("Initialized length of data to send");
 948
 949   /***************************************************************************/
 950   /* Now run off and do what you've been told!                               */
 951   /***************************************************************************/
 952   http_response_code = 0;
 953
 954   curl_rc = curl_easy_perform(curl_handle);
 955   if (curl_rc != CURLE_OK)
 956   {
 957     rc = EVEL_CURL_LIBRARY_FAIL;
 958     log_error_state("Failed to transfer an event to Vendor Event Listener! "
 959                     "Error code=%d (%s)", curl_rc, curl_err_string);
 960     EVEL_ERROR("Dropped event: %s", msg);
 961     goto exit_label;
 962   }
 963
 964   /***************************************************************************/
 965   /* See what response we got - any 2XX response is good.                    */
 966   /***************************************************************************/
 967   curl_easy_getinfo(curl_handle, CURLINFO_RESPONSE_CODE, &http_response_code);
 968   EVEL_DEBUG("HTTP response code: %d", http_response_code);
 969   if ((http_response_code / 100) == 2)
 970   {
 971     /*************************************************************************/
 972     /* If the server responded with data it may be interesting but not a     */
 973     /* problem.                                                              */
 974     /*************************************************************************/
 975     if ((rx_chunk.size > 0) && (rx_chunk.memory != NULL))
 976     {
 977       EVEL_DEBUG("Server returned data = %d (%s)",
 978                  rx_chunk.size,
 979                  rx_chunk.memory);
 980
 981       /***********************************************************************/
 982       /* If this is a response to priority post, then we're not interested.  */
 983       /***********************************************************************/
 984       if (priority_post.memory != NULL)
 985       {
 986         EVEL_ERROR("Ignoring priority post response");
 987       }
 988       else
 989       {
 990         evel_handle_event_response(&rx_chunk, &priority_post);
 991       }
 992     }
 993   }
 994   else
 995   {
 996     EVEL_ERROR("Unexpected HTTP response code: %d with data size %d (%s)",
 997                 http_response_code,
 998                 rx_chunk.size,
 999                 rx_chunk.size > 0 ? rx_chunk.memory : "NONE");
1000     EVEL_ERROR("Potentially dropped event: %s", msg);
1001   }
1002
1003 exit_label:
1004   free(rx_chunk.memory);
1005   EVEL_EXIT();
1006   return(rc);
1007 }
1008
1009 /**************************************************************************//**
1010  * Send event to another collector
1011  *
1012  * Identify the next collector and try sending the event to that collector
1013  ****************************************************************************/
1014 static EVEL_ERR_CODES evel_send_to_another_collector(
1015                         const EVEL_EVENT_DOMAINS evel_domain,
1016                         char * json_body,
1017                         size_t json_size)
1018 {
1019   int rc = EVEL_SUCCESS;
1020   CURLcode curl_rc;
1021
1022   EVEL_ENTER();
1023
1024   if ((evel_collector_id == 1) && (curr_global_handles == 2))
1025   {
1026      evel_collector_id =2;
1027   }
1028   else if (evel_collector_id == 2)
1029   {
1030      evel_collector_id =1;
1031   }
1032
1033   rc = evel_setup_curl();
1034
1035   if ( rc == EVEL_SUCCESS)
1036   {
1037      if (evel_collector_id == 1)
1038      {
1039         if (evel_domain == EVEL_DOMAIN_BATCH)
1040            curl_rc = curl_easy_setopt(curl_handle, CURLOPT_URL, evel_batch_api_url);
1041         else
1042            curl_rc = curl_easy_setopt(curl_handle, CURLOPT_URL, evel_event_api_url);
1043      }
1044      else if (evel_collector_id == 2)
1045      {
1046         if (evel_domain == EVEL_DOMAIN_BATCH)
1047            curl_rc = curl_easy_setopt(curl_handle, CURLOPT_URL, evel_bbatch_api_url);
1048         else
1049            curl_rc = curl_easy_setopt(curl_handle, CURLOPT_URL, evel_bevent_api_url);
1050      }
1051
1052      rc = evel_post_api(json_body, json_size);
1053   }
1054
1055   EVEL_EXIT();
1056
1057   return rc;
1058 }
1059
1060 /**************************************************************************//**
1061  * Callback function to provide data to send.
1062  *
1063  * Copy data into the supplied buffer, read_callback::ptr, checking size
1064  * limits.
1065  *
1066  * @returns   Number of bytes placed into read_callback::ptr. 0 for EOF.
1067  *****************************************************************************/
1068 static size_t read_callback(void *ptr, size_t size, size_t nmemb, void *userp)
1069 {
1070   size_t rtn = 0;
1071   size_t bytes_to_write = 0;
1072   MEMORY_CHUNK *tx_chunk = (MEMORY_CHUNK *)userp;
1073
1074   EVEL_ENTER();
1075
1076   bytes_to_write = min(size*nmemb, tx_chunk->size);
1077
1078   if (bytes_to_write > 0)
1079   {
1080     EVEL_DEBUG("Going to try to write %d bytes", bytes_to_write);
1081     strncpy((char *)ptr, tx_chunk->memory, bytes_to_write);
1082     tx_chunk->memory += bytes_to_write;
1083     tx_chunk->size -= bytes_to_write;
1084     rtn = bytes_to_write;
1085   }
1086   else
1087   {
1088     EVEL_DEBUG("Reached EOF");
1089   }
1090
1091   EVEL_EXIT();
1092   return rtn;
1093 }
1094
1095 /**************************************************************************//**
1096  * Callback function to provide returned data.
1097  *
1098  * Copy data into the supplied buffer, write_callback::ptr, checking size
1099  * limits.
1100  *
1101  * @returns   Number of bytes placed into write_callback::ptr. 0 for EOF.
1102  *****************************************************************************/
1103 size_t evel_write_callback(void *contents,
1104                              size_t size,
1105                              size_t nmemb,
1106                              void *userp)
1107 {
1108   size_t realsize = size * nmemb;
1109   MEMORY_CHUNK * rx_chunk = (MEMORY_CHUNK *)userp;
1110
1111   EVEL_ENTER();
1112
1113   EVEL_DEBUG("Called with %d chunks of %d size = %d", nmemb, size, realsize);
1114   EVEL_DEBUG("rx chunk size is %d", rx_chunk->size);
1115
1116   rx_chunk->memory = realloc(rx_chunk->memory, rx_chunk->size + realsize + 1);
1117   if(rx_chunk->memory == NULL) {
1118     /* out of memory! */
1119     printf("not enough memory (realloc returned NULL)\n");
1120     return 0;
1121   }
1122
1123   memcpy(&(rx_chunk->memory[rx_chunk->size]), contents, realsize);
1124   rx_chunk->size += realsize;
1125   rx_chunk->memory[rx_chunk->size] = 0;
1126
1127   EVEL_DEBUG("Rx data: %s", rx_chunk->memory);
1128   EVEL_DEBUG("Returning: %d", realsize);
1129
1130   EVEL_EXIT();
1131   return realsize;
1132 }
1133
1134 /**************************************************************************//**
1135  * Event Handler.
1136  *
1137  * Watch for messages coming on the internal queue and send them to the
1138  * listener.
1139  *
1140  * param[in]  arg  Argument - unused.
1141  *****************************************************************************/
1142 static void * event_handler(void * arg __attribute__ ((unused)))
1143 {
1144   int old_type = 0;
1145   EVENT_HEADER * msg = NULL;
1146   EVENT_INTERNAL * internal_msg = NULL;
1147   int json_size = 0;
1148   char json_body[EVEL_MAX_JSON_BODY];
1149   int rc = EVEL_SUCCESS;
1150   CURLcode curl_rc;
1151   int collector_down_count = 0;
1152   int switch_coll = 0;
1153
1154   EVEL_INFO("Event handler thread started");
1155
1156   /***************************************************************************/
1157   /* Set this thread to be cancellable immediately.                          */
1158   /***************************************************************************/
1159   pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, &old_type);
1160
1161   /***************************************************************************/
1162   /* Set the handler as active, defending against weird situations like      */
1163   /* immediately shutting down after initializing the library so the         */
1164   /* handler never gets started up properly.                                 */
1165   /***************************************************************************/
1166   if (evt_handler_state == EVT_HANDLER_INACTIVE)
1167   {
1168     evt_handler_state = EVT_HANDLER_ACTIVE;
1169   }
1170   else
1171   {
1172     EVEL_ERROR("Event Handler State was not INACTIVE at start-up - "
1173                "Handler will exit immediately!");
1174   }
1175   /***************************************************************************/
1176   /* Set the connection to collector                                         */
1177   /***************************************************************************/
1178   while (true)
1179   {
1180      evel_collector_id = 1;
1181      rc = evel_setup_curl();
1182
1183      if ( rc != EVEL_SUCCESS)
1184      {
1185         EVEL_ERROR("Failed to setup the first collector. Error code=%d", rc);
1186         if (curr_global_handles == 2)
1187         {
1188            EVEL_DEBUG("Switching to other collector");
1189
1190            evel_collector_id = 2;
1191
1192            rc = evel_setup_curl();
1193            if ( rc != EVEL_SUCCESS)
1194            {
1195               EVEL_ERROR("Failed to setup the connection to second collector also, Error code%d", rc);
1196               sleep(EVEL_COLLECTOR_RECONNECTION_WAIT_TIME);
1197               collector_down_count = collector_down_count + 1;
1198               EVEL_ERROR("Collectors setup issue- retry count=%d", collector_down_count);
1199            }
1200            else
1201            {
1202               collector_down_count = 0;
1203               break;
1204            }
1205         }
1206         else
1207         {
1208            sleep(EVEL_COLLECTOR_RECONNECTION_WAIT_TIME);
1209            collector_down_count = collector_down_count + 1;
1210            EVEL_ERROR("Collector setup issue-retry count=%d", collector_down_count);
1211          }
1212      }
1213      else
1214      {
1215         collector_down_count = 0;
1216         break;
1217      }
1218   }
1219
1220   while (evt_handler_state == EVT_HANDLER_ACTIVE)
1221   {
1222     /*************************************************************************/
1223     /* Wait for a message to be received.                                    */
1224     /*************************************************************************/
1225     EVEL_DEBUG("Event handler getting any messages");
1226     msg = ring_buffer_read(&event_buffer);
1227
1228     /*************************************************************************/
1229     /* Internal events get special treatment while regular events get posted */
1230     /* to the far side.                                                      */
1231     /*************************************************************************/
1232     if (msg->event_domain == EVEL_DOMAIN_BATCH )
1233     {
1234       EVEL_DEBUG("Batch event received");
1235
1236       /***********************************************************************/
1237       /* Encode the event in JSON.                                           */
1238       /***********************************************************************/
1239       json_size = evel_json_encode_batch_event(json_body, EVEL_MAX_JSON_BODY, msg);
1240
1241       /***************************************************************************/
1242       /* Set the URL for the API.                                                */
1243       /***************************************************************************/
1244       if (evel_collector_id == 1)
1245       {
1246          curl_rc = curl_easy_setopt(curl_handle, CURLOPT_URL, evel_batch_api_url);
1247       }
1248       else
1249       {
1250          curl_rc = curl_easy_setopt(curl_handle, CURLOPT_URL, evel_bbatch_api_url);
1251       }
1252
1253       if (curl_rc != CURLE_OK)
1254       {
1255         rc = EVEL_CURL_LIBRARY_FAIL;
1256         log_error_state("Failed to initialize libCURL with the Batch API URL. "
1257                     "Error code=%d (%s)", curl_rc, curl_err_string);
1258       }
1259
1260       /***********************************************************************/
1261       /* Send the JSON across the API.                                       */
1262       /***********************************************************************/
1263       EVEL_DEBUG("Sending Batch JSON of size %d is: %s", json_size, json_body);
1264       rc = evel_post_api(json_body, json_size);
1265
1266       switch_coll = 0;
1267       if ((rc == EVEL_SUCCESS) && ((http_response_code / 100) != 2))
1268       {
1269          switch_coll = 1;
1270          if (http_response_code == 400) // 400 - Bad JSON related return code
1271             switch_coll = 0;
1272       }
1273
1274       if ((rc != EVEL_SUCCESS)  || (switch_coll == 1))
1275       {
1276         EVEL_ERROR("Failed to transfer the data. Error code=%d", rc);
1277         EVEL_DEBUG("Switching to other collector if any");
1278
1279         while (true)
1280         {
1281            if (curr_global_handles == 2)
1282            {
1283               rc = evel_send_to_another_collector(msg->event_domain, json_body, json_size);
1284
1285               switch_coll = 0;
1286               if ((rc == EVEL_SUCCESS) && ((http_response_code / 100) != 2))
1287               {
1288                  switch_coll = 1;
1289                  if (http_response_code == 400) // 400 - Bad JSON related return code
1290                     switch_coll = 0;
1291               }
1292               if ((rc != EVEL_SUCCESS)  || (switch_coll == 1))
1293               {
1294                  sleep(EVEL_COLLECTOR_RECONNECTION_WAIT_TIME);
1295                  collector_down_count = collector_down_count + 1;
1296                  EVEL_ERROR("Collector setup issue-retry count=%d", collector_down_count);
1297               }
1298               else
1299               {
1300                  break;
1301               }
1302            }
1303            else
1304            {
1305               sleep(EVEL_COLLECTOR_RECONNECTION_WAIT_TIME);
1306               collector_down_count = collector_down_count + 1;
1307               EVEL_ERROR("Collector setup issue-retry count=%d", collector_down_count);
1308            }
1309
1310            rc = evel_send_to_another_collector(msg->event_domain, json_body, json_size);
1311
1312            switch_coll = 0;
1313            if ((rc == EVEL_SUCCESS) && ((http_response_code / 100) != 2))
1314            {
1315               switch_coll = 1;
1316               if (http_response_code == 400) // 400 - Bad JSON related return code
1317                  switch_coll = 0;
1318            }
1319            if ((rc != EVEL_SUCCESS)  || (switch_coll == 1))
1320            {
1321               collector_down_count = collector_down_count + 1;
1322               EVEL_ERROR("Collector setup issue-retry count=%d", collector_down_count);
1323            }
1324            else
1325            {
1326               break;
1327            }
1328         }
1329       }
1330     }
1331     else if (msg->event_domain != EVEL_DOMAIN_INTERNAL )
1332     {
1333       EVEL_DEBUG("External event received");
1334
1335       /***********************************************************************/
1336       /* Encode the event in JSON.                                           */
1337       /***********************************************************************/
1338       json_size = evel_json_encode_event(json_body, EVEL_MAX_JSON_BODY, msg);
1339
1340       /***************************************************************************/
1341       /* Set the URL for the API.                                                */
1342       /***************************************************************************/
1343       if (evel_collector_id == 1)
1344          curl_rc = curl_easy_setopt(curl_handle, CURLOPT_URL, evel_event_api_url);
1345       else
1346          curl_rc = curl_easy_setopt(curl_handle, CURLOPT_URL, evel_bevent_api_url);
1347       if (curl_rc != CURLE_OK)
1348       {
1349         rc = EVEL_CURL_LIBRARY_FAIL;
1350         log_error_state("Failed to initialize libCURL with the API URL. "
1351                     "Error code=%d (%s)", curl_rc, curl_err_string);
1352       }
1353
1354       /***********************************************************************/
1355       /* Send the JSON across the API.                                       */
1356       /***********************************************************************/
1357       EVEL_DEBUG("Sending JSON of size %d is: %s", json_size, json_body);
1358       rc = evel_post_api(json_body, json_size);
1359
1360       switch_coll = 0;
1361       if ((rc == EVEL_SUCCESS) && ((http_response_code / 100) != 2))
1362       {
1363          switch_coll = 1;
1364          if (http_response_code == 400) // 400 - Bad JSON related return code
1365             switch_coll = 0;
1366       }
1367
1368       if ((rc != EVEL_SUCCESS)  || (switch_coll == 1))
1369       {
1370         EVEL_ERROR("Failed to transfer the data. Error code=%d", rc);
1371         EVEL_DEBUG("Switching to other collector if any");
1372
1373         while (true)
1374         {
1375            if (curr_global_handles == 2)
1376            {
1377               rc = evel_send_to_another_collector(msg->event_domain, json_body, json_size);
1378
1379               switch_coll = 0;
1380               if ((rc == EVEL_SUCCESS) && ((http_response_code / 100) != 2))
1381               {
1382                  switch_coll = 1;
1383                  if (http_response_code == 400) // 400 - Bad JSON related return code
1384                     switch_coll = 0;
1385               }
1386               if ((rc != EVEL_SUCCESS)  || (switch_coll == 1))
1387               {
1388                  sleep(EVEL_COLLECTOR_RECONNECTION_WAIT_TIME);
1389                  collector_down_count = collector_down_count + 1;
1390                  EVEL_ERROR("Collector setup issue-retry count=%d", collector_down_count);
1391               }
1392               else
1393               {
1394                  break;
1395               }
1396            }
1397            else
1398            {
1399               sleep(EVEL_COLLECTOR_RECONNECTION_WAIT_TIME);
1400               collector_down_count = collector_down_count + 1;
1401               EVEL_ERROR("Collector setup issue-retry count=%d", collector_down_count);
1402            }
1403
1404            rc = evel_send_to_another_collector(msg->event_domain, json_body, json_size);
1405
1406            switch_coll = 0;
1407            if ((rc == EVEL_SUCCESS) && ((http_response_code / 100) != 2))
1408            {
1409               switch_coll = 1;
1410               if (http_response_code == 400) // 400 - Bad JSON related return code
1411                  switch_coll = 0;
1412            }
1413            if ((rc != EVEL_SUCCESS)  || (switch_coll == 1))
1414            {
1415               collector_down_count = collector_down_count + 1;
1416               EVEL_ERROR("Collector setup issue-retry count=%d", collector_down_count);
1417            }
1418            else
1419            {
1420               break;
1421            }
1422         }
1423       }
1424     }
1425     else
1426     {
1427       EVEL_DEBUG("Internal event received");
1428       internal_msg = (EVENT_INTERNAL *) msg;
1429       assert(internal_msg->command == EVT_CMD_TERMINATE);
1430       evt_handler_state = EVT_HANDLER_TERMINATING;
1431     }
1432
1433     /*************************************************************************/
1434     /* We are responsible for freeing the memory.                            */
1435     /*************************************************************************/
1436     evel_free_event(msg);
1437     msg = NULL;
1438
1439     /*************************************************************************/
1440     /* There may be a single priority post to be sent.                       */
1441     /*************************************************************************/
1442     if (priority_post.memory != NULL)
1443     {
1444       EVEL_DEBUG("Priority Post");
1445
1446       /***********************************************************************/
1447       /* Set the URL for the throttling API.                                 */
1448       /***********************************************************************/
1449       curl_rc = curl_easy_setopt(curl_handle, CURLOPT_URL, evel_throt_api_url);
1450       if (curl_rc != CURLE_OK)
1451       {
1452         /*********************************************************************/
1453         /* This is only likely to happen with CURLE_OUT_OF_MEMORY, in which  */
1454         /* case we carry on regardless.                                      */
1455         /*********************************************************************/
1456         EVEL_ERROR("Failed to set throttling URL. Error code=%d", rc);
1457       }
1458       else
1459       {
1460         rc = evel_post_api(priority_post.memory, priority_post.size);
1461         if (rc != EVEL_SUCCESS)
1462         {
1463           EVEL_ERROR("Failed to transfer priority post. Error code=%d", rc);
1464         }
1465       }
1466
1467       /***********************************************************************/
1468       /* Reinstate the URL for the event API.                                */
1469       /***********************************************************************/
1470       curl_rc = curl_easy_setopt(curl_handle, CURLOPT_URL, evel_event_api_url);
1471       if (curl_rc != CURLE_OK)
1472       {
1473         /*********************************************************************/
1474         /* This is only likely to happen with CURLE_OUT_OF_MEMORY, in which  */
1475         /* case we carry on regardless.                                      */
1476         /*********************************************************************/
1477         EVEL_ERROR("Failed to reinstate events URL. Error code=%d", rc);
1478       }
1479
1480       /***********************************************************************/
1481       /* We are responsible for freeing the memory.                          */
1482       /***********************************************************************/
1483       free(priority_post.memory);
1484       priority_post.memory = NULL;
1485     }
1486   }
1487
1488   /***************************************************************************/
1489   /* The event handler is now exiting. The ring-buffer could contain events  */
1490   /* which have not been processed, so deplete those.  Because we've been    */
1491   /* asked to exit we can be confident that the foreground will have stopped */
1492   /* sending events in so we know that this process will conclude!           */
1493   /***************************************************************************/
1494   evt_handler_state = EVT_HANDLER_TERMINATING;
1495   while (!ring_buffer_is_empty(&event_buffer))
1496   {
1497     EVEL_DEBUG("Reading event from buffer");
1498     msg = ring_buffer_read(&event_buffer);
1499     evel_free_event(msg);
1500   }
1501   evt_handler_state = EVT_HANDLER_TERMINATED;
1502   EVEL_INFO("Event handler thread stopped");
1503
1504   return (NULL);
1505 }
1506
1507 /**************************************************************************//**
1508  * Handle a JSON response from the listener, contained in a ::MEMORY_CHUNK.
1509  *
1510  * Tokenize the response, and decode any tokens found.
1511  *
1512  * @param chunk         The memory chunk containing the response.
1513  * @param post          The memory chunk in which to place any resulting POST.
1514  *****************************************************************************/
1515 void evel_handle_event_response(const MEMORY_CHUNK * const chunk,
1516                                 MEMORY_CHUNK * const post)
1517 {
1518   jsmn_parser json_parser;
1519   jsmntok_t json_tokens[EVEL_MAX_RESPONSE_TOKENS];
1520   int num_tokens = 0;
1521
1522   EVEL_ENTER();
1523
1524   /***************************************************************************/
1525   /* Check preconditions.                                                    */
1526   /***************************************************************************/
1527   assert(chunk != NULL);
1528   assert(priority_post.memory == NULL);
1529
1530   EVEL_DEBUG("Response size = %d", chunk->size);
1531   EVEL_DEBUG("Response = %s", chunk->memory);
1532
1533   /***************************************************************************/
1534   /* Initialize the parser and tokenize the response.                        */
1535   /***************************************************************************/
1536   jsmn_init(&json_parser);
1537   num_tokens = jsmn_parse(&json_parser,
1538                           chunk->memory,
1539                           chunk->size,
1540                           json_tokens,
1541                           EVEL_MAX_RESPONSE_TOKENS);
1542
1543   if (num_tokens < 0)
1544   {
1545     EVEL_ERROR("Failed to parse JSON response.  "
1546                "Error code=%d", num_tokens);
1547   }
1548   else if (num_tokens == 0)
1549   {
1550     EVEL_DEBUG("No tokens found in JSON response");
1551   }
1552   else
1553   {
1554     EVEL_DEBUG("Decode JSON response tokens");
1555     if (!evel_handle_response_tokens(chunk, json_tokens, num_tokens, post))
1556     {
1557       EVEL_ERROR("Failed to handle JSON response.");
1558     }
1559   }
1560
1561   EVEL_EXIT();
1562 }
1563
1564 /**************************************************************************//**
1565  * Handle a JSON response from the listener, as a list of tokens from JSMN.
1566  *
1567  * @param chunk         Memory chunk containing the JSON buffer.
1568  * @param json_tokens   Array of tokens to handle.
1569  * @param num_tokens    The number of tokens to handle.
1570  * @param post          The memory chunk in which to place any resulting POST.
1571  * @return true if we handled the response, false otherwise.
1572  *****************************************************************************/
1573 bool evel_handle_response_tokens(const MEMORY_CHUNK * const chunk,
1574                                  const jsmntok_t * const json_tokens,
1575                                  const int num_tokens,
1576                                  MEMORY_CHUNK * const post)
1577 {
1578   bool json_ok = true;
1579
1580   EVEL_ENTER();
1581
1582   /***************************************************************************/
1583   /* Check preconditions.                                                    */
1584   /***************************************************************************/
1585   assert(chunk != NULL);
1586   assert(json_tokens != NULL);
1587   assert(num_tokens < EVEL_MAX_RESPONSE_TOKENS);
1588
1589   /***************************************************************************/
1590   /* Peek at the tokens to decide what the response it, then call the        */
1591   /* appropriate handler to handle it.  There is only one handler at this    */
1592   /* point.                                                                  */
1593   /***************************************************************************/
1594   if (evel_tokens_match_command_list(chunk, json_tokens, num_tokens))
1595   {
1596     json_ok = evel_handle_command_list(chunk, json_tokens, num_tokens, post);
1597   }
1598
1599   EVEL_EXIT();
1600
1601   return json_ok;
1602 }
1603
1604 /**************************************************************************//**
1605  * Determine whether a list of tokens looks like a "commandList" response.
1606  *
1607  * @param chunk         Memory chunk containing the JSON buffer.
1608  * @param json_tokens   Token to check.
1609  * @param num_tokens    The number of tokens to handle.
1610  * @return true if the tokens look like a "commandList" match, or false.
1611  *****************************************************************************/
1612 bool evel_tokens_match_command_list(const MEMORY_CHUNK * const chunk,
1613                                     const jsmntok_t * const json_tokens,
1614                                     const int num_tokens)
1615 {
1616   bool result = false;
1617
1618   EVEL_ENTER();
1619
1620   /***************************************************************************/
1621   /* Make some checks on the basic layout of the commandList.                */
1622   /***************************************************************************/
1623   if ((num_tokens > 3) &&
1624       (json_tokens[0].type == JSMN_OBJECT) &&
1625       (json_tokens[1].type == JSMN_STRING) &&
1626       (json_tokens[2].type == JSMN_ARRAY) &&
1627       (evel_token_equals_string(chunk, &json_tokens[1], "commandList")))
1628   {
1629     result = true;
1630   }
1631
1632   EVEL_EXIT();
1633
1634   return result;
1635 }
1636
1637 /**************************************************************************//**
1638  * Check that a string token matches a given input string.
1639  *
1640  * @param chunk         Memory chunk containing the JSON buffer.
1641  * @param json_token    Token to check.
1642  * @param check_string  String to check it against.
1643  * @return true if the strings match, or false.
1644  *****************************************************************************/
1645 bool evel_token_equals_string(const MEMORY_CHUNK * const chunk,
1646                               const jsmntok_t * json_token,
1647                               const char * check_string)
1648 {
1649   bool result = false;
1650
1651   EVEL_ENTER();
1652
1653   const int token_length = json_token->end - json_token->start;
1654   const char * const token_string = chunk->memory + json_token->start;
1655
1656   if (token_length == (int)strlen(check_string))
1657   {
1658     result = (strncmp(token_string, check_string, token_length) == 0);
1659   }
1660
1661   EVEL_EXIT();
1662
1663   return result;
1664 }