From: Rajamohan Raj Date: Thu, 20 Jun 2019 17:40:13 +0000 (-0700) Subject: Add support for query_range in promql_api X-Git-Tag: 1.5.0~58 X-Git-Url: https://gerrit.onap.org/r/gitweb?p=demo.git;a=commitdiff_plain;h=876fd6b17a11c3ee29aa62cd1227c9ccc0a495c7 Add support for query_range in promql_api Added support for query_range in promql_api Issue-ID: ONAPARC-452 Signed-off-by: Rajamohan Raj Change-Id: If4870c8a8ea2fd4e6b23237cb8fd0f34a17fe3e0 --- diff --git a/vnfs/DAaaS/lib/promql_api/README.md b/vnfs/DAaaS/lib/promql_api/README.md index 89f996af..594e1279 100644 --- a/vnfs/DAaaS/lib/promql_api/README.md +++ b/vnfs/DAaaS/lib/promql_api/README.md @@ -1,47 +1,106 @@ ## What does this API do ? -This api as of now provides a function which takes in a list of 'QUERY_STRINGs' with respect to prometheus -and returns the corresponding result_sets in a list. -For eg: -If the QUERY_STRING is +This API has support for two PROMQL functions as of now : + +1. 'query' +2. 'query_range' + +You can directly call the above functions with required parameters, and API shall +give out a result set. + +## How to use this API ? + +### Using 'query' + +``` +1. Copy the directory 'promql_api' to your working directory. +``` ``` +2. Import the API function: query +from promql_api.prom_ql_api import query +``` + +``` +3. have a global or local variable as 'QUERY_STRING' QUERY_STRING = ['irate(collectd_cpufreq{exported_instance="otconap7",cpufreq="1"}[2m])'] ``` -The return is: +``` +4. Store the result set in a list: +list_of_result_sets = query(QUERY_STRING) +``` ``` +The return is : + [{'metric': {'cpufreq': '1', 'endpoint': 'collectd-prometheus', 'exported_instance': 'otconap7', - 'instance': '172.25.103.1:9103', + 'instance': 'ZZ.QQQ.XXX.YY:9103', 'job': 'collectd', 'namespace': 'edge1', 'pod': 'plundering-liger-collectd-wz7xg', 'service': 'collectd'}, 'value': [1559177169.415, '119727200']}] ``` - -## How to use this API ? +### Using 'query_range' ``` 1. Copy the directory 'promql_api' to your working directory. ``` ``` -2. Import the API function: query -from promql_api.prom_ql_api import query +2. Import the API function: query_range +from promql_api.prom_ql_api import query_range ``` ``` -3. have a global or local variable as 'QUERY_STRING' -QUERY_STRING = ['irate(collectd_cpufreq{exported_instance="otconap7",cpufreq="1"}[2m])'] +3. Its the calling application's responsibility to pass on the correct set of paramters. Form a map of required parameters: + +map_of_parameters = {'query': 'up', 'start': '2019-06-19T20:10:30.781Z', 'end': '2019-06-19T20:10:45.781Z', 'step': '15s'} ``` ``` 4. Store the result set in a list: -list_of_result_sets = query(QUERY_STRING) +list_of_result_sets = query_range(map_of_parameters) +``` + +``` +The return is: + +[[{'metric': {'__name__': 'up', + 'endpoint': 'cadvisor-prometheus', + 'instance': 'ZZ.QQQ.XXX.YY:8080', + 'job': 'cadvisor', + 'namespace': 'default', + 'pod': 'cp-cadvisor-69287', + 'service': 'cadvisor'}, + 'values': [[1560975030.781, '1'], [1560975045.781, '1']]}, + {'metric': {'__name__': 'up', + 'endpoint': 'collectd-prometheus', + 'instance': 'ZZ.QQQ.XXX.YY:9103', + 'job': 'collectd', + 'namespace': 'default', + 'pod': 'vcmts-stats-daemonset-lsqjk', + 'service': 'collectd'}, + 'values': [[1560975030.781, '1'], [1560975045.781, '1']]}, + {'metric': {'__name__': 'up', + 'endpoint': 'metrics', + 'instance': 'ZZ.QQQ.XXX.YY:9100', + 'job': 'cp-prometheus-node-exporter', + 'namespace': 'default', + 'pod': 'cp-prometheus-node-exporter-sbbb9', + 'service': 'cp-prometheus-node-exporter'}, + 'values': [[1560975030.781, '1'], [1560975045.781, '1']]}, + {'metric': {'__name__': 'up', + 'endpoint': 'metrics', + 'instance': 'ZZ.QQQ.XXX.YY:9100', + 'job': 'cp-prometheus-node-exporter', + 'namespace': 'default', + 'pod': 'cp-prometheus-node-exporter-mcqtp', + 'service': 'cp-prometheus-node-exporter'}, + 'values': [[1560975030.781, '1'], [1560975045.781, '1']]}]] ``` ## Troubleshooting tips @@ -58,11 +117,11 @@ list_of_result_sets = query(QUERY_STRING) 05-30-2019 08:47:53PM ::prom_ql_api.py :: load_and_validate_env_vars :: INFO :: Loading the env variables ... 05-30-2019 08:47:53PM ::prom_ql_api.py :: load_and_validate_env_vars :: ERROR :: Env var: DATA_ENDPOINT not found ! 05-30-2019 08:47:53PM ::prom_ql_api.py :: query :: INFO :: Forming the get request ... - 05-30-2019 08:47:53PM ::prom_ql_api.py :: query :: INFO :: API request::: URL: http://172.25.103.1:30090/api/v1/ query + 05-30-2019 08:47:53PM ::prom_ql_api.py :: query :: INFO :: API request::: URL: http://ZZ.QQQ.XXX.YY:30090/api/v1/ query 05-30-2019 08:47:53PM ::prom_ql_api.py :: query :: INFO :: API request::: params: {'query': 'irate(collectd_cpufreq{exported_instance="otconap7",cpufreq="1"}[2m])'} - 05-30-2019 08:47:53PM ::connectionpool.py :: _new_conn :: DEBUG :: Starting new HTTP connection (1): 172.25.103.1 05-30-2019 08:47:53PM ::connectionpool.py :: _make_request :: DEBUG :: http://172.25.103.1:30090 "GET /api/v1/ query?query=irate%28collectd_cpufreq%7Bexported_instance%3D%22otconap7%22%2Ccpufreq%3D%221%22%7D%5B2m%5D%29 HTTP/1. 1" 200 370 + 05-30-2019 08:47:53PM ::connectionpool.py :: _new_conn :: DEBUG :: Starting new HTTP connection (1): ZZ.QQQ.XXX.YY 05-30-2019 08:47:53PM ::connectionpool.py :: _make_request :: DEBUG :: http://ZZ.QQQ.XXX.YY:30090 "GET /api/v1/ query?query=irate%28collectd_cpufreq%7Bexported_instance%3D%22otconap7%22%2Ccpufreq%3D%221%22%7D%5B2m%5D%29 HTTP/1. 1" 200 370 05-30-2019 08:47:53PM ::prom_ql_api.py :: query :: INFO :: ::::::::::RESULTS::::::::::::: irate(collectd_cpufreq{exported_instance="otconap7",cpufreq="1"}[2m]) - 05-30-2019 08:47:53PM ::prom_ql_api.py :: query :: INFO :: {'metric': {'cpufreq': '1', 'endpoint': 'collectd- prometheus', 'exported_instance': 'otconap7', 'instance': '172.25.103.1:9103', 'job': 'collectd', 'namespace': 'edge1', 'pod': 'plundering-liger-collectd-wz7xg', 'service': 'collectd'}, 'value': [1559249299.084, '236300']} + 05-30-2019 08:47:53PM ::prom_ql_api.py :: query :: INFO :: {'metric': {'cpufreq': '1', 'endpoint': 'collectd- prometheus', 'exported_instance': 'otconap7', 'instance': 'ZZ.QQQ.XXX.YY:9103', 'job': 'collectd', 'namespace': 'edge1', 'pod': 'plundering-liger-collectd-wz7xg', 'service': 'collectd'}, 'value': [1559249299.084, '236300']} ``` * Tested Error scenario: Configure QUERY_STRING as : @@ -71,11 +130,11 @@ list_of_result_sets = query(QUERY_STRING) ``` O/P : ``` - Check logs..HTTP error occurred: 400 Client Error: Bad Request for url: http://172.25.103.1:30090/api/v1/query?query=collectd_cpu_percent%7Bjob%3D%22collectd%22+exported_instance%3D%22an11-31%22%7D%5B1m%5D + Check logs..HTTP error occurred: 400 Client Error: Bad Request for url: http://ZZ.QQQ.XXX.YY:30090/api/v1/query?query=collectd_cpu_percent%7Bjob%3D%22collectd%22+exported_instance%3D%22an11-31%22%7D%5B1m%5D [{'metric': {'cpufreq': '1', 'endpoint': 'collectd-prometheus', 'exported_instance': 'otconap7', - 'instance': '172.25.103.1:9103', + 'instance': 'ZZ.QQQ.XXX.YY:9103', 'job': 'collectd', 'namespace': 'edge1', 'pod': 'plundering-liger-collectd-wz7xg', diff --git a/vnfs/DAaaS/lib/promql_api/prom_ql_api.py b/vnfs/DAaaS/lib/promql_api/prom_ql_api.py index 8a5ce508..1dc578e9 100644 --- a/vnfs/DAaaS/lib/promql_api/prom_ql_api.py +++ b/vnfs/DAaaS/lib/promql_api/prom_ql_api.py @@ -23,7 +23,8 @@ import requests from requests.exceptions import HTTPError -API_VERSION = '/api/v1/query' +QUERY_API_VERSION = '/api/v1/query' +QUERY_RANGE_API_VERSION = '/api/v1/query_range' LIST_OF_ENV_VARIABLES = ["DATA_ENDPOINT"] MAP_ENV_VARIABLES = dict() LOG = logging.getLogger(__name__) @@ -34,7 +35,7 @@ def set_log_config(): datefmt='%m-%d-%Y %I:%M:%S%p', level=logging.DEBUG, filename='promql_api.log', - filemode='w') + filemode='a') LOG.info("Set the log configs.") @@ -45,7 +46,7 @@ def load_and_validate_env_vars(list_of_env_vars): LOG.info("Found env variable: {} ".format(env_var.upper())) MAP_ENV_VARIABLES[env_var.upper()] = environ.get(env_var) else: - #MAP_ENV_VARIABLES['DATA_ENDPOINT']='http://127.0.0.1:30090' # to be deleted + #MAP_ENV_VARIABLES['DATA_ENDPOINT']='http://127.0.0.1:9090' # to be deleted LOG.error("Env var: {} not found ! ".format(env_var.upper())) raise KeyError("Env variable: {} not found ! ".format(env_var.upper())) @@ -74,7 +75,7 @@ def query(QUERY_STRING): params_map = {} list_of_result_sets = [] list_of_substrings.append(MAP_ENV_VARIABLES['DATA_ENDPOINT']) - list_of_substrings.append(API_VERSION) + list_of_substrings.append(QUERY_API_VERSION) url = ''.join(list_of_substrings) for each_query_string in QUERY_STRING: @@ -111,3 +112,60 @@ def query(QUERY_STRING): LOG.info(each_result) list_of_result_sets.append(results) return list_of_result_sets + + +def validate_parameters(map_of_parameters): + for k,v in map_of_parameters.items(): + if k not in ['query', 'start', 'end', 'step', 'timeout']: + LOG.error('Parameter : \'{}\' not supported by query_range'.format(k)) + LOG.info('Valid parameters :: \'query\', \'start\', \'end\', \'step\', \'timeout\'') + raise Exception('Parameter : \'{}\' not supported by query_range. Check logs'.format(k)) + if not isinstance(k,str): + LOG.error(':: Key Paramter : \'{}\' NOT a string! Keys should be string ::'.format(k)) + raise Exception(':: Key Paramter : \'{}\' NOT a string! Keys should be string ::'.format(k)) + if not isinstance(v,str): + LOG.error(':: Value Paramter of key: \'{}\' NOT a string! Values should be string ::'.format(k)) + raise Exception(':: Value Paramter of key: \'{}\' NOT a string! Values should be string ::'.format(k)) + return True + + +def query_range(map_of_parameters): + list_of_result_sets = [] + set_log_config() + if validate_parameters(map_of_parameters): + LOG.info(':::Validation of map_of_parameters done::') + load_and_validate_env_vars(LIST_OF_ENV_VARIABLES) + LOG.info("Forming the query_range request ...") + + list_of_substrings = [] + list_of_substrings.append(MAP_ENV_VARIABLES['DATA_ENDPOINT']) + list_of_substrings.append(QUERY_RANGE_API_VERSION) + url = ''.join(list_of_substrings) + + try: + LOG.info('API request::: URL: {} '.format(url)) + LOG.info('API request::: params: {} '.format(map_of_parameters)) + response = requests.get(url, params=map_of_parameters) + response.raise_for_status() # This might raise HTTPError which is handled in except block + except HTTPError as http_err: + if response.json()['status'] == "error": + LOG.error("::::ERROR OCCURED::::") + LOG.error("::::ERROR TYPE:::: {}".format(response.json()['errorType'])) + LOG.error("::::ERROR:::: {}".format(response.json()['error'])) + print(f'Check logs..HTTP error occurred: {http_err}') + except Exception as err: + print(f'Check logs..Other error occurred: {err}') + else: + if response.json()['status'] == "error": + LOG.error("::::ERROR OCCURED!::::") + LOG.error("::::ERROR TYPE:::: {}".format(response.json()['errorType'])) + LOG.error("::::ERROR:::: {}".format(response.json()['error'])) + list_of_result_sets.append(response.json()['error']) + list_of_result_sets.append(dict({'error':response.json()['error'], + 'errorType' : response.json()['errorType']})) + else: + results = response.json()['data']['result'] + LOG.info('::::::::::RESULTS OF QUERY_RANGE::::::::::::: {}'.format(map_of_parameters)) + list_of_result_sets.append(results) + + return list_of_result_sets \ No newline at end of file diff --git a/vnfs/DAaaS/sample-apps/m3db_promql/build/sample_promql_query.py b/vnfs/DAaaS/sample-apps/m3db_promql/build/sample_promql_query.py index fbbf497e..2c85f1f8 100644 --- a/vnfs/DAaaS/sample-apps/m3db_promql/build/sample_promql_query.py +++ b/vnfs/DAaaS/sample-apps/m3db_promql/build/sample_promql_query.py @@ -17,19 +17,32 @@ from promql_api.prom_ql_api import query +from promql_api.prom_ql_api import query_range import pprint QUERY_STRING = ['irate(collectd_cpufreq{exported_instance="otconap7",cpufreq="1"}[2m])'] +MAP_OF_PARAMETERS = {'query': 'up', 'start': '2019-06-19T20:10:30.781Z', 'end': '2019-06-19T20:10:45.781Z', 'step': '15s'} #Other examples #QUERY_STRING = [ 'irate(http_requests_total{code="200"}[1m])', 'collectd_cpu_percent{job="collectd", exported_instance="otconap7"}[1m]' ] #QUERY_STRING = ['irate(collectd_cpufreq{exported_instance="otconap7",cpufreq="1"}[2m])', 'go_info'] +#MAP_OF_PARAMETERS = {'query': 'up', 'start': '2019-06-19T20:10:30.781Z', 'end': '2019-06-19T20:10:45.781Z', 'step': '15s', 'timeout':'600s'} + +def demo_query(): + list_of_result_sets = query(QUERY_STRING) + if list_of_result_sets: + for each_result in list_of_result_sets: + pprint.pprint(each_result) + +def demo_query_range(): + list_of_result_sets = query_range(MAP_OF_PARAMETERS) + pprint.pprint(list_of_result_sets) + def main(): - list_of_result_sets = query(QUERY_STRING) - if list_of_result_sets: - for each_result in list_of_result_sets: - pprint.pprint(each_result) + demo_query() + demo_query_range() + if __name__ == "__main__": - main() + main()