dcae-cli/dcae_cli/util/tests/test_discovery.py

   1 # ============LICENSE_START=======================================================
   2 # org.onap.dcae
   3 # ================================================================================
   4 # Copyright (c) 2017-2018 AT&T Intellectual Property. All rights reserved.
   5 # ================================================================================
   6 # Licensed under the Apache License, Version 2.0 (the "License");
   7 # you may not use this file except in compliance with the License.
   8 # You may obtain a copy of the License at
   9 #
  10 #      http://www.apache.org/licenses/LICENSE-2.0
  11 #
  12 # Unless required by applicable law or agreed to in writing, software
  13 # distributed under the License is distributed on an "AS IS" BASIS,
  14 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15 # See the License for the specific language governing permissions and
  16 # limitations under the License.
  17 # ============LICENSE_END=========================================================
  18 #
  19 # ECOMP is a trademark and service mark of AT&T Intellectual Property.
  20
  21 # -*- coding: utf-8 -*-
  22 '''
  23 Provides tests for the discovery module
  24 '''
  25 import json
  26 from functools import partial
  27 from copy import deepcopy
  28
  29 import pytest
  30
  31 from dcae_cli.util import discovery as dis
  32 from dcae_cli.util.discovery import create_config, Consul, config_context, DiscoveryNoDownstreamComponentError
  33
  34
  35 user = 'bob'
  36 cname = 'asimov.test_comp'
  37 cver = '0.0.0'
  38 inst_pref = 'abc123'
  39 params = {'param0': 12345}
  40
  41
  42 def test_create_config():
  43     '''
  44     Test explanation:
  45     1. param1 in the component spec has 2 compatible component types, comp1 and comp2. however infrastructure
  46     support only allows for 1. thus comp2 shouldn't make it to the rels.
  47     2. comp1 has two instances, so both should make it to the rels
  48     3. param2 is compatible with comp3, but there are no comp3 instances. thus it's missing from rels.
  49     '''
  50     expected_ckey = 'bob.abc123.0-0-0.asimov-test_comp'
  51     expected_conf = {'param1': '{{1-1-1.foo-bar-comp1}}', 'param0': 12345, 'param2': '{{3-3-3.foo-bar-comp3}}'}
  52     expected_rkey = 'bob.abc123.0-0-0.asimov-test_comp:rel'
  53     expected_rels = ['bob.aaa111.1-1-1.foo-bar-comp1.suffix',
  54             'bob.bbb222.1-1-1.foo-bar-comp1.suffix',
  55             'bob.ddd444.3-3-3.foo-bar-comp3.suffix']
  56     expected_dmaap_key = 'bob.abc123.0-0-0.asimov-test_comp:dmaap'
  57     expected_dmaap_map = {}
  58
  59     interface_map = {'param1': [('foo.bar.comp1', '1.1.1'),
  60                                 ('foo.bar.comp2', '2.2.2')],
  61                      'param2': [('foo.bar.comp3', '3.3.3')]
  62                      }
  63     instance_map = {('foo.bar.comp1', '1.1.1'): ['bob.aaa111.1-1-1.foo-bar-comp1.suffix',
  64                                                  'bob.bbb222.1-1-1.foo-bar-comp1.suffix'],
  65                     ('foo.bar.comp2', '2.2.2'): ['bob.ccc333.2-2-2.foo-bar-comp2.suffix'],
  66                     ('foo.bar.comp3', '3.3.3'): ['bob.ddd444.3-3-3.foo-bar-comp3.suffix']}
  67
  68     ckey, conf, rkey, rels, dmaap_key, dmaap_map  = create_config(user, cname, cver,
  69             params, interface_map, instance_map, expected_dmaap_map, inst_pref)
  70
  71     assert ckey == expected_ckey
  72     assert conf == expected_conf
  73     assert rkey == expected_rkey
  74     assert sorted(rels) == sorted(expected_rels)
  75     assert dmaap_key == expected_dmaap_key
  76     assert dmaap_map == expected_dmaap_map
  77
  78     #
  79     # Fail cases: When a downstream dependency does not exist
  80     #
  81
  82     # (1) Case when there's no actual instance
  83     instance_map_missing_3 = deepcopy(instance_map)
  84     instance_map_missing_3[('foo.bar.comp3', '3.3.3')] = []
  85
  86     with pytest.raises(DiscoveryNoDownstreamComponentError):
  87         create_config(user, cname, cver, params, interface_map, instance_map_missing_3,
  88                 expected_dmaap_map, inst_pref)
  89
  90     # (2) Case when there's no existence in instance_map
  91     interface_map_extra = deepcopy(interface_map)
  92     interface_map_extra["param_not_exist"] = []
  93
  94     with pytest.raises(DiscoveryNoDownstreamComponentError):
  95         create_config(user, cname, cver, params, interface_map_extra, instance_map,
  96                 expected_dmaap_map, inst_pref)
  97
  98     #
  99     # Force the fail cases to succeed
 100     #
 101
 102     # (1)
 103     ckey, conf, rkey, rels, dmaap_key, dmaap_map = create_config(user, cname, cver,
 104             params, interface_map, instance_map_missing_3, expected_dmaap_map, inst_pref,
 105             force=True)
 106
 107     assert ckey == expected_ckey
 108     assert conf == expected_conf
 109     assert rkey == expected_rkey
 110     # Remove the foo.bar.comp3:3.3.3 instance because we are simulating when that
 111     # instance does not exist
 112     assert sorted(rels) == sorted(expected_rels[:2])
 113     assert dmaap_key == expected_dmaap_key
 114     assert dmaap_map == expected_dmaap_map
 115
 116     # (2)
 117     ckey, conf, rkey, rels, dmaap_key, dmaap_map = create_config(user, cname, cver,
 118             params, interface_map_extra, instance_map, expected_dmaap_map, inst_pref,
 119             force=True)
 120
 121     expected_conf["param_not_exist"] = "{{}}"
 122
 123     assert ckey == expected_ckey
 124     assert conf == expected_conf
 125     assert rkey == expected_rkey
 126     assert sorted(rels) == sorted(expected_rels)
 127     assert dmaap_key == expected_dmaap_key
 128     assert dmaap_map == expected_dmaap_map
 129
 130     #
 131     # Test differnt dashes scenario
 132     #
 133
 134     # Component has been added with dashes but the instance comes back with dots
 135     # because the discovery layer always brings back instances with dots
 136     interface_map_dashes = {'param1': [('foo-bar-comp1', '1.1.1')]}
 137     instance_map_dashes = {('foo.bar.comp1', '1.1.1'):
 138             ['bob.aaa111.1-1-1.foo-bar-comp1.suffix']}
 139
 140     with pytest.raises(DiscoveryNoDownstreamComponentError):
 141         create_config(user, cname, cver, params, interface_map_dashes, instance_map_dashes,
 142                 expected_dmaap_map, inst_pref)
 143
 144     # The fix in v2.3.2 was to have the caller to send in instances with dots and
 145     # with dashes
 146     instance_map_dashes = {
 147             ('foo.bar.comp1', '1.1.1'): ['bob.aaa111.1-1-1.foo-bar-comp1.suffix'],
 148             ('foo-bar-comp1', '1.1.1'): ['bob.aaa111.1-1-1.foo-bar-comp1.suffix'] }
 149
 150     ckey, conf, rkey, rels, dmaap_key, dmaap_map = create_config(user, cname, cver,
 151             params, interface_map_dashes, instance_map_dashes, expected_dmaap_map, inst_pref)
 152
 153     # The expecteds have changed because the inputs have been narrowed to just
 154     # one
 155     assert ckey == expected_ckey
 156     assert conf == {'param1': '{{1-1-1.foo-bar-comp1}}', 'param0': 12345}
 157     assert rkey == expected_rkey
 158     assert sorted(rels) == sorted(['bob.aaa111.1-1-1.foo-bar-comp1.suffix'])
 159     assert dmaap_key == expected_dmaap_key
 160     assert dmaap_map == expected_dmaap_map
 161
 162     # Pass in a non-empty dmaap map
 163     dmaap_map_input = { "some-config-key": { "type": "message_router",
 164         "dmaap_info": {"topic_url": "http://some-topic-url.com/abc"} } }
 165     del expected_conf["param_not_exist"]
 166     expected_conf["some-config-key"] = { "type": "message_router",
 167             "dmaap_info": "<<some-config-key>>" }
 168
 169     ckey, conf, rkey, rels, dmaap_key, dmaap_map  = create_config(user, cname, cver,
 170             params, interface_map, instance_map, dmaap_map_input, inst_pref)
 171
 172     assert ckey == expected_ckey
 173     assert conf == expected_conf
 174     assert rkey == expected_rkey
 175     assert sorted(rels) == sorted(expected_rels)
 176     assert dmaap_key == expected_dmaap_key
 177     assert dmaap_map == {'some-config-key': {'topic_url': 'http://some-topic-url.com/abc'}}
 178
 179
 180 @pytest.mark.skip(reason="Not a pure unit test")
 181 def test_config_context(mock_cli_config):
 182     interface_map = {'param1': [('foo.bar.comp1', '1.1.1'),
 183                                 ('foo.bar.comp2', '2.2.2')],
 184                      'param2': [('foo.bar.comp3', '3.3.3')]
 185                      }
 186     instance_map = {('foo.bar.comp1', '1.1.1'): ['bob.aaa111.1-1-1.foo-bar-comp1.suffix',
 187                                                  'bob.bbb222.1-1-1.foo-bar-comp1.suffix'],
 188                     ('foo.bar.comp2', '2.2.2'): ['bob.ccc333.2-2-2.foo-bar-comp2.suffix'],
 189                     ('foo.bar.comp3', '3.3.3'): ['bob.ddd444.3-3-3.foo-bar-comp3.suffix']}
 190
 191     config_key_map = {"param1": {"group": "streams_publishes", "type": "http"},
 192             "param2": {"group": "services_calls", "type": "http"}}
 193
 194     ckey = 'bob.abc123.0-0-0.asimov-test_comp'
 195     rkey = 'bob.abc123.0-0-0.asimov-test_comp:rel'
 196     expected_conf = {"streams_publishes": {'param1': '{{1-1-1.foo-bar-comp1}}'},
 197             'param0': 12345, "streams_subscribes": {},
 198             "services_calls": {'param2': '{{3-3-3.foo-bar-comp3}}'}}
 199     expected_rels = ['bob.aaa111.1-1-1.foo-bar-comp1.suffix',
 200             'bob.bbb222.1-1-1.foo-bar-comp1.suffix',
 201             'bob.ddd444.3-3-3.foo-bar-comp3.suffix']
 202
 203     c = Consul(dis.default_consul_host())
 204     with config_context(user, cname, cver, params, interface_map, instance_map,
 205             config_key_map, instance_prefix=inst_pref) as (instance,_):
 206         assert json.loads(c.kv.get(ckey)[1]['Value'].decode('utf-8')) == expected_conf
 207         assert sorted(json.loads(c.kv.get(rkey)[1]['Value'].decode('utf-8'))) \
 208                 == sorted(expected_rels)
 209         assert instance == ckey
 210
 211     assert c.kv.get(ckey)[1] is None
 212     assert c.kv.get(rkey)[1] is None
 213
 214     # Fail case: When a downstream dependency does not exist
 215     interface_map_extra = deepcopy(interface_map)
 216     interface_map_extra["param_not_exist"] = []
 217
 218     with pytest.raises(DiscoveryNoDownstreamComponentError):
 219         with config_context(user, cname, cver, params, interface_map_extra,
 220                 instance_map, config_key_map, instance_prefix=inst_pref) as (instance,_):
 221             pass
 222
 223     # Force fail case to succeed
 224     expected_conf["param_not_exist"] = "{{}}"
 225
 226     with config_context(user, cname, cver, params, interface_map_extra,
 227             instance_map, config_key_map, instance_prefix=inst_pref,
 228             force_config=True) as (instance,_):
 229         assert json.loads(c.kv.get(ckey)[1]['Value'].decode('utf-8')) == expected_conf
 230         assert sorted(json.loads(c.kv.get(rkey)[1]['Value'].decode('utf-8'))) \
 231             == sorted(expected_rels)
 232         assert instance == ckey
 233
 234
 235 def test_inst_regex():
 236     ckey = 'bob.abc123.0-0-0.asimov-test_comp'
 237     match = dis._inst_re.match(ckey)
 238     assert match != None
 239
 240     # Big version case
 241
 242     ckey = 'bob.abc123.100-100-100.asimov-test_comp'
 243     match = dis._inst_re.match(ckey)
 244     assert match != None
 245
 246
 247 def test_is_healthy_pure():
 248     component = { 'CreateIndex': 204546, 'Flags': 0,
 249             'Key': 'mike.21fbcabd-fac1-4b9b-9d18-2f624bfa44a5.0-4-0.sandbox-platform-dummy_subscriber', 'LockIndex': 0, 'ModifyIndex': 204546,
 250             'Value': b'{}' }
 251
 252     component_health_good = ('262892',
 253             [{'Checks': [{'CheckID': 'serfHealth',
 254                    'CreateIndex': 3,
 255                    'ModifyIndex': 3,
 256                    'Name': 'Serf Health Status',
 257                    'Node': 'agent-one',
 258                    'Notes': '',
 259                    'Output': 'Agent alive and reachable',
 260                    'ServiceID': '',
 261                    'ServiceName': '',
 262                    'Status': 'passing'},
 263                   {'CheckID': 'service:rework-central-swarm-master:mike.21fbcabd-fac1-4b9b-9d18-2f624bfa44a5.0-4-0.sandbox-platform-dummy_subscriber:8080',
 264                    'CreateIndex': 204550,
 265                    'ModifyIndex': 204551,
 266                    'Name': 'Service '
 267                            "'mike.21fbcabd-fac1-4b9b-9d18-2f624bfa44a5.0-4-0.sandbox-platform-dummy_subscriber' "
 268                            'check',
 269                    'Node': 'agent-one',
 270                    'Notes': '',
 271                    'Output': '',
 272                    'ServiceID': 'rework-central-swarm-master:mike.21fbcabd-fac1-4b9b-9d18-2f624bfa44a5.0-4-0.sandbox-platform-dummy_subscriber:8080',
 273                    'ServiceName': 'mike.21fbcabd-fac1-4b9b-9d18-2f624bfa44a5.0-4-0.sandbox-platform-dummy_subscriber',
 274                    'Status': 'passing'}],
 275        'Node': {'Address': '10.170.2.17',
 276                 'CreateIndex': 3,
 277                 'ModifyIndex': 262877,
 278                 'Node': 'agent-one',
 279                 'TaggedAddresses': {'wan': '10.170.2.17'}},
 280        'Service': {'Address': '196.207.170.175',
 281                    'CreateIndex': 204550,
 282                    'EnableTagOverride': False,
 283                    'ID': 'rework-central-swarm-master:mike.21fbcabd-fac1-4b9b-9d18-2f624bfa44a5.0-4-0.sandbox-platform-dummy_subscriber:8080',
 284                    'ModifyIndex': 204551,
 285                    'Port': 33064,
 286                    'Service': 'mike.21fbcabd-fac1-4b9b-9d18-2f624bfa44a5.0-4-0.sandbox-platform-dummy_subscriber',
 287                    'Tags': None}}])
 288
 289     assert True == dis._is_healthy_pure(lambda name: component_health_good, component)
 290
 291     # Case: Check is failing
 292
 293     component_health_bad = deepcopy(component_health_good)
 294     # NOTE: The failed status here. Not sure if this is what Consul actually sends
 295     # but at least its not "passing"
 296     component_health_bad[1][0]["Checks"][0]["Status"] = "failing"
 297
 298     assert False == dis._is_healthy_pure(lambda name: component_health_bad, component)
 299
 300     # Case: No health for a component
 301
 302     component_health_nothing = ('262892', [])
 303     assert False == dis._is_healthy_pure(lambda name: component_health_nothing, component)
 304
 305
 306 def test_get_instances_from_kv():
 307
 308     def get_from_kv_fake(result, user, recurse=True):
 309         return "don't care about first arg", result
 310
 311     user = "jane"
 312     kvs_nothing = []
 313
 314     assert dis._get_instances_from_kv(partial(get_from_kv_fake, kvs_nothing), user) == []
 315
 316     kvs_success = [ { "Value": "some value", "Key": "jane.1344a03a-06a8-4b92-bfac-d8f89df0c0cd.1-0-0.dcae-controller-ves-collector:rel"
 317         },
 318         { "Value": "some value", "Key": "jane.1344a03a-06a8-4b92-bfac-d8f89df0c0cd.1-0-0.dcae-controller-ves-collector" } ]
 319
 320     assert dis._get_instances_from_kv(partial(get_from_kv_fake, kvs_success), user) == ["jane.1344a03a-06a8-4b92-bfac-d8f89df0c0cd.1-0-0.dcae-controller-ves-collector"]
 321
 322     kvs_partial = [ { "Value": "some value", "Key": "jane.1344a03a-06a8-4b92-bfac-d8f89df0c0cd.1-0-0.dcae-controller-ves-collector:rel"
 323         } ]
 324
 325     assert dis._get_instances_from_kv(partial(get_from_kv_fake, kvs_partial), user) == ["jane.1344a03a-06a8-4b92-bfac-d8f89df0c0cd.1-0-0.dcae-controller-ves-collector"]
 326
 327
 328 def test_get_instances_from_catalog():
 329
 330     def get_from_catalog_fake(result):
 331         return ("some Consul index", result)
 332
 333     user = "jane"
 334     services_nothing = {}
 335
 336     assert dis._get_instances_from_catalog(
 337             partial(get_from_catalog_fake, services_nothing), user) == []
 338
 339     services_no_matching = { '4f09bb72-8578-4e82-a6a4-9b7d679bd711.cdap_app_hello_world.hello-world-cloudify-test': [],
 340   '666.fake_testing_service.rework-central.com': [],
 341   'Platform_Dockerhost_Solutioning_Test': [],
 342   'jack.2271ec6b-9224-4f42-b0b0-bfa91b41218f.1-0-1.cdap-event-proc-map-app': [],
 343   'jack.bca28c8c-a352-41f1-81bc-63ff46db2582.1-0-1.cdap-event-proc-supplement-app':
 344   [] }
 345
 346     assert dis._get_instances_from_catalog(
 347             partial(get_from_catalog_fake, services_no_matching), user) == []
 348
 349     services_success = { '4f09bb72-8578-4e82-a6a4-9b7d679bd711.cdap_app_hello_world.hello-world-cloudify-test': [],
 350   '666.fake_testing_service.rework-central.com': [],
 351   'Platform_Dockerhost_Solutioning_Test': [],
 352   'jack.2271ec6b-9224-4f42-b0b0-bfa91b41218f.1-0-1.cdap-event-proc-map-app': [],
 353   'jane.bca28c8c-a352-41f1-81bc-63ff46db2582.1-0-1.cdap-event-proc-supplement-app':
 354   [] }
 355
 356     assert dis._get_instances_from_catalog(
 357             partial(get_from_catalog_fake, services_success), user) == ['jane.bca28c8c-a352-41f1-81bc-63ff46db2582.1-0-1.cdap-event-proc-supplement-app']
 358
 359
 360 def test_merge_instances():
 361     user = "somebody"
 362     group_one = [ "123", "456" ]
 363     group_two = [ "123", "abc" ]
 364     group_three = []
 365
 366     assert sorted(dis._merge_instances(user, lambda user: group_one, lambda user: group_two,
 367             lambda user: group_three)) == sorted([ "123", "456", "abc" ])
 368
 369
 370 def test_make_instance_map():
 371     instances_latest_format = ["mike.112e4faa-2ac8-4b13-93e9-8924150538d5.0-5-0.sandbox-platform-laika"]
 372
 373     instances_map = dis._make_instances_map(instances_latest_format)
 374     assert instances_map.get(("sandbox.platform.laika", "0.5.0")) == set(instances_latest_format)
 375
 376
 377 def test_get_component_instances(monkeypatch):
 378     instances = [
 379             'jane.b493b48b-5fdf-4c1d-bd2a-8ce747b918ba.1-0-0.dcae-controller-ves-collector',
 380             'jane.2455ec5c-67e6-4d4d-8581-79037c7b5f8e.1-0-0.dcae-controller-ves-collector.rework-central.dcae.com',
 381             'jane.bfbb1356-d703-4007-8799-759a9e1fc8c2.1-0-0.dcae-controller-ves-collector.rework-central.dcae.com',
 382             'jane.89d82ff6-1482-4c01-8758-db9325aad085.1-0-0.dcae-controller-ves-collector'
 383             ]
 384
 385     instances_map =  { ('dcae.controller.ves.collector', '1.0.0'): set(instances) }
 386
 387     def get_user_instances_mock(user, consul_host=None, filter_instances_func=None):
 388         return instances_map
 389
 390     monkeypatch.setattr(dis, 'get_user_instances', get_user_instances_mock)
 391
 392     def always_true_filter(consul_host, instance):
 393         return True
 394
 395     # Test base case
 396
 397     user = "jane"
 398     cname = "dcae.controller.ves.collector"
 399     cver = "1.0.0"
 400     consul_host = "bogus"
 401
 402     assert sorted(dis._get_component_instances(always_true_filter, user, cname, cver,
 403         consul_host)) == sorted(instances)
 404
 405     # Test for dashes
 406
 407     cname = "dcae-controller-ves-collector"
 408
 409     assert sorted(dis._get_component_instances(always_true_filter, user, cname, cver,
 410         consul_host)) == sorted(instances)
 411
 412
 413 def test_group_config():
 414     config_key_map = {'call1': {'group': 'services_calls'}, 'pub1': {'type': 'http', 'group': 'streams_publishes'}, 'sub2': {'type': 'message_router', 'group': 'streams_subscribes'}, 'pub2': {'type': 'message_router', 'group': 'streams_publishes'}}
 415
 416     config = { "call1": "{{yo}}", "pub1": "{{target}}", "some-param": 123,
 417             "sub2": { "dmaap_info": "<<sub2>>" }, "pub2": { "dmaap_info": "<<pub2>>" } }
 418
 419     gc = dis._group_config(config, config_key_map)
 420     expected = {'services_calls': {'call1': '{{yo}}'}, 'streams_publishes': {'pub2': {'dmaap_info': '<<pub2>>'}, 'pub1': '{{target}}'}, 'some-param': 123, 'streams_subscribes': {'sub2': {'dmaap_info': '<<sub2>>'}}}
 421
 422     assert gc == expected
 423
 424
 425 def test_parse_instance_lookup():
 426     results = [{"ServiceAddress": "192.168.1.100", "ServicePort": "8080"},
 427             {"ServiceAddress": "10.100.1.100", "ServicePort": "8081"}]
 428     assert dis.parse_instance_lookup(results) == "192.168.1.100:8080"
 429
 430
 431 def test_apply_inputs():
 432     updated_config = dis._apply_inputs({"foo": "bar"}, {"foo": "baz"})
 433     assert updated_config == {"foo": "baz"}
 434
 435
 436 def test_choose_consul_host(monkeypatch):
 437     def fake_default_consul_host():
 438         return "default-consul-host"
 439
 440     monkeypatch.setattr(dis, "default_consul_host", fake_default_consul_host)
 441     assert "default-consul-host" == dis._choose_consul_host(None)
 442     assert "provided-consul-host" == dis._choose_consul_host("provided-consul-host")
 443
 444
 445 if __name__ == '__main__':
 446     '''Test area'''
 447     pytest.main([__file__, ])