1 # ============LICENSE_START=======================================================
3 # ================================================================================
4 # Copyright (c) 2017-2018 AT&T Intellectual Property. All rights reserved.
5 # ================================================================================
6 # Licensed under the Apache License, Version 2.0 (the "License");
7 # you may not use this file except in compliance with the License.
8 # You may obtain a copy of the License at
10 # http://www.apache.org/licenses/LICENSE-2.0
12 # Unless required by applicable law or agreed to in writing, software
13 # distributed under the License is distributed on an "AS IS" BASIS,
14 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 # See the License for the specific language governing permissions and
16 # limitations under the License.
17 # ============LICENSE_END=========================================================
19 # ECOMP is a trademark and service mark of AT&T Intellectual Property.
21 # -*- coding: utf-8 -*-
23 Provides tests for the discovery module
26 from functools import partial
27 from copy import deepcopy
31 from dcae_cli.util import discovery as dis
32 from dcae_cli.util.discovery import create_config, Consul, config_context, DiscoveryNoDownstreamComponentError
36 cname = 'asimov.test_comp'
39 params = {'param0': 12345}
42 def test_create_config():
45 1. param1 in the component spec has 2 compatible component types, comp1 and comp2. however infrastructure
46 support only allows for 1. thus comp2 shouldn't make it to the rels.
47 2. comp1 has two instances, so both should make it to the rels
48 3. param2 is compatible with comp3, but there are no comp3 instances. thus it's missing from rels.
50 expected_ckey = 'bob.abc123.0-0-0.asimov-test_comp'
51 expected_conf = {'param1': '{{1-1-1.foo-bar-comp1}}', 'param0': 12345, 'param2': '{{3-3-3.foo-bar-comp3}}'}
52 expected_rkey = 'bob.abc123.0-0-0.asimov-test_comp:rel'
53 expected_rels = ['bob.aaa111.1-1-1.foo-bar-comp1.suffix',
54 'bob.bbb222.1-1-1.foo-bar-comp1.suffix',
55 'bob.ddd444.3-3-3.foo-bar-comp3.suffix']
56 expected_dmaap_key = 'bob.abc123.0-0-0.asimov-test_comp:dmaap'
57 expected_dmaap_map = {}
59 interface_map = {'param1': [('foo.bar.comp1', '1.1.1'),
60 ('foo.bar.comp2', '2.2.2')],
61 'param2': [('foo.bar.comp3', '3.3.3')]
63 instance_map = {('foo.bar.comp1', '1.1.1'): ['bob.aaa111.1-1-1.foo-bar-comp1.suffix',
64 'bob.bbb222.1-1-1.foo-bar-comp1.suffix'],
65 ('foo.bar.comp2', '2.2.2'): ['bob.ccc333.2-2-2.foo-bar-comp2.suffix'],
66 ('foo.bar.comp3', '3.3.3'): ['bob.ddd444.3-3-3.foo-bar-comp3.suffix']}
68 ckey, conf, rkey, rels, dmaap_key, dmaap_map = create_config(user, cname, cver,
69 params, interface_map, instance_map, expected_dmaap_map, inst_pref)
71 assert ckey == expected_ckey
72 assert conf == expected_conf
73 assert rkey == expected_rkey
74 assert sorted(rels) == sorted(expected_rels)
75 assert dmaap_key == expected_dmaap_key
76 assert dmaap_map == expected_dmaap_map
79 # Fail cases: When a downstream dependency does not exist
82 # (1) Case when there's no actual instance
83 instance_map_missing_3 = deepcopy(instance_map)
84 instance_map_missing_3[('foo.bar.comp3', '3.3.3')] = []
86 with pytest.raises(DiscoveryNoDownstreamComponentError):
87 create_config(user, cname, cver, params, interface_map, instance_map_missing_3,
88 expected_dmaap_map, inst_pref)
90 # (2) Case when there's no existence in instance_map
91 interface_map_extra = deepcopy(interface_map)
92 interface_map_extra["param_not_exist"] = []
94 with pytest.raises(DiscoveryNoDownstreamComponentError):
95 create_config(user, cname, cver, params, interface_map_extra, instance_map,
96 expected_dmaap_map, inst_pref)
99 # Force the fail cases to succeed
103 ckey, conf, rkey, rels, dmaap_key, dmaap_map = create_config(user, cname, cver,
104 params, interface_map, instance_map_missing_3, expected_dmaap_map, inst_pref,
107 assert ckey == expected_ckey
108 assert conf == expected_conf
109 assert rkey == expected_rkey
110 # Remove the foo.bar.comp3:3.3.3 instance because we are simulating when that
111 # instance does not exist
112 assert sorted(rels) == sorted(expected_rels[:2])
113 assert dmaap_key == expected_dmaap_key
114 assert dmaap_map == expected_dmaap_map
117 ckey, conf, rkey, rels, dmaap_key, dmaap_map = create_config(user, cname, cver,
118 params, interface_map_extra, instance_map, expected_dmaap_map, inst_pref,
121 expected_conf["param_not_exist"] = "{{}}"
123 assert ckey == expected_ckey
124 assert conf == expected_conf
125 assert rkey == expected_rkey
126 assert sorted(rels) == sorted(expected_rels)
127 assert dmaap_key == expected_dmaap_key
128 assert dmaap_map == expected_dmaap_map
131 # Test differnt dashes scenario
134 # Component has been added with dashes but the instance comes back with dots
135 # because the discovery layer always brings back instances with dots
136 interface_map_dashes = {'param1': [('foo-bar-comp1', '1.1.1')]}
137 instance_map_dashes = {('foo.bar.comp1', '1.1.1'):
138 ['bob.aaa111.1-1-1.foo-bar-comp1.suffix']}
140 with pytest.raises(DiscoveryNoDownstreamComponentError):
141 create_config(user, cname, cver, params, interface_map_dashes, instance_map_dashes,
142 expected_dmaap_map, inst_pref)
144 # The fix in v2.3.2 was to have the caller to send in instances with dots and
146 instance_map_dashes = {
147 ('foo.bar.comp1', '1.1.1'): ['bob.aaa111.1-1-1.foo-bar-comp1.suffix'],
148 ('foo-bar-comp1', '1.1.1'): ['bob.aaa111.1-1-1.foo-bar-comp1.suffix'] }
150 ckey, conf, rkey, rels, dmaap_key, dmaap_map = create_config(user, cname, cver,
151 params, interface_map_dashes, instance_map_dashes, expected_dmaap_map, inst_pref)
153 # The expecteds have changed because the inputs have been narrowed to just
155 assert ckey == expected_ckey
156 assert conf == {'param1': '{{1-1-1.foo-bar-comp1}}', 'param0': 12345}
157 assert rkey == expected_rkey
158 assert sorted(rels) == sorted(['bob.aaa111.1-1-1.foo-bar-comp1.suffix'])
159 assert dmaap_key == expected_dmaap_key
160 assert dmaap_map == expected_dmaap_map
162 # Pass in a non-empty dmaap map
163 dmaap_map_input = { "some-config-key": { "type": "message_router",
164 "dmaap_info": {"topic_url": "http://some-topic-url.com/abc"} } }
165 del expected_conf["param_not_exist"]
166 expected_conf["some-config-key"] = { "type": "message_router",
167 "dmaap_info": "<<some-config-key>>" }
169 ckey, conf, rkey, rels, dmaap_key, dmaap_map = create_config(user, cname, cver,
170 params, interface_map, instance_map, dmaap_map_input, inst_pref)
172 assert ckey == expected_ckey
173 assert conf == expected_conf
174 assert rkey == expected_rkey
175 assert sorted(rels) == sorted(expected_rels)
176 assert dmaap_key == expected_dmaap_key
177 assert dmaap_map == {'some-config-key': {'topic_url': 'http://some-topic-url.com/abc'}}
180 @pytest.mark.skip(reason="Not a pure unit test")
181 def test_config_context(mock_cli_config):
182 interface_map = {'param1': [('foo.bar.comp1', '1.1.1'),
183 ('foo.bar.comp2', '2.2.2')],
184 'param2': [('foo.bar.comp3', '3.3.3')]
186 instance_map = {('foo.bar.comp1', '1.1.1'): ['bob.aaa111.1-1-1.foo-bar-comp1.suffix',
187 'bob.bbb222.1-1-1.foo-bar-comp1.suffix'],
188 ('foo.bar.comp2', '2.2.2'): ['bob.ccc333.2-2-2.foo-bar-comp2.suffix'],
189 ('foo.bar.comp3', '3.3.3'): ['bob.ddd444.3-3-3.foo-bar-comp3.suffix']}
191 config_key_map = {"param1": {"group": "streams_publishes", "type": "http"},
192 "param2": {"group": "services_calls", "type": "http"}}
194 ckey = 'bob.abc123.0-0-0.asimov-test_comp'
195 rkey = 'bob.abc123.0-0-0.asimov-test_comp:rel'
196 expected_conf = {"streams_publishes": {'param1': '{{1-1-1.foo-bar-comp1}}'},
197 'param0': 12345, "streams_subscribes": {},
198 "services_calls": {'param2': '{{3-3-3.foo-bar-comp3}}'}}
199 expected_rels = ['bob.aaa111.1-1-1.foo-bar-comp1.suffix',
200 'bob.bbb222.1-1-1.foo-bar-comp1.suffix',
201 'bob.ddd444.3-3-3.foo-bar-comp3.suffix']
203 c = Consul(dis.default_consul_host())
204 with config_context(user, cname, cver, params, interface_map, instance_map,
205 config_key_map, instance_prefix=inst_pref) as (instance,_):
206 assert json.loads(c.kv.get(ckey)[1]['Value'].decode('utf-8')) == expected_conf
207 assert sorted(json.loads(c.kv.get(rkey)[1]['Value'].decode('utf-8'))) \
208 == sorted(expected_rels)
209 assert instance == ckey
211 assert c.kv.get(ckey)[1] is None
212 assert c.kv.get(rkey)[1] is None
214 # Fail case: When a downstream dependency does not exist
215 interface_map_extra = deepcopy(interface_map)
216 interface_map_extra["param_not_exist"] = []
218 with pytest.raises(DiscoveryNoDownstreamComponentError):
219 with config_context(user, cname, cver, params, interface_map_extra,
220 instance_map, config_key_map, instance_prefix=inst_pref) as (instance,_):
223 # Force fail case to succeed
224 expected_conf["param_not_exist"] = "{{}}"
226 with config_context(user, cname, cver, params, interface_map_extra,
227 instance_map, config_key_map, instance_prefix=inst_pref,
228 force_config=True) as (instance,_):
229 assert json.loads(c.kv.get(ckey)[1]['Value'].decode('utf-8')) == expected_conf
230 assert sorted(json.loads(c.kv.get(rkey)[1]['Value'].decode('utf-8'))) \
231 == sorted(expected_rels)
232 assert instance == ckey
235 def test_inst_regex():
236 ckey = 'bob.abc123.0-0-0.asimov-test_comp'
237 match = dis._inst_re.match(ckey)
242 ckey = 'bob.abc123.100-100-100.asimov-test_comp'
243 match = dis._inst_re.match(ckey)
247 def test_is_healthy_pure():
248 component = { 'CreateIndex': 204546, 'Flags': 0,
249 'Key': 'mike.21fbcabd-fac1-4b9b-9d18-2f624bfa44a5.0-4-0.sandbox-platform-dummy_subscriber', 'LockIndex': 0, 'ModifyIndex': 204546,
252 component_health_good = ('262892',
253 [{'Checks': [{'CheckID': 'serfHealth',
256 'Name': 'Serf Health Status',
259 'Output': 'Agent alive and reachable',
262 'Status': 'passing'},
263 {'CheckID': 'service:rework-central-swarm-master:mike.21fbcabd-fac1-4b9b-9d18-2f624bfa44a5.0-4-0.sandbox-platform-dummy_subscriber:8080',
264 'CreateIndex': 204550,
265 'ModifyIndex': 204551,
267 "'mike.21fbcabd-fac1-4b9b-9d18-2f624bfa44a5.0-4-0.sandbox-platform-dummy_subscriber' "
272 'ServiceID': 'rework-central-swarm-master:mike.21fbcabd-fac1-4b9b-9d18-2f624bfa44a5.0-4-0.sandbox-platform-dummy_subscriber:8080',
273 'ServiceName': 'mike.21fbcabd-fac1-4b9b-9d18-2f624bfa44a5.0-4-0.sandbox-platform-dummy_subscriber',
274 'Status': 'passing'}],
275 'Node': {'Address': '10.170.2.17',
277 'ModifyIndex': 262877,
279 'TaggedAddresses': {'wan': '10.170.2.17'}},
280 'Service': {'Address': '196.207.170.175',
281 'CreateIndex': 204550,
282 'EnableTagOverride': False,
283 'ID': 'rework-central-swarm-master:mike.21fbcabd-fac1-4b9b-9d18-2f624bfa44a5.0-4-0.sandbox-platform-dummy_subscriber:8080',
284 'ModifyIndex': 204551,
286 'Service': 'mike.21fbcabd-fac1-4b9b-9d18-2f624bfa44a5.0-4-0.sandbox-platform-dummy_subscriber',
289 assert True == dis._is_healthy_pure(lambda name: component_health_good, component)
291 # Case: Check is failing
293 component_health_bad = deepcopy(component_health_good)
294 # NOTE: The failed status here. Not sure if this is what Consul actually sends
295 # but at least its not "passing"
296 component_health_bad[1][0]["Checks"][0]["Status"] = "failing"
298 assert False == dis._is_healthy_pure(lambda name: component_health_bad, component)
300 # Case: No health for a component
302 component_health_nothing = ('262892', [])
303 assert False == dis._is_healthy_pure(lambda name: component_health_nothing, component)
306 def test_get_instances_from_kv():
308 def get_from_kv_fake(result, user, recurse=True):
309 return "don't care about first arg", result
314 assert dis._get_instances_from_kv(partial(get_from_kv_fake, kvs_nothing), user) == []
316 kvs_success = [ { "Value": "some value", "Key": "jane.1344a03a-06a8-4b92-bfac-d8f89df0c0cd.1-0-0.dcae-controller-ves-collector:rel"
318 { "Value": "some value", "Key": "jane.1344a03a-06a8-4b92-bfac-d8f89df0c0cd.1-0-0.dcae-controller-ves-collector" } ]
320 assert dis._get_instances_from_kv(partial(get_from_kv_fake, kvs_success), user) == ["jane.1344a03a-06a8-4b92-bfac-d8f89df0c0cd.1-0-0.dcae-controller-ves-collector"]
322 kvs_partial = [ { "Value": "some value", "Key": "jane.1344a03a-06a8-4b92-bfac-d8f89df0c0cd.1-0-0.dcae-controller-ves-collector:rel"
325 assert dis._get_instances_from_kv(partial(get_from_kv_fake, kvs_partial), user) == ["jane.1344a03a-06a8-4b92-bfac-d8f89df0c0cd.1-0-0.dcae-controller-ves-collector"]
328 def test_get_instances_from_catalog():
330 def get_from_catalog_fake(result):
331 return ("some Consul index", result)
334 services_nothing = {}
336 assert dis._get_instances_from_catalog(
337 partial(get_from_catalog_fake, services_nothing), user) == []
339 services_no_matching = { '4f09bb72-8578-4e82-a6a4-9b7d679bd711.cdap_app_hello_world.hello-world-cloudify-test': [],
340 '666.fake_testing_service.rework-central.com': [],
341 'Platform_Dockerhost_Solutioning_Test': [],
342 'jack.2271ec6b-9224-4f42-b0b0-bfa91b41218f.1-0-1.cdap-event-proc-map-app': [],
343 'jack.bca28c8c-a352-41f1-81bc-63ff46db2582.1-0-1.cdap-event-proc-supplement-app':
346 assert dis._get_instances_from_catalog(
347 partial(get_from_catalog_fake, services_no_matching), user) == []
349 services_success = { '4f09bb72-8578-4e82-a6a4-9b7d679bd711.cdap_app_hello_world.hello-world-cloudify-test': [],
350 '666.fake_testing_service.rework-central.com': [],
351 'Platform_Dockerhost_Solutioning_Test': [],
352 'jack.2271ec6b-9224-4f42-b0b0-bfa91b41218f.1-0-1.cdap-event-proc-map-app': [],
353 'jane.bca28c8c-a352-41f1-81bc-63ff46db2582.1-0-1.cdap-event-proc-supplement-app':
356 assert dis._get_instances_from_catalog(
357 partial(get_from_catalog_fake, services_success), user) == ['jane.bca28c8c-a352-41f1-81bc-63ff46db2582.1-0-1.cdap-event-proc-supplement-app']
360 def test_merge_instances():
362 group_one = [ "123", "456" ]
363 group_two = [ "123", "abc" ]
366 assert sorted(dis._merge_instances(user, lambda user: group_one, lambda user: group_two,
367 lambda user: group_three)) == sorted([ "123", "456", "abc" ])
370 def test_make_instance_map():
371 instances_latest_format = ["mike.112e4faa-2ac8-4b13-93e9-8924150538d5.0-5-0.sandbox-platform-laika"]
373 instances_map = dis._make_instances_map(instances_latest_format)
374 assert instances_map.get(("sandbox.platform.laika", "0.5.0")) == set(instances_latest_format)
377 def test_get_component_instances(monkeypatch):
379 'jane.b493b48b-5fdf-4c1d-bd2a-8ce747b918ba.1-0-0.dcae-controller-ves-collector',
380 'jane.2455ec5c-67e6-4d4d-8581-79037c7b5f8e.1-0-0.dcae-controller-ves-collector.rework-central.dcae.com',
381 'jane.bfbb1356-d703-4007-8799-759a9e1fc8c2.1-0-0.dcae-controller-ves-collector.rework-central.dcae.com',
382 'jane.89d82ff6-1482-4c01-8758-db9325aad085.1-0-0.dcae-controller-ves-collector'
385 instances_map = { ('dcae.controller.ves.collector', '1.0.0'): set(instances) }
387 def get_user_instances_mock(user, consul_host=None, filter_instances_func=None):
390 monkeypatch.setattr(dis, 'get_user_instances', get_user_instances_mock)
392 def always_true_filter(consul_host, instance):
398 cname = "dcae.controller.ves.collector"
400 consul_host = "bogus"
402 assert sorted(dis._get_component_instances(always_true_filter, user, cname, cver,
403 consul_host)) == sorted(instances)
407 cname = "dcae-controller-ves-collector"
409 assert sorted(dis._get_component_instances(always_true_filter, user, cname, cver,
410 consul_host)) == sorted(instances)
413 def test_group_config():
414 config_key_map = {'call1': {'group': 'services_calls'}, 'pub1': {'type': 'http', 'group': 'streams_publishes'}, 'sub2': {'type': 'message_router', 'group': 'streams_subscribes'}, 'pub2': {'type': 'message_router', 'group': 'streams_publishes'}}
416 config = { "call1": "{{yo}}", "pub1": "{{target}}", "some-param": 123,
417 "sub2": { "dmaap_info": "<<sub2>>" }, "pub2": { "dmaap_info": "<<pub2>>" } }
419 gc = dis._group_config(config, config_key_map)
420 expected = {'services_calls': {'call1': '{{yo}}'}, 'streams_publishes': {'pub2': {'dmaap_info': '<<pub2>>'}, 'pub1': '{{target}}'}, 'some-param': 123, 'streams_subscribes': {'sub2': {'dmaap_info': '<<sub2>>'}}}
422 assert gc == expected
425 def test_parse_instance_lookup():
426 results = [{"ServiceAddress": "192.168.1.100", "ServicePort": "8080"},
427 {"ServiceAddress": "10.100.1.100", "ServicePort": "8081"}]
428 assert dis.parse_instance_lookup(results) == "192.168.1.100:8080"
431 def test_apply_inputs():
432 updated_config = dis._apply_inputs({"foo": "bar"}, {"foo": "baz"})
433 assert updated_config == {"foo": "baz"}
436 def test_choose_consul_host(monkeypatch):
437 def fake_default_consul_host():
438 return "default-consul-host"
440 monkeypatch.setattr(dis, "default_consul_host", fake_default_consul_host)
441 assert "default-consul-host" == dis._choose_consul_host(None)
442 assert "provided-consul-host" == dis._choose_consul_host("provided-consul-host")
445 if __name__ == '__main__':
447 pytest.main([__file__, ])