1 # ============LICENSE_START=======================================================
3 # ================================================================================
4 # Copyright (c) 2017 AT&T Intellectual Property. All rights reserved.
5 # ================================================================================
6 # Licensed under the Apache License, Version 2.0 (the "License");
7 # you may not use this file except in compliance with the License.
8 # You may obtain a copy of the License at
10 # http://www.apache.org/licenses/LICENSE-2.0
12 # Unless required by applicable law or agreed to in writing, software
13 # distributed under the License is distributed on an "AS IS" BASIS,
14 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 # See the License for the specific language governing permissions and
16 # limitations under the License.
17 # ============LICENSE_END=========================================================
19 # ECOMP is a trademark and service mark of AT&T Intellectual Property.
21 # -*- coding: utf-8 -*-
23 Tests the mock catalog
26 from copy import deepcopy
27 from functools import partial
31 from sqlalchemy.exc import IntegrityError
33 from dcae_cli.catalog.mock.catalog import MockCatalog, MissingEntry, DuplicateEntry, _get_unique_format_things
34 from dcae_cli.catalog.mock import catalog
37 _c1_spec = {'self': {'name': 'std.comp_one',
39 'description': 'comp1',
40 'component_type': 'docker'},
41 'streams': {'publishes': [{'format': 'std.format_one',
45 'subscribes': [{'format': 'std.format_one',
49 'services': {'calls': [{'request': {'format': 'std.format_one',
51 'response': {'format': 'std.format_one',
53 'config_key': 'call1'}],
54 'provides': [{'request': {'format': 'std.format_one',
56 'response': {'format': 'std.format_one',
59 'parameters': [{"name": "foo",
61 "description": "the foo thing"},
64 "description": "the bar thing"}
66 'artifacts': [{ "uri": "foo-image", "type": "docker image" }],
70 "endpoint": "/health",
77 _c2_spec = {'self': {'name': 'std.comp_two',
79 'description': 'comp2',
80 'component_type': 'docker'},
81 'streams': {'publishes': [],
82 'subscribes': [{'format': 'std.format_one',
86 'services': {'calls': [],
87 'provides': [{'request': {'format': 'std.format_one',
89 'response': {'format': 'std.format_one',
93 'artifacts': [{ "uri": "bar-image", "type": "docker image" }],
97 "endpoint": "/health",
105 _c2v2_spec = {'self': {'name': 'std.comp_two',
107 'description': 'comp2',
108 'component_type': 'docker'},
109 'streams': {'publishes': [],
110 'subscribes': [{'format': 'std.format_one',
114 'services': {'calls': [],
115 'provides': [{'request': {'format': 'std.format_one',
117 'response': {'format': 'std.format_one',
119 'route': '/prov1'}]},
121 'artifacts': [{ "uri": "baz-image", "type": "docker image" }],
125 "endpoint": "/health",
133 _c3_spec = {'self': {'name': 'std.comp_three',
135 'description': 'comp3',
136 'component_type': 'docker'},
137 'streams': {'publishes': [],
138 'subscribes': [{'format': 'std.format_two',
142 'services': {'calls': [],
143 'provides': [{'request': {'format': 'std.format_one',
145 'response': {'format': 'std.format_two',
147 'route': '/prov1'}]},
149 'artifacts': [{ "uri": "bazinga-image", "type": "docker image" }],
153 "endpoint": "/health",
163 "name": "std.format_one",
167 "dataformatversion": "1.0.0",
169 "$schema": "http://json-schema.org/draft-04/schema#",
176 "required": ["raw-text"],
177 "additionalProperties": False
182 "name": "std.format_two",
186 "dataformatversion": "1.0.0",
188 "$schema": "http://json-schema.org/draft-04/schema#",
195 "required": ["raw-text"],
196 "additionalProperties": False
201 "name": "std.format_two",
205 "dataformatversion": "1.0.0",
207 "$schema": "http://json-schema.org/draft-04/schema#",
214 "required": ["raw-text"],
215 "additionalProperties": False
221 "name":"std.cdap_comp",
223 "description":"cdap test component",
224 "component_type":"cdap"
229 "format":"std.format_one",
237 "format":"std.format_two",
251 "format":"std.format_one",
255 "format":"std.format_two",
258 "service_name":"baphomet",
259 "service_endpoint":"rises",
266 "app_preferences" : [],
267 "program_preferences" : []
269 "artifacts": [{"uri": "bahpomet.com", "type": "jar"}],
271 "streamname":"streamname",
272 "artifact_version":"6.6.6",
273 "artifact_name": "test_name",
274 "programs" : [{"program_type" : "flows", "program_id" : "flow_id"}]
280 def test_component_basic(catalog=None):
281 '''Tests basic component usage of MockCatalog'''
283 mc = MockCatalog(db_name='dcae_cli.test.db', purge_existing=True, enforce_image=False)
287 c1_spec = deepcopy(_c1_spec)
288 df1_spec = deepcopy(_df1_spec)
289 df2_spec = deepcopy(_df2_spec)
291 user = "test_component_basic"
294 mc.add_format(df2_spec, user)
297 with pytest.raises(DuplicateEntry):
298 mc.add_format(df2_spec, user)
300 # component relies on df1_spec which hasn't been added
301 with pytest.raises(MissingEntry):
302 mc.add_component(user, c1_spec)
305 mc.add_format(df1_spec, user)
306 mc.add_component(user, c1_spec)
308 with pytest.raises(DuplicateEntry):
309 mc.add_component(user, c1_spec)
311 cname, cver = mc.verify_component('std.comp_one', version=None)
312 assert cver == '1.0.0'
315 def test_format_basic(catalog=None):
316 '''Tests basic data format usage of MockCatalog'''
318 mc = MockCatalog(db_name='dcae_cli.test.db', purge_existing=True)
322 user = "test_format_basic"
324 df1_spec = deepcopy(_df1_spec)
325 df2_spec = deepcopy(_df2_spec)
328 mc.add_format(df1_spec, user)
331 with pytest.raises(DuplicateEntry):
332 mc.add_format(df1_spec, user)
334 # allow update of same version
335 new_descr = 'a new description'
336 df1_spec['self']['description'] = new_descr
337 mc.add_format(df1_spec, user, update=True)
339 # adding a new version is kosher
341 df1_spec['self']['version'] = new_ver
342 mc.add_format(df1_spec, user)
344 # can't update a format that doesn't exist
345 with pytest.raises(MissingEntry):
346 mc.add_format(df2_spec, user, update=True)
348 # get spec and make sure it's updated
349 spec = mc.get_format_spec(df1_spec['self']['name'], version=None)
350 assert spec['self']['version'] == new_ver
351 assert spec['self']['description'] == new_descr
354 def test_discovery(catalog=None):
355 '''Tests creation of discovery objects'''
357 mc = MockCatalog(db_name='dcae_cli.test.db', purge_existing=True, enforce_image=False)
361 user = "test_discovery"
363 c1_spec = deepcopy(_c1_spec)
364 df1_spec = deepcopy(_df1_spec)
365 c2_spec = deepcopy(_c2_spec)
367 mc.add_format(df1_spec, user)
368 mc.add_component(user, c1_spec)
369 mc.add_component(user, c2_spec)
371 params, interfaces = mc.get_discovery_for_docker(c1_spec['self']['name'], c1_spec['self']['version'])
372 assert params == {'bar': 2, 'foo': 1}
373 assert interfaces == {'call1': [('std.comp_two', '1.0.0')], 'pub1': [('std.comp_two', '1.0.0')]}
377 '''Returns a (name, version, component type) tuple from a given component spec dict'''
378 return dd['self']['name'], dd['self']['version'], dd['self']['component_type']
381 def _comp_tuple_set(*dds):
382 '''Runs a set of component spec tuples'''
383 return set(map(_spec_tuple, dds))
386 def _format_tuple(dd):
387 '''Returns a (name, version) tuple from a given data format spec dict'''
388 return dd['self']['name'], dd['self']['version']
391 def _format_tuple_set(*dds):
392 '''Runs a set of data format spec tuples'''
393 return set(map(_format_tuple, dds))
396 def test_comp_list(catalog=None):
397 '''Tests the list functionality of the catalog'''
399 mc = MockCatalog(db_name='dcae_cli.test.db', purge_existing=True, enforce_image=False)
403 user = "test_comp_list"
405 df1_spec = deepcopy(_df1_spec)
406 df2_spec = deepcopy(_df2_spec)
407 df2v2_spec = deepcopy(_df2v2_spec)
409 c1_spec = deepcopy(_c1_spec)
410 c2_spec = deepcopy(_c2_spec)
411 c2v2_spec = deepcopy(_c2v2_spec)
412 c3_spec = deepcopy(_c3_spec)
414 mc.add_format(df1_spec, user)
415 mc.add_format(df2_spec, user)
416 mc.add_format(df2v2_spec, user)
417 mc.add_component(user, c1_spec)
418 mc.add_component(user, c2_spec)
419 mc.add_component(user, c2v2_spec)
420 mc.add_component(user, c3_spec)
422 mc.add_component(user,_cdap_spec)
424 def components_to_specs(components):
425 return [ json.loads(c["spec"]) for c in components ]
427 # latest by default. only v2 of c2
428 components = mc.list_components()
429 specs = components_to_specs(components)
430 assert _comp_tuple_set(*specs) == _comp_tuple_set(c1_spec, c2v2_spec, c3_spec, _cdap_spec)
433 components = mc.list_components(latest=False)
434 specs = components_to_specs(components)
435 assert _comp_tuple_set(*specs) == _comp_tuple_set(c1_spec, c2_spec, c2v2_spec, c3_spec, _cdap_spec)
437 components = mc.list_components(subscribes=[('std.format_one', None)])
438 specs = components_to_specs(components)
439 assert _comp_tuple_set(*specs) == _comp_tuple_set(c1_spec, c2v2_spec)
441 # no comps subscribe to latest std.format_two
442 components = mc.list_components(subscribes=[('std.format_two', None)])
443 assert not components
445 components = mc.list_components(subscribes=[('std.format_two', '1.5.0')])
446 specs = components_to_specs(components)
447 assert _comp_tuple_set(*specs) == _comp_tuple_set(c3_spec, _cdap_spec)
449 # raise if format doesn't exist
450 with pytest.raises(MissingEntry):
451 mc.list_components(subscribes=[('std.format_two', '5.0.0')])
453 components = mc.list_components(publishes=[('std.format_one', None)])
454 specs = components_to_specs(components)
455 assert _comp_tuple_set(*specs) == _comp_tuple_set(c1_spec, _cdap_spec)
457 components = mc.list_components(calls=[(('std.format_one', None), ('std.format_one', None)), ])
458 specs = components_to_specs(components)
459 assert _comp_tuple_set(*specs) == _comp_tuple_set(c1_spec)
461 # raise if format doesn't exist
462 with pytest.raises(MissingEntry):
463 mc.list_components(calls=[(('std.format_one', '5.0.0'), ('std.format_one', None)), ])
465 components = mc.list_components(provides=[(('std.format_one', '1.0.0'), ('std.format_two', '1.5.0')), ])
466 specs = components_to_specs(components)
467 assert _comp_tuple_set(*specs) == _comp_tuple_set(c3_spec, _cdap_spec)
469 # test for listing published components
471 name_pub = c1_spec["self"]["name"]
472 version_pub = c1_spec["self"]["version"]
473 mc.publish_component(user, name_pub, version_pub)
474 components = mc.list_components(only_published=True)
475 specs = components_to_specs(components)
476 assert _comp_tuple_set(*specs) == _comp_tuple_set(c1_spec)
478 components = mc.list_components(only_published=False)
479 assert len(components) == 4
482 def test_format_list(catalog=None):
483 '''Tests the list functionality of the catalog'''
485 mc = MockCatalog(db_name='dcae_cli.test.db', purge_existing=True, enforce_image=False)
489 user = "test_format_list"
491 df1_spec = deepcopy(_df1_spec)
492 df2_spec = deepcopy(_df2_spec)
493 df2v2_spec = deepcopy(_df2v2_spec)
495 mc.add_format(df1_spec, user)
496 mc.add_format(df2_spec, user)
497 mc.add_format(df2v2_spec, user)
499 def formats_to_specs(components):
500 return [ json.loads(c["spec"]) for c in components ]
502 # latest by default. ensure only v2 of df2 makes it
503 formats = mc.list_formats()
504 specs = formats_to_specs(formats)
505 assert _format_tuple_set(*specs) == _format_tuple_set(df1_spec, df2v2_spec)
508 formats = mc.list_formats(latest=False)
509 specs = formats_to_specs(formats)
510 assert _format_tuple_set(*specs) == _format_tuple_set(df1_spec, df2_spec, df2v2_spec)
512 # test listing of published formats
514 name_pub = df1_spec["self"]["name"]
515 version_pub = df1_spec["self"]["version"]
517 mc.publish_format(user, name_pub, version_pub)
518 formats = mc.list_formats(only_published=True)
519 specs = formats_to_specs(formats)
520 assert _format_tuple_set(*specs) == _format_tuple_set(df1_spec)
522 formats = mc.list_formats(only_published=False)
523 assert len(formats) == 2
526 def test_component_add_cdap(catalog=None):
527 '''Adds a mock CDAP application'''
529 mc = MockCatalog(db_name='dcae_cli.test.db', purge_existing=True)
533 user = "test_component_add_cdap"
535 df1_spec = deepcopy(_df1_spec)
536 df2_spec = deepcopy(_df2_spec)
538 mc.add_format(df1_spec, user)
539 mc.add_format(df2_spec, user)
541 mc.add_component(user, _cdap_spec)
543 name, version, _ = _spec_tuple(_cdap_spec)
544 jar_out, cdap_config_out, spec_out = mc.get_cdap(name, version)
546 assert _cdap_spec["artifacts"][0]["uri"] == jar_out
547 assert _cdap_spec["auxilary"] == cdap_config_out
548 assert _cdap_spec == spec_out
551 def test_get_discovery_from_spec():
552 mc = MockCatalog(db_name='dcae_cli.test.db', purge_existing=True,
555 user = "test_get_discovery_from_spec"
557 c1_spec_updated = deepcopy(_c1_spec)
558 c1_spec_updated["streams"]["publishes"][0] = {
559 'format': 'std.format_one',
561 'config_key': 'pub1',
564 c1_spec_updated["streams"]["subscribes"][0] = {
565 'format': 'std.format_one',
571 # Case when c1 doesn't exist
573 mc.add_format(_df1_spec, user)
574 mc.add_component(user, _c2_spec)
575 actual_params, actual_interface_map, actual_dmaap_config_keys \
576 = mc.get_discovery_from_spec(user, c1_spec_updated, None)
578 assert actual_params == {'bar': 2, 'foo': 1}
579 assert actual_interface_map == { 'pub1': [('std.comp_two', '1.0.0')],
580 'call1': [('std.comp_two', '1.0.0')] }
581 assert actual_dmaap_config_keys == ([], [])
583 # Case when c1 already exist
585 mc.add_component(user,_c1_spec)
587 c1_spec_updated["services"]["calls"][0]["config_key"] = "callme"
588 actual_params, actual_interface_map, actual_dmaap_config_keys \
589 = mc.get_discovery_from_spec(user, c1_spec_updated, None)
591 assert actual_params == {'bar': 2, 'foo': 1}
592 assert actual_interface_map == { 'pub1': [('std.comp_two', '1.0.0')],
593 'callme': [('std.comp_two', '1.0.0')] }
594 assert actual_dmaap_config_keys == ([], [])
596 # Case where add in dmaap streams
597 # TODO: Add in subscribes test case after spec gets pushed
599 c1_spec_updated["streams"]["publishes"][0] = {
600 'format': 'std.format_one',
602 'config_key': 'pub1',
603 'type': 'message router'
606 actual_params, actual_interface_map, actual_dmaap_config_keys \
607 = mc.get_discovery_from_spec(user, c1_spec_updated, None)
609 assert actual_params == {'bar': 2, 'foo': 1}
610 assert actual_interface_map == { 'callme': [('std.comp_two', '1.0.0')] }
611 assert actual_dmaap_config_keys == (["pub1"], [])
613 # Case when cdap spec doesn't exist
615 cdap_spec = deepcopy(_cdap_spec)
616 cdap_spec["streams"]["publishes"][0] = {
617 'format': 'std.format_one',
619 'config_key': 'pub1',
622 cdap_spec["streams"]["subscribes"][0] = {
623 'format': 'std.format_two',
629 mc.add_format(_df2_spec, user)
630 actual_params, actual_interface_map, actual_dmaap_config_keys \
631 = mc.get_discovery_from_spec(user, cdap_spec, None)
633 assert actual_params == {'program_preferences': [], 'app_config': {}, 'app_preferences': {}}
634 assert actual_interface_map == {'pub1': [('std.comp_two', '1.0.0'), ('std.comp_one', '1.0.0')]}
635 assert actual_dmaap_config_keys == ([], [])
638 def test_get_unpublished_formats(catalog=None):
640 mc = MockCatalog(db_name='dcae_cli.test.db', purge_existing=True, enforce_image=False)
644 user = "test_get_unpublished_formats"
646 mc.add_format(_df1_spec, user)
647 mc.add_component(user, _c1_spec)
649 # detect unpublished formats
651 name_to_pub = _c1_spec["self"]["name"]
652 version_to_pub = _c1_spec["self"]["version"]
653 formats = mc.get_unpublished_formats(name_to_pub, version_to_pub)
654 assert [('std.format_one', '1.0.0')] == formats
656 # all formats published
658 mc.publish_format(user, _df1_spec["self"]["name"], _df1_spec["self"]["version"])
659 formats = mc.get_unpublished_formats(name_to_pub, version_to_pub)
660 assert len(formats) == 0
663 def test_get_unique_format_things():
664 def create_tuple(entry):
665 return (entry["name"], entry["version"])
667 def get_orm(name, version):
668 return ("ORM", name, version)
670 entries = [{"name": "abc", "version": 123},
671 {"name": "abc", "version": 123},
672 {"name": "abc", "version": 123},
673 {"name": "def", "version": 456},
674 {"name": "def", "version": 456}]
676 get_unique_fake_format = partial(_get_unique_format_things, create_tuple,
678 expected = [("ORM", "abc", 123), ("ORM", "def", 456)]
680 assert sorted(expected) == sorted(get_unique_fake_format(entries))
683 def test_filter_latest():
684 orms = [('std.empty.get', '1.0.0'), ('std.unknown', '1.0.0'),
685 ('std.unknown', '1.0.1'), ('std.empty.get', '1.0.1')]
687 assert list(catalog._filter_latest(orms)) == [('std.empty.get', '1.0.1'), \
688 ('std.unknown', '1.0.1')]
691 def test_raise_if_duplicate():
692 class FakeOrig(object):
693 args = ["unique", "duplicate"]
697 error = IntegrityError("Error about uniqueness", None, orig)
699 with pytest.raises(catalog.DuplicateEntry):
700 catalog._raise_if_duplicate(url, error)
702 # Couldn't find psycopg2.IntegrityError constructor nor way
703 # to set pgcode so decided to mock it.
704 class FakeOrigPostgres(object):
708 orig = FakeOrigPostgres()
709 error = IntegrityError("Error about uniqueness", None, orig)
711 with pytest.raises(catalog.DuplicateEntry):
712 catalog._raise_if_duplicate(url, error)
715 def test_get_docker_image_from_spec():
716 assert "foo-image" == catalog._get_docker_image_from_spec(_c1_spec)
718 def test_get_cdap_jar_from_spec():
719 assert "bahpomet.com" == catalog._get_cdap_jar_from_spec(_cdap_spec)
722 def test_build_config_keys_map():
726 {'format': 'std.format_one', 'version': '1.0.0',
727 'config_key': 'pub1', 'type': 'http'},
728 {'format': 'std.format_one', 'version': '1.0.0',
729 'config_key': 'pub2', 'type': 'message_router'}
732 {'format': 'std.format_one', 'version': '1.0.0', 'route': '/sub1',
734 {'format': 'std.format_one', 'version': '1.0.0',
735 'config_key': 'sub2', 'type': 'message_router'}
740 {'request': {'format': 'std.format_one', 'version': '1.0.0'},
741 'response': {'format': 'std.format_one', 'version': '1.0.0'},
742 'config_key': 'call1'}
745 {'request': {'format': 'std.format_one', 'version': '1.0.0'},
746 'response': {'format': 'std.format_one', 'version': '1.0.0'},
752 grouping = catalog.build_config_keys_map(stub_spec)
753 expected = {'call1': {'group': 'services_calls'}, 'pub1': {'type': 'http', 'group': 'streams_publishes'}, 'sub2': {'type': 'message_router', 'group': 'streams_subscribes'}, 'pub2': {'type': 'message_router', 'group': 'streams_publishes'}}
754 assert expected == grouping
757 def test_get_data_router_subscriber_route():
758 spec = {"streams": {"subscribes": [ { "type": "data_router", "config_key":
759 "alpha", "route": "/alpha" }, { "type": "message_router", "config_key":
762 assert "/alpha" == catalog.get_data_router_subscriber_route(spec, "alpha")
764 with pytest.raises(catalog.MissingEntry):
765 catalog.get_data_router_subscriber_route(spec, "beta")
767 with pytest.raises(catalog.MissingEntry):
768 catalog.get_data_router_subscriber_route(spec, "gamma")
771 if __name__ == '__main__':
773 pytest.main([__file__, ])