1 # ============LICENSE_START=======================================================
3 # ================================================================================
4 # Copyright (c) 2017-2018 AT&T Intellectual Property. All rights reserved.
5 # ================================================================================
6 # Licensed under the Apache License, Version 2.0 (the "License");
7 # you may not use this file except in compliance with the License.
8 # You may obtain a copy of the License at
10 # http://www.apache.org/licenses/LICENSE-2.0
12 # Unless required by applicable law or agreed to in writing, software
13 # distributed under the License is distributed on an "AS IS" BASIS,
14 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 # See the License for the specific language governing permissions and
16 # limitations under the License.
17 # ============LICENSE_END=========================================================
19 # ECOMP is a trademark and service mark of AT&T Intellectual Property.
21 # -*- coding: utf-8 -*-
23 Tests the mock catalog
26 from copy import deepcopy
27 from functools import partial
31 from sqlalchemy.exc import IntegrityError
33 from dcae_cli.catalog.mock.catalog import MockCatalog, MissingEntry, DuplicateEntry, _get_unique_format_things
34 from dcae_cli.catalog.mock import catalog
37 _c1_spec = {'self': {'name': 'std.comp_one',
39 'description': 'comp1',
40 'component_type': 'docker'},
41 'streams': {'publishes': [{'format': 'std.format_one',
45 'subscribes': [{'format': 'std.format_one',
49 'services': {'calls': [{'request': {'format': 'std.format_one',
51 'response': {'format': 'std.format_one',
53 'config_key': 'call1'}],
54 'provides': [{'request': {'format': 'std.format_one',
56 'response': {'format': 'std.format_one',
59 'parameters': [{"name": "foo",
61 "description": "the foo thing",
62 "designer_editable": False,
63 "sourced_at_deployment": False,
64 "policy_editable": False},
67 "description": "the bar thing",
68 "designer_editable": False,
69 "sourced_at_deployment": False,
70 "policy_editable": False}
72 'artifacts': [{ "uri": "foo-image", "type": "docker image" }],
76 "endpoint": "/health",
83 _c2_spec = {'self': {'name': 'std.comp_two',
85 'description': 'comp2',
86 'component_type': 'docker'},
87 'streams': {'publishes': [],
88 'subscribes': [{'format': 'std.format_one',
92 'services': {'calls': [],
93 'provides': [{'request': {'format': 'std.format_one',
95 'response': {'format': 'std.format_one',
99 'artifacts': [{ "uri": "bar-image", "type": "docker image" }],
103 "endpoint": "/health",
111 _c2v2_spec = {'self': {'name': 'std.comp_two',
113 'description': 'comp2',
114 'component_type': 'docker'},
115 'streams': {'publishes': [],
116 'subscribes': [{'format': 'std.format_one',
120 'services': {'calls': [],
121 'provides': [{'request': {'format': 'std.format_one',
123 'response': {'format': 'std.format_one',
125 'route': '/prov1'}]},
127 'artifacts': [{ "uri": "baz-image", "type": "docker image" }],
131 "endpoint": "/health",
139 _c3_spec = {'self': {'name': 'std.comp_three',
141 'description': 'comp3',
142 'component_type': 'docker'},
143 'streams': {'publishes': [],
144 'subscribes': [{'format': 'std.format_two',
148 'services': {'calls': [],
149 'provides': [{'request': {'format': 'std.format_one',
151 'response': {'format': 'std.format_two',
153 'route': '/prov1'}]},
155 'artifacts': [{ "uri": "bazinga-image", "type": "docker image" }],
159 "endpoint": "/health",
169 "name": "std.format_one",
173 "dataformatversion": "1.0.0",
175 "$schema": "http://json-schema.org/draft-04/schema#",
182 "required": ["raw-text"],
183 "additionalProperties": False
188 "name": "std.format_two",
192 "dataformatversion": "1.0.0",
194 "$schema": "http://json-schema.org/draft-04/schema#",
201 "required": ["raw-text"],
202 "additionalProperties": False
207 "name": "std.format_two",
211 "dataformatversion": "1.0.0",
213 "$schema": "http://json-schema.org/draft-04/schema#",
220 "required": ["raw-text"],
221 "additionalProperties": False
227 "name":"std.cdap_comp",
229 "description":"cdap test component",
230 "component_type":"cdap"
235 "format":"std.format_one",
243 "format":"std.format_two",
257 "format":"std.format_one",
261 "format":"std.format_two",
264 "service_name":"baphomet",
265 "service_endpoint":"rises",
272 "app_preferences" : [],
273 "program_preferences" : []
275 "artifacts": [{"uri": "bahpomet.com", "type": "jar"}],
277 "streamname":"streamname",
278 "artifact_version":"6.6.6",
279 "artifact_name": "test_name",
280 "programs" : [{"program_type" : "flows", "program_id" : "flow_id"}]
286 def test_component_basic(mock_cli_config, mock_db_url, catalog=None):
287 '''Tests basic component usage of MockCatalog'''
289 mc = MockCatalog(db_name='dcae_cli.test.db', purge_existing=True,
290 enforce_image=False, db_url=mock_db_url)
294 c1_spec = deepcopy(_c1_spec)
295 df1_spec = deepcopy(_df1_spec)
296 df2_spec = deepcopy(_df2_spec)
298 user = "test_component_basic"
301 mc.add_format(df2_spec, user)
304 with pytest.raises(DuplicateEntry):
305 mc.add_format(df2_spec, user)
307 # component relies on df1_spec which hasn't been added
308 with pytest.raises(MissingEntry):
309 mc.add_component(user, c1_spec)
312 mc.add_format(df1_spec, user)
313 mc.add_component(user, c1_spec)
315 with pytest.raises(DuplicateEntry):
316 mc.add_component(user, c1_spec)
318 cname, cver = mc.verify_component('std.comp_one', version=None)
319 assert cver == '1.0.0'
322 def test_format_basic(mock_cli_config, mock_db_url, catalog=None):
323 '''Tests basic data format usage of MockCatalog'''
325 mc = MockCatalog(db_name='dcae_cli.test.db', purge_existing=True,
330 user = "test_format_basic"
332 df1_spec = deepcopy(_df1_spec)
333 df2_spec = deepcopy(_df2_spec)
336 mc.add_format(df1_spec, user)
339 with pytest.raises(DuplicateEntry):
340 mc.add_format(df1_spec, user)
342 # allow update of same version
343 new_descr = 'a new description'
344 df1_spec['self']['description'] = new_descr
345 mc.add_format(df1_spec, user, update=True)
347 # adding a new version is kosher
349 df1_spec['self']['version'] = new_ver
350 mc.add_format(df1_spec, user)
352 # can't update a format that doesn't exist
353 with pytest.raises(MissingEntry):
354 mc.add_format(df2_spec, user, update=True)
356 # get spec and make sure it's updated
357 spec = mc.get_format_spec(df1_spec['self']['name'], version=None)
358 assert spec['self']['version'] == new_ver
359 assert spec['self']['description'] == new_descr
362 def test_discovery(mock_cli_config, mock_db_url, catalog=None):
363 '''Tests creation of discovery objects'''
365 mc = MockCatalog(db_name='dcae_cli.test.db', purge_existing=True,
366 enforce_image=False, db_url=mock_db_url)
370 user = "test_discovery"
372 c1_spec = deepcopy(_c1_spec)
373 df1_spec = deepcopy(_df1_spec)
374 c2_spec = deepcopy(_c2_spec)
376 mc.add_format(df1_spec, user)
377 mc.add_component(user, c1_spec)
378 mc.add_component(user, c2_spec)
380 params, interfaces = mc.get_discovery_for_docker(c1_spec['self']['name'], c1_spec['self']['version'])
381 assert params == {'bar': 2, 'foo': 1}
382 assert interfaces == {'call1': [('std.comp_two', '1.0.0')], 'pub1': [('std.comp_two', '1.0.0')]}
386 '''Returns a (name, version, component type) tuple from a given component spec dict'''
387 return dd['self']['name'], dd['self']['version'], dd['self']['component_type']
390 def _comp_tuple_set(*dds):
391 '''Runs a set of component spec tuples'''
392 return set(map(_spec_tuple, dds))
395 def _format_tuple(dd):
396 '''Returns a (name, version) tuple from a given data format spec dict'''
397 return dd['self']['name'], dd['self']['version']
400 def _format_tuple_set(*dds):
401 '''Runs a set of data format spec tuples'''
402 return set(map(_format_tuple, dds))
405 def test_comp_list(mock_cli_config, mock_db_url, catalog=None):
406 '''Tests the list functionality of the catalog'''
408 mc = MockCatalog(db_name='dcae_cli.test.db', purge_existing=True,
409 enforce_image=False, db_url=mock_db_url)
413 user = "test_comp_list"
415 df1_spec = deepcopy(_df1_spec)
416 df2_spec = deepcopy(_df2_spec)
417 df2v2_spec = deepcopy(_df2v2_spec)
419 c1_spec = deepcopy(_c1_spec)
420 c2_spec = deepcopy(_c2_spec)
421 c2v2_spec = deepcopy(_c2v2_spec)
422 c3_spec = deepcopy(_c3_spec)
424 mc.add_format(df1_spec, user)
425 mc.add_format(df2_spec, user)
426 mc.add_format(df2v2_spec, user)
427 mc.add_component(user, c1_spec)
428 mc.add_component(user, c2_spec)
429 mc.add_component(user, c2v2_spec)
430 mc.add_component(user, c3_spec)
432 mc.add_component(user,_cdap_spec)
434 def components_to_specs(components):
435 return [ json.loads(c["spec"]) for c in components ]
437 # latest by default. only v2 of c2
438 components = mc.list_components()
439 specs = components_to_specs(components)
440 assert _comp_tuple_set(*specs) == _comp_tuple_set(c1_spec, c2v2_spec, c3_spec, _cdap_spec)
443 components = mc.list_components(latest=False)
444 specs = components_to_specs(components)
445 assert _comp_tuple_set(*specs) == _comp_tuple_set(c1_spec, c2_spec, c2v2_spec, c3_spec, _cdap_spec)
447 components = mc.list_components(subscribes=[('std.format_one', None)])
448 specs = components_to_specs(components)
449 assert _comp_tuple_set(*specs) == _comp_tuple_set(c1_spec, c2v2_spec)
451 # no comps subscribe to latest std.format_two
452 components = mc.list_components(subscribes=[('std.format_two', None)])
453 assert not components
455 components = mc.list_components(subscribes=[('std.format_two', '1.5.0')])
456 specs = components_to_specs(components)
457 assert _comp_tuple_set(*specs) == _comp_tuple_set(c3_spec, _cdap_spec)
459 # raise if format doesn't exist
460 with pytest.raises(MissingEntry):
461 mc.list_components(subscribes=[('std.format_two', '5.0.0')])
463 components = mc.list_components(publishes=[('std.format_one', None)])
464 specs = components_to_specs(components)
465 assert _comp_tuple_set(*specs) == _comp_tuple_set(c1_spec, _cdap_spec)
467 components = mc.list_components(calls=[(('std.format_one', None), ('std.format_one', None)), ])
468 specs = components_to_specs(components)
469 assert _comp_tuple_set(*specs) == _comp_tuple_set(c1_spec)
471 # raise if format doesn't exist
472 with pytest.raises(MissingEntry):
473 mc.list_components(calls=[(('std.format_one', '5.0.0'), ('std.format_one', None)), ])
475 components = mc.list_components(provides=[(('std.format_one', '1.0.0'), ('std.format_two', '1.5.0')), ])
476 specs = components_to_specs(components)
477 assert _comp_tuple_set(*specs) == _comp_tuple_set(c3_spec, _cdap_spec)
479 # test for listing published components
481 name_pub = c1_spec["self"]["name"]
482 version_pub = c1_spec["self"]["version"]
483 mc.publish_component(user, name_pub, version_pub)
484 components = mc.list_components(only_published=True)
485 specs = components_to_specs(components)
486 assert _comp_tuple_set(*specs) == _comp_tuple_set(c1_spec)
488 components = mc.list_components(only_published=False)
489 assert len(components) == 4
492 def test_format_list(mock_cli_config, mock_db_url, catalog=None):
493 '''Tests the list functionality of the catalog'''
495 mc = MockCatalog(db_name='dcae_cli.test.db', purge_existing=True,
496 enforce_image=False, db_url=mock_db_url)
500 user = "test_format_list"
502 df1_spec = deepcopy(_df1_spec)
503 df2_spec = deepcopy(_df2_spec)
504 df2v2_spec = deepcopy(_df2v2_spec)
506 mc.add_format(df1_spec, user)
507 mc.add_format(df2_spec, user)
508 mc.add_format(df2v2_spec, user)
510 def formats_to_specs(components):
511 return [ json.loads(c["spec"]) for c in components ]
513 # latest by default. ensure only v2 of df2 makes it
514 formats = mc.list_formats()
515 specs = formats_to_specs(formats)
516 assert _format_tuple_set(*specs) == _format_tuple_set(df1_spec, df2v2_spec)
519 formats = mc.list_formats(latest=False)
520 specs = formats_to_specs(formats)
521 assert _format_tuple_set(*specs) == _format_tuple_set(df1_spec, df2_spec, df2v2_spec)
523 # test listing of published formats
525 name_pub = df1_spec["self"]["name"]
526 version_pub = df1_spec["self"]["version"]
528 mc.publish_format(user, name_pub, version_pub)
529 formats = mc.list_formats(only_published=True)
530 specs = formats_to_specs(formats)
531 assert _format_tuple_set(*specs) == _format_tuple_set(df1_spec)
533 formats = mc.list_formats(only_published=False)
534 assert len(formats) == 2
537 def test_component_add_cdap(mock_cli_config, mock_db_url, catalog=None):
538 '''Adds a mock CDAP application'''
540 mc = MockCatalog(db_name='dcae_cli.test.db', purge_existing=True,
545 user = "test_component_add_cdap"
547 df1_spec = deepcopy(_df1_spec)
548 df2_spec = deepcopy(_df2_spec)
550 mc.add_format(df1_spec, user)
551 mc.add_format(df2_spec, user)
553 mc.add_component(user, _cdap_spec)
555 name, version, _ = _spec_tuple(_cdap_spec)
556 jar_out, cdap_config_out, spec_out = mc.get_cdap(name, version)
558 assert _cdap_spec["artifacts"][0]["uri"] == jar_out
559 assert _cdap_spec["auxilary"] == cdap_config_out
560 assert _cdap_spec == spec_out
563 def test_get_discovery_from_spec(mock_cli_config, mock_db_url):
564 mc = MockCatalog(db_name='dcae_cli.test.db', purge_existing=True,
565 enforce_image=False, db_url=mock_db_url)
567 user = "test_get_discovery_from_spec"
569 c1_spec_updated = deepcopy(_c1_spec)
570 c1_spec_updated["streams"]["publishes"][0] = {
571 'format': 'std.format_one',
573 'config_key': 'pub1',
576 c1_spec_updated["streams"]["subscribes"][0] = {
577 'format': 'std.format_one',
583 # Case when c1 doesn't exist
585 mc.add_format(_df1_spec, user)
586 mc.add_component(user, _c2_spec)
587 actual_params, actual_interface_map, actual_dmaap_config_keys \
588 = mc.get_discovery_from_spec(user, c1_spec_updated, None)
590 assert actual_params == {'bar': 2, 'foo': 1}
591 assert actual_interface_map == { 'pub1': [('std.comp_two', '1.0.0')],
592 'call1': [('std.comp_two', '1.0.0')] }
593 assert actual_dmaap_config_keys == ([], [])
595 # Case when c1 already exist
597 mc.add_component(user,_c1_spec)
599 c1_spec_updated["services"]["calls"][0]["config_key"] = "callme"
600 actual_params, actual_interface_map, actual_dmaap_config_keys \
601 = mc.get_discovery_from_spec(user, c1_spec_updated, None)
603 assert actual_params == {'bar': 2, 'foo': 1}
604 assert actual_interface_map == { 'pub1': [('std.comp_two', '1.0.0')],
605 'callme': [('std.comp_two', '1.0.0')] }
606 assert actual_dmaap_config_keys == ([], [])
608 # Case where add in dmaap streams
609 # TODO: Add in subscribes test case after spec gets pushed
611 c1_spec_updated["streams"]["publishes"][0] = {
612 'format': 'std.format_one',
614 'config_key': 'pub1',
615 'type': 'message router'
618 actual_params, actual_interface_map, actual_dmaap_config_keys \
619 = mc.get_discovery_from_spec(user, c1_spec_updated, None)
621 assert actual_params == {'bar': 2, 'foo': 1}
622 assert actual_interface_map == { 'callme': [('std.comp_two', '1.0.0')] }
623 assert actual_dmaap_config_keys == (["pub1"], [])
625 # Case when cdap spec doesn't exist
627 cdap_spec = deepcopy(_cdap_spec)
628 cdap_spec["streams"]["publishes"][0] = {
629 'format': 'std.format_one',
631 'config_key': 'pub1',
634 cdap_spec["streams"]["subscribes"][0] = {
635 'format': 'std.format_two',
641 mc.add_format(_df2_spec, user)
642 actual_params, actual_interface_map, actual_dmaap_config_keys \
643 = mc.get_discovery_from_spec(user, cdap_spec, None)
645 assert actual_params == {'program_preferences': [], 'app_config': {}, 'app_preferences': {}}
646 assert actual_interface_map == {'pub1': [('std.comp_two', '1.0.0'), ('std.comp_one', '1.0.0')]}
647 assert actual_dmaap_config_keys == ([], [])
650 def test_get_unpublished_formats(mock_cli_config, mock_db_url, catalog=None):
652 mc = MockCatalog(db_name='dcae_cli.test.db', purge_existing=True,
653 enforce_image=False, db_url=mock_db_url)
657 user = "test_get_unpublished_formats"
659 mc.add_format(_df1_spec, user)
660 mc.add_component(user, _c1_spec)
662 # detect unpublished formats
664 name_to_pub = _c1_spec["self"]["name"]
665 version_to_pub = _c1_spec["self"]["version"]
666 formats = mc.get_unpublished_formats(name_to_pub, version_to_pub)
667 assert [('std.format_one', '1.0.0')] == formats
669 # all formats published
671 mc.publish_format(user, _df1_spec["self"]["name"], _df1_spec["self"]["version"])
672 formats = mc.get_unpublished_formats(name_to_pub, version_to_pub)
673 assert len(formats) == 0
676 def test_get_unique_format_things():
677 def create_tuple(entry):
678 return (entry["name"], entry["version"])
680 def get_orm(name, version):
681 return ("ORM", name, version)
683 entries = [{"name": "abc", "version": 123},
684 {"name": "abc", "version": 123},
685 {"name": "abc", "version": 123},
686 {"name": "def", "version": 456},
687 {"name": "def", "version": 456}]
689 get_unique_fake_format = partial(_get_unique_format_things, create_tuple,
691 expected = [("ORM", "abc", 123), ("ORM", "def", 456)]
693 assert sorted(expected) == sorted(get_unique_fake_format(entries))
696 def test_filter_latest():
697 orms = [('std.empty.get', '1.0.0'), ('std.unknown', '1.0.0'),
698 ('std.unknown', '1.0.1'), ('std.empty.get', '1.0.1')]
700 assert list(catalog._filter_latest(orms)) == [('std.empty.get', '1.0.1'), \
701 ('std.unknown', '1.0.1')]
704 def test_raise_if_duplicate():
705 class FakeOrig(object):
706 args = ["unique", "duplicate"]
710 error = IntegrityError("Error about uniqueness", None, orig)
712 with pytest.raises(catalog.DuplicateEntry):
713 catalog._raise_if_duplicate(url, error)
715 # Couldn't find psycopg2.IntegrityError constructor nor way
716 # to set pgcode so decided to mock it.
717 class FakeOrigPostgres(object):
721 orig = FakeOrigPostgres()
722 error = IntegrityError("Error about uniqueness", None, orig)
724 with pytest.raises(catalog.DuplicateEntry):
725 catalog._raise_if_duplicate(url, error)
728 def test_get_docker_image_from_spec():
729 assert "foo-image" == catalog._get_docker_image_from_spec(_c1_spec)
731 def test_get_cdap_jar_from_spec():
732 assert "bahpomet.com" == catalog._get_cdap_jar_from_spec(_cdap_spec)
735 def test_build_config_keys_map():
739 {'format': 'std.format_one', 'version': '1.0.0',
740 'config_key': 'pub1', 'type': 'http'},
741 {'format': 'std.format_one', 'version': '1.0.0',
742 'config_key': 'pub2', 'type': 'message_router'}
745 {'format': 'std.format_one', 'version': '1.0.0', 'route': '/sub1',
747 {'format': 'std.format_one', 'version': '1.0.0',
748 'config_key': 'sub2', 'type': 'message_router'}
753 {'request': {'format': 'std.format_one', 'version': '1.0.0'},
754 'response': {'format': 'std.format_one', 'version': '1.0.0'},
755 'config_key': 'call1'}
758 {'request': {'format': 'std.format_one', 'version': '1.0.0'},
759 'response': {'format': 'std.format_one', 'version': '1.0.0'},
765 grouping = catalog.build_config_keys_map(stub_spec)
766 expected = {'call1': {'group': 'services_calls'}, 'pub1': {'type': 'http', 'group': 'streams_publishes'}, 'sub2': {'type': 'message_router', 'group': 'streams_subscribes'}, 'pub2': {'type': 'message_router', 'group': 'streams_publishes'}}
767 assert expected == grouping
770 def test_get_data_router_subscriber_route():
771 spec = {"streams": {"subscribes": [ { "type": "data_router", "config_key":
772 "alpha", "route": "/alpha" }, { "type": "message_router", "config_key":
775 assert "/alpha" == catalog.get_data_router_subscriber_route(spec, "alpha")
777 with pytest.raises(catalog.MissingEntry):
778 catalog.get_data_router_subscriber_route(spec, "beta")
780 with pytest.raises(catalog.MissingEntry):
781 catalog.get_data_router_subscriber_route(spec, "gamma")
784 if __name__ == '__main__':
786 pytest.main([__file__, ])