1 # ============LICENSE_START=======================================================
3 # ================================================================================
4 # Copyright (c) 2017-2018 AT&T Intellectual Property. All rights reserved.
5 # ================================================================================
6 # Licensed under the Apache License, Version 2.0 (the "License");
7 # you may not use this file except in compliance with the License.
8 # You may obtain a copy of the License at
10 # http://www.apache.org/licenses/LICENSE-2.0
12 # Unless required by applicable law or agreed to in writing, software
13 # distributed under the License is distributed on an "AS IS" BASIS,
14 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 # See the License for the specific language governing permissions and
16 # limitations under the License.
17 # ============LICENSE_END=========================================================
19 # ECOMP is a trademark and service mark of AT&T Intellectual Property.
21 # -*- coding: utf-8 -*-
23 Provides utilities for running components
27 from functools import partial
29 from dcae_cli.util import docker_util as du
30 from dcae_cli.util import dmaap, inputs
31 from dcae_cli.util.cdap_util import run_component as run_cdap_component
32 from dcae_cli.util.exc import DcaeException
33 from dcae_cli.util import discovery as dis
34 from dcae_cli.util.discovery import get_user_instances, config_context, \
36 import dcae_cli.util.profiles as profiles
37 from dcae_cli.util.logger import get_logger
38 from dcae_cli.catalog.mock.catalog import build_config_keys_map, \
39 get_data_router_subscriber_route
40 # This seems to be an abstraction leak
41 from dcae_cli.catalog.mock.schema import apply_defaults_docker_config
44 log = get_logger('Run')
47 def _get_instances(user, additional_user=None):
48 instance_map = get_user_instances(user)
51 # Merge current user with another user's instance map to be available to
53 instance_map_additional = get_user_instances(additional_user)
54 log.info("#Components for {0}: {1}".format(additional_user,
55 len(instance_map_additional)))
56 instance_map.update(instance_map_additional)
58 # REVIEW: Getting instances always returns back component names with dots
59 # even though the component name could originally have dots or dashes.
60 # To put this dot vs dash headache to rest, we have to understand what the
61 # discovery abstraction should be. Should the discovery be aware of this type
62 # of naming magic? If so then the discovery abstraction may need to be
63 # enhanced to be catalog aware to do name verfication queries. If not then
64 # the dot-to-dash transformation might not belong inside of the discovery
65 # abstraction and the higher level should do that.
67 # Another possible fix is to map the dots to something that's less likely to
68 # be used multiple dashes. This would help disambiguate between a forced
69 # mapping vs component name with dashes.
71 # In the meantime, here is a fix to address the issue where a downstream component
72 # can't be matched when the downstream component uses dashes. This affects
73 # the subsequent calls:
75 # - catalog.get_discovery* query
78 # The instance map will contain entries where the names will be with dots and
79 # with dashes. There should be no harm because only one set should match. The
80 # assumption is that people won't have the same name as dots and as dashes.
81 instance_map_dashes = { (replace_dots(k[0]), k[1]): v
82 for k, v in six.iteritems(instance_map) }
83 instance_map.update(instance_map_dashes)
88 def _update_delivery_urls(spec, target_host, dmaap_map):
89 """Updates the delivery urls for data router subscribers"""
90 # Try to stick in the more appropriate delivery url which is not realized
91 # until after deployment because you need the ip, port.
92 # Realized that this is not actually needed by the component but kept it because
93 # it might be useful for component developers to **see** this info.
94 get_route_func = partial(get_data_router_subscriber_route, spec)
95 target_base_url = "http://{0}".format(target_host)
96 return dmaap.update_delivery_urls(get_route_func, target_base_url,
100 def _verify_component(name, max_wait, consul_host):
101 """Verify that the component is healthy
105 max_wait (integer): limit to how may attempts to make which translates to
106 seconds because each sleep is one second. 0 means infinite.
110 True if component is healthy else returns False
115 if dis.is_healthy(consul_host, name):
120 if max_wait > 0 and max_wait < num_attempts:
126 def run_component(user, cname, cver, catalog, additional_user, attached, force,
127 dmaap_map, inputs_map, external_ip=None):
128 '''Runs a component based on the component type
133 Continue to run even when there are no valid downstream components when
134 this flag is set to True.
135 dmaap_map: (dict) config_key to message router or data router connections.
136 Used as a manual way to make available this information for the component.
137 inputs_map: (dict) config_key to value that is intended to be provided at
138 deployment time as an input
140 cname, cver = catalog.verify_component(cname, cver)
141 ctype = catalog.get_component_type(cname, cver)
142 profile = profiles.get_profile()
144 instance_map = _get_instances(user, additional_user)
145 neighbors = six.iterkeys(instance_map)
148 dmaap_config_keys = catalog.get_discovery_for_dmaap(cname, cver)
150 if not dmaap.validate_dmaap_map_entries(dmaap_map, *dmaap_config_keys):
153 if ctype == 'docker':
154 params, interface_map = catalog.get_discovery_for_docker(cname, cver, neighbors)
155 should_wait = attached
157 spec = catalog.get_component_spec(cname, cver)
158 config_key_map = build_config_keys_map(spec)
159 inputs_map = inputs.filter_entries(inputs_map, spec)
161 dmaap_map = _update_delivery_urls(spec, profile.docker_host.split(":")[0],
164 with config_context(user, cname, cver, params, interface_map,
165 instance_map, config_key_map, dmaap_map=dmaap_map, inputs_map=inputs_map,
166 always_cleanup=should_wait, force_config=force) as (instance_name, _):
167 image = catalog.get_docker_image(cname, cver)
168 docker_config = catalog.get_docker_config(cname, cver)
170 docker_logins = dis.get_docker_logins()
173 du.deploy_component(profile, image, instance_name, docker_config,
174 should_wait=True, logins=docker_logins)
176 result = du.deploy_component(profile, image, instance_name, docker_config,
177 logins=docker_logins)
181 log.info("Deployed {0}. Verifying..".format(instance_name))
183 # TODO: Be smarter here but for now wait longer i.e. 5min
184 max_wait = 300 # 300s == 5min
186 if _verify_component(instance_name, max_wait,
187 dis.default_consul_host()):
188 log.info("Container is up and healthy")
190 # This block of code is used to construct the delivery
191 # urls for data router subscribers and to display it for
192 # users to help with manually provisioning feeds.
193 results = dis.lookup_instance(dis.default_consul_host(),
195 target_host = dis.parse_instance_lookup(results)
197 dmaap_map = _update_delivery_urls(spec, target_host, dmaap_map)
198 delivery_urls = dmaap.list_delivery_urls(dmaap_map)
201 msg = "\n".join(["\t{k}: {url}".format(k=k, url=url)
202 for k, url in delivery_urls])
203 msg = "\n\n{0}\n".format(msg)
204 log.warn("Your component is a data router subscriber. Here are the delivery urls: {0}".format(msg))
206 log.warn("Container never became healthy")
208 raise DcaeException("Failed to deploy docker component")
211 (jar, config, spec) = catalog.get_cdap(cname, cver)
212 config_key_map = build_config_keys_map(spec)
213 inputs_map = inputs.filter_entries(inputs_map, spec)
215 params, interface_map = catalog.get_discovery_for_cdap(cname, cver, neighbors)
217 with config_context(user, cname, cver, params, interface_map, instance_map,
218 config_key_map, dmaap_map=dmaap_map, inputs_map=inputs_map, always_cleanup=False,
219 force_config=force) as (instance_name, templated_conf):
220 run_cdap_component(catalog, params, instance_name, profile, jar, config, spec, templated_conf)
222 raise DcaeException("Unsupported component type for run")
225 def dev_component(user, catalog, specification, additional_user, force, dmaap_map,
227 '''Sets up the discovery layer for in development component
229 The passed-in component specification is
231 * Generates the corresponding application config
232 * Pushes the application config and rels key into Consul
234 This allows developers to play with their spec and the resulting configuration
235 outside of being in the catalog and in a container.
239 user: (string) user name
240 catalog: (object) instance of MockCatalog
241 specification: (dict) experimental component specification
242 additional_user: (string) another user name used to source additional
245 Continue to run even when there are no valid downstream components when
246 this flag is set to True.
247 dmaap_map: (dict) config_key to message router connections. Used as a
248 manual way to make available this information for the component.
249 inputs_map: (dict) config_key to value that is intended to be provided at
250 deployment time as an input
252 instance_map = _get_instances(user, additional_user)
253 neighbors = six.iterkeys(instance_map)
255 params, interface_map, dmaap_config_keys = catalog.get_discovery_from_spec(
256 user, specification, neighbors)
258 if not dmaap.validate_dmaap_map_entries(dmaap_map, *dmaap_config_keys):
261 cname = specification["self"]["name"]
262 cver = specification["self"]["version"]
263 config_key_map = build_config_keys_map(specification)
264 inputs_map = inputs.filter_entries(inputs_map, specification)
266 dmaap_map = _update_delivery_urls(specification, "localhost", dmaap_map)
268 with config_context(user, cname, cver, params, interface_map, instance_map,
269 config_key_map, dmaap_map, inputs_map=inputs_map, always_cleanup=True,
270 force_config=force) \
271 as (instance_name, templated_conf):
273 click.echo("Ready for component development")
275 if specification["self"]["component_type"] == "docker":
276 # The env building is only for docker right now
277 docker_config = apply_defaults_docker_config(specification["auxilary"])
278 envs = du.build_envs(profiles.get_profile(), docker_config, instance_name)
279 envs_message = "\n".join(["export {0}={1}".format(k, v) for k,v in envs.items()])
280 envs_filename = "env_{0}".format(profiles.get_active_name())
282 with open(envs_filename, "w") as f:
283 f.write(envs_message)
286 click.echo("Setup these environment varibles. Run \"source {0}\":".format(envs_filename))
288 click.echo(envs_message)
291 click.echo("Set the following as your HOSTNAME:\n {0}".format(instance_name))
293 input("Press any key to stop and to clean up")