1 # ============LICENSE_START=======================================================
3 # ================================================================================
4 # Copyright (c) 2017 AT&T Intellectual Property. All rights reserved.
5 # ================================================================================
6 # Licensed under the Apache License, Version 2.0 (the "License");
7 # you may not use this file except in compliance with the License.
8 # You may obtain a copy of the License at
10 # http://www.apache.org/licenses/LICENSE-2.0
12 # Unless required by applicable law or agreed to in writing, software
13 # distributed under the License is distributed on an "AS IS" BASIS,
14 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 # See the License for the specific language governing permissions and
16 # limitations under the License.
17 # ============LICENSE_END=========================================================
19 # ECOMP is a trademark and service mark of AT&T Intellectual Property.
21 # -*- coding: utf-8 -*-
23 Provides utilities for running components
27 from functools import partial
29 from dcae_cli.util import docker_util as du
30 from dcae_cli.util import dmaap, inputs
31 from dcae_cli.util.cdap_util import run_component as run_cdap_component
32 from dcae_cli.util.exc import DcaeException
33 from dcae_cli.util import discovery as dis
34 from dcae_cli.util.discovery import get_user_instances, config_context, \
36 import dcae_cli.util.profiles as profiles
37 from dcae_cli.util.logger import get_logger
38 from dcae_cli.catalog.mock.catalog import build_config_keys_map, \
39 get_data_router_subscriber_route
40 # This seems to be an abstraction leak
41 from dcae_cli.catalog.mock.schema import apply_defaults_docker_config
44 log = get_logger('Run')
47 def _get_instances(user, additional_user=None):
48 instance_map = get_user_instances(user)
51 # Merge current user with another user's instance map to be available to
53 instance_map_additional = get_user_instances(additional_user)
54 log.info("#Components for {0}: {1}".format(additional_user,
55 len(instance_map_additional)))
56 instance_map.update(instance_map_additional)
58 # REVIEW: Getting instances always returns back component names with dots
59 # even though the component name could originally have dots or dashes.
60 # To put this dot vs dash headache to rest, we have to understand what the
61 # discovery abstraction should be. Should the discovery be aware of this type
62 # of naming magic? If so then the discovery abstraction may need to be
63 # enhanced to be catalog aware to do name verfication queries. If not then
64 # the dot-to-dash transformation might not belong inside of the discovery
65 # abstraction and the higher level should do that.
67 # Another possible fix is to map the dots to something that's less likely to
68 # be used multiple dashes. This would help disambiguate between a forced
69 # mapping vs component name with dashes.
71 # In the meantime, here is a fix to address the issue where a downstream component
72 # can't be matched when the downstream component uses dashes. This affects
73 # the subsequent calls:
75 # - catalog.get_discovery* query
78 # The instance map will contain entries where the names will be with dots and
79 # with dashes. There should be no harm because only one set should match. The
80 # assumption is that people won't have the same name as dots and as dashes.
81 instance_map_dashes = { (replace_dots(k[0]), k[1]): v
82 for k, v in six.iteritems(instance_map) }
83 instance_map.update(instance_map_dashes)
88 def _update_delivery_urls(spec, target_host, dmaap_map):
89 """Updates the delivery urls for data router subscribers"""
90 # Try to stick in the more appropriate delivery url which is not realized
91 # until after deployment because you need the ip, port.
92 # Realized that this is not actually needed by the component but kept it because
93 # it might be useful for component developers to **see** this info.
94 get_route_func = partial(get_data_router_subscriber_route, spec)
95 target_base_url = "http://{0}".format(target_host)
96 return dmaap.update_delivery_urls(get_route_func, target_base_url,
100 def _verify_component(name, max_wait, consul_host):
101 """Verify that the component is healthy
105 max_wait (integer): limit to how may attempts to make which translates to
106 seconds because each sleep is one second. 0 means infinite.
110 True if component is healthy else returns False
115 if dis.is_healthy(consul_host, name):
120 if max_wait > 0 and max_wait < num_attempts:
126 def run_component(user, cname, cver, catalog, additional_user, attached, force,
127 dmaap_map, inputs_map, external_ip=None):
128 '''Runs a component based on the component type
133 Continue to run even when there are no valid downstream components when
134 this flag is set to True.
135 dmaap_map: (dict) config_key to message router or data router connections.
136 Used as a manual way to make available this information for the component.
137 inputs_map: (dict) config_key to value that is intended to be provided at
138 deployment time as an input
140 cname, cver = catalog.verify_component(cname, cver)
141 ctype = catalog.get_component_type(cname, cver)
142 profile = profiles.get_profile()
144 instance_map = _get_instances(user, additional_user)
145 neighbors = six.iterkeys(instance_map)
148 dmaap_config_keys = catalog.get_discovery_for_dmaap(cname, cver)
150 if not dmaap.validate_dmaap_map_entries(dmaap_map, *dmaap_config_keys):
153 if ctype == 'docker':
154 params, interface_map = catalog.get_discovery_for_docker(cname, cver, neighbors)
155 should_wait = attached
157 spec = catalog.get_component_spec(cname, cver)
158 config_key_map = build_config_keys_map(spec)
159 inputs_map = inputs.filter_entries(inputs_map, spec)
161 dmaap_map = _update_delivery_urls(spec, profile.docker_host.split(":")[0],
164 with config_context(user, cname, cver, params, interface_map,
165 instance_map, config_key_map, dmaap_map=dmaap_map, inputs_map=inputs_map,
166 always_cleanup=should_wait, force_config=force) as (instance_name, _):
167 image = catalog.get_docker_image(cname, cver)
168 docker_config = catalog.get_docker_config(cname, cver)
171 du.deploy_component(profile, image, instance_name, docker_config,
174 result = du.deploy_component(profile, image, instance_name, docker_config)
178 log.info("Deployed {0}. Verifying..".format(instance_name))
182 if _verify_component(instance_name, max_wait, dis.consul_host):
183 log.info("Container is up and healthy")
185 # This block of code is used to construct the delivery
186 # urls for data router subscribers and to display it for
187 # users to help with manually provisioning feeds.
188 results = dis.lookup_instance(dis.consul_host, instance_name)
189 target_host = dis.parse_instance_lookup(results)
191 dmaap_map = _update_delivery_urls(spec, target_host, dmaap_map)
192 delivery_urls = dmaap.list_delivery_urls(dmaap_map)
195 msg = "\n".join(["\t{k}: {url}".format(k=k, url=url)
196 for k, url in delivery_urls])
197 msg = "\n\n{0}\n".format(msg)
198 log.warn("Your component is a data router subscriber. Here are the delivery urls: {0}".format(msg))
200 log.warn("Container never became healthy")
202 raise DcaeException("Failed to deploy docker component")
205 (jar, config, spec) = catalog.get_cdap(cname, cver)
206 config_key_map = build_config_keys_map(spec)
207 inputs_map = inputs.filter_entries(inputs_map, spec)
209 params, interface_map = catalog.get_discovery_for_cdap(cname, cver, neighbors)
211 with config_context(user, cname, cver, params, interface_map, instance_map,
212 config_key_map, dmaap_map=dmaap_map, inputs_map=inputs_map, always_cleanup=False,
213 force_config=force) as (instance_name, templated_conf):
214 run_cdap_component(catalog, params, instance_name, profile, jar, config, spec, templated_conf)
216 raise DcaeException("Unsupported component type for run")
219 def dev_component(user, catalog, specification, additional_user, force, dmaap_map,
221 '''Sets up the discovery layer for in development component
223 The passed-in component specification is
225 * Generates the corresponding application config
226 * Pushes the application config and rels key into Consul
228 This allows developers to play with their spec and the resulting configuration
229 outside of being in the catalog and in a container.
233 user: (string) user name
234 catalog: (object) instance of MockCatalog
235 specification: (dict) experimental component specification
236 additional_user: (string) another user name used to source additional
239 Continue to run even when there are no valid downstream components when
240 this flag is set to True.
241 dmaap_map: (dict) config_key to message router connections. Used as a
242 manual way to make available this information for the component.
243 inputs_map: (dict) config_key to value that is intended to be provided at
244 deployment time as an input
246 instance_map = _get_instances(user, additional_user)
247 neighbors = six.iterkeys(instance_map)
249 params, interface_map, dmaap_config_keys = catalog.get_discovery_from_spec(
250 user, specification, neighbors)
252 if not dmaap.validate_dmaap_map_entries(dmaap_map, *dmaap_config_keys):
255 cname = specification["self"]["name"]
256 cver = specification["self"]["version"]
257 config_key_map = build_config_keys_map(specification)
258 inputs_map = inputs.filter_entries(inputs_map, specification)
260 dmaap_map = _update_delivery_urls(specification, "localhost", dmaap_map)
262 with config_context(user, cname, cver, params, interface_map, instance_map,
263 config_key_map, dmaap_map, inputs_map=inputs_map, always_cleanup=True,
264 force_config=force) \
265 as (instance_name, templated_conf):
267 click.echo("Ready for component development")
269 if specification["self"]["component_type"] == "docker":
270 # The env building is only for docker right now
271 docker_config = apply_defaults_docker_config(specification["auxilary"])
272 envs = du.build_envs(profiles.get_profile(), docker_config, instance_name)
273 envs_message = "\n".join(["export {0}={1}".format(k, v) for k,v in envs.items()])
274 envs_filename = "env_{0}".format(profiles.get_active_name())
276 with open(envs_filename, "w") as f:
277 f.write(envs_message)
280 click.echo("Setup these environment varibles. Run \"source {0}\":".format(envs_filename))
282 click.echo(envs_message)
285 click.echo("Set the following as your HOSTNAME:\n {0}".format(instance_name))
287 input("Press any key to stop and to clean up")