"type": "boolean",
"default": false
},
+ "sourced_at_deployment": {
+ "description": "An optional key that declares a parameter's value to be assigned at deployment time (true). Default is false.",
+ "type": "boolean",
+ "default": false
+ },
"policy_schema" :{
"type": "array",
"uniqueItems": true,
--- /dev/null
+{
+ "$schema": "http://json-schema.org/draft-04/schema#",
+ "title": "Schema for dmaap inputs",
+ "type": "object",
+ "oneOf": [
+ { "$ref": "#/definitions/message_router" }
+ ],
+ "definitions": {
+ "message_router": {
+ "type": "object",
+ "properties": {
+ "type": {
+ "type": "string",
+ "enum": ["message_router"]
+ },
+ "aaf_username": {
+ "type": "string"
+ },
+ "aaf_password": {
+ "type": "string"
+ },
+ "dmaap_info": {
+ "type": "object",
+ "properties": {
+ "client_role": {
+ "type": "string"
+ },
+ "client_id": {
+ "type": "string"
+ },
+ "location": {
+ "type": "string"
+ },
+ "topic_url": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "topic_url"
+ ],
+ "additionalProperties": false
+ }
+ },
+ "required": [
+ "type",
+ "dmaap_info"
+ ],
+ "additionalProperties": false
+ }
+ }
+}
--- /dev/null
+{
+ "type": "message_router",
+ "aaf_username": "foo3",
+ "aaf_password": "bar3",
+ "something_else": "boo",
+ "dmaap_info":{
+ "client_role":"some.dcae.member",
+ "client_id":"123456",
+ "location":"mtc5",
+ "topic_url":"https://message-router-url/some-topic"
+ }
+}
--- /dev/null
+{
+ "type": "message_router",
+ "aaf_username": "foo3",
+ "aaf_password": "bar3",
+ "dmaap_info":{
+ "client_role":"some.dcae.member",
+ "client_id":"123456",
+ "location":"mtc5"
+ }
+}
--- /dev/null
+{
+ "type": "message_router",
+ "aaf_username": "foo3",
+ "aaf_password": "bar3",
+ "dmaap_info":{
+ "client_role":"some.dcae.member",
+ "client_id":"123456",
+ "location":"mtc5",
+ "topic_url":"https://message-router-url/some-topic"
+ }
+}
The format is based on [Keep a Changelog](http://keepachangelog.com/)
and this project adheres to [Semantic Versioning](http://semver.org/).
+## [2.9.0]
+
+* Add data format generate command
+* Fix issue with data router config keys
+
+## [2.8.1]
+
+* Improve error message when inputs map is missing item. Show the specific parameters that are causing issues.
+
+## [2.8.0]
+
+* Enhance to support parameters that are sourced at deployment
+* Provide new command line arg --inputs-file
+* Use inputs file to bind values to generated configuration for parameters that have been specified to be `sourced_at_deployment` true.
+
## [2.7.0]
* Rip out Docker related code and use common python-dockering library
# ECOMP is a trademark and service mark of AT&T Intellectual Property.
# -*- coding: utf-8 -*-
-__version__ = "2.7.0"
+__version__ = "2.9.0"
from discovery_client import resolve_name
-from dcae_cli.util import profiles, load_json, dmaap
+from dcae_cli.util import profiles, load_json, dmaap, inputs
from dcae_cli.util.run import run_component, dev_component
from dcae_cli.util import discovery as dis
from dcae_cli.util.discovery import DiscoveryNoDownstreamComponentError
message = "Problems with parsing the dmaap file. Check to make sure that it is a valid json and is in the expected structure."
raise DcaeException(message)
+_help_inputs_file = """
+Path to a file that contains a json that contains values to be used to bind to configuration parameters that have been marked as "sourced_at_deployment". The structure of the json is expected to be:
+
+ {
+ <parameter1 name>: value,
+ <parameter2 name>: value
+ }
+
+The "parameter name" is the value of the "name" property for the given configuration parameter.
+"""
+
+def _parse_inputs_file(inputs_file):
+ try:
+ with open(inputs_file, 'r+') as f:
+ inputs_map = json.load(f)
+ # TODO: Validation of schema in the future? Skipping this because
+ # dti_payload is not being intended to be used.
+ return inputs_map
+ except Exception as e:
+ message = "Problems with parsing the inputs file. Check to make sure that it is a valid json and is in the expected structure."
+ raise DcaeException(message)
+
@component.command()
@click.option('--external-ip', '-ip', default=None, help='The external IP address of the Docker host. Only used for Docker components.')
@click.option('--force', is_flag=True, help='Force component to run without valid downstream dependencies')
@click.option('--dmaap-file', type=click.Path(resolve_path=True, exists=True, dir_okay=False),
help=_help_dmaap_file)
+@click.option('--inputs-file', type=click.Path(resolve_path=True, exists=True, dir_okay=False),
+ help=_help_inputs_file)
@click.argument('component')
@click.pass_obj
-def run(obj, external_ip, additional_user, attached, force, dmaap_file, component):
+def run(obj, external_ip, additional_user, attached, force, dmaap_file, component,
+ inputs_file):
'''Runs the latest version of COMPONENT. You may optionally specify version via COMPONENT:VERSION'''
cname, cver = parse_input(component)
user, catalog = obj['config']['user'], obj['catalog']
dmaap_map = _parse_dmaap_file(dmaap_file) if dmaap_file else {}
+ inputs_map = _parse_inputs_file(inputs_file) if inputs_file else {}
try:
run_component(user, cname, cver, catalog, additional_user, attached, force,
- dmaap_map, external_ip)
+ dmaap_map, inputs_map, external_ip)
except DiscoveryNoDownstreamComponentError as e:
message = "Either run a compatible downstream component first or run with the --force flag to ignore this error"
raise DcaeException(message)
+ except inputs.InputsValidationError as e:
+ click.echo("There is a problem. {0}".format(e))
+ message = "Component requires inputs. Please look at the use of --inputs-file and make sure the format is correct"
+ raise DcaeException(message)
@component.command()
@click.argument('component')
@click.option('--force', is_flag=True, help='Force component to run without valid downstream dependencies')
@click.option('--dmaap-file', type=click.Path(resolve_path=True, exists=True, dir_okay=False),
help=_help_dmaap_file)
+@click.option('--inputs-file', type=click.Path(resolve_path=True, exists=True, dir_okay=False),
+ help=_help_inputs_file)
@click.pass_obj
-def dev(obj, specification, additional_user, force, dmaap_file):
+def dev(obj, specification, additional_user, force, dmaap_file, inputs_file):
'''Set up component in development for discovery, use for local development'''
user, catalog = obj['config']['user'], obj['catalog']
dmaap_map = _parse_dmaap_file(dmaap_file) if dmaap_file else {}
+ inputs_map = _parse_inputs_file(inputs_file) if inputs_file else {}
with open(specification, 'r+') as f:
spec = json.loads(f.read())
try:
- dev_component(user, catalog, spec, additional_user, force, dmaap_map)
+ dev_component(user, catalog, spec, additional_user, force, dmaap_map,
+ inputs_map)
except DiscoveryNoDownstreamComponentError as e:
message = "Either run a compatible downstream component first or run with the --force flag to ignore this error"
raise DcaeException(message)
+ except inputs.InputsValidationError as e:
+ click.echo("There is a problem. {0}".format(e))
+ message = "Component requires inputs. Please look at the use of --inputs-file and make sure the format is correct"
+ raise DcaeException(message)
@component.command()
import click
+import genson
+
+import sys
+
+import os
+
+from jsonschema import Draft4Validator
+
from dcae_cli.util import load_json
from dcae_cli.util.logger import get_logger
+
from dcae_cli.commands import util
from dcae_cli.commands.util import create_table, parse_input
from dcae_cli.catalog.exc import MissingEntry
+from dcae_cli.catalog.exc import DcaeException
logger = get_logger('DataFormatCommand')
click.echo("Data format has been published")
else:
click.echo("Data format could not be published")
+
+@data_format.command()
+@click.option('--keywords', is_flag=True, help='Adds a template of possible descriptive keywords', default=False)
+@click.argument('name_version', metavar="name:version", required = True)
+@click.argument('file-or-dir-path', type=click.Path(resolve_path=True, exists=True, dir_okay=True, file_okay=True, readable=True), metavar="file-or-dir-path")
+@click.pass_obj
+def generate(obj, name_version, file_or_dir_path, keywords):
+ '''Create schema from a file or directory examples'''
+ name, version = parse_input(name_version)
+ if version == None:
+ version = ""
+ schema = genson.Schema()
+ if os.path.isfile(file_or_dir_path):
+ addfile(file_or_dir_path, schema)
+ else:
+ foundJSON = False
+ for root, dirs, files in os.walk(file_or_dir_path):
+ for filename in files:
+ fullfilename = os.path.join(file_or_dir_path, filename)
+ addfile(fullfilename,schema)
+ foundJSON = True
+ if foundJSON == False:
+ raise DcaeException('No JSON files found in ' + file_or_dir_path)
+
+ json_obj = json.loads(schema.to_json())
+ json_obj['$schema'] = "http://json-schema.org/draft-04/schema#"
+ jschema = json.dumps(json_obj)
+ jschema = jschema.replace('"required":', '"additionalproperties": true, "required":')
+ jschema = jschema.replace('"type":', ' "description": "", "type":')
+
+ if (keywords):
+ jschema = jschema.replace('"type": "string"', ' "maxLength": 0, "minLength": 0, "pattern": "", "type": "string"')
+ jschema = jschema.replace('"type": "integer"', ' "maximum": 0, "mininimum": 0, "multipleOf": 0, "type": "integer"')
+ jschema = jschema.replace('"type": "array"', ' "maxItems": 0, "minItems": 0, "uniqueItems": "false", "type": "array"')
+
+ jschema = '{ "self": { "name": "' + name + '", "version": "' + version + '", "description": ""} , "dataformatversion": "1.0.0", "jsonschema": ' + jschema + '}'
+ #Draft4Validator.check_schema(json.loads(jschema))
+ try:
+ print(json.dumps(json.loads(jschema), sort_keys=True, indent=4 ))
+ except ValueError:
+ raise DcaeException('Problem with JSON generation')
+
+def addfile(filename, schema):
+ try:
+ fileadd = open(filename, "r")
+ except IOError:
+ raise DcaeException('Cannot open' + filename)
+ try:
+ json_object = json.loads(fileadd.read())
+ schema.add_object(json_object)
+ except ValueError:
+ raise DcaeException('Bad JSON file: ' + filename)
+ finally:
+ fileadd.close()
+
+
--- /dev/null
+{
+ "foobar": "test 1"
+}
--- /dev/null
+{
+ "foobar2": "test 1"
+}
+
spec_str = runner.invoke(cli, cmd, obj=obj).output
assert df_spec == json.loads(spec_str)
+ # test of generate
+ bad_dir = os.path.join(TEST_DIR, 'mocked_components', 'model', 'baddir')
+ cmd = "data_format generate --keywords \"name:1.0.2\" {:}".format(bad_dir).split()
+ err_str = runner.invoke(cli, cmd, obj=obj).output
+ assert "does not exist" in err_str
+
+ empty_dir = os.path.join(TEST_DIR, 'mocked_components', 'model', 'emptydir')
+ try:
+ os.stat(empty_dir)
+ except:
+ os.mkdir(empty_dir)
+ cmd = "data_format generate --keywords \"name:1.0.2\" {:}".format(empty_dir).split()
+ err_str = runner.invoke(cli, cmd, obj=obj).output
+ assert "No JSON files found" in err_str
+
+ bad_json = os.path.join(TEST_DIR, 'mocked_components', 'model', 'badjson')
+ cmd = "data_format generate --keywords \"name:1.0.2\" {:}".format(bad_json).split()
+ err_str = runner.invoke(cli, cmd, obj=obj).output
+ assert "Bad JSON file" in err_str
+
+ generate_dir = os.path.join(TEST_DIR, 'mocked_components', 'model', 'generatedir')
+ cmd = "data_format generate --keywords name:1.0.2 {:} ".format(generate_dir).split()
+ out_str = runner.invoke(cli, cmd, obj=obj).output
+ assert '{\n "dataformatversion": "1.0.0", \n "jsonschema": {\n "$schema": "http://json-schema.org/draft-04/schema#", \n "description": "", \n "properties": {\n "foobar": {\n "description": "", \n "maxLength": 0, \n "minLength": 0, \n "pattern": "", \n "type": "string"\n }, \n "foobar2": {\n "description": "", \n "maxLength": 0, \n "minLength": 0, \n "pattern": "", \n "type": "string"\n }\n }, \n "type": "object"\n }, \n "self": {\n "description": "", \n "name": "name", \n "version": "1.0.2"\n }\n}\n' == out_str
+
+ generate_dir = os.path.join(TEST_DIR, 'mocked_components', 'model', 'generatedir')
+ cmd = "data_format generate name:1.0.2 {:} ".format(generate_dir).split()
+ out_str = runner.invoke(cli, cmd, obj=obj).output
+ assert '{\n "dataformatversion": "1.0.0", \n "jsonschema": {\n "$schema": "http://json-schema.org/draft-04/schema#", \n "description": "", \n "properties": {\n "foobar": {\n "description": "", \n "type": "string"\n }, \n "foobar2": {\n "description": "", \n "type": "string"\n }\n }, \n "type": "object"\n }, \n "self": {\n "description": "", \n "name": "name", \n "version": "1.0.2"\n }\n}\n' == out_str
+
+ generate_dir = os.path.join(TEST_DIR, 'mocked_components', 'model', 'generatedir', 'ex1.json')
+ cmd = "data_format generate name:1.0.2 {:} ".format(generate_dir).split()
+ out_str = runner.invoke(cli, cmd, obj=obj).output
+ assert '{\n "dataformatversion": "1.0.0", \n "jsonschema": {\n "$schema": "http://json-schema.org/draft-04/schema#", \n "additionalproperties": true, \n "description": "", \n "properties": {\n "foobar": {\n "description": "", \n "type": "string"\n }\n }, \n "required": [\n "foobar"\n ], \n "type": "object"\n }, \n "self": {\n "description": "", \n "name": "name", \n "version": "1.0.2"\n }\n}\n' == out_str
+
if __name__ == '__main__':
'''Test area'''
return grouped_conf
+def _apply_inputs(config, inputs_map):
+ """Update configuration with inputs
+
+ This method updates the values of the configuration parameters using values
+ from the inputs map.
+ """
+ config.update(inputs_map)
+ return config
+
+
@contextlib.contextmanager
def config_context(user, cname, cver, params, interface_map, instance_map,
- config_key_map, dmaap_map={}, instance_prefix=None, host=consul_host,
- always_cleanup=True, force_config=False):
+ config_key_map, dmaap_map={}, inputs_map={}, instance_prefix=None,
+ host=consul_host, always_cleanup=True, force_config=False):
'''Convenience utility for creating configs and cleaning them up
Args
user, cname, cver, params, interface_map, instance_map, dmaap_map,
instance_prefix, force=force_config)
+ conf = _apply_inputs(conf, inputs_map)
conf = _group_config(conf, config_key_map)
push_config(conf_key, conf, rels_key, rels, dmaap_key, dmaap_map, host)
logger.error("Please use the \"--dmaap-file\" option")
return False
+ config_keys = dr_config_keys + mr_config_keys
# Look for missing keys
is_missing = lambda config_key: config_key not in dmaap_map
- missing_keys = list(filter(is_missing, mr_config_keys))
+ missing_keys = list(filter(is_missing, config_keys))
if missing_keys:
logger.error("Missing config keys in dmaap json: {0}".format(
return False
# Look for unexpected keys
- is_unexpected = lambda config_key: config_key not in mr_config_keys
+ is_unexpected = lambda config_key: config_key not in config_keys
unexpected_keys = list(filter(is_unexpected, dmaap_map.keys()))
if unexpected_keys:
--- /dev/null
+# ============LICENSE_START=======================================================
+# org.onap.dcae
+# ================================================================================
+# Copyright (c) 2017 AT&T Intellectual Property. All rights reserved.
+# ================================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============LICENSE_END=========================================================
+#
+# ECOMP is a trademark and service mark of AT&T Intellectual Property.
+
+"""
+Functions for handling inputs
+"""
+
+class InputsValidationError(RuntimeError):
+ pass
+
+def filter_entries(inputs_map, spec):
+ """Filter inputs entries that are not in the spec"""
+ param_names = [ p["name"] for p in spec["parameters"] \
+ if "sourced_at_deployment" in p and p["sourced_at_deployment"] ]
+
+ # Identify any missing parameters from inputs_map
+ missing = list(filter(lambda pn: pn not in inputs_map, param_names))
+
+ if missing:
+ raise InputsValidationError(
+ "Inputs map is missing keys: {0}".format(missing))
+
+ return { pn: inputs_map[pn] for pn in param_names }
from functools import partial
import click
from dcae_cli.util import docker_util as du
-from dcae_cli.util import dmaap
+from dcae_cli.util import dmaap, inputs
from dcae_cli.util.cdap_util import run_component as run_cdap_component
from dcae_cli.util.exc import DcaeException
from dcae_cli.util import discovery as dis
def run_component(user, cname, cver, catalog, additional_user, attached, force,
- dmaap_map, external_ip=None):
+ dmaap_map, inputs_map, external_ip=None):
'''Runs a component based on the component type
Args
this flag is set to True.
dmaap_map: (dict) config_key to message router or data router connections.
Used as a manual way to make available this information for the component.
+ inputs_map: (dict) config_key to value that is intended to be provided at
+ deployment time as an input
'''
cname, cver = catalog.verify_component(cname, cver)
ctype = catalog.get_component_type(cname, cver)
spec = catalog.get_component_spec(cname, cver)
config_key_map = build_config_keys_map(spec)
+ inputs_map = inputs.filter_entries(inputs_map, spec)
dmaap_map = _update_delivery_urls(spec, profile.docker_host.split(":")[0],
dmaap_map)
with config_context(user, cname, cver, params, interface_map,
- instance_map, config_key_map, dmaap_map=dmaap_map,
+ instance_map, config_key_map, dmaap_map=dmaap_map, inputs_map=inputs_map,
always_cleanup=should_wait, force_config=force) as (instance_name, _):
image = catalog.get_docker_image(cname, cver)
docker_config = catalog.get_docker_config(cname, cver)
elif ctype =='cdap':
(jar, config, spec) = catalog.get_cdap(cname, cver)
config_key_map = build_config_keys_map(spec)
+ inputs_map = inputs.filter_entries(inputs_map, spec)
+
params, interface_map = catalog.get_discovery_for_cdap(cname, cver, neighbors)
with config_context(user, cname, cver, params, interface_map, instance_map,
- config_key_map, dmaap_map=dmaap_map, always_cleanup=False,
+ config_key_map, dmaap_map=dmaap_map, inputs_map=inputs_map, always_cleanup=False,
force_config=force) as (instance_name, templated_conf):
run_cdap_component(catalog, params, instance_name, profile, jar, config, spec, templated_conf)
else:
raise DcaeException("Unsupported component type for run")
-def dev_component(user, catalog, specification, additional_user, force, dmaap_map):
+def dev_component(user, catalog, specification, additional_user, force, dmaap_map,
+ inputs_map):
'''Sets up the discovery layer for in development component
The passed-in component specification is
this flag is set to True.
dmaap_map: (dict) config_key to message router connections. Used as a
manual way to make available this information for the component.
+ inputs_map: (dict) config_key to value that is intended to be provided at
+ deployment time as an input
'''
instance_map = _get_instances(user, additional_user)
neighbors = six.iterkeys(instance_map)
cname = specification["self"]["name"]
cver = specification["self"]["version"]
config_key_map = build_config_keys_map(specification)
+ inputs_map = inputs.filter_entries(inputs_map, specification)
dmaap_map = _update_delivery_urls(specification, "localhost", dmaap_map)
with config_context(user, cname, cver, params, interface_map, instance_map,
- config_key_map, dmaap_map, always_cleanup=True, force_config=force) \
+ config_key_map, dmaap_map, inputs_map=inputs_map, always_cleanup=True,
+ force_config=force) \
as (instance_name, templated_conf):
click.echo("Ready for component development")
assert dis.parse_instance_lookup(results) == "192.168.1.100:8080"
+def test_apply_inputs():
+ updated_config = dis._apply_inputs({"foo": "bar"}, {"foo": "baz"})
+ assert updated_config == {"foo": "baz"}
+
+
if __name__ == '__main__':
'''Test area'''
pytest.main([__file__, ])
--- /dev/null
+# ============LICENSE_START=======================================================
+# org.onap.dcae
+# ================================================================================
+# Copyright (c) 2017 AT&T Intellectual Property. All rights reserved.
+# ================================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============LICENSE_END=========================================================
+#
+# ECOMP is a trademark and service mark of AT&T Intellectual Property.
+
+"""
+Tests for inputs module
+"""
+import pytest
+from dcae_cli.util import inputs
+
+
+def test_filter_entries():
+ spec = { "parameters": [{"name": "foo"}, {"name": "bar",
+ "sourced_at_deployment": False}, {"name": "baz", "sourced_at_deployment": True}] }
+
+ with pytest.raises(inputs.InputsValidationError):
+ inputs.filter_entries({}, spec)
+
+ inputs_map = { "foo": "do not copy", "baz": "hello world", "extra": "do not copy" }
+
+ assert len(inputs.filter_entries(inputs_map, spec)) == 1
docker-py==1.10.6
terminaltables==3.1.0
python-discovery-client==2.0.0
-python-dockering==1.2.0
+python-dockering==1.2.1
psycopg2==2.7.1
+genson==0.2.2
'docker-py>=1.10,<2',
'terminaltables',
'psycopg2',
+ 'genson',
'python-discovery-client>=2.0.0',
'python-dockering>=1.0.0,<2.0.0'
],
Code Review / dcaegen2 / platform / cli.git / commitdiff