From 7676ca5e12557f72226fac162e5f1530964906cb Mon Sep 17 00:00:00 2001 From: Lianhao Lu Date: Thu, 22 Mar 2018 20:39:04 +0800 Subject: [PATCH] Adjusted for pypi support We need to adjust the python module structure meet the pypi requirements. This has been tested on test pypi https://test.pypi.org/project/vnfsdk. 1. move 3 directories cli/ validator/ packager/ into vnfsdk_pkgtools. so now the python module for vnfsdk pkgtools would be "vnfsdk_pkgtool.*" 2. Added missing README.rst, LICENSE.txt according to pypi requirement. 3. Added new version mechanism accroding onap community suggestions. 4. Other clean sweep job like dos2unix. Change-Id: If90df33673bff045d85d67c29a1d0ab44d0c8858 Issue-ID: VNFSDK-143 Signed-off-by: Lianhao Lu --- LICENSE => LICENSE.txt | 0 MANIFEST.in | 2 +- Makefile | 3 - README.rst | 16 + assembly.xml | 4 +- pom.xml | 4 +- setup.py | 190 ++++--- tests/cli/test_cli.py | 2 +- tests/packager/test_package.py | 2 +- tox.ini | 2 +- __init__.py => vnfsdk_pkgtools/__init__.py | 0 {cli => vnfsdk_pkgtools/cli}/__init__.py | 0 {cli => vnfsdk_pkgtools/cli}/__main__.py | 240 ++++----- {packager => vnfsdk_pkgtools/packager}/__init__.py | 0 {packager => vnfsdk_pkgtools/packager}/csar.py | 570 ++++++++++----------- .../validator}/__init__.py | 2 +- .../validator}/aria_validator.py | 2 +- vnfsdk_pkgtools/version.py | 3 + 18 files changed, 526 insertions(+), 516 deletions(-) rename LICENSE => LICENSE.txt (100%) create mode 100644 README.rst rename __init__.py => vnfsdk_pkgtools/__init__.py (100%) rename {cli => vnfsdk_pkgtools/cli}/__init__.py (100%) rename {cli => vnfsdk_pkgtools/cli}/__main__.py (95%) rename {packager => vnfsdk_pkgtools/packager}/__init__.py (100%) rename {packager => vnfsdk_pkgtools/packager}/csar.py (97%) rename {validator => vnfsdk_pkgtools/validator}/__init__.py (96%) rename {validator => vnfsdk_pkgtools/validator}/aria_validator.py (97%) create mode 100644 vnfsdk_pkgtools/version.py diff --git a/LICENSE b/LICENSE.txt similarity index 100% rename from LICENSE rename to LICENSE.txt diff --git a/MANIFEST.in b/MANIFEST.in index 1e4434f..ff36802 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,3 +1,3 @@ include COPYRIGHT -include LICENSE +include LICENSE.txt include requirements.txt diff --git a/Makefile b/Makefile index f16e167..a1f7365 100644 --- a/Makefile +++ b/Makefile @@ -14,9 +14,6 @@ # under the License. # -PACKAGER_CLI=cli -PACKAGER_LIB=packager - .PHONY: clean requirements .DEFAULT_GOAL = requirements diff --git a/README.rst b/README.rst new file mode 100644 index 0000000..cd13d63 --- /dev/null +++ b/README.rst @@ -0,0 +1,16 @@ +ONAP VNFSDK CSAR Package Tool +======================== +VNFSDK package tool provides VNF product DevOps engineers with the tools to manage the VNF package content. The tools are provided in a form of a shared library (Python module) that can be used in other projects. A CLI is also provided out-of-the box for DevOps to use the library with their scripts and automation framework. + +Source Code: https://git.onap.org/vnfsdk/pkgtools + +Usage +----- +- Create CSAR package + $ vnfsdk csar-create -d DESTINATION [–manifest MANIFEST] [–history HISTORY] [–tests TESTS] [–licenses LICENSES] source entry +- Extract CSAR package + $ vnfsdk csar-open -d DESTINATION source +- Validate CSAR package + $ vnfsdk csar-validate source + +All commands have -h switch which displays help and description of all parameters. diff --git a/assembly.xml b/assembly.xml index 2c62ae3..41bb7f9 100644 --- a/assembly.xml +++ b/assembly.xml @@ -28,11 +28,9 @@ under the License. LICENSE MANIFEST.in Makefile - cli/** - packager/** + vnfsdk_pkgtools/** requirements.txt setup.py - validator/** **/*.pyc diff --git a/pom.xml b/pom.xml index 413f2d8..a64eff7 100644 --- a/pom.xml +++ b/pom.xml @@ -33,13 +33,13 @@ under the License. CSAR manipulation shared library with CLI UTF-8 - cli,packager,validator + vnfsdk_pkgtools tests xunit-results.xml coverage.xml py python - cli/**.py,packager/**.py,validator/**.py + vnfsdk_pkgtools/**.py **/tests/**,**/.tox/py27/** **/tests/**.py setup.py,.tox/py27/** diff --git a/setup.py b/setup.py index 61c8ec1..c6ba237 100644 --- a/setup.py +++ b/setup.py @@ -1,97 +1,93 @@ -#!/usr/bin/env python - -# -# Copyright (c) 2016-2017 GigaSpaces Technologies Ltd. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -# - -import os -from setuptools import setup -import sys - -if sys.version_info < (2, 7): - sys.exit('VNF SDK requires Python 2.7+') -if sys.version_info >= (3, 0): - sys.exit('VNF SDK does not support Python 3') - - -root_dir = os.path.dirname(__file__) -install_requires = [] -extras_require = {} - -with open(os.path.join(root_dir, 'requirements.txt')) as requirements: - for requirement in requirements.readlines(): - # get rid of comments or trailing comments - requirement = requirement.split('#')[0].strip() - if not requirement: - continue # skip empty and comment lines - # dependencies which use environment markers have to go in as - # conditional dependencies under "extra_require", see more at: - # https://wheel.readthedocs.io/en/latest/index.html#defining-conditional-dependencies - if ';' in requirement: - package, condition = requirement.split(';') - cond_name = ':{0}'.format(condition.strip()) - extras_require.setdefault(cond_name, []) - extras_require[cond_name].append(package.strip()) - else: - install_requires.append(requirement) - -setup( - name='vnfsdk', - version='0.1', - description='VNF SDK CSAR package tool', - license='Apache License Version 2.0', - - author='GigaSpaces', - author_email='info@gigaspaces.com', - - url='http://onap.org/', - - classifiers=[ - 'Development Status :: 4 - Beta', - 'Environment :: Console', - 'Environment :: Web Environment', - 'Intended Audience :: Developers', - 'Intended Audience :: System Administrators', - 'License :: OSI Approved :: Apache Software License', - 'Operating System :: OS Independent', - 'Programming Language :: Python', - 'Topic :: Software Development :: Libraries :: Python Modules', - 'Topic :: System :: Networking', - 'Topic :: System :: Systems Administration'], - - packages=[ - 'packager', - 'cli', - 'validator' - ], - - package_dir={ - 'packager': 'packager', - 'cli': 'cli', - 'validator': 'validator' - }, - - entry_points={ - 'console_scripts': [ - 'vnfsdk = cli.__main__:main'], - 'vnfsdk.validator': [ - 'aria = validator.aria_validator:AriaValidator' - ] - }, - - include_package_data=True, - install_requires=install_requires, - extras_require=extras_require) - +#!/usr/bin/env python + +# +# Copyright (c) 2016-2017 GigaSpaces Technologies Ltd. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# + +import os +from setuptools import find_packages, setup +import sys + +if sys.version_info < (2, 7): + sys.exit('VNF SDK requires Python 2.7+') +if sys.version_info >= (3, 0): + sys.exit('VNF SDK does not support Python 3') + + +root_dir = os.path.dirname(__file__) +install_requires = [] +extras_require = {} + +with open(os.path.join(root_dir, 'requirements.txt')) as requirements: + for requirement in requirements.readlines(): + # get rid of comments or trailing comments + requirement = requirement.split('#')[0].strip() + if not requirement: + continue # skip empty and comment lines + # dependencies which use environment markers have to go in as + # conditional dependencies under "extra_require", see more at: + # https://wheel.readthedocs.io/en/latest/index.html#defining-conditional-dependencies + if ';' in requirement: + package, condition = requirement.split(';') + cond_name = ':{0}'.format(condition.strip()) + extras_require.setdefault(cond_name, []) + extras_require[cond_name].append(package.strip()) + else: + install_requires.append(requirement) + +version = { } +with open(os.path.join(root_dir, 'vnfsdk_pkgtools/version.py')) as fp: + exec(fp.read(), version) + +setup( + name='vnfsdk', + version=version['__version__'], + description='VNFSDK CSAR package tool', + long_description="VNFSDK CSAR package tool in ONAP", + license='Apache License Version 2.0', + + author='ONAP', + + url='https://git.onap.org/vnfsdk/pkgtools', + + classifiers=[ + 'Development Status :: 4 - Beta', + 'Environment :: Console', + 'Environment :: Web Environment', + 'Intended Audience :: Developers', + 'Intended Audience :: System Administrators', + 'License :: OSI Approved :: Apache Software License', + 'Operating System :: OS Independent', + 'Programming Language :: Python', + 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 2.7', + 'Topic :: Software Development :: Libraries :: Python Modules', + 'Topic :: System :: Networking', + 'Topic :: System :: Systems Administration'], + + packages=find_packages(exclude=['tests*']), + + entry_points={ + 'console_scripts': [ + 'vnfsdk = vnfsdk_pkgtools.cli.__main__:main'], + 'vnfsdk.pkgtools.validator': [ + 'aria = vnfsdk_pkgtools.validator.aria_validator:AriaValidator' + ] + }, + + include_package_data=True, + install_requires=install_requires, + extras_require=extras_require) + diff --git a/tests/cli/test_cli.py b/tests/cli/test_cli.py index 0be9ec6..b3fb8f0 100644 --- a/tests/cli/test_cli.py +++ b/tests/cli/test_cli.py @@ -15,7 +15,7 @@ # import pytest -from cli import __main__ +from vnfsdk_pkgtools.cli import __main__ def test_main(capsys): with pytest.raises(SystemExit): diff --git a/tests/packager/test_package.py b/tests/packager/test_package.py index b4c4526..662d004 100644 --- a/tests/packager/test_package.py +++ b/tests/packager/test_package.py @@ -20,7 +20,7 @@ import os import tempfile import shutil -from packager import csar +from vnfsdk_pkgtools.packager import csar CSAR_RESOURCE_DIR = 'tests/resources/csar' CSAR_ENTRY_FILE = 'test_entry.yaml' diff --git a/tox.ini b/tox.ini index 393ea42..17f33c7 100644 --- a/tox.ini +++ b/tox.ini @@ -36,5 +36,5 @@ commands = coverage run --module pytest --junitxml xunit-results.xml coverage xml --omit=".tox/py27/*","tests/*" coverage report --omit=".tox/py27/*","tests/*" - #pytest tests --cov-report term-missing --cov packager --cov cli + #pytest tests --cov-report term-missing --cov vnfsdk_pkgtools diff --git a/__init__.py b/vnfsdk_pkgtools/__init__.py similarity index 100% rename from __init__.py rename to vnfsdk_pkgtools/__init__.py diff --git a/cli/__init__.py b/vnfsdk_pkgtools/cli/__init__.py similarity index 100% rename from cli/__init__.py rename to vnfsdk_pkgtools/cli/__init__.py diff --git a/cli/__main__.py b/vnfsdk_pkgtools/cli/__main__.py similarity index 95% rename from cli/__main__.py rename to vnfsdk_pkgtools/cli/__main__.py index ff11bca..005a1ac 100644 --- a/cli/__main__.py +++ b/vnfsdk_pkgtools/cli/__main__.py @@ -1,120 +1,120 @@ -# -# Copyright (c) 2016-2017 GigaSpaces Technologies Ltd. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -# - -from packager import csar -import sys -import logging -import argparse -from aria import install_aria_extensions -import os -import shutil -import tempfile - -import validator - -def csar_create_func(namespace): - csar.write(namespace.source, - namespace.entry, - namespace.destination, - logging, - args=namespace) -def csar_open_func(namespace): - csar.read(namespace.source, - namespace.destination, - logging) -def csar_validate_func(namespace): - workdir = tempfile.mkdtemp() - try: - reader = None - reader = csar.read(namespace.source, - workdir, - logging) - - driver = validator.get_validator(namespace.parser) - driver.validate(reader) - finally: - shutil.rmtree(workdir, ignore_errors=True) - - -def parse_args(args_list): - """ - CLI entry point - """ - install_aria_extensions() - - parser = argparse.ArgumentParser(description='VNF SDK CSAR manipulation tool') - - subparsers = parser.add_subparsers(help='csar-create') - csar_create = subparsers.add_parser('csar-create') - csar_create.set_defaults(func=csar_create_func) - csar_create.add_argument('-v', '--verbose', - dest='verbosity', - action='count', - default=0, - help='Set verbosity level (can be passed multiple times)') - csar_create.add_argument( - 'source', - help='Service template directory') - csar_create.add_argument( - 'entry', - help='Entry definition file relative to service template directory') - csar_create.add_argument( - '-d', '--destination', - help='Output CSAR zip destination', - required=True) - csar_create.add_argument( - '--manifest', - help='Manifest file relative to service template directory') - csar_create.add_argument( - '--history', - help='Change history file relative to service template directory') - csar_create.add_argument( - '--tests', - help='Directory containing test information, relative to service template directory') - csar_create.add_argument( - '--licenses', - help='Directory containing license information, relative to service template directory') - - - csar_open = subparsers.add_parser('csar-open') - csar_open.set_defaults(func=csar_open_func) - csar_open.add_argument( - 'source', - help='CSAR file location') - csar_open.add_argument( - '-d', '--destination', - help='Output directory to extract the CSAR into', - required=True) - - csar_validate = subparsers.add_parser('csar-validate') - csar_validate.set_defaults(func=csar_validate_func) - csar_validate.add_argument( - 'source', - help='CSAR file location') - csar_validate.add_argument( - '-p', '--parser', - default='aria', - help='use which csar parser to validate') - - return parser.parse_args(args_list) - -def main(): - args = parse_args(sys.argv[1:]) - args.func(args) - - -if __name__ == '__main__': - main() +# +# Copyright (c) 2016-2017 GigaSpaces Technologies Ltd. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# + +from vnfsdk_pkgtools.packager import csar +import sys +import logging +import argparse +from aria import install_aria_extensions +import os +import shutil +import tempfile + +from vnfsdk_pkgtools import validator + +def csar_create_func(namespace): + csar.write(namespace.source, + namespace.entry, + namespace.destination, + logging, + args=namespace) +def csar_open_func(namespace): + csar.read(namespace.source, + namespace.destination, + logging) +def csar_validate_func(namespace): + workdir = tempfile.mkdtemp() + try: + reader = None + reader = csar.read(namespace.source, + workdir, + logging) + + driver = validator.get_validator(namespace.parser) + driver.validate(reader) + finally: + shutil.rmtree(workdir, ignore_errors=True) + + +def parse_args(args_list): + """ + CLI entry point + """ + install_aria_extensions() + + parser = argparse.ArgumentParser(description='VNF SDK CSAR manipulation tool') + + subparsers = parser.add_subparsers(help='csar-create') + csar_create = subparsers.add_parser('csar-create') + csar_create.set_defaults(func=csar_create_func) + csar_create.add_argument('-v', '--verbose', + dest='verbosity', + action='count', + default=0, + help='Set verbosity level (can be passed multiple times)') + csar_create.add_argument( + 'source', + help='Service template directory') + csar_create.add_argument( + 'entry', + help='Entry definition file relative to service template directory') + csar_create.add_argument( + '-d', '--destination', + help='Output CSAR zip destination', + required=True) + csar_create.add_argument( + '--manifest', + help='Manifest file relative to service template directory') + csar_create.add_argument( + '--history', + help='Change history file relative to service template directory') + csar_create.add_argument( + '--tests', + help='Directory containing test information, relative to service template directory') + csar_create.add_argument( + '--licenses', + help='Directory containing license information, relative to service template directory') + + + csar_open = subparsers.add_parser('csar-open') + csar_open.set_defaults(func=csar_open_func) + csar_open.add_argument( + 'source', + help='CSAR file location') + csar_open.add_argument( + '-d', '--destination', + help='Output directory to extract the CSAR into', + required=True) + + csar_validate = subparsers.add_parser('csar-validate') + csar_validate.set_defaults(func=csar_validate_func) + csar_validate.add_argument( + 'source', + help='CSAR file location') + csar_validate.add_argument( + '-p', '--parser', + default='aria', + help='use which csar parser to validate') + + return parser.parse_args(args_list) + +def main(): + args = parse_args(sys.argv[1:]) + args.func(args) + + +if __name__ == '__main__': + main() diff --git a/packager/__init__.py b/vnfsdk_pkgtools/packager/__init__.py similarity index 100% rename from packager/__init__.py rename to vnfsdk_pkgtools/packager/__init__.py diff --git a/packager/csar.py b/vnfsdk_pkgtools/packager/csar.py similarity index 97% rename from packager/csar.py rename to vnfsdk_pkgtools/packager/csar.py index 0f4af5e..b4bee29 100644 --- a/packager/csar.py +++ b/vnfsdk_pkgtools/packager/csar.py @@ -1,285 +1,285 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import pprint -import tempfile -import zipfile - -import requests -from ruamel import yaml # @UnresolvedImport - - -META_FILE = 'TOSCA-Metadata/TOSCA.meta' -META_FILE_VERSION_KEY = 'TOSCA-Meta-File-Version' -META_FILE_VERSION_VALUE = '1.0' -META_CSAR_VERSION_KEY = 'CSAR-Version' -META_CSAR_VERSION_VALUE = '1.1' -META_CREATED_BY_KEY = 'Created-By' -META_CREATED_BY_VALUE = 'ONAP' -META_ENTRY_DEFINITIONS_KEY = 'Entry-Definitions' -META_ENTRY_MANIFEST_FILE_KEY = 'Entry-Manifest' -META_ENTRY_HISTORY_FILE_KEY = 'Entry-Change-Log' -META_ENTRY_TESTS_DIR_KEY = 'Entry-Tests' -META_ENTRY_LICENSES_DIR_KEY = 'Entry-Licenses' - -BASE_METADATA = { - META_FILE_VERSION_KEY: META_FILE_VERSION_VALUE, - META_CSAR_VERSION_KEY: META_CSAR_VERSION_VALUE, - META_CREATED_BY_KEY: META_CREATED_BY_VALUE, -} - - -def check_file_dir(root, entry, msg, check_for_non=False, check_dir=False): - path = os.path.join(root, entry) - if check_for_non: - ret = not os.path.exists(path) - error_msg = '{0} already exists. ' + msg - elif check_dir: - ret = os.path.isdir(path) - error_msg = '{0} is not an existing directory. ' + msg - else: - ret = os.path.isfile(path) - error_msg = '{0} is not an existing file. ' + msg - if not ret: - raise ValueError(error_msg.format(path)) - - -def write(source, entry, destination, logger, args): - source = os.path.expanduser(source) - destination = os.path.expanduser(destination) - metadata = BASE_METADATA.copy() - - check_file_dir(root=source, - entry='', - msg='Please specify the service template directory.', - check_dir=True) - - check_file_dir(root=source, - entry=entry, - msg='Please specify a valid entry point.', - check_dir=False) - metadata[META_ENTRY_DEFINITIONS_KEY] = entry - - check_file_dir(root='', - entry=destination, - msg='Please provide a path to where the CSAR should be created.', - check_for_non=True) - - check_file_dir(root=source, - entry=META_FILE, - msg='This commands generates a meta file for you. Please ' - 'remove the existing metafile.', - check_for_non=True) - - if(args.manifest): - check_file_dir(root=source, - entry=args.manifest, - msg='Please specify a valid manifest file.', - check_dir=False) - metadata[META_ENTRY_MANIFEST_FILE_KEY] = args.manifest - - if(args.history): - check_file_dir(root=source, - entry=args.history, - msg='Please specify a valid change history file.', - check_dir=False) - metadata[META_ENTRY_HISTORY_FILE_KEY] = args.history - - if(args.tests): - check_file_dir(root=source, - entry=args.tests, - msg='Please specify a valid test directory.', - check_dir=True) - metadata[META_ENTRY_TESTS_DIR_KEY] = args.tests - - if(args.licenses): - check_file_dir(root=source, - entry=args.licenses, - msg='Please specify a valid license directory.', - check_dir=True) - metadata[META_ENTRY_LICENSES_DIR_KEY] = args.licenses - - logger.debug('Compressing root directory to ZIP') - with zipfile.ZipFile(destination, 'w', zipfile.ZIP_DEFLATED) as f: - for root, dirs, files in os.walk(source): - for file in files: - file_full_path = os.path.join(root, file) - file_relative_path = os.path.relpath(file_full_path, source) - logger.debug('Writing to archive: {0}'.format(file_relative_path)) - f.write(file_full_path, file_relative_path) - # add empty dir - for dir in dirs: - dir_full_path = os.path.join(root, dir) - if len(os.listdir(dir_full_path)) == 0: - dir_relative_path = os.path.relpath(dir_full_path, source) + os.sep - logger.debug('Writing to archive: {0}'.format(dir_relative_path)) - f.write(dir_full_path + os.sep, dir_relative_path) - - logger.debug('Writing new metadata file to {0}'.format(META_FILE)) - f.writestr(META_FILE, yaml.dump(metadata, default_flow_style=False)) - - -class _CSARReader(object): - - def __init__(self, source, destination, logger): - self.logger = logger - if os.path.isdir(destination) and os.listdir(destination): - raise ValueError('{0} already exists and is not empty. ' - 'Please specify the location where the CSAR ' - 'should be extracted.'.format(destination)) - downloaded_csar = '://' in source - if downloaded_csar: - file_descriptor, download_target = tempfile.mkstemp() - os.close(file_descriptor) - self._download(source, download_target) - source = download_target - self.source = os.path.expanduser(source) - self.destination = os.path.expanduser(destination) - self.metadata = {} - try: - if not os.path.exists(self.source): - raise ValueError('{0} does not exists. Please specify a valid CSAR path.' - .format(self.source)) - if not zipfile.is_zipfile(self.source): - raise ValueError('{0} is not a valid CSAR.'.format(self.source)) - self._extract() - self._read_metadata() - self._validate() - finally: - if downloaded_csar: - os.remove(self.source) - - @property - def created_by(self): - return self.metadata.get(META_CREATED_BY_KEY) - - @property - def csar_version(self): - return self.metadata.get(META_CSAR_VERSION_KEY) - - @property - def meta_file_version(self): - return self.metadata.get(META_FILE_VERSION_KEY) - - @property - def entry_definitions(self): - return self.metadata.get(META_ENTRY_DEFINITIONS_KEY) - - @property - def entry_definitions_yaml(self): - with open(os.path.join(self.destination, self.entry_definitions)) as f: - return yaml.load(f) - - @property - def entry_manifest_file(self): - return self.metadata.get(META_ENTRY_MANIFEST_FILE_KEY) - - @property - def entry_history_file(self): - return self.metadata.get(META_ENTRY_HISTORY_FILE_KEY) - - @property - def entry_tests_dir(self): - return self.metadata.get(META_ENTRY_TESTS_DIR_KEY) - - @property - def entry_licenses_dir(self): - return self.metadata.get(META_ENTRY_LICENSES_DIR_KEY) - - def _extract(self): - self.logger.debug('Extracting CSAR contents') - if not os.path.exists(self.destination): - os.mkdir(self.destination) - with zipfile.ZipFile(self.source) as f: - f.extractall(self.destination) - self.logger.debug('CSAR contents successfully extracted') - - def _read_metadata(self): - csar_metafile = os.path.join(self.destination, META_FILE) - if not os.path.exists(csar_metafile): - raise ValueError('Metadata file {0} is missing from the CSAR'.format(csar_metafile)) - self.logger.debug('CSAR metadata file: {0}'.format(csar_metafile)) - self.logger.debug('Attempting to parse CSAR metadata YAML') - with open(csar_metafile) as f: - self.metadata.update(yaml.load(f)) - self.logger.debug('CSAR metadata:\n{0}'.format(pprint.pformat(self.metadata))) - - def _validate(self): - def validate_key(key, expected=None): - if not self.metadata.get(key): - raise ValueError('{0} is missing from the metadata file.'.format(key)) - actual = str(self.metadata[key]) - if expected and actual != expected: - raise ValueError('{0} is expected to be {1} in the metadata file while it is in ' - 'fact {2}.'.format(key, expected, actual)) - validate_key(META_FILE_VERSION_KEY, expected=META_FILE_VERSION_VALUE) - validate_key(META_CSAR_VERSION_KEY, expected=META_CSAR_VERSION_VALUE) - validate_key(META_CREATED_BY_KEY) - validate_key(META_ENTRY_DEFINITIONS_KEY) - self.logger.debug('CSAR entry definitions: {0}'.format(self.entry_definitions)) - self.logger.debug('CSAR manifest file: {0}'.format(self.entry_manifest_file)) - self.logger.debug('CSAR change history file: {0}'.format(self.entry_history_file)) - self.logger.debug('CSAR tests directory: {0}'.format(self.entry_tests_dir)) - self.logger.debug('CSAR licenses directory: {0}'.format(self.entry_licenses_dir)) - - check_file_dir(self.destination, - self.entry_definitions, - 'The entry definitions {0} referenced by the metadata ' - 'file does not exist.'.format(self.entry_definitions), - check_dir=False) - - if(self.entry_manifest_file): - check_file_dir(self.destination, - self.entry_manifest_file, - 'The manifest file {0} referenced by the metadata ' - 'file does not exist.'.format(self.entry_manifest_file), - check_dir=False) - - if(self.entry_history_file): - check_file_dir(self.destination, - self.entry_history_file, - 'The change history file {0} referenced by the metadata ' - 'file does not exist.'.format(self.entry_history_file), - check_dir=False) - - if(self.entry_tests_dir): - check_file_dir(self.destination, - self.entry_tests_dir, - 'The test directory {0} referenced by the metadata ' - 'file does not exist.'.format(self.entry_tests_dir), - check_dir=True) - - if(self.entry_licenses_dir): - check_file_dir(self.destination, - self.entry_licenses_dir, - 'The license directory {0} referenced by the metadata ' - 'file does not exist.'.format(self.entry_licenses_dir), - check_dir=True) - - def _download(self, url, target): - response = requests.get(url, stream=True) - if response.status_code != 200: - raise ValueError('Server at {0} returned a {1} status code' - .format(url, response.status_code)) - self.logger.info('Downloading {0} to {1}'.format(url, target)) - with open(target, 'wb') as f: - for chunk in response.iter_content(chunk_size=8192): - if chunk: - f.write(chunk) - - -def read(source, destination, logger): - return _CSARReader(source=source, destination=destination, logger=logger) +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import pprint +import tempfile +import zipfile + +import requests +from ruamel import yaml # @UnresolvedImport + + +META_FILE = 'TOSCA-Metadata/TOSCA.meta' +META_FILE_VERSION_KEY = 'TOSCA-Meta-File-Version' +META_FILE_VERSION_VALUE = '1.0' +META_CSAR_VERSION_KEY = 'CSAR-Version' +META_CSAR_VERSION_VALUE = '1.1' +META_CREATED_BY_KEY = 'Created-By' +META_CREATED_BY_VALUE = 'ONAP' +META_ENTRY_DEFINITIONS_KEY = 'Entry-Definitions' +META_ENTRY_MANIFEST_FILE_KEY = 'Entry-Manifest' +META_ENTRY_HISTORY_FILE_KEY = 'Entry-Change-Log' +META_ENTRY_TESTS_DIR_KEY = 'Entry-Tests' +META_ENTRY_LICENSES_DIR_KEY = 'Entry-Licenses' + +BASE_METADATA = { + META_FILE_VERSION_KEY: META_FILE_VERSION_VALUE, + META_CSAR_VERSION_KEY: META_CSAR_VERSION_VALUE, + META_CREATED_BY_KEY: META_CREATED_BY_VALUE, +} + + +def check_file_dir(root, entry, msg, check_for_non=False, check_dir=False): + path = os.path.join(root, entry) + if check_for_non: + ret = not os.path.exists(path) + error_msg = '{0} already exists. ' + msg + elif check_dir: + ret = os.path.isdir(path) + error_msg = '{0} is not an existing directory. ' + msg + else: + ret = os.path.isfile(path) + error_msg = '{0} is not an existing file. ' + msg + if not ret: + raise ValueError(error_msg.format(path)) + + +def write(source, entry, destination, logger, args): + source = os.path.expanduser(source) + destination = os.path.expanduser(destination) + metadata = BASE_METADATA.copy() + + check_file_dir(root=source, + entry='', + msg='Please specify the service template directory.', + check_dir=True) + + check_file_dir(root=source, + entry=entry, + msg='Please specify a valid entry point.', + check_dir=False) + metadata[META_ENTRY_DEFINITIONS_KEY] = entry + + check_file_dir(root='', + entry=destination, + msg='Please provide a path to where the CSAR should be created.', + check_for_non=True) + + check_file_dir(root=source, + entry=META_FILE, + msg='This commands generates a meta file for you. Please ' + 'remove the existing metafile.', + check_for_non=True) + + if(args.manifest): + check_file_dir(root=source, + entry=args.manifest, + msg='Please specify a valid manifest file.', + check_dir=False) + metadata[META_ENTRY_MANIFEST_FILE_KEY] = args.manifest + + if(args.history): + check_file_dir(root=source, + entry=args.history, + msg='Please specify a valid change history file.', + check_dir=False) + metadata[META_ENTRY_HISTORY_FILE_KEY] = args.history + + if(args.tests): + check_file_dir(root=source, + entry=args.tests, + msg='Please specify a valid test directory.', + check_dir=True) + metadata[META_ENTRY_TESTS_DIR_KEY] = args.tests + + if(args.licenses): + check_file_dir(root=source, + entry=args.licenses, + msg='Please specify a valid license directory.', + check_dir=True) + metadata[META_ENTRY_LICENSES_DIR_KEY] = args.licenses + + logger.debug('Compressing root directory to ZIP') + with zipfile.ZipFile(destination, 'w', zipfile.ZIP_DEFLATED) as f: + for root, dirs, files in os.walk(source): + for file in files: + file_full_path = os.path.join(root, file) + file_relative_path = os.path.relpath(file_full_path, source) + logger.debug('Writing to archive: {0}'.format(file_relative_path)) + f.write(file_full_path, file_relative_path) + # add empty dir + for dir in dirs: + dir_full_path = os.path.join(root, dir) + if len(os.listdir(dir_full_path)) == 0: + dir_relative_path = os.path.relpath(dir_full_path, source) + os.sep + logger.debug('Writing to archive: {0}'.format(dir_relative_path)) + f.write(dir_full_path + os.sep, dir_relative_path) + + logger.debug('Writing new metadata file to {0}'.format(META_FILE)) + f.writestr(META_FILE, yaml.dump(metadata, default_flow_style=False)) + + +class _CSARReader(object): + + def __init__(self, source, destination, logger): + self.logger = logger + if os.path.isdir(destination) and os.listdir(destination): + raise ValueError('{0} already exists and is not empty. ' + 'Please specify the location where the CSAR ' + 'should be extracted.'.format(destination)) + downloaded_csar = '://' in source + if downloaded_csar: + file_descriptor, download_target = tempfile.mkstemp() + os.close(file_descriptor) + self._download(source, download_target) + source = download_target + self.source = os.path.expanduser(source) + self.destination = os.path.expanduser(destination) + self.metadata = {} + try: + if not os.path.exists(self.source): + raise ValueError('{0} does not exists. Please specify a valid CSAR path.' + .format(self.source)) + if not zipfile.is_zipfile(self.source): + raise ValueError('{0} is not a valid CSAR.'.format(self.source)) + self._extract() + self._read_metadata() + self._validate() + finally: + if downloaded_csar: + os.remove(self.source) + + @property + def created_by(self): + return self.metadata.get(META_CREATED_BY_KEY) + + @property + def csar_version(self): + return self.metadata.get(META_CSAR_VERSION_KEY) + + @property + def meta_file_version(self): + return self.metadata.get(META_FILE_VERSION_KEY) + + @property + def entry_definitions(self): + return self.metadata.get(META_ENTRY_DEFINITIONS_KEY) + + @property + def entry_definitions_yaml(self): + with open(os.path.join(self.destination, self.entry_definitions)) as f: + return yaml.load(f) + + @property + def entry_manifest_file(self): + return self.metadata.get(META_ENTRY_MANIFEST_FILE_KEY) + + @property + def entry_history_file(self): + return self.metadata.get(META_ENTRY_HISTORY_FILE_KEY) + + @property + def entry_tests_dir(self): + return self.metadata.get(META_ENTRY_TESTS_DIR_KEY) + + @property + def entry_licenses_dir(self): + return self.metadata.get(META_ENTRY_LICENSES_DIR_KEY) + + def _extract(self): + self.logger.debug('Extracting CSAR contents') + if not os.path.exists(self.destination): + os.mkdir(self.destination) + with zipfile.ZipFile(self.source) as f: + f.extractall(self.destination) + self.logger.debug('CSAR contents successfully extracted') + + def _read_metadata(self): + csar_metafile = os.path.join(self.destination, META_FILE) + if not os.path.exists(csar_metafile): + raise ValueError('Metadata file {0} is missing from the CSAR'.format(csar_metafile)) + self.logger.debug('CSAR metadata file: {0}'.format(csar_metafile)) + self.logger.debug('Attempting to parse CSAR metadata YAML') + with open(csar_metafile) as f: + self.metadata.update(yaml.load(f)) + self.logger.debug('CSAR metadata:\n{0}'.format(pprint.pformat(self.metadata))) + + def _validate(self): + def validate_key(key, expected=None): + if not self.metadata.get(key): + raise ValueError('{0} is missing from the metadata file.'.format(key)) + actual = str(self.metadata[key]) + if expected and actual != expected: + raise ValueError('{0} is expected to be {1} in the metadata file while it is in ' + 'fact {2}.'.format(key, expected, actual)) + validate_key(META_FILE_VERSION_KEY, expected=META_FILE_VERSION_VALUE) + validate_key(META_CSAR_VERSION_KEY, expected=META_CSAR_VERSION_VALUE) + validate_key(META_CREATED_BY_KEY) + validate_key(META_ENTRY_DEFINITIONS_KEY) + self.logger.debug('CSAR entry definitions: {0}'.format(self.entry_definitions)) + self.logger.debug('CSAR manifest file: {0}'.format(self.entry_manifest_file)) + self.logger.debug('CSAR change history file: {0}'.format(self.entry_history_file)) + self.logger.debug('CSAR tests directory: {0}'.format(self.entry_tests_dir)) + self.logger.debug('CSAR licenses directory: {0}'.format(self.entry_licenses_dir)) + + check_file_dir(self.destination, + self.entry_definitions, + 'The entry definitions {0} referenced by the metadata ' + 'file does not exist.'.format(self.entry_definitions), + check_dir=False) + + if(self.entry_manifest_file): + check_file_dir(self.destination, + self.entry_manifest_file, + 'The manifest file {0} referenced by the metadata ' + 'file does not exist.'.format(self.entry_manifest_file), + check_dir=False) + + if(self.entry_history_file): + check_file_dir(self.destination, + self.entry_history_file, + 'The change history file {0} referenced by the metadata ' + 'file does not exist.'.format(self.entry_history_file), + check_dir=False) + + if(self.entry_tests_dir): + check_file_dir(self.destination, + self.entry_tests_dir, + 'The test directory {0} referenced by the metadata ' + 'file does not exist.'.format(self.entry_tests_dir), + check_dir=True) + + if(self.entry_licenses_dir): + check_file_dir(self.destination, + self.entry_licenses_dir, + 'The license directory {0} referenced by the metadata ' + 'file does not exist.'.format(self.entry_licenses_dir), + check_dir=True) + + def _download(self, url, target): + response = requests.get(url, stream=True) + if response.status_code != 200: + raise ValueError('Server at {0} returned a {1} status code' + .format(url, response.status_code)) + self.logger.info('Downloading {0} to {1}'.format(url, target)) + with open(target, 'wb') as f: + for chunk in response.iter_content(chunk_size=8192): + if chunk: + f.write(chunk) + + +def read(source, destination, logger): + return _CSARReader(source=source, destination=destination, logger=logger) diff --git a/validator/__init__.py b/vnfsdk_pkgtools/validator/__init__.py similarity index 96% rename from validator/__init__.py rename to vnfsdk_pkgtools/validator/__init__.py index d33dcdf..f6d9073 100644 --- a/validator/__init__.py +++ b/vnfsdk_pkgtools/validator/__init__.py @@ -19,7 +19,7 @@ import six from stevedore import driver -VALIDATOR_NS = "vnfsdk.validator" +VALIDATOR_NS = "vnfsdk.pkgtools.validator" def get_validator(params): """Get validate driver and load it. diff --git a/validator/aria_validator.py b/vnfsdk_pkgtools/validator/aria_validator.py similarity index 97% rename from validator/aria_validator.py rename to vnfsdk_pkgtools/validator/aria_validator.py index 6149790..83d7dfe 100644 --- a/validator/aria_validator.py +++ b/vnfsdk_pkgtools/validator/aria_validator.py @@ -25,7 +25,7 @@ from aria.parser.consumption import ( ServiceInstance ) -import validator +from vnfsdk_pkgtools import validator class AriaValidator(validator.ValidatorBase): diff --git a/vnfsdk_pkgtools/version.py b/vnfsdk_pkgtools/version.py new file mode 100644 index 0000000..9960302 --- /dev/null +++ b/vnfsdk_pkgtools/version.py @@ -0,0 +1,3 @@ +global __version__ + +__version__='0.2' -- 2.16.6