Added file digest computation

Support to compute file digest and save it to manifest file. Also extend
the manifest file module to be able to write content into temporary
file.

Change-Id: If1f3f42799eb527bdeac418d1a40aa203641f628
Issue-ID: VNFSDK-174
Signed-off-by: Lianhao Lu <lianhao.lu@intel.com>

diff --git a/tests/packager/test_package.py b/tests/packager/test_csar.py
similarity index 78%
rename from tests/packager/test_package.py
rename to tests/packager/test_csar.py
index 662d004..fcf034e 100644 (file)
--- a/tests/packager/test_package.py
+++ b/tests/packager/test_csar.py
@@ -27,7 +27,7 @@ CSAR_ENTRY_FILE = 'test_entry.yaml'
 CSAR_OUTPUT_FILE = 'output.csar'
 
 Args = collections.namedtuple('Args',
-        ['source', 'entry', 'manifest', 'history', 'tests', 'licenses'])
+        ['source', 'entry', 'manifest', 'history', 'tests', 'licenses', 'digest'])
 
 ARGS_MANIFEST = {
             'source': CSAR_RESOURCE_DIR,
@@ -36,8 +36,20 @@ ARGS_MANIFEST = {
             'history': 'ChangeLog.txt',
             'tests': 'Tests',
             'licenses': 'Licenses',
+            'digest': None
         }
 
+ARGS_MANIFEST_DIGEST = {
+            'source': CSAR_RESOURCE_DIR,
+            'entry': CSAR_ENTRY_FILE,
+            'manifest': 'test_entry.mf',
+            'history': 'ChangeLog.txt',
+            'tests': 'Tests',
+            'licenses': 'Licenses',
+            'digest': 'sha256'
+        }
+
+
 ARGS_NO_MANIFEST = {
             'source': CSAR_RESOURCE_DIR,
             'entry': CSAR_ENTRY_FILE,
@@ -45,6 +57,7 @@ ARGS_NO_MANIFEST = {
             'history': None,
             'tests': None,
             'licenses': None,
+            'digest': None,
         }
 
 
@@ -55,7 +68,7 @@ def csar_write_test(args):
         csar.write(args.source, args.entry, csar_target_dir + '/' + CSAR_OUTPUT_FILE, logging, args)
         csar.read(csar_target_dir + '/' + CSAR_OUTPUT_FILE, csar_extract_dir, logging)
         assert filecmp.cmp(args.source + '/' + args.entry, csar_extract_dir + '/' + args.entry)
-        if(args.manifest):
+        if(args.manifest and not args.digest):
             assert filecmp.cmp(args.source + '/' + args.manifest,
                                csar_extract_dir + '/' + args.manifest)
         if(args.history):
@@ -76,3 +89,11 @@ def test_CSARWrite_manifest():
     if not os.path.exists(license_path):
         os.makedirs(license_path)
     csar_write_test(Args(**ARGS_MANIFEST))
+
+
+def test_CSARWrite_manifest_digest():
+    # Because git can not store emptry directory, we need to create manually here
+    license_path = ARGS_MANIFEST['source'] + '/' + ARGS_MANIFEST['licenses']
+    if not os.path.exists(license_path):
+        os.makedirs(license_path)
+    csar_write_test(Args(**ARGS_MANIFEST_DIGEST))

diff --git a/tests/resources/csar/test_entry.mf b/tests/resources/csar/test_entry.mf
index 710d1a2..4441457 100644 (file)
--- a/tests/resources/csar/test_entry.mf
+++ b/tests/resources/csar/test_entry.mf
@@ -1,5 +1,5 @@
 metadata:
 vnf_product_name: test
 vnf_provider_id: test
-vnf_pacakage_version: 1.0
-vnf_release_date_time: 2017.09.15T15:00+8:00
+vnf_package_version: 1.0
+vnf_release_data_time: 2017-09-15T15:00:03+08:00

diff --git a/vnfsdk_pkgtools/cli/__main__.py b/vnfsdk_pkgtools/cli/__main__.py
index 005a1ac..88776de 100644 (file)
--- a/vnfsdk_pkgtools/cli/__main__.py
+++ b/vnfsdk_pkgtools/cli/__main__.py
@@ -87,6 +87,10 @@ def parse_args(args_list):
     csar_create.add_argument(
         '--licenses',
         help='Directory containing license information, relative to service template directory')
+    csar_create.add_argument(
+        '--digest',
+        choices=['SHA256', 'SHA512'],
+        help='If present, means to check the file deigest in manifest;  compute the digest using the specified hash algorithm of all files in the csar package to be put into the manifest file')
 
 
     csar_open = subparsers.add_parser('csar-open')

diff --git a/vnfsdk_pkgtools/packager/csar.py b/vnfsdk_pkgtools/packager/csar.py
index b4bee29..31fba8b 100644 (file)
--- a/vnfsdk_pkgtools/packager/csar.py
+++ b/vnfsdk_pkgtools/packager/csar.py
@@ -21,6 +21,7 @@ import zipfile
 import requests
 from ruamel import yaml # @UnresolvedImport
 
+from vnfsdk_pkgtools.packager import manifest
 
 META_FILE = 'TOSCA-Metadata/TOSCA.meta'
 META_FILE_VERSION_KEY = 'TOSCA-Meta-File-Version'
@@ -90,6 +91,12 @@ def write(source, entry, destination, logger, args):
                        msg='Please specify a valid manifest file.',
                        check_dir=False)
         metadata[META_ENTRY_MANIFEST_FILE_KEY] = args.manifest
+        manifest_file = manifest.Manifest(source, args.manifest) 
+        manifest_file_full_path = os.path.join(source, args.manifest)
+    else:
+        manifest_file = None
+        manifest_file_full_path = None
+
 
     if(args.history):
         check_file_dir(root=source,
@@ -117,9 +124,14 @@ def write(source, entry, destination, logger, args):
         for root, dirs, files in os.walk(source):
             for file in files:
                 file_full_path = os.path.join(root, file)
-                file_relative_path = os.path.relpath(file_full_path, source)
-                logger.debug('Writing to archive: {0}'.format(file_relative_path))
-                f.write(file_full_path, file_relative_path)
+                # skip manifest file here in case we need to generate digest
+                if file_full_path!=manifest_file_full_path:
+                    file_relative_path = os.path.relpath(file_full_path, source)
+                    logger.debug('Writing to archive: {0}'.format(file_relative_path))
+                    f.write(file_full_path, file_relative_path)
+                    if manifest_file and args.digest:
+                        logger.debug('Update file digest: {0}'.format(file_relative_path))
+                        manifest_file.add_file(file_relative_path, args.digest)
             # add empty dir
             for dir in dirs:
                 dir_full_path = os.path.join(root, dir)
@@ -128,6 +140,13 @@ def write(source, entry, destination, logger, args):
                     logger.debug('Writing to archive: {0}'.format(dir_relative_path))
                     f.write(dir_full_path + os.sep, dir_relative_path)
 
+        if manifest_file:
+            if args.digest:
+                logger.debug('Update manifest file to temporary file')
+                manifest_file_full_path = manifest_file.update_to_file(True)
+            logger.debug('Writing to archive: {0}'.format(args.manifest))
+            f.write(manifest_file_full_path, args.manifest)
+
         logger.debug('Writing new metadata file to {0}'.format(META_FILE))
         f.writestr(META_FILE, yaml.dump(metadata, default_flow_style=False))
 
@@ -149,6 +168,7 @@ class _CSARReader(object):
         self.source = os.path.expanduser(source)
         self.destination = os.path.expanduser(destination)
         self.metadata = {}
+        self.manifest = None
         try:
             if not os.path.exists(self.source):
                 raise ValueError('{0} does not exists. Please specify a valid CSAR path.'
@@ -247,6 +267,9 @@ class _CSARReader(object):
                             'The manifest file {0} referenced by the metadata '
                             'file does not exist.'.format(self.entry_manifest_file),
                             check_dir=False)
+             self.manifest = manifest.Manifest(self.destination,
+                                               self.entry_manifest_file)
+
 
         if(self.entry_history_file):
              check_file_dir(self.destination,

diff --git a/vnfsdk_pkgtools/packager/manifest.py b/vnfsdk_pkgtools/packager/manifest.py
index a2d9d70..d819a70 100644 (file)
--- a/vnfsdk_pkgtools/packager/manifest.py
+++ b/vnfsdk_pkgtools/packager/manifest.py
@@ -15,6 +15,7 @@
 
 from collections import namedtuple
 import os
+import tempfile
 
 import udatetime
 
@@ -158,7 +159,13 @@ class Manifest(object):
             ret += "Hash: %s\n" % digest[1]
         return ret
 
-    def update_to_file(self):
+    def update_to_file(self, temporary=False):
         content = self.return_as_string()
-        with open(os.path.join(self.root, self.path), 'w') as fp:
+        if temporary:
+            abs_path = tempfile.mktemp()
+        else:
+            abs_path = os.path.abspath(os.path.join(self.root, self.path))
+
+        with open(abs_path, 'w') as fp:
             fp.write(content)
+        return abs_path