From 2006f67db860e755b599acda36cfc85a49d8c8f3 Mon Sep 17 00:00:00 2001 From: Ruslan Kashapov Date: Tue, 19 Jan 2021 17:57:00 +0200 Subject: [PATCH] ZIP archive support for multiple YANG files delivery on Schema Set creation using REST Issue-ID: CPS-180 Change-Id: I7e78a595593b170b981746e9aed1a7e5a45b202a Signed-off-by: Ruslan Kashapov --- .../org/onap/cps/rest/utils/MultipartFileUtil.java | 88 ++++++++++++++++++--- .../rest/controller/AdminRestControllerSpec.groovy | 69 +++++++++++++--- .../cps/rest/utils/MultipartFileUtilSpec.groovy | 67 +++++++++++++--- cps-rest/src/test/resources/no-yang-files.zip | Bin 0 -> 390 bytes .../test/resources/yang-files-multiple-sets.zip | Bin 0 -> 1304 bytes cps-rest/src/test/resources/yang-files-set.zip | Bin 0 -> 655 bytes 6 files changed, 197 insertions(+), 27 deletions(-) create mode 100644 cps-rest/src/test/resources/no-yang-files.zip create mode 100644 cps-rest/src/test/resources/yang-files-multiple-sets.zip create mode 100644 cps-rest/src/test/resources/yang-files-set.zip diff --git a/cps-rest/src/main/java/org/onap/cps/rest/utils/MultipartFileUtil.java b/cps-rest/src/main/java/org/onap/cps/rest/utils/MultipartFileUtil.java index c53d1a42a..532a0ca84 100644 --- a/cps-rest/src/main/java/org/onap/cps/rest/utils/MultipartFileUtil.java +++ b/cps-rest/src/main/java/org/onap/cps/rest/utils/MultipartFileUtil.java @@ -19,13 +19,18 @@ package org.onap.cps.rest.utils; -import static com.google.common.base.Preconditions.checkNotNull; import static org.opendaylight.yangtools.yang.common.YangConstants.RFC6020_YANG_FILE_EXTENSION; import com.google.common.collect.ImmutableMap; +import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.io.InputStream; import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Locale; import java.util.Map; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; import lombok.AccessLevel; import lombok.NoArgsConstructor; import org.onap.cps.spi.exceptions.CpsException; @@ -35,26 +40,81 @@ import org.springframework.web.multipart.MultipartFile; @NoArgsConstructor(access = AccessLevel.PRIVATE) public class MultipartFileUtil { + private static final String ZIP_FILE_EXTENSION = ".zip"; + private static final String YANG_FILE_EXTENSION = RFC6020_YANG_FILE_EXTENSION; + private static final int READ_BUFFER_SIZE = 1024; + /** * Extracts yang resources from multipart file instance. * * @param multipartFile the yang file uploaded * @return yang resources as {map} where the key is original file name, and the value is file content - * @throws ModelValidationException if the file name extension is not '.yang' + * @throws ModelValidationException if the file name extension is not '.yang' or '.zip' + * or if zip archive contain no yang files * @throws CpsException if the file content cannot be read */ public static Map extractYangResourcesMap(final MultipartFile multipartFile) { - return ImmutableMap.of(extractYangResourceName(multipartFile), extractYangResourceContent(multipartFile)); + final String originalFileName = multipartFile.getOriginalFilename(); + if (resourceNameEndsWithExtension(originalFileName, YANG_FILE_EXTENSION)) { + return ImmutableMap.of(originalFileName, extractYangResourceContent(multipartFile)); + } + if (resourceNameEndsWithExtension(originalFileName, ZIP_FILE_EXTENSION)) { + return extractYangResourcesMapFromZipArchive(multipartFile); + } + throw new ModelValidationException("Unsupported file type.", + String.format("Filename %s matches none of expected extensions: %s", originalFileName, + Arrays.asList(YANG_FILE_EXTENSION, ZIP_FILE_EXTENSION))); + } + + private static Map extractYangResourcesMapFromZipArchive(final MultipartFile multipartFile) { + final ImmutableMap.Builder yangResourceMapBuilder = ImmutableMap.builder(); + + try ( + final InputStream inputStream = multipartFile.getInputStream(); + final ZipInputStream zipInputStream = new ZipInputStream(inputStream); + ) { + ZipEntry zipEntry; + while ((zipEntry = zipInputStream.getNextEntry()) != null) { + extractZipEntryToMapIfApplicable(yangResourceMapBuilder, zipEntry, zipInputStream); + } + zipInputStream.closeEntry(); + + } catch (final IOException e) { + throw new CpsException("Cannot extract resources from zip archive.", e.getMessage(), e); + } + + try { + final Map yangResourceMap = yangResourceMapBuilder.build(); + if (yangResourceMap.isEmpty()) { + throw new ModelValidationException("Archive contains no YANG resources.", + String.format("Archive contains no files having %s extension.", YANG_FILE_EXTENSION)); + } + return yangResourceMap; + + } catch (final IllegalArgumentException e) { + throw new ModelValidationException("Invalid ZIP archive content.", + "Multiple resources with same name detected.", e); + } } - private static String extractYangResourceName(final MultipartFile multipartFile) { - final String fileName = checkNotNull(multipartFile.getOriginalFilename(), "Missing filename."); - if (!fileName.endsWith(RFC6020_YANG_FILE_EXTENSION)) { - throw new ModelValidationException("Unsupported file type.", - String.format("Filename %s does not end with '%s'", fileName, RFC6020_YANG_FILE_EXTENSION)); + private static void extractZipEntryToMapIfApplicable( + final ImmutableMap.Builder yangResourceMapBuilder, final ZipEntry zipEntry, + final ZipInputStream zipInputStream) throws IOException { + + final String yangResourceName = extractResourceNameFromPath(zipEntry.getName()); + if (zipEntry.isDirectory() || !resourceNameEndsWithExtension(yangResourceName, YANG_FILE_EXTENSION)) { + return; } - return fileName; + yangResourceMapBuilder.put(yangResourceName, extractYangResourceContent(zipInputStream)); + } + + private static boolean resourceNameEndsWithExtension(final String resourceName, final String extension) { + return resourceName != null && resourceName.toLowerCase(Locale.ENGLISH).endsWith(extension); + } + + private static String extractResourceNameFromPath(final String path) { + return path == null ? "" : path.replaceAll("^.*[\\\\/]", ""); } private static String extractYangResourceContent(final MultipartFile multipartFile) { @@ -65,4 +125,14 @@ public class MultipartFileUtil { } } + private static String extractYangResourceContent(final ZipInputStream zipInputStream) throws IOException { + try (final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream()) { + final byte[] buffer = new byte[READ_BUFFER_SIZE]; + int numberOfBytesRead; + while ((numberOfBytesRead = zipInputStream.read(buffer, 0, READ_BUFFER_SIZE)) > 0) { + byteArrayOutputStream.write(buffer, 0, numberOfBytesRead); + } + return byteArrayOutputStream.toString(StandardCharsets.UTF_8); + } + } } diff --git a/cps-rest/src/test/groovy/org/onap/cps/rest/controller/AdminRestControllerSpec.groovy b/cps-rest/src/test/groovy/org/onap/cps/rest/controller/AdminRestControllerSpec.groovy index a95d606a3..60f54bfa7 100644 --- a/cps-rest/src/test/groovy/org/onap/cps/rest/controller/AdminRestControllerSpec.groovy +++ b/cps-rest/src/test/groovy/org/onap/cps/rest/controller/AdminRestControllerSpec.groovy @@ -37,6 +37,7 @@ import org.springframework.test.web.servlet.MockMvc import org.springframework.util.LinkedMultiValueMap import org.springframework.util.MultiValueMap import spock.lang.Specification +import spock.lang.Unroll import static org.onap.cps.spi.CascadeDeleteAllowed.CASCADE_DELETE_PROHIBITED import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.delete @@ -85,29 +86,66 @@ class AdminRestControllerSpec extends Specification { response.status == HttpStatus.BAD_REQUEST.value() } - def 'Create schema set from yang file'() { + def 'Create schema set from yang file.'() { def yangResourceMapCapture - given: + given: 'single yang file' def multipartFile = createMultipartFile("filename.yang", "content") - when: + when: 'file uploaded with schema set create request' def response = performCreateSchemaSetRequest(multipartFile) - then: 'Service method is invoked with expected parameters' + then: 'associated service method is invoked with expected parameters' 1 * mockCpsModuleService.createSchemaSet('test-dataspace', 'test-schema-set', _) >> { args -> yangResourceMapCapture = args[2] } yangResourceMapCapture['filename.yang'] == 'content' - and: 'Response code indicates success' + and: 'response code indicates success' response.status == HttpStatus.CREATED.value() } - def 'Create schema set from file with invalid filename extension'() { - given: + def 'Create schema set from zip archive.'() { + def yangResourceMapCapture + given: 'zip archive with multiple .yang files inside' + def multipartFile = createZipMultipartFileFromResource("/yang-files-set.zip") + when: 'file uploaded with schema set create request' + def response = performCreateSchemaSetRequest(multipartFile) + then: 'associated service method is invoked with expected parameters' + 1 * mockCpsModuleService.createSchemaSet('test-dataspace', 'test-schema-set', _) >> + { args -> yangResourceMapCapture = args[2] } + yangResourceMapCapture['assembly.yang'] == "fake assembly content 1\n" + yangResourceMapCapture['component.yang'] == "fake component content 1\n" + and: 'response code indicates success' + response.status == HttpStatus.CREATED.value() + } + + @Unroll + def 'Create schema set from zip archive having #caseDescriptor.'() { + when: 'zip archive having #caseDescriptor is uploaded with create schema set request' + def response = performCreateSchemaSetRequest(multipartFile) + then: 'create schema set rejected' + response.status == HttpStatus.BAD_REQUEST.value() + where: 'following cases are tested' + caseDescriptor | multipartFile + 'no .yang files inside' | createZipMultipartFileFromResource("/no-yang-files.zip") + 'multiple .yang files with same name' | createZipMultipartFileFromResource("/yang-files-multiple-sets.zip") + } + + def 'Create schema set from file with unsupported filename extension.'() { + given: 'file with unsupported filename extension (.doc)' def multipartFile = createMultipartFile("filename.doc", "content") - when: + when: 'file uploaded with schema set create request' def response = performCreateSchemaSetRequest(multipartFile) - then: 'Create schema fails' + then: 'create schema set rejected' response.status == HttpStatus.BAD_REQUEST.value() } + @Unroll + def 'Create schema set from #fileType file with IOException occurrence on processing.'() { + when: 'file uploaded with schema set create request' + def response = performCreateSchemaSetRequest(createMultipartFileForIOException(fileType)) + then: 'the error response returned indicating internal server error occurrence' + response.status == HttpStatus.INTERNAL_SERVER_ERROR.value() + where: 'following file types are used' + fileType << ['YANG', 'ZIP'] + } + def 'Delete schema set.'() { when: 'delete schema set endpoint is invoked' def response = performDeleteRequest(schemaSetEndpoint) @@ -138,6 +176,19 @@ class AdminRestControllerSpec extends Specification { return new MockMultipartFile("file", filename, "text/plain", content.getBytes()) } + def createZipMultipartFileFromResource(resourcePath) { + return new MockMultipartFile("file", "test.zip", "application/zip", + getClass().getResource(resourcePath).getBytes()) + } + + def createMultipartFileForIOException(extension) { + def multipartFile = Mock(MockMultipartFile) + multipartFile.getOriginalFilename() >> "TEST." + extension + multipartFile.getBytes() >> { throw new IOException() } + multipartFile.getInputStream() >> { throw new IOException() } + return multipartFile + } + def performCreateSchemaSetRequest(multipartFile) { return mvc.perform( multipart(schemaSetsEndpoint) diff --git a/cps-rest/src/test/groovy/org/onap/cps/rest/utils/MultipartFileUtilSpec.groovy b/cps-rest/src/test/groovy/org/onap/cps/rest/utils/MultipartFileUtilSpec.groovy index ba5aa4cac..3e2bdec37 100644 --- a/cps-rest/src/test/groovy/org/onap/cps/rest/utils/MultipartFileUtilSpec.groovy +++ b/cps-rest/src/test/groovy/org/onap/cps/rest/utils/MultipartFileUtilSpec.groovy @@ -19,30 +19,79 @@ package org.onap.cps.rest.utils +import org.onap.cps.spi.exceptions.CpsException import org.onap.cps.spi.exceptions.ModelValidationException import org.springframework.mock.web.MockMultipartFile +import org.springframework.web.multipart.MultipartFile import spock.lang.Specification +import spock.lang.Unroll class MultipartFileUtilSpec extends Specification { - def 'Extract yang resource from multipart file'() { - given: + def 'Extract yang resource from yang file.'() { + given: 'uploaded yang file' def multipartFile = new MockMultipartFile("file", "filename.yang", "text/plain", "content".getBytes()) - when: + when: 'resources are extracted from the file' def result = MultipartFileUtil.extractYangResourcesMap(multipartFile) - then: - assert result != null + then: 'the expected name and content are extracted as result' assert result.size() == 1 assert result.get("filename.yang") == "content" } - def 'Extract yang resource from file with invalid filename extension'() { - given: + def 'Extract yang resources from zip archive.'() { + given: 'uploaded zip archive containing 2 yang files and 1 not yang (json) file' + def multipartFile = new MockMultipartFile("file", "TEST.ZIP", "application/zip", + getClass().getResource("/yang-files-set.zip").getBytes()) + when: 'resources are extracted from zip file' + def result = MultipartFileUtil.extractYangResourcesMap(multipartFile) + then: 'information from yang files is extracted, not yang file (json) is ignored' + assert result.size() == 2 + assert result["assembly.yang"] == "fake assembly content 1\n" + assert result["component.yang"] == "fake component content 1\n" + } + + @Unroll + def 'Extract resources from zip archive having #caseDescriptor.'() { + when: 'attempt to extract resources from zip file is performed' + MultipartFileUtil.extractYangResourcesMap(multipartFile) + then: 'the validation exception is thrown indicating invalid zip file content' + thrown(ModelValidationException) + where: 'following cases are tested' + caseDescriptor | multipartFile + 'text files only' | multipartZipFileFromResource("/no-yang-files.zip") + 'multiple yang file with same name' | multipartZipFileFromResource("/yang-files-multiple-sets.zip") + } + + def 'Extract yang resource from a file with invalid filename extension.'() { + given: 'uploaded file with unsupported (.doc) exception' def multipartFile = new MockMultipartFile("file", "filename.doc", "text/plain", "content".getBytes()) - when: + when: 'attempt to extract resources from the file is performed' MultipartFileUtil.extractYangResourcesMap(multipartFile) - then: + then: 'validation exception is thrown indicating the file type is not supported' thrown(ModelValidationException) } + @Unroll + def 'IOException thrown during yang resources extraction from #fileType file.'() { + when: 'attempt to extract resources from the file is performed' + MultipartFileUtil.extractYangResourcesMap(multipartFileForIOException(fileType)) + then: 'CpsException is thrown indicating the internal error occurrence' + thrown(CpsException) + where: 'following file types are used' + fileType << ['YANG', 'ZIP'] + } + + def multipartZipFileFromResource(resourcePath) { + return new MockMultipartFile("file", "TEST.ZIP", "application/zip", + getClass().getResource(resourcePath).getBytes()) + } + + def multipartFileForIOException(extension) { + def multipartFile = Mock(MultipartFile) + multipartFile.getOriginalFilename() >> "TEST." + extension + multipartFile.getBytes() >> { throw new IOException() } + multipartFile.getInputStream() >> { throw new IOException() } + return multipartFile + } + } diff --git a/cps-rest/src/test/resources/no-yang-files.zip b/cps-rest/src/test/resources/no-yang-files.zip new file mode 100644 index 0000000000000000000000000000000000000000..83f6963b1856efa46e2aebf89bccc9224cf2d9f9 GIT binary patch literal 390 zcmWIWW@Zs#00Gr5;~+2tO7H>cynNlt#JqIfw9K5;V*LQ1S}w5cu}APv%VB!AH0ogJMfd~a0*EQte1I?|3#%y*{{(omvVl}G1K|lEy#mBx006pt BQaAtr literal 0 HcmV?d00001 diff --git a/cps-rest/src/test/resources/yang-files-multiple-sets.zip b/cps-rest/src/test/resources/yang-files-multiple-sets.zip new file mode 100644 index 0000000000000000000000000000000000000000..855e87bb2700cf0e3c46985022c9b5078c692c63 GIT binary patch literal 1304 zcmWIWW@Zs#00EW+lOQkyO7H{e%EY{M-L%Y{)MDM@)Drywuwo_#4u(&)Re{^4PksQ> zaR!KGFcl{j7pLYXPPnGzW=Y3BU)B}|bbE7plEiCDSAq;rF4f$NVPM-8X#9jm8t zGl1N}1+SZI)&u?ypq(s5(Pspgzu7pCiZ18 zJBb5LlLTT}AVv#{JXdb{>z~FdPUTz5sVEnGZ9mGaRL7WdX)7M%r6Bfks*nCJhkOe`3 z%*bTVj6Ik^X&46nI)Yee(FoIwD^0^RZ)w!U)Ql1V5R-7HNQ6l{@t6d1996=LJcyQQ6Meo`5Tl3Vc@T0EQV3w1PKXRY@q{61Te6ru?L%WoZ*8lK_SeNVnX*7 k&?#`U&_f7YLPD68hs`YXFk)o`Y2W}t2cYspK+M1Z0HZxvZU6uP literal 0 HcmV?d00001 diff --git a/cps-rest/src/test/resources/yang-files-set.zip b/cps-rest/src/test/resources/yang-files-set.zip new file mode 100644 index 0000000000000000000000000000000000000000..09236ce474349db7f3e28233cf057077405fee31 GIT binary patch literal 655 zcmWIWW@Zs#0D;nYlOQkyO7H{e%EY{M-L%Y{)MDM@)Drywuwo_#4u(&)Re{^4PksQ> zaR!KGFcl{j7pLYXPPnGzW=Y3BU)B}|bbE7plEiCDSAq;rF4f$NVPM-8X#9jm8t zGl1N}1+cMm6b!i#zDow0*q6cV zBn~u95{PAi7%eE0^K%RG^MD%lKoZ{GXZ%7v&Yjmiao+o^HshvGUJhFn#er^QWU^<* z9lSte!QihWh(fagrWsdoL$oq5Y-!ZR)C}_;IPf7R;SMZ>NjveF1acm-o3I5E!X$1+ Z3=g3OS%5by8%Q||5Xt}*#{w|}0|3XnszU$( literal 0 HcmV?d00001 -- 2.16.6