gerrit.onap Code Review - dcaegen2/collectors/datafile.git/blob

   1 /*-
   2  * ============LICENSE_START======================================================================
   3  * Copyright (C) 2018-2019 Nordix Foundation. All rights reserved.
   4  * Copyright (C) 2020 Nokia. All rights reserved.
   5  * ===============================================================================================
   6  * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
   7  * in compliance with the License. You may obtain a copy of the License at
   8  *
   9  * http://www.apache.org/licenses/LICENSE-2.0
  10  *
  11  * Unless required by applicable law or agreed to in writing, software distributed under the License
  12  * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
  13  * or implied. See the License for the specific language governing permissions and limitations under
  14  * the License.
  15  * ============LICENSE_END========================================================================
  16  */
  17
  18 package org.onap.dcaegen2.collectors.datafile.tasks;
  19
  20 import static org.junit.jupiter.api.Assertions.assertEquals;
  21 import static org.mockito.ArgumentMatchers.any;
  22 import static org.mockito.Mockito.doReturn;
  23 import static org.mockito.Mockito.doThrow;
  24 import static org.mockito.Mockito.mock;
  25 import static org.mockito.Mockito.spy;
  26 import static org.mockito.Mockito.times;
  27 import static org.mockito.Mockito.verify;
  28 import static org.mockito.Mockito.verifyNoMoreInteractions;
  29 import static org.mockito.Mockito.when;
  30
  31 import java.nio.file.Path;
  32 import java.nio.file.Paths;
  33 import java.time.Duration;
  34 import java.util.HashMap;
  35 import java.util.Map;
  36
  37 import org.junit.jupiter.api.BeforeAll;
  38 import org.junit.jupiter.api.BeforeEach;
  39 import org.junit.jupiter.api.Test;
  40 import org.onap.dcaegen2.collectors.datafile.configuration.AppConfig;
  41 import org.onap.dcaegen2.collectors.datafile.configuration.CertificateConfig;
  42 import org.onap.dcaegen2.collectors.datafile.exceptions.DatafileTaskException;
  43 import org.onap.dcaegen2.collectors.datafile.exceptions.NonRetryableDatafileTaskException;
  44 import org.onap.dcaegen2.collectors.datafile.ftp.FtpesClient;
  45 import org.onap.dcaegen2.collectors.datafile.commons.Scheme;
  46 import org.onap.dcaegen2.collectors.datafile.ftp.SftpClient;
  47 import org.onap.dcaegen2.collectors.datafile.http.DfcHttpClient;
  48 import org.onap.dcaegen2.collectors.datafile.http.DfcHttpsClient;
  49 import org.onap.dcaegen2.collectors.datafile.model.Counters;
  50 import org.onap.dcaegen2.collectors.datafile.model.FileData;
  51 import org.onap.dcaegen2.collectors.datafile.model.FilePublishInformation;
  52 import org.onap.dcaegen2.collectors.datafile.model.ImmutableFileData;
  53 import org.onap.dcaegen2.collectors.datafile.model.ImmutableFilePublishInformation;
  54 import org.onap.dcaegen2.collectors.datafile.model.ImmutableMessageMetaData;
  55 import org.onap.dcaegen2.collectors.datafile.model.MessageMetaData;
  56 import reactor.test.StepVerifier;
  57
  58 public class FileCollectorTest {
  59     private static final String PRODUCT_NAME = "NrRadio";
  60     private static final String VENDOR_NAME = "Ericsson";
  61     private static final String LAST_EPOCH_MICROSEC = "8745745764578";
  62     private static final String SOURCE_NAME = "oteNB5309";
  63     private static final String START_EPOCH_MICROSEC = "8745745764578";
  64     private static final String TIME_ZONE_OFFSET = "UTC+05:00";
  65     private static final String PM_MEAS_CHANGE_IDENTIFIER = "PM_MEAS_FILES";
  66     private static final String FILE_READY_CHANGE_TYPE = "FileReady";
  67     private static final String FTPES_SCHEME = "ftpes://";
  68     private static final String SFTP_SCHEME = "sftp://";
  69     private static final String HTTP_SCHEME = "http://";
  70     private static final String HTTPS_SCHEME = "https://";
  71     private static final String SERVER_ADDRESS = "192.168.0.101";
  72     private static final int PORT_22 = 22;
  73     private static final String PM_FILE_NAME = "A20161224.1030-1045.bin.gz";
  74     private static final Path LOCAL_FILE_LOCATION = Paths.get(FileData.DATAFILE_TMPDIR, PM_FILE_NAME);
  75     private static final String REMOTE_FILE_LOCATION = "/ftp/rop/" + PM_FILE_NAME;
  76     private static final String USER = "usr";
  77     private static final String PWD = "pwd";
  78     private static final String FTPES_LOCATION =
  79         FTPES_SCHEME + USER + ":" + PWD + "@" + SERVER_ADDRESS + ":" + PORT_22 + REMOTE_FILE_LOCATION;
  80
  81     private static final String FTPES_LOCATION_NO_PORT =
  82         FTPES_SCHEME + USER + ":" + PWD + "@" + SERVER_ADDRESS + REMOTE_FILE_LOCATION;
  83     private static final String SFTP_LOCATION = SFTP_SCHEME + SERVER_ADDRESS + ":" + PORT_22 + REMOTE_FILE_LOCATION;
  84     private static final String SFTP_LOCATION_NO_PORT = SFTP_SCHEME + SERVER_ADDRESS + REMOTE_FILE_LOCATION;
  85
  86     private static final String HTTP_LOCATION =
  87             HTTP_SCHEME + USER + ":" + PWD + "@" + SERVER_ADDRESS + ":" + PORT_22 + REMOTE_FILE_LOCATION;
  88     private static final String HTTP_LOCATION_NO_PORT =
  89             HTTP_SCHEME + USER + ":" + PWD + "@" + SERVER_ADDRESS + REMOTE_FILE_LOCATION;
  90     private static final String HTTPS_LOCATION =
  91             HTTP_SCHEME + USER + ":" + PWD + "@" + SERVER_ADDRESS + ":" + PORT_22 + REMOTE_FILE_LOCATION;
  92     private static final String HTTPS_LOCATION_NO_PORT =
  93             HTTP_SCHEME + USER + ":" + PWD + "@" + SERVER_ADDRESS + REMOTE_FILE_LOCATION;
  94
  95     private static final String GZIP_COMPRESSION = "gzip";
  96     private static final String MEAS_COLLECT_FILE_FORMAT_TYPE = "org.3GPP.32.435#measCollec";
  97     private static final String FILE_FORMAT_VERSION = "V10";
  98
  99     private static final String CERTIFICATE_KEY_PATH = "certificateKeyPath";
 100     private static final String CERTIFICATE_KEY_PASSWORD_PATH = "certificateKeyPassword";
 101     private static final String TRUSTED_CA_PATH = "trustedCAPath";
 102     private static final String TRUSTED_CA_PASSWORD_PATH = "trustedCAPassword";
 103     private static final String CHANGE_IDENTIFIER = "PM_MEAS_FILES";
 104
 105     private static AppConfig appConfigMock = mock(AppConfig.class);
 106     private static CertificateConfig certificateConfigMock = mock(CertificateConfig.class);
 107
 108     private FtpesClient ftpesClientMock = mock(FtpesClient.class);
 109
 110     private SftpClient sftpClientMock = mock(SftpClient.class);
 111     private final Map<String, String> contextMap = new HashMap<>();
 112
 113     private DfcHttpClient dfcHttpClientMock = mock(DfcHttpClient.class);
 114     private DfcHttpsClient dfcHttpsClientMock = mock(DfcHttpsClient.class);
 115
 116     private Counters counters;
 117
 118     private MessageMetaData createMessageMetaData() {
 119         return ImmutableMessageMetaData.builder() //
 120             .productName(PRODUCT_NAME) //
 121             .vendorName(VENDOR_NAME) //
 122             .lastEpochMicrosec(LAST_EPOCH_MICROSEC) //
 123             .sourceName(SOURCE_NAME) //
 124             .startEpochMicrosec(START_EPOCH_MICROSEC) //
 125             .timeZoneOffset(TIME_ZONE_OFFSET) //
 126             .changeIdentifier(PM_MEAS_CHANGE_IDENTIFIER) //
 127             .changeType(FILE_READY_CHANGE_TYPE) //
 128             .build();
 129     }
 130
 131     private FileData createFileData(String location, Scheme scheme) {
 132         return ImmutableFileData.builder() //
 133             .name(PM_FILE_NAME) //
 134             .location(location) //
 135             .compression(GZIP_COMPRESSION) //
 136             .fileFormatType(MEAS_COLLECT_FILE_FORMAT_TYPE) //
 137             .fileFormatVersion(FILE_FORMAT_VERSION) //
 138             .scheme(scheme) //
 139             .messageMetaData(createMessageMetaData()) //
 140             .build();
 141     }
 142
 143     private FilePublishInformation createExpectedFilePublishInformation(String location) {
 144         return ImmutableFilePublishInformation.builder() //
 145             .productName(PRODUCT_NAME) //
 146             .vendorName(VENDOR_NAME) //
 147             .lastEpochMicrosec(LAST_EPOCH_MICROSEC) //
 148             .sourceName(SOURCE_NAME) //
 149             .startEpochMicrosec(START_EPOCH_MICROSEC) //
 150             .timeZoneOffset(TIME_ZONE_OFFSET) //
 151             .name(PM_FILE_NAME) //
 152             .location(location) //
 153             .internalLocation(LOCAL_FILE_LOCATION) //
 154             .compression(GZIP_COMPRESSION) //
 155             .fileFormatType(MEAS_COLLECT_FILE_FORMAT_TYPE) //
 156             .fileFormatVersion(FILE_FORMAT_VERSION) //
 157             .context(new HashMap<String, String>()) //
 158             .changeIdentifier(CHANGE_IDENTIFIER) //
 159             .build();
 160     }
 161
 162     @BeforeAll
 163     static void setUpConfiguration() {
 164         when(appConfigMock.getCertificateConfiguration()).thenReturn(certificateConfigMock);
 165         when(certificateConfigMock.keyCert()).thenReturn(CERTIFICATE_KEY_PATH);
 166         when(certificateConfigMock.keyPasswordPath()).thenReturn(CERTIFICATE_KEY_PASSWORD_PATH);
 167         when(certificateConfigMock.trustedCa()).thenReturn(TRUSTED_CA_PATH);
 168         when(certificateConfigMock.trustedCaPasswordPath()).thenReturn(TRUSTED_CA_PASSWORD_PATH);
 169     }
 170
 171     @BeforeEach
 172     void setUpTest() {
 173         counters = new Counters();
 174     }
 175
 176     @Test
 177     public void whenFtpesFile_returnCorrectResponse() throws Exception {
 178         FileCollector collectorUndetTest = spy(new FileCollector(appConfigMock, counters));
 179         doReturn(ftpesClientMock).when(collectorUndetTest).createFtpesClient(any());
 180
 181         FileData fileData = createFileData(FTPES_LOCATION_NO_PORT, Scheme.FTPES);
 182
 183         FilePublishInformation expectedfilePublishInformation =
 184             createExpectedFilePublishInformation(FTPES_LOCATION_NO_PORT);
 185
 186         StepVerifier.create(collectorUndetTest.collectFile(fileData, 3, Duration.ofSeconds(0), contextMap))
 187             .expectNext(expectedfilePublishInformation) //
 188             .verifyComplete();
 189
 190         verify(ftpesClientMock, times(1)).open();
 191         verify(ftpesClientMock, times(1)).collectFile(REMOTE_FILE_LOCATION, LOCAL_FILE_LOCATION);
 192         verify(ftpesClientMock, times(1)).close();
 193         verifyNoMoreInteractions(ftpesClientMock);
 194
 195         assertEquals(1, counters.getNoOfCollectedFiles(),"collectedFiles should have been 1");
 196         assertEquals(0, counters.getNoOfFailedFtpAttempts(),"failedFtpAttempts should have been 0");
 197         assertEquals(0, counters.getNoOfFailedHttpAttempts(),"failedHttpAttempts should have been 0");
 198     }
 199
 200     @Test
 201     public void whenSftpFile_returnCorrectResponse() throws Exception {
 202         FileCollector collectorUndetTest = spy(new FileCollector(appConfigMock, counters));
 203         doReturn(sftpClientMock).when(collectorUndetTest).createSftpClient(any());
 204
 205         FileData fileData = createFileData(SFTP_LOCATION_NO_PORT, Scheme.SFTP);
 206         FilePublishInformation expectedfilePublishInformation =
 207             createExpectedFilePublishInformation(SFTP_LOCATION_NO_PORT);
 208
 209         StepVerifier.create(collectorUndetTest.collectFile(fileData, 3, Duration.ofSeconds(0), contextMap))
 210             .expectNext(expectedfilePublishInformation) //
 211             .verifyComplete();
 212
 213         // The same again, but with port
 214         fileData = createFileData(SFTP_LOCATION, Scheme.SFTP);
 215         expectedfilePublishInformation = createExpectedFilePublishInformation(SFTP_LOCATION);
 216
 217         StepVerifier.create(collectorUndetTest.collectFile(fileData, 3, Duration.ofSeconds(0), contextMap))
 218             .expectNext(expectedfilePublishInformation) //
 219             .verifyComplete();
 220
 221         verify(sftpClientMock, times(2)).open();
 222         verify(sftpClientMock, times(2)).collectFile(REMOTE_FILE_LOCATION, LOCAL_FILE_LOCATION);
 223         verify(sftpClientMock, times(2)).close();
 224         verifyNoMoreInteractions(sftpClientMock);
 225
 226         assertEquals(2, counters.getNoOfCollectedFiles(),"collectedFiles should have been 2");
 227     }
 228
 229     @Test
 230     public void whenHttpFile_returnCorrectResponse() throws Exception {
 231         FileCollector collectorUndetTest = spy(new FileCollector(appConfigMock, counters));
 232         doReturn(dfcHttpClientMock).when(collectorUndetTest).createHttpClient(any());
 233
 234         FileData fileData = createFileData(HTTP_LOCATION_NO_PORT, Scheme.HTTP);
 235
 236         FilePublishInformation expectedfilePublishInformation =
 237                 createExpectedFilePublishInformation(HTTP_LOCATION_NO_PORT);
 238
 239         StepVerifier.create(collectorUndetTest.collectFile(fileData, 3, Duration.ofSeconds(0), contextMap))
 240                 .expectNext(expectedfilePublishInformation) //
 241                 .verifyComplete();
 242
 243         // The same again, but with port
 244         fileData = createFileData(HTTP_LOCATION, Scheme.HTTP);
 245         expectedfilePublishInformation = createExpectedFilePublishInformation(HTTP_LOCATION);
 246
 247         StepVerifier.create(collectorUndetTest.collectFile(fileData, 3, Duration.ofSeconds(0), contextMap))
 248                 .expectNext(expectedfilePublishInformation) //
 249                 .verifyComplete();
 250
 251         verify(dfcHttpClientMock, times(2)).open();
 252         verify(dfcHttpClientMock, times(2)).collectFile(REMOTE_FILE_LOCATION, LOCAL_FILE_LOCATION);
 253         verify(dfcHttpClientMock, times(2)).close();
 254         verifyNoMoreInteractions(dfcHttpClientMock);
 255
 256         assertEquals(2, counters.getNoOfCollectedFiles(),"collectedFiles should have been 1");
 257         assertEquals(0, counters.getNoOfFailedFtpAttempts(),"failedFtpAttempts should have been 0");
 258         assertEquals(0, counters.getNoOfFailedHttpAttempts(),"failedHttpAttempts should have been 0");
 259     }
 260
 261     @Test
 262     public void whenHttpsFile_returnCorrectResponse() throws Exception {
 263         FileCollector collectorUndetTest = spy(new FileCollector(appConfigMock, counters));
 264         doReturn(dfcHttpsClientMock).when(collectorUndetTest).createHttpsClient(any());
 265
 266         FileData fileData = createFileData(HTTPS_LOCATION_NO_PORT, Scheme.HTTPS);
 267
 268         FilePublishInformation expectedfilePublishInformation =
 269                 createExpectedFilePublishInformation(HTTPS_LOCATION_NO_PORT);
 270
 271         StepVerifier.create(collectorUndetTest.collectFile(fileData, 3, Duration.ofSeconds(0), contextMap))
 272                 .expectNext(expectedfilePublishInformation) //
 273                 .verifyComplete();
 274
 275         // The same again, but with port
 276         fileData = createFileData(HTTPS_LOCATION, Scheme.HTTPS);
 277         expectedfilePublishInformation = createExpectedFilePublishInformation(HTTPS_LOCATION);
 278
 279         StepVerifier.create(collectorUndetTest.collectFile(fileData, 3, Duration.ofSeconds(0), contextMap))
 280                 .expectNext(expectedfilePublishInformation) //
 281                 .verifyComplete();
 282
 283         verify(dfcHttpsClientMock, times(2)).open();
 284         verify(dfcHttpsClientMock, times(2)).collectFile(REMOTE_FILE_LOCATION, LOCAL_FILE_LOCATION);
 285         verify(dfcHttpsClientMock, times(2)).close();
 286         verifyNoMoreInteractions(dfcHttpsClientMock);
 287
 288         assertEquals(2, counters.getNoOfCollectedFiles(),"collectedFiles should have been 1");
 289         assertEquals(0, counters.getNoOfFailedFtpAttempts(),"failedFtpAttempts should have been 0");
 290         assertEquals(0, counters.getNoOfFailedHttpAttempts(),"failedHttpAttempts should have been 0");
 291     }
 292
 293     @Test
 294     public void whenFtpesFileAlwaysFail_retryAndFail() throws Exception {
 295         FileCollector collectorUndetTest = spy(new FileCollector(appConfigMock, counters));
 296         doReturn(ftpesClientMock).when(collectorUndetTest).createFtpesClient(any());
 297
 298         FileData fileData = createFileData(FTPES_LOCATION, Scheme.FTPES);
 299         doThrow(new DatafileTaskException("Unable to collect file.")).when(ftpesClientMock)
 300             .collectFile(REMOTE_FILE_LOCATION, LOCAL_FILE_LOCATION);
 301
 302         StepVerifier.create(collectorUndetTest.collectFile(fileData, 3, Duration.ofSeconds(0), contextMap))
 303             .expectErrorMessage("Retries exhausted: 3/3") //
 304             .verify();
 305
 306         verify(ftpesClientMock, times(4)).collectFile(REMOTE_FILE_LOCATION, LOCAL_FILE_LOCATION);
 307
 308         assertEquals(0, counters.getNoOfCollectedFiles(),"collectedFiles should have been 0");
 309         assertEquals(4, counters.getNoOfFailedFtpAttempts(),"failedFtpAttempts should have been 4");
 310         assertEquals(0, counters.getNoOfFailedHttpAttempts(),"failedHttpAttempts should have been 0");
 311     }
 312
 313     @Test
 314     public void whenFtpesFileAlwaysFail_failWithoutRetry() throws Exception {
 315         FileCollector collectorUndetTest = spy(new FileCollector(appConfigMock, counters));
 316         doReturn(ftpesClientMock).when(collectorUndetTest).createFtpesClient(any());
 317
 318         FileData fileData = createFileData(FTPES_LOCATION, Scheme.FTPES);
 319         doThrow(new NonRetryableDatafileTaskException("Unable to collect file.")).when(ftpesClientMock)
 320             .collectFile(REMOTE_FILE_LOCATION, LOCAL_FILE_LOCATION);
 321
 322         StepVerifier.create(collectorUndetTest.collectFile(fileData, 3, Duration.ofSeconds(0), contextMap))
 323             .expectErrorMessage("Non retryable file transfer failure") //
 324             .verify();
 325
 326         verify(ftpesClientMock, times(1)).collectFile(REMOTE_FILE_LOCATION, LOCAL_FILE_LOCATION);
 327
 328         assertEquals(0, counters.getNoOfCollectedFiles(),"collectedFiles should have been 0");
 329         assertEquals(1, counters.getNoOfFailedFtpAttempts(),"failedFtpAttempts should have been 1");
 330         assertEquals(0, counters.getNoOfFailedHttpAttempts(),"failedHttpAttempts should have been 0");
 331     }
 332
 333     @Test
 334     public void whenFtpesFileFailOnce_retryAndReturnCorrectResponse() throws Exception {
 335         FileCollector collectorUndetTest = spy(new FileCollector(appConfigMock, counters));
 336         doReturn(ftpesClientMock).when(collectorUndetTest).createFtpesClient(any());
 337         doThrow(new DatafileTaskException("Unable to collect file.")).doNothing().when(ftpesClientMock)
 338             .collectFile(REMOTE_FILE_LOCATION, LOCAL_FILE_LOCATION);
 339
 340         FilePublishInformation expectedfilePublishInformation =
 341             createExpectedFilePublishInformation(FTPES_LOCATION_NO_PORT);
 342
 343         FileData fileData = createFileData(FTPES_LOCATION_NO_PORT, Scheme.FTPES);
 344
 345         StepVerifier.create(collectorUndetTest.collectFile(fileData, 3, Duration.ofSeconds(0), contextMap))
 346             .expectNext(expectedfilePublishInformation) //
 347             .verifyComplete();
 348
 349         verify(ftpesClientMock, times(2)).collectFile(REMOTE_FILE_LOCATION, LOCAL_FILE_LOCATION);
 350
 351         assertEquals(1, counters.getNoOfCollectedFiles(),"collectedFiles should have been 1");
 352         assertEquals(1, counters.getNoOfFailedFtpAttempts(),"failedFtpAttempts should have been 1");
 353         assertEquals(0, counters.getNoOfFailedHttpAttempts(),"failedHttpAttempts should have been 0");
 354     }
 355 }