gerrit.onap Code Review - dcaegen2/collectors/datafile.git/blob

   1 /*-
   2  * ============LICENSE_START======================================================================
   3  * Copyright (C) 2018-2019 Nordix Foundation. All rights reserved.
   4  * Copyright (C) 2020 Nokia. All rights reserved.
   5  * ===============================================================================================
   6  * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
   7  * in compliance with the License. You may obtain a copy of the License at
   8  *
   9  * http://www.apache.org/licenses/LICENSE-2.0
  10  *
  11  * Unless required by applicable law or agreed to in writing, software distributed under the License
  12  * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
  13  * or implied. See the License for the specific language governing permissions and limitations under
  14  * the License.
  15  * ============LICENSE_END========================================================================
  16  */
  17
  18 package org.onap.dcaegen2.collectors.datafile.tasks;
  19
  20 import static org.junit.jupiter.api.Assertions.assertEquals;
  21 import static org.mockito.ArgumentMatchers.any;
  22 import static org.mockito.Mockito.doReturn;
  23 import static org.mockito.Mockito.doThrow;
  24 import static org.mockito.Mockito.mock;
  25 import static org.mockito.Mockito.spy;
  26 import static org.mockito.Mockito.times;
  27 import static org.mockito.Mockito.verify;
  28 import static org.mockito.Mockito.verifyNoMoreInteractions;
  29 import static org.mockito.Mockito.when;
  30
  31 import java.nio.file.Path;
  32 import java.nio.file.Paths;
  33 import java.time.Duration;
  34 import java.util.HashMap;
  35 import java.util.Map;
  36
  37 import org.junit.jupiter.api.BeforeAll;
  38 import org.junit.jupiter.api.BeforeEach;
  39 import org.junit.jupiter.api.Test;
  40 import org.onap.dcaegen2.collectors.datafile.configuration.AppConfig;
  41 import org.onap.dcaegen2.collectors.datafile.configuration.FtpesConfig;
  42 import org.onap.dcaegen2.collectors.datafile.exceptions.DatafileTaskException;
  43 import org.onap.dcaegen2.collectors.datafile.exceptions.NonRetryableDatafileTaskException;
  44 import org.onap.dcaegen2.collectors.datafile.ftp.FtpesClient;
  45 import org.onap.dcaegen2.collectors.datafile.ftp.Scheme;
  46 import org.onap.dcaegen2.collectors.datafile.ftp.SftpClient;
  47 import org.onap.dcaegen2.collectors.datafile.model.Counters;
  48 import org.onap.dcaegen2.collectors.datafile.model.FileData;
  49 import org.onap.dcaegen2.collectors.datafile.model.FilePublishInformation;
  50 import org.onap.dcaegen2.collectors.datafile.model.ImmutableFileData;
  51 import org.onap.dcaegen2.collectors.datafile.model.ImmutableFilePublishInformation;
  52 import org.onap.dcaegen2.collectors.datafile.model.ImmutableMessageMetaData;
  53 import org.onap.dcaegen2.collectors.datafile.model.MessageMetaData;
  54 import reactor.test.StepVerifier;
  55
  56 public class FileCollectorTest {
  57     private static final String PRODUCT_NAME = "NrRadio";
  58     private static final String VENDOR_NAME = "Ericsson";
  59     private static final String LAST_EPOCH_MICROSEC = "8745745764578";
  60     private static final String SOURCE_NAME = "oteNB5309";
  61     private static final String START_EPOCH_MICROSEC = "8745745764578";
  62     private static final String TIME_ZONE_OFFSET = "UTC+05:00";
  63     private static final String PM_MEAS_CHANGE_IDENTIFIER = "PM_MEAS_FILES";
  64     private static final String FILE_READY_CHANGE_TYPE = "FileReady";
  65     private static final String FTPES_SCHEME = "ftpes://";
  66     private static final String SFTP_SCHEME = "sftp://";
  67     private static final String SERVER_ADDRESS = "192.168.0.101";
  68     private static final int PORT_22 = 22;
  69     private static final String PM_FILE_NAME = "A20161224.1030-1045.bin.gz";
  70     private static final Path LOCAL_FILE_LOCATION = Paths.get(FileData.DATAFILE_TMPDIR, PM_FILE_NAME);
  71     private static final String REMOTE_FILE_LOCATION = "/ftp/rop/" + PM_FILE_NAME;
  72     private static final String USER = "usr";
  73     private static final String PWD = "pwd";
  74     private static final String FTPES_LOCATION =
  75         FTPES_SCHEME + USER + ":" + PWD + "@" + SERVER_ADDRESS + ":" + PORT_22 + REMOTE_FILE_LOCATION;
  76
  77     private static final String FTPES_LOCATION_NO_PORT =
  78         FTPES_SCHEME + USER + ":" + PWD + "@" + SERVER_ADDRESS + REMOTE_FILE_LOCATION;
  79     private static final String SFTP_LOCATION = SFTP_SCHEME + SERVER_ADDRESS + ":" + PORT_22 + REMOTE_FILE_LOCATION;
  80     private static final String SFTP_LOCATION_NO_PORT = SFTP_SCHEME + SERVER_ADDRESS + REMOTE_FILE_LOCATION;
  81
  82     private static final String GZIP_COMPRESSION = "gzip";
  83     private static final String MEAS_COLLECT_FILE_FORMAT_TYPE = "org.3GPP.32.435#measCollec";
  84     private static final String FILE_FORMAT_VERSION = "V10";
  85
  86     private static final String FTP_KEY_PATH = "ftpKeyPath";
  87     private static final String FTP_KEY_PASSWORD_PATH = "ftpKeyPassword";
  88     private static final String TRUSTED_CA_PATH = "trustedCAPath";
  89     private static final String TRUSTED_CA_PASSWORD_PATH = "trustedCAPassword";
  90     private static final String CHANGE_IDENTIFIER = "PM_MEAS_FILES";
  91
  92     private static AppConfig appConfigMock = mock(AppConfig.class);
  93     private static FtpesConfig ftpesConfigMock = mock(FtpesConfig.class);
  94
  95     private FtpesClient ftpesClientMock = mock(FtpesClient.class);
  96
  97     private SftpClient sftpClientMock = mock(SftpClient.class);
  98     private final Map<String, String> contextMap = new HashMap<>();
  99
 100     private Counters counters;
 101
 102     private MessageMetaData createMessageMetaData() {
 103         return ImmutableMessageMetaData.builder() //
 104             .productName(PRODUCT_NAME) //
 105             .vendorName(VENDOR_NAME) //
 106             .lastEpochMicrosec(LAST_EPOCH_MICROSEC) //
 107             .sourceName(SOURCE_NAME) //
 108             .startEpochMicrosec(START_EPOCH_MICROSEC) //
 109             .timeZoneOffset(TIME_ZONE_OFFSET) //
 110             .changeIdentifier(PM_MEAS_CHANGE_IDENTIFIER) //
 111             .changeType(FILE_READY_CHANGE_TYPE) //
 112             .build();
 113     }
 114
 115     private FileData createFileData(String location, Scheme scheme) {
 116         return ImmutableFileData.builder() //
 117             .name(PM_FILE_NAME) //
 118             .location(location) //
 119             .compression(GZIP_COMPRESSION) //
 120             .fileFormatType(MEAS_COLLECT_FILE_FORMAT_TYPE) //
 121             .fileFormatVersion(FILE_FORMAT_VERSION) //
 122             .scheme(scheme) //
 123             .messageMetaData(createMessageMetaData()) //
 124             .build();
 125     }
 126
 127     private FilePublishInformation createExpectedFilePublishInformation(String location) {
 128         return ImmutableFilePublishInformation.builder() //
 129             .productName(PRODUCT_NAME) //
 130             .vendorName(VENDOR_NAME) //
 131             .lastEpochMicrosec(LAST_EPOCH_MICROSEC) //
 132             .sourceName(SOURCE_NAME) //
 133             .startEpochMicrosec(START_EPOCH_MICROSEC) //
 134             .timeZoneOffset(TIME_ZONE_OFFSET) //
 135             .name(PM_FILE_NAME) //
 136             .location(location) //
 137             .internalLocation(LOCAL_FILE_LOCATION) //
 138             .compression(GZIP_COMPRESSION) //
 139             .fileFormatType(MEAS_COLLECT_FILE_FORMAT_TYPE) //
 140             .fileFormatVersion(FILE_FORMAT_VERSION) //
 141             .context(new HashMap<String, String>()) //
 142             .changeIdentifier(CHANGE_IDENTIFIER) //
 143             .build();
 144     }
 145
 146     @BeforeAll
 147     static void setUpConfiguration() {
 148         when(appConfigMock.getFtpesConfiguration()).thenReturn(ftpesConfigMock);
 149         when(ftpesConfigMock.keyCert()).thenReturn(FTP_KEY_PATH);
 150         when(ftpesConfigMock.keyPasswordPath()).thenReturn(FTP_KEY_PASSWORD_PATH);
 151         when(ftpesConfigMock.trustedCa()).thenReturn(TRUSTED_CA_PATH);
 152         when(ftpesConfigMock.trustedCaPasswordPath()).thenReturn(TRUSTED_CA_PASSWORD_PATH);
 153     }
 154
 155     @BeforeEach
 156     void setUpTest() {
 157         counters = new Counters();
 158     }
 159
 160     @Test
 161     public void whenFtpesFile_returnCorrectResponse() throws Exception {
 162         FileCollector collectorUndetTest = spy(new FileCollector(appConfigMock, counters));
 163         doReturn(ftpesClientMock).when(collectorUndetTest).createFtpesClient(any());
 164
 165         FileData fileData = createFileData(FTPES_LOCATION_NO_PORT, Scheme.FTPES);
 166
 167         FilePublishInformation expectedfilePublishInformation =
 168             createExpectedFilePublishInformation(FTPES_LOCATION_NO_PORT);
 169
 170         StepVerifier.create(collectorUndetTest.collectFile(fileData, 3, Duration.ofSeconds(0), contextMap))
 171             .expectNext(expectedfilePublishInformation) //
 172             .verifyComplete();
 173
 174         verify(ftpesClientMock, times(1)).open();
 175         verify(ftpesClientMock, times(1)).collectFile(REMOTE_FILE_LOCATION, LOCAL_FILE_LOCATION);
 176         verify(ftpesClientMock, times(1)).close();
 177         verifyNoMoreInteractions(ftpesClientMock);
 178
 179         assertEquals(1, counters.getNoOfCollectedFiles(),"collectedFiles should have been 1");
 180         assertEquals(0, counters.getNoOfFailedFtpAttempts(),"failedFtpAttempts should have been 0");
 181     }
 182
 183     @Test
 184     public void whenSftpFile_returnCorrectResponse() throws Exception {
 185         FileCollector collectorUndetTest = spy(new FileCollector(appConfigMock, counters));
 186         doReturn(sftpClientMock).when(collectorUndetTest).createSftpClient(any());
 187
 188         FileData fileData = createFileData(SFTP_LOCATION_NO_PORT, Scheme.SFTP);
 189         FilePublishInformation expectedfilePublishInformation =
 190             createExpectedFilePublishInformation(SFTP_LOCATION_NO_PORT);
 191
 192         StepVerifier.create(collectorUndetTest.collectFile(fileData, 3, Duration.ofSeconds(0), contextMap))
 193             .expectNext(expectedfilePublishInformation) //
 194             .verifyComplete();
 195
 196         // The same again, but with port
 197         fileData = createFileData(SFTP_LOCATION, Scheme.SFTP);
 198         expectedfilePublishInformation = createExpectedFilePublishInformation(SFTP_LOCATION);
 199
 200         StepVerifier.create(collectorUndetTest.collectFile(fileData, 3, Duration.ofSeconds(0), contextMap))
 201             .expectNext(expectedfilePublishInformation) //
 202             .verifyComplete();
 203
 204         verify(sftpClientMock, times(2)).open();
 205         verify(sftpClientMock, times(2)).collectFile(REMOTE_FILE_LOCATION, LOCAL_FILE_LOCATION);
 206         verify(sftpClientMock, times(2)).close();
 207         verifyNoMoreInteractions(sftpClientMock);
 208
 209         assertEquals(2, counters.getNoOfCollectedFiles(),"collectedFiles should have been 2");
 210     }
 211
 212     @Test
 213     public void whenFtpesFileAlwaysFail_retryAndFail() throws Exception {
 214         FileCollector collectorUndetTest = spy(new FileCollector(appConfigMock, counters));
 215         doReturn(ftpesClientMock).when(collectorUndetTest).createFtpesClient(any());
 216
 217         FileData fileData = createFileData(FTPES_LOCATION, Scheme.FTPES);
 218         doThrow(new DatafileTaskException("Unable to collect file.")).when(ftpesClientMock)
 219             .collectFile(REMOTE_FILE_LOCATION, LOCAL_FILE_LOCATION);
 220
 221         StepVerifier.create(collectorUndetTest.collectFile(fileData, 3, Duration.ofSeconds(0), contextMap))
 222             .expectErrorMessage("Retries exhausted: 3/3") //
 223             .verify();
 224
 225         verify(ftpesClientMock, times(4)).collectFile(REMOTE_FILE_LOCATION, LOCAL_FILE_LOCATION);
 226
 227         assertEquals(0, counters.getNoOfCollectedFiles(),"collectedFiles should have been 0");
 228         assertEquals(4, counters.getNoOfFailedFtpAttempts(),"failedFtpAttempts should have been 4");
 229     }
 230
 231     @Test
 232     public void whenFtpesFileAlwaysFail_failWithoutRetry() throws Exception {
 233         FileCollector collectorUndetTest = spy(new FileCollector(appConfigMock, counters));
 234         doReturn(ftpesClientMock).when(collectorUndetTest).createFtpesClient(any());
 235
 236         FileData fileData = createFileData(FTPES_LOCATION, Scheme.FTPES);
 237         doThrow(new NonRetryableDatafileTaskException("Unable to collect file.")).when(ftpesClientMock)
 238             .collectFile(REMOTE_FILE_LOCATION, LOCAL_FILE_LOCATION);
 239
 240         StepVerifier.create(collectorUndetTest.collectFile(fileData, 3, Duration.ofSeconds(0), contextMap))
 241             .expectErrorMessage("Non retryable file transfer failure") //
 242             .verify();
 243
 244         verify(ftpesClientMock, times(1)).collectFile(REMOTE_FILE_LOCATION, LOCAL_FILE_LOCATION);
 245
 246         assertEquals(0, counters.getNoOfCollectedFiles(),"collectedFiles should have been 0");
 247         assertEquals(1, counters.getNoOfFailedFtpAttempts(),"failedFtpAttempts should have been 1");
 248     }
 249
 250     @Test
 251     public void whenFtpesFileFailOnce_retryAndReturnCorrectResponse() throws Exception {
 252         FileCollector collectorUndetTest = spy(new FileCollector(appConfigMock, counters));
 253         doReturn(ftpesClientMock).when(collectorUndetTest).createFtpesClient(any());
 254         doThrow(new DatafileTaskException("Unable to collect file.")).doNothing().when(ftpesClientMock)
 255             .collectFile(REMOTE_FILE_LOCATION, LOCAL_FILE_LOCATION);
 256
 257         FilePublishInformation expectedfilePublishInformation =
 258             createExpectedFilePublishInformation(FTPES_LOCATION_NO_PORT);
 259
 260         FileData fileData = createFileData(FTPES_LOCATION_NO_PORT, Scheme.FTPES);
 261
 262         StepVerifier.create(collectorUndetTest.collectFile(fileData, 3, Duration.ofSeconds(0), contextMap))
 263             .expectNext(expectedfilePublishInformation) //
 264             .verifyComplete();
 265
 266         verify(ftpesClientMock, times(2)).collectFile(REMOTE_FILE_LOCATION, LOCAL_FILE_LOCATION);
 267
 268         assertEquals(1, counters.getNoOfCollectedFiles(),"collectedFiles should have been 1");
 269         assertEquals(1, counters.getNoOfFailedFtpAttempts(),"failedFtpAttempts should have been 1");
 270     }
 271 }