When the Datafile collector is unable to download the file from an xNF it now retries to collect the file.
Change-Id: I61f68f9cf7af1a7fab160b0e936daafd1a23aaf8
Issue-ID: DCAEGEN2-864
Signed-off-by: elinuxhenrik <henrik.b.andersson@est.tech>
<parent>
<groupId>org.onap.dcaegen2.collectors</groupId>
<artifactId>datafile</artifactId>
- <version>1.0.0-SNAPSHOT</version>
+ <version>1.0.2-SNAPSHOT</version>
</parent>
<groupId>org.onap.dcaegen2.collectors.datafile</groupId>
<artifactId>datafile-app-server</artifactId>
- <version>1.0.1-SNAPSHOT</version>
<packaging>jar</packaging>
<properties>
--- /dev/null
+/*
+ * ============LICENSE_START======================================================================
+ * Copyright (C) 2018 Nordix Foundation. All rights reserved.
+ * ===============================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END========================================================================
+ */
+
+package org.onap.dcaegen2.collectors.datafile.tasks;
+
+public class RetryTimer {
+ public void waitRetryTime() {
+ try {
+ Thread.sleep(60000);
+ } catch (InterruptedException e) {
+ // Nothing, no one will interrupt.
+ }
+
+ }
+}
import org.onap.dcaegen2.collectors.datafile.configuration.AppConfig;
import org.onap.dcaegen2.collectors.datafile.configuration.Config;
import org.onap.dcaegen2.collectors.datafile.configuration.FtpesConfig;
+import org.onap.dcaegen2.collectors.datafile.ftp.FileCollectClient;
+import org.onap.dcaegen2.collectors.datafile.ftp.FileCollectResult;
import org.onap.dcaegen2.collectors.datafile.ftp.FileServerData;
import org.onap.dcaegen2.collectors.datafile.ftp.FtpsClient;
import org.onap.dcaegen2.collectors.datafile.ftp.ImmutableFileServerData;
private final FtpsClient ftpsClient;
private final SftpClient sftpClient;
+ private RetryTimer retryTimer;
@Autowired
protected XnfCollectorTaskImpl(AppConfig datafileAppConfig, FtpsClient ftpsCleint, SftpClient sftpClient) {
FileServerData fileServerData = getFileServerData(uri);
String remoteFile = uri.getPath();
String localFile = "target" + File.separator + fileData.name();
- String scheme = uri.getScheme();
- boolean fileDownloaded = false;
- if (FTPES.equals(scheme) || FTPS.equals(scheme)) {
- fileDownloaded = ftpsClient.collectFile(fileServerData, remoteFile, localFile);
- } else if (SFTP.equals(scheme)) {
- fileDownloaded = sftpClient.collectFile(fileServerData, remoteFile, localFile);
- } else {
- logger.error("DFC does not support protocol {}. Supported protocols are {}, {}, and {}. Data: {}", scheme,
- FTPES, FTPS, SFTP, fileData);
- localFile = null;
- }
- if (!fileDownloaded) {
+ FileCollectClient currentClient = selectClient(fileData, uri);
+
+ if (currentClient != null) {
+ FileCollectResult fileCollectResult = currentClient.collectFile(fileServerData, remoteFile, localFile);
+ if (!fileCollectResult.downloadSuccessful()) {
+ fileCollectResult = retry(fileCollectResult, currentClient);
+ }
+ if (!fileCollectResult.downloadSuccessful()) {
+ localFile = null;
+ logger.error("Download of file aborted after maximum number of retries. Data: {} Error causes {}",
+ fileServerData, fileCollectResult.getErrorData());
+ }
+ } else {
localFile = null;
}
return localFile;
return userInfo;
}
+ private FileCollectClient selectClient(FileData fileData, URI uri) {
+ FileCollectClient selectedClient = null;
+ String scheme = uri.getScheme();
+ if (FTPES.equals(scheme) || FTPS.equals(scheme)) {
+ selectedClient = ftpsClient;
+ } else if (SFTP.equals(scheme)) {
+ selectedClient = sftpClient;
+ } else {
+ logger.error("DFC does not support protocol {}. Supported protocols are {}, {}, and {}. Data: {}", scheme,
+ FTPES, FTPS, SFTP, fileData);
+ }
+ return selectedClient;
+ }
+
+ private FileCollectResult retry(FileCollectResult fileCollectResult, FileCollectClient fileCollectClient) {
+ int retryCount = 1;
+ FileCollectResult newResult = fileCollectResult;
+ while (!newResult.downloadSuccessful() && retryCount++ < 3) {
+ getRetryTimer().waitRetryTime();
+ newResult = fileCollectClient.retryCollectFile();
+ }
+ return newResult;
+ }
+
private ConsumerDmaapModel getConsumerDmaapModel(FileData fileData, String localFile) {
String name = fileData.name();
String compression = fileData.compression();
return ImmutableConsumerDmaapModel.builder().name(name).location(localFile).compression(compression)
.fileFormatType(fileFormatType).fileFormatVersion(fileFormatVersion).build();
}
+
+ private RetryTimer getRetryTimer() {
+ if (retryTimer == null) {
+ retryTimer = new RetryTimer();
+ }
+ return retryTimer;
+ }
+
+ protected void setRetryTimer(RetryTimer retryTimer) {
+ this.retryTimer = retryTimer;
+ }
}
package org.onap.dcaegen2.collectors.datafile.tasks;
+import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import org.junit.jupiter.api.Test;
import org.onap.dcaegen2.collectors.datafile.configuration.AppConfig;
import org.onap.dcaegen2.collectors.datafile.configuration.FtpesConfig;
+import org.onap.dcaegen2.collectors.datafile.ftp.ErrorData;
+import org.onap.dcaegen2.collectors.datafile.ftp.FileCollectResult;
import org.onap.dcaegen2.collectors.datafile.ftp.FileServerData;
import org.onap.dcaegen2.collectors.datafile.ftp.FtpsClient;
import org.onap.dcaegen2.collectors.datafile.ftp.ImmutableFileServerData;
private FtpsClient ftpsClientMock = mock(FtpsClient.class);
private SftpClient sftpClientMock = mock(SftpClient.class);
+ private RetryTimer retryTimerMock = mock(RetryTimer.class);
- private XnfCollectorTask collectorUndetTest = new XnfCollectorTaskImpl(appConfigMock, ftpsClientMock, sftpClientMock);
@BeforeAll
public static void setUpConfiguration() {
@Test
public void whenFtpesFile_returnCorrectResponse() {
+ XnfCollectorTaskImpl collectorUndetTest = new XnfCollectorTaskImpl(appConfigMock, ftpsClientMock, sftpClientMock);
+
FileData fileData = ImmutableFileData.builder().changeIdentifier(PM_MEAS_CHANGE_IDINTIFIER)
.changeType(FILE_READY_CHANGE_TYPE).name(PM_FILE_NAME).location(FTPES_LOCATION)
.compression(GZIP_COMPRESSION).fileFormatType(MEAS_COLLECT_FILE_FORMAT_TYPE)
FileServerData fileServerData = ImmutableFileServerData.builder().serverAddress(SERVER_ADDRESS).userId(USER)
.password(PWD).port(PORT_22).build();
when(ftpsClientMock.collectFile(fileServerData, REMOTE_FILE_LOCATION, LOCAL_FILE_LOCATION))
- .thenReturn(Boolean.TRUE);
+ .thenReturn(new FileCollectResult());
ConsumerDmaapModel expectedConsumerDmaapModel = ImmutableConsumerDmaapModel.builder().name(PM_FILE_NAME)
.location(LOCAL_FILE_LOCATION).compression(GZIP_COMPRESSION)
@Test
public void whenSftpFile_returnCorrectResponse() {
+ XnfCollectorTaskImpl collectorUndetTest = new XnfCollectorTaskImpl(appConfigMock, ftpsClientMock, sftpClientMock);
+
FileData fileData = ImmutableFileData.builder().changeIdentifier(PM_MEAS_CHANGE_IDINTIFIER)
.changeType(FILE_READY_CHANGE_TYPE).name(PM_FILE_NAME).location(SFTP_LOCATION)
.compression(GZIP_COMPRESSION).fileFormatType(MEAS_COLLECT_FILE_FORMAT_TYPE)
FileServerData fileServerData = ImmutableFileServerData.builder().serverAddress(SERVER_ADDRESS).userId("")
.password("").port(PORT_22).build();
when(sftpClientMock.collectFile(fileServerData, REMOTE_FILE_LOCATION, LOCAL_FILE_LOCATION))
- .thenReturn(Boolean.TRUE);
+ .thenReturn(new FileCollectResult());
ConsumerDmaapModel expectedConsumerDmaapModel = ImmutableConsumerDmaapModel.builder().name(PM_FILE_NAME)
.location(LOCAL_FILE_LOCATION).compression(GZIP_COMPRESSION)
.verifyComplete();
verify(sftpClientMock, times(1)).collectFile(fileServerData, REMOTE_FILE_LOCATION, LOCAL_FILE_LOCATION);
- verify(ftpsClientMock).setKeyCertPath(FTP_KEY_PATH);
- verify(ftpsClientMock).setKeyCertPassword(FTP_KEY_PASSWORD);
- verify(ftpsClientMock).setTrustedCAPath(TRUSTED_CA_PATH);
- verify(ftpsClientMock).setTrustedCAPassword(TRUSTED_CA_PASSWORD);
- verifyNoMoreInteractions(ftpsClientMock);
+ verifyNoMoreInteractions(sftpClientMock);
+ }
+
+ @Test
+ public void whenFtpesFileAlwaysFail_retryAndReturnEmpty() {
+ XnfCollectorTaskImpl collectorUndetTest = new XnfCollectorTaskImpl(appConfigMock, ftpsClientMock, sftpClientMock);
+ collectorUndetTest.setRetryTimer(retryTimerMock);
+
+ FileData fileData = ImmutableFileData.builder().changeIdentifier(PM_MEAS_CHANGE_IDINTIFIER)
+ .changeType(FILE_READY_CHANGE_TYPE).name(PM_FILE_NAME).location(FTPES_LOCATION)
+ .compression(GZIP_COMPRESSION).fileFormatType(MEAS_COLLECT_FILE_FORMAT_TYPE)
+ .fileFormatVersion(FILE_FORMAT_VERSION).build();
+
+ FileServerData fileServerData = ImmutableFileServerData.builder().serverAddress(SERVER_ADDRESS).userId(USER)
+ .password(PWD).port(PORT_22).build();
+ ErrorData errorData = new ErrorData();
+ errorData.addError("Unable to collect file.", new Exception());
+ when(ftpsClientMock.collectFile(fileServerData, REMOTE_FILE_LOCATION, LOCAL_FILE_LOCATION))
+ .thenReturn(new FileCollectResult(errorData));
+ doReturn(new FileCollectResult(errorData)).when(ftpsClientMock).retryCollectFile();
+
+ StepVerifier.create(collectorUndetTest.execute(fileData)).expectNextCount(0).verifyComplete();
+
+ verify(ftpsClientMock, times(1)).collectFile(fileServerData, REMOTE_FILE_LOCATION, LOCAL_FILE_LOCATION);
+ verify(ftpsClientMock, times(2)).retryCollectFile();
+ }
+
+ @Test
+ public void whenFtpesFileFailOnce_retryAndReturnCorrectResponse() {
+ XnfCollectorTaskImpl collectorUndetTest = new XnfCollectorTaskImpl(appConfigMock, ftpsClientMock, sftpClientMock);
+ collectorUndetTest.setRetryTimer(retryTimerMock);
+
+ FileData fileData = ImmutableFileData.builder().changeIdentifier(PM_MEAS_CHANGE_IDINTIFIER)
+ .changeType(FILE_READY_CHANGE_TYPE).name(PM_FILE_NAME).location(FTPES_LOCATION)
+ .compression(GZIP_COMPRESSION).fileFormatType(MEAS_COLLECT_FILE_FORMAT_TYPE)
+ .fileFormatVersion(FILE_FORMAT_VERSION).build();
+
+ FileServerData fileServerData = ImmutableFileServerData.builder().serverAddress(SERVER_ADDRESS).userId(USER)
+ .password(PWD).port(PORT_22).build();
+ ErrorData errorData = new ErrorData();
+ errorData.addError("Unable to collect file.", new Exception());
+ when(ftpsClientMock.collectFile(fileServerData, REMOTE_FILE_LOCATION, LOCAL_FILE_LOCATION))
+ .thenReturn(new FileCollectResult(errorData));
+ doReturn(new FileCollectResult()).when(ftpsClientMock).retryCollectFile();
+
+ ConsumerDmaapModel expectedConsumerDmaapModel = ImmutableConsumerDmaapModel.builder().name(PM_FILE_NAME)
+ .location(LOCAL_FILE_LOCATION).compression(GZIP_COMPRESSION)
+ .fileFormatType(MEAS_COLLECT_FILE_FORMAT_TYPE).fileFormatVersion(FILE_FORMAT_VERSION).build();
+
+ StepVerifier.create(collectorUndetTest.execute(fileData)).expectNext(expectedConsumerDmaapModel)
+ .verifyComplete();
+
+
+ verify(ftpsClientMock, times(1)).collectFile(fileServerData, REMOTE_FILE_LOCATION, LOCAL_FILE_LOCATION);
+ verify(ftpsClientMock, times(1)).retryCollectFile();
}
@Test
public void whenWrongScheme_returnEmpty() {
+ XnfCollectorTaskImpl collectorUndetTest = new XnfCollectorTaskImpl(appConfigMock, ftpsClientMock, sftpClientMock);
+
FileData fileData = ImmutableFileData.builder().changeIdentifier(PM_MEAS_CHANGE_IDINTIFIER)
.changeType(FILE_READY_CHANGE_TYPE).name(PM_FILE_NAME).location("http://host.com/file.zip")
.compression(GZIP_COMPRESSION).fileFormatType(MEAS_COLLECT_FILE_FORMAT_TYPE)
FileServerData fileServerData = ImmutableFileServerData.builder().serverAddress(SERVER_ADDRESS).userId("")
.password("").port(PORT_22).build();
when(sftpClientMock.collectFile(fileServerData, REMOTE_FILE_LOCATION, LOCAL_FILE_LOCATION))
- .thenReturn(Boolean.TRUE);
+ .thenReturn(new FileCollectResult());
StepVerifier.create(collectorUndetTest.execute(fileData)).expectNextCount(0).verifyComplete();
- verify(ftpsClientMock).setKeyCertPath(FTP_KEY_PATH);
- verify(ftpsClientMock).setKeyCertPassword(FTP_KEY_PASSWORD);
- verify(ftpsClientMock).setTrustedCAPath(TRUSTED_CA_PATH);
- verify(ftpsClientMock).setTrustedCAPassword(TRUSTED_CA_PASSWORD);
- verifyNoMoreInteractions(ftpsClientMock);
+ verifyNoMoreInteractions(sftpClientMock);
}
}
<parent>
<groupId>org.onap.dcaegen2.collectors</groupId>
<artifactId>datafile</artifactId>
- <version>1.0.0-SNAPSHOT</version>
+ <version>1.0.2-SNAPSHOT</version>
</parent>
<groupId>org.onap.dcaegen2.collectors.datafile</groupId>
<parent>
<groupId>org.onap.dcaegen2.collectors</groupId>
<artifactId>datafile</artifactId>
- <version>1.0.0-SNAPSHOT</version>
+ <version>1.0.2-SNAPSHOT</version>
</parent>
<groupId>org.onap.dcaegen2.collectors.datafile</groupId>
--- /dev/null
+/*
+ * ============LICENSE_START======================================================================
+ * Copyright (C) 2018 Nordix Foundation. All rights reserved.
+ * ===============================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END========================================================================
+ */
+
+package org.onap.dcaegen2.collectors.datafile.ftp;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class ErrorData {
+ private List<String> errorMessages = new ArrayList<>();
+ private List<Throwable> errorCauses = new ArrayList<>();
+
+ public void addError(String errorMessage, Throwable errorCause) {
+ errorMessages.add(errorMessage);
+ errorCauses.add(errorCause);
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder message = new StringBuilder();
+ for (int i = 0; i < errorMessages.size(); i++) {
+ message.append(errorMessages.get(i)).append(" Cause: ").append(errorCauses.get(i)).append("\n");
+ }
+ return message.toString();
+ }
+}
--- /dev/null
+/*
+ * ============LICENSE_START======================================================================
+ * Copyright (C) 2018 Nordix Foundation. All rights reserved.
+ * ===============================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ * ============LICENSE_END========================================================================
+ */
+
+package org.onap.dcaegen2.collectors.datafile.ftp;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * @author
+ *
+ */
+public abstract class FileCollectClient {
+ protected static final Logger logger = LoggerFactory.getLogger(FtpsClient.class);
+
+ protected FileServerData fileServerData;
+ protected String remoteFile;
+ protected String localFile;
+ protected ErrorData errorData;
+
+ public FileCollectResult collectFile(FileServerData fileServerData, String remoteFile, String localFile) {
+ logger.trace("collectFile called with fileServerData: {}, remoteFile: {}, localFile: {}", fileServerData,
+ remoteFile, localFile);
+
+ this.fileServerData = fileServerData;
+ this.remoteFile = remoteFile;
+ this.localFile = localFile;
+
+ return retryCollectFile();
+ }
+
+ public abstract FileCollectResult retryCollectFile();
+
+ protected void addError(String errorMessage, Throwable errorCause) {
+ if (errorData == null) {
+ errorData = new ErrorData();
+ }
+ errorData.addError(errorMessage, errorCause);
+ }
+}
--- /dev/null
+/*
+ * ============LICENSE_START======================================================================
+ * Copyright (C) 2018 Nordix Foundation. All rights reserved.
+ * ===============================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END========================================================================
+ */
+
+package org.onap.dcaegen2.collectors.datafile.ftp;
+
+public class FileCollectResult {
+ private boolean result;
+ private ErrorData errorData;
+
+ public FileCollectResult() {
+ this.result = true;
+ }
+
+ public FileCollectResult(ErrorData errorData) {
+ this.errorData = errorData;
+ result = false;
+ }
+
+ public boolean downloadSuccessful() {
+ return result;
+ }
+
+ public String getErrorData() {
+ return errorData.toString();
+ }
+
+ @Override
+ public String toString() {
+ return "Download successful: " + result + " Error data: " + getErrorData();
+ }
+}
import org.onap.dcaegen2.collectors.datafile.ssl.KeyManagerUtilsWrapper;
import org.onap.dcaegen2.collectors.datafile.ssl.KeyStoreWrapper;
import org.onap.dcaegen2.collectors.datafile.ssl.TrustManagerFactoryWrapper;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Component;
/**
* @author <a href="mailto:martin.c.yan@est.tech">Martin Yan</a>
*/
@Component
-public class FtpsClient {
- private static final Logger logger = LoggerFactory.getLogger(FtpsClient.class);
-
+public class FtpsClient extends FileCollectClient {
private String keyCertPath;
private String keyCertPassword;
private String trustedCAPath;
private IKeyManagerUtils kmu;
private IKeyStore keyStore;
private ITrustManagerFactory trustManagerFactory;
- private IFile localFile;
+ private IFile lf;
private IFileSystemResource fileResource;
private IOutputStream os;
- public boolean collectFile(FileServerData fileServerData, String remoteFile, String localFile) {
- logger.trace("collectFile called with fileServerData: {}, remoteFile: {}, localFile: {}", fileServerData,
- remoteFile, localFile);
- boolean result = true;
+ @Override
+ public FileCollectResult retryCollectFile() {
+ logger.trace("retryCollectFile called");
+
+ FileCollectResult fileCollectResult;
+
IFTPSClient ftps = getFtpsClient();
ftps.setNeedClientAuth(true);
- if (setUpKeyManager(ftps) && setUpTrustedCA(ftps) && setUpConnection(fileServerData, ftps)) {
- result = getFileFromxNF(remoteFile, localFile, ftps, fileServerData);
-
- closeDownConnection(ftps);
+ if (setUpKeyManager(ftps) && setUpTrustedCA(ftps) && setUpConnection(ftps)) {
+ if (getFileFromxNF(ftps)) {
+ closeDownConnection(ftps);
+ fileCollectResult = new FileCollectResult();
+ } else {
+ fileCollectResult = new FileCollectResult(errorData);
+ }
} else {
- result = false;
+ fileCollectResult = new FileCollectResult(errorData);
}
- logger.trace("collectFile left with result: {}", result);
- return result;
+ logger.trace("retryCollectFile left with result: {}", fileCollectResult);
+ return fileCollectResult;
}
private boolean setUpKeyManager(IFTPSClient ftps) {
keyManagerUtils.setCredentials(keyCertPath, keyCertPassword);
ftps.setKeyManager(keyManagerUtils.getClientKeyManager());
} catch (GeneralSecurityException | IOException e) {
- logger.error("Unable to use own key store {}", keyCertPath, e);
+ addError("Unable to use own key store " + keyCertPath, e);
result = false;
}
return result;
ftps.setTrustManager(tmf.getTrustManagers()[0]);
} catch (Exception e) {
- logger.error("Unable to trust xNF's CA, {}", trustedCAPath, e);
+ addError("Unable to trust xNF's CA, " + trustedCAPath, e);
result = false;
}
return result;
}
- private boolean setUpConnection(FileServerData fileServerData, IFTPSClient ftps) {
+ private boolean setUpConnection(IFTPSClient ftps) {
boolean result = true;
try {
ftps.connect(fileServerData.serverAddress(), fileServerData.port());
boolean loginSuccesful = ftps.login(fileServerData.userId(), fileServerData.password());
if (!loginSuccesful) {
ftps.logout();
- logger.error("Unable to log in to xNF. {}", fileServerData);
+ addError("Unable to log in to xNF. " + fileServerData, null);
result = false;
}
ftps.execPROT("P");
} else {
ftps.disconnect();
- logger.error("Unable to connect to xNF. {}", fileServerData);
+ addError("Unable to connect to xNF. " + fileServerData + "xNF reply code: " + ftps.getReplyCode(), null);
result = false;
}
} catch (Exception ex) {
- logger.error("Unable to connect to xNF. Data: {}", fileServerData, ex);
+ addError("Unable to connect to xNF. Data: " + fileServerData, ex);
result = false;
}
logger.trace("setUpConnection return value: {}", result);
return result;
}
- private boolean getFileFromxNF(String remoteFile, String localFilePath, IFTPSClient ftps,
- FileServerData fileServerData) {
+ private boolean getFileFromxNF(IFTPSClient ftps) {
logger.trace("starting to getFile");
boolean result = true;
+ IFile outfile = getFile();
try {
- IFile outfile = getFile();
- outfile.setPath(localFilePath);
+ outfile.setPath(localFile);
outfile.createNewFile();
IOutputStream outputStream = getOutputStream();
ftps.retrieveFile(remoteFile, output);
output.close();
- logger.debug("File {} Download Successfull from xNF", localFilePath);
+ logger.debug("File {} Download Successfull from xNF", localFile);
} catch (IOException ex) {
- logger.error("Unable to collect file from xNF. Data: {}", fileServerData, ex);
+ addError("Unable to collect file from xNF. Data: " + fileServerData, ex);
+ try {
+ outfile.delete();
+ } catch (Exception e) {
+ // Nothing
+ }
result = false;
}
return result;
}
private IFile getFile() {
- if (localFile == null) {
- localFile = new FileWrapper();
+ if (lf == null) {
+ lf = new FileWrapper();
}
- return localFile;
+ return lf;
}
private IOutputStream getOutputStream() {
}
protected void setFile(IFile file) {
- localFile = file;
+ lf = file;
}
protected void setOutputStream(IOutputStream outputStream) {
import com.jcraft.jsch.SftpException;
import org.apache.commons.io.FilenameUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Component;
/**
*
*/
@Component
-public class SftpClient {
- private static final Logger logger = LoggerFactory.getLogger(SftpClient.class);
+public class SftpClient extends FileCollectClient {
+ @Override
+ public FileCollectResult retryCollectFile() {
+ logger.trace("retryCollectFile called");
- public boolean collectFile(FileServerData fileServerData, String remoteFile, String localFile) {
- boolean result = true;
+ FileCollectResult result;
Session session = setUpSession(fileServerData);
if (session != null) {
if (sftpChannel != null) {
try {
sftpChannel.get(remoteFile, localFile);
+ result = new FileCollectResult();
logger.debug("File {} Download Successfull from xNF", FilenameUtils.getName(localFile));
} catch (SftpException e) {
- logger.error("Unable to get file from xNF. Data: {}", fileServerData, e);
- result = false;
+ addError("Unable to get file from xNF. Data: " + fileServerData, e);
+ result = new FileCollectResult(errorData);
}
sftpChannel.exit();
} else {
- result = false;
+ result = new FileCollectResult(errorData);
}
session.disconnect();
} else {
- result = false;
+ result = new FileCollectResult(errorData);
}
+ logger.trace("retryCollectFile left with result: {}", result);
return result;
}
session.setPassword(fileServerData.password());
session.connect();
} catch (JSchException e) {
- logger.error("Unable to set up SFTP connection to xNF. Data: {}", fileServerData, e);
+ addError("Unable to set up SFTP connection to xNF. Data: " + fileServerData, e);
}
return session;
}
channel.connect();
sftpChannel = (ChannelSftp) channel;
} catch (JSchException e) {
- logger.error("Unable to get sftp channel to xNF. Data: {}", fileServerData, e);
+ addError("Unable to get sftp channel to xNF. Data: " + fileServerData, e);
}
return sftpChannel;
}
public File getFile() {
return file;
}
+
+ @Override
+ public boolean delete() {
+ return file.delete();
+ }
}
public boolean createNewFile() throws IOException;
public File getFile();
+
+ public boolean delete();
}
ImmutableFileServerData fileServerData = ImmutableFileServerData.builder().serverAddress(XNF_ADDRESS)
.userId(USERNAME).password(PASSWORD).port(PORT).build();
- boolean result = clientUnderTest.collectFile(fileServerData, REMOTE_FILE_PATH, LOCAL_FILE_PATH);
+ FileCollectResult result = clientUnderTest.collectFile(fileServerData, REMOTE_FILE_PATH, LOCAL_FILE_PATH);
- assertTrue(result);
+ assertTrue(result.downloadSuccessful());
verify(ftpsClientMock).setNeedClientAuth(true);
verify(keyManagerUtilsMock).setCredentials(FTP_KEY_PATH, FTP_KEY_PASSWORD);
verify(ftpsClientMock).setKeyManager(keyManagerMock);
ImmutableFileServerData fileServerData = ImmutableFileServerData.builder().serverAddress(XNF_ADDRESS)
.userId(USERNAME).password(PASSWORD).port(PORT).build();
- boolean result = clientUnderTest.collectFile(fileServerData, REMOTE_FILE_PATH, LOCAL_FILE_PATH);
+ FileCollectResult result = clientUnderTest.collectFile(fileServerData, REMOTE_FILE_PATH, LOCAL_FILE_PATH);
- assertFalse(result);
+ assertFalse(result.downloadSuccessful());
}
@Test
ImmutableFileServerData fileServerData = ImmutableFileServerData.builder().serverAddress(XNF_ADDRESS)
.userId(USERNAME).password(PASSWORD).port(PORT).build();
- boolean result = clientUnderTest.collectFile(fileServerData, REMOTE_FILE_PATH, LOCAL_FILE_PATH);
+ FileCollectResult result = clientUnderTest.collectFile(fileServerData, REMOTE_FILE_PATH, LOCAL_FILE_PATH);
- assertFalse(result);
+ assertFalse(result.downloadSuccessful());
}
@Test
ImmutableFileServerData fileServerData = ImmutableFileServerData.builder().serverAddress(XNF_ADDRESS)
.userId(USERNAME).password(PASSWORD).port(PORT).build();
- boolean result = clientUnderTest.collectFile(fileServerData, REMOTE_FILE_PATH, LOCAL_FILE_PATH);
+ FileCollectResult result = clientUnderTest.collectFile(fileServerData, REMOTE_FILE_PATH, LOCAL_FILE_PATH);
verify(ftpsClientMock, times(1)).logout();
- assertFalse(result);
+ assertFalse(result.downloadSuccessful());
}
@Test
ImmutableFileServerData fileServerData = ImmutableFileServerData.builder().serverAddress(XNF_ADDRESS)
.userId(USERNAME).password(PASSWORD).port(PORT).build();
- boolean result = clientUnderTest.collectFile(fileServerData, REMOTE_FILE_PATH, LOCAL_FILE_PATH);
+ FileCollectResult result = clientUnderTest.collectFile(fileServerData, REMOTE_FILE_PATH, LOCAL_FILE_PATH);
verify(ftpsClientMock, times(1)).disconnect();
- assertFalse(result);
+ assertFalse(result.downloadSuccessful());
}
@Test
ImmutableFileServerData fileServerData = ImmutableFileServerData.builder().serverAddress(XNF_ADDRESS)
.userId(USERNAME).password(PASSWORD).port(PORT).build();
- boolean result = clientUnderTest.collectFile(fileServerData, REMOTE_FILE_PATH, LOCAL_FILE_PATH);
+ FileCollectResult result = clientUnderTest.collectFile(fileServerData, REMOTE_FILE_PATH, LOCAL_FILE_PATH);
- assertFalse(result);
+ assertFalse(result.downloadSuccessful());
}
@Test
ImmutableFileServerData fileServerData = ImmutableFileServerData.builder().serverAddress(XNF_ADDRESS)
.userId(USERNAME).password(PASSWORD).port(PORT).build();
- boolean result = clientUnderTest.collectFile(fileServerData, REMOTE_FILE_PATH, LOCAL_FILE_PATH);
+ FileCollectResult result = clientUnderTest.collectFile(fileServerData, REMOTE_FILE_PATH, LOCAL_FILE_PATH);
- assertFalse(result);
+ assertFalse(result.downloadSuccessful());
+ verify(localFileMock, times(1)).delete();
}
}
\ No newline at end of file
<groupId>org.onap.dcaegen2.collectors</groupId>
<artifactId>datafile</artifactId>
- <version>1.0.0-SNAPSHOT</version>
+ <version>1.0.2-SNAPSHOT</version>
<name>dcaegen2-collectors.datafile</name>
<description>datafile collector</description>
major=1\r
minor=0\r
-patch=0\r
+patch=2\r
base_version=${major}.${minor}.${patch}\r
release_version=${base_version}\r
snapshot_version=${base_version}-SNAPSHOT\r
Code Review / dcaegen2 / collectors / datafile.git / commitdiff