950fbae92cacd98aec01bfd8ed9d142bff685023
[dcaegen2/collectors/datafile.git] /
1 /*-
2  * ============LICENSE_START=======================================================
3  *  Copyright (C) 2019 Nordix Foundation.
4  * ================================================================================
5  * Licensed under the Apache License, Version 2.0 (the "License");
6  * you may not use this file except in compliance with the License.
7  * You may obtain a copy of the License at
8  *
9  *      http://www.apache.org/licenses/LICENSE-2.0
10  *
11  * Unless required by applicable law or agreed to in writing, software
12  * distributed under the License is distributed on an "AS IS" BASIS,
13  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14  * See the License for the specific language governing permissions and
15  * limitations under the License.
16  *
17  * SPDX-License-Identifier: Apache-2.0
18  * ============LICENSE_END=========================================================
19  */
20
21 package org.onap.dcaegen2.collectors.datafile.tasks;
22
23 import com.google.gson.JsonElement;
24 import com.google.gson.JsonParser;
25
26 import java.io.IOException;
27 import java.io.InputStream;
28 import java.net.URI;
29 import java.nio.file.Path;
30 import java.time.Duration;
31
32 import org.apache.commons.io.IOUtils;
33 import org.apache.http.HttpResponse;
34 import org.apache.http.client.methods.HttpPut;
35 import org.apache.http.entity.ByteArrayEntity;
36 import org.onap.dcaegen2.collectors.datafile.configuration.AppConfig;
37 import org.onap.dcaegen2.collectors.datafile.model.CommonFunctions;
38 import org.onap.dcaegen2.collectors.datafile.model.FilePublishInformation;
39 import org.onap.dcaegen2.collectors.datafile.model.logging.MappedDiagnosticContext;
40 import org.onap.dcaegen2.collectors.datafile.service.HttpUtils;
41 import org.onap.dcaegen2.collectors.datafile.service.producer.DmaapProducerHttpClient;
42 import org.onap.dcaegen2.services.sdk.rest.services.dmaap.client.config.DmaapPublisherConfiguration;
43 import org.slf4j.Logger;
44 import org.slf4j.LoggerFactory;
45 import org.slf4j.MDC;
46 import org.springframework.core.io.FileSystemResource;
47 import org.springframework.http.HttpHeaders;
48 import org.springframework.http.HttpStatus;
49
50 import reactor.core.publisher.Mono;
51
52 /**
53  * Publishes a file to the DataRouter.
54  *
55  * @author <a href="mailto:przemyslaw.wasala@nokia.com">Przemysław Wąsala</a> on 4/13/18
56  * @author <a href="mailto:henrik.b.andersson@est.tech">Henrik Andersson</a>
57  */
58 public class DataRouterPublisher {
59     private static final String X_DMAAP_DR_META = "X-DMAAP-DR-META";
60     private static final String CONTENT_TYPE = "application/octet-stream";
61     private static final String NAME_JSON_TAG = "name";
62     private static final String INTERNAL_LOCATION_JSON_TAG = "internalLocation";
63     private static final String CONTEXT_JSON_TAG = "context";
64     private static final String PUBLISH_TOPIC = "publish";
65     private static final String DEFAULT_FEED_ID = "1";
66
67     private static final Logger logger = LoggerFactory.getLogger(DataRouterPublisher.class);
68     private final AppConfig datafileAppConfig;
69     private DmaapProducerHttpClient dmaapProducerReactiveHttpClient;
70
71     public DataRouterPublisher(AppConfig datafileAppConfig) {
72         this.datafileAppConfig = datafileAppConfig;
73     }
74
75     /**
76      * Publish one file.
77      *
78      * @param publishInfo information about the file to publish
79      * @param numRetries the maximal number of retries if the publishing fails
80      * @param firstBackoff the time to delay the first retry
81      * @return the (same) filePublishInformation
82      */
83     public Mono<FilePublishInformation> publishFile(FilePublishInformation publishInfo, long numRetries,
84             Duration firstBackoff) {
85         MDC.setContextMap(publishInfo.getContext());
86         logger.trace("publishFile called with arg {}", publishInfo);
87         dmaapProducerReactiveHttpClient = resolveClient();
88
89         return Mono.just(publishInfo) //
90                 .cache() //
91                 .flatMap(this::publishFile) //
92                 .flatMap(httpStatus -> handleHttpResponse(httpStatus, publishInfo)) //
93                 .retryBackoff(numRetries, firstBackoff);
94     }
95
96     private Mono<HttpStatus> publishFile(FilePublishInformation publishInfo
97             ) {
98         logger.trace("Entering publishFile with {}", publishInfo);
99         try {
100             HttpPut put = new HttpPut();
101             prepareHead(publishInfo, put);
102             prepareBody(publishInfo, put);
103             dmaapProducerReactiveHttpClient.addUserCredentialsToHead(put);
104
105             HttpResponse response =
106                     dmaapProducerReactiveHttpClient.getDmaapProducerResponseWithRedirect(put, publishInfo.getContext());
107             logger.trace("{}", response);
108             return Mono.just(HttpStatus.valueOf(response.getStatusLine().getStatusCode()));
109         } catch (Exception e) {
110             logger.warn("Unable to send file to DataRouter. Data: {}", publishInfo.getInternalLocation(), e);
111             return Mono.error(e);
112         }
113     }
114
115     private void prepareHead(FilePublishInformation publishInfo, HttpPut put) {
116         put.addHeader(HttpHeaders.CONTENT_TYPE, CONTENT_TYPE);
117         JsonElement metaData = new JsonParser().parse(CommonFunctions.createJsonBody(publishInfo));
118         metaData.getAsJsonObject().remove(NAME_JSON_TAG).getAsString();
119         metaData.getAsJsonObject().remove(INTERNAL_LOCATION_JSON_TAG);
120         metaData.getAsJsonObject().remove(CONTEXT_JSON_TAG);
121
122         put.addHeader(X_DMAAP_DR_META, metaData.toString());
123         put.setURI(getPublishUri(publishInfo.getName()));
124         MappedDiagnosticContext.appendTraceInfo(put);
125     }
126
127     private void prepareBody(FilePublishInformation publishInfo, HttpPut put) throws IOException {
128         Path fileLocation = publishInfo.getInternalLocation();
129         try (InputStream fileInputStream = createInputStream(fileLocation)) {
130             put.setEntity(new ByteArrayEntity(IOUtils.toByteArray(fileInputStream)));
131         }
132     }
133
134     private URI getPublishUri(String fileName) {
135         return dmaapProducerReactiveHttpClient.getBaseUri() //
136                 .pathSegment(PUBLISH_TOPIC) //
137                 .pathSegment(DEFAULT_FEED_ID) //
138                 .pathSegment(fileName).build();
139     }
140
141     private Mono<FilePublishInformation> handleHttpResponse(HttpStatus response, FilePublishInformation publishInfo) {
142         MDC.setContextMap(publishInfo.getContext());
143         if (HttpUtils.isSuccessfulResponseCode(response.value())) {
144             logger.trace("Publish to DR successful!");
145             return Mono.just(publishInfo);
146         } else {
147             logger.warn("Publish to DR unsuccessful, response code: {}", response);
148             return Mono.error(new Exception("Publish to DR unsuccessful, response code: " + response));
149         }
150     }
151
152     InputStream createInputStream(Path filePath) throws IOException {
153         FileSystemResource realResource = new FileSystemResource(filePath);
154         return realResource.getInputStream();
155     }
156
157     DmaapPublisherConfiguration resolveConfiguration() {
158         return datafileAppConfig.getDmaapPublisherConfiguration();
159     }
160
161     DmaapProducerHttpClient resolveClient() {
162         return new DmaapProducerHttpClient(resolveConfiguration());
163     }
164 }