gerrit.onap Code Review - dcaegen2/collectors/datafile.git/blob

   1 /*-
   2  * ============LICENSE_START=======================================================
   3  *  Copyright (C) 2019 Nordix Foundation.
   4  * ================================================================================
   5  * Licensed under the Apache License, Version 2.0 (the "License");
   6  * you may not use this file except in compliance with the License.
   7  * You may obtain a copy of the License at
   8  *
   9  *      http://www.apache.org/licenses/LICENSE-2.0
  10  *
  11  * Unless required by applicable law or agreed to in writing, software
  12  * distributed under the License is distributed on an "AS IS" BASIS,
  13  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14  * See the License for the specific language governing permissions and
  15  * limitations under the License.
  16  *
  17  * SPDX-License-Identifier: Apache-2.0
  18  * ============LICENSE_END=========================================================
  19  */
  20
  21 package org.onap.dcaegen2.collectors.datafile.tasks;
  22
  23 import static org.onap.dcaegen2.collectors.datafile.model.logging.MdcVariables.REQUEST_ID;
  24 import static org.onap.dcaegen2.collectors.datafile.model.logging.MdcVariables.X_INVOCATION_ID;
  25 import static org.onap.dcaegen2.collectors.datafile.model.logging.MdcVariables.X_ONAP_REQUEST_ID;
  26
  27 import com.google.gson.JsonElement;
  28 import com.google.gson.JsonParser;
  29
  30 import java.io.IOException;
  31 import java.io.InputStream;
  32 import java.net.URI;
  33 import java.nio.file.Path;
  34 import java.time.Duration;
  35 import java.util.Map;
  36 import java.util.UUID;
  37
  38 import org.apache.commons.io.IOUtils;
  39 import org.apache.http.HttpResponse;
  40 import org.apache.http.client.methods.HttpPut;
  41 import org.apache.http.entity.ByteArrayEntity;
  42 import org.onap.dcaegen2.collectors.datafile.configuration.AppConfig;
  43 import org.onap.dcaegen2.collectors.datafile.model.CommonFunctions;
  44 import org.onap.dcaegen2.collectors.datafile.model.ConsumerDmaapModel;
  45 import org.onap.dcaegen2.collectors.datafile.model.logging.MdcVariables;
  46 import org.onap.dcaegen2.collectors.datafile.service.HttpUtils;
  47 import org.onap.dcaegen2.collectors.datafile.service.producer.DmaapProducerReactiveHttpClient;
  48 import org.onap.dcaegen2.services.sdk.rest.services.dmaap.client.config.DmaapPublisherConfiguration;
  49 import org.slf4j.Logger;
  50 import org.slf4j.LoggerFactory;
  51 import org.slf4j.MDC;
  52 import org.springframework.core.io.FileSystemResource;
  53 import org.springframework.http.HttpHeaders;
  54 import org.springframework.http.HttpStatus;
  55
  56 import reactor.core.publisher.Mono;
  57
  58 /**
  59  * Publishes a file to the DataRouter.
  60  *
  61  * @author <a href="mailto:przemyslaw.wasala@nokia.com">Przemysław Wąsala</a> on 4/13/18
  62  * @author <a href="mailto:henrik.b.andersson@est.tech">Henrik Andersson</a>
  63  */
  64 public class DataRouterPublisher {
  65     private static final String X_DMAAP_DR_META = "X-DMAAP-DR-META";
  66     private static final String CONTENT_TYPE = "application/octet-stream";
  67     private static final String NAME_JSON_TAG = "name";
  68     private static final String INTERNAL_LOCATION_JSON_TAG = "internalLocation";
  69     private static final String PUBLISH_TOPIC = "publish";
  70     private static final String DEFAULT_FEED_ID = "1";
  71
  72     private static final Logger logger = LoggerFactory.getLogger(DataRouterPublisher.class);
  73     private final AppConfig datafileAppConfig;
  74     private DmaapProducerReactiveHttpClient dmaapProducerReactiveHttpClient;
  75
  76     public DataRouterPublisher(AppConfig datafileAppConfig) {
  77         this.datafileAppConfig = datafileAppConfig;
  78     }
  79
  80
  81     /**
  82      * Publish one file.
  83      *
  84      * @param model information about the file to publish
  85      * @param numRetries the maximal number of retries if the publishing fails
  86      * @param firstBackoff the time to delay the first retry
  87      * @param contextMap tracing context variables
  88      * @return the (same) ConsumerDmaapModel
  89      */
  90     public Mono<ConsumerDmaapModel> execute(ConsumerDmaapModel model, long numRetries, Duration firstBackoff,
  91             Map<String, String> contextMap) {
  92         MdcVariables.setMdcContextMap(contextMap);
  93         logger.trace("Publish called with arg {}", model);
  94         dmaapProducerReactiveHttpClient = resolveClient();
  95
  96         return Mono.just(model)
  97                 .cache()
  98                 .flatMap(m -> publishFile(m, contextMap)) //
  99                 .flatMap(httpStatus -> handleHttpResponse(httpStatus, model, contextMap)) //
 100                 .retryBackoff(numRetries, firstBackoff);
 101     }
 102
 103     private Mono<HttpStatus> publishFile(ConsumerDmaapModel consumerDmaapModel, Map<String, String> contextMap) {
 104         logger.trace("Entering publishFile with {}", consumerDmaapModel);
 105         try {
 106             HttpPut put = new HttpPut();
 107             String requestId = MDC.get(REQUEST_ID);
 108             put.addHeader(X_ONAP_REQUEST_ID, requestId);
 109             String invocationId = UUID.randomUUID().toString();
 110             put.addHeader(X_INVOCATION_ID, invocationId);
 111
 112             prepareHead(consumerDmaapModel, put);
 113             prepareBody(consumerDmaapModel, put);
 114             dmaapProducerReactiveHttpClient.addUserCredentialsToHead(put);
 115
 116             HttpResponse response =
 117                     dmaapProducerReactiveHttpClient.getDmaapProducerResponseWithRedirect(put, contextMap);
 118             logger.trace(response.toString());
 119             return Mono.just(HttpStatus.valueOf(response.getStatusLine().getStatusCode()));
 120         } catch (Exception e) {
 121             logger.warn("Unable to send file to DataRouter. Data: {}", consumerDmaapModel.getInternalLocation(), e);
 122             return Mono.error(e);
 123         }
 124     }
 125
 126     private void prepareHead(ConsumerDmaapModel model, HttpPut put) {
 127         put.addHeader(HttpHeaders.CONTENT_TYPE, CONTENT_TYPE);
 128         JsonElement metaData = new JsonParser().parse(CommonFunctions.createJsonBody(model));
 129         metaData.getAsJsonObject().remove(NAME_JSON_TAG).getAsString();
 130         metaData.getAsJsonObject().remove(INTERNAL_LOCATION_JSON_TAG);
 131         put.addHeader(X_DMAAP_DR_META, metaData.toString());
 132         put.setURI(getPublishUri(model.getInternalLocation().getFileName().toString()));
 133     }
 134
 135     private void prepareBody(ConsumerDmaapModel model, HttpPut put) throws IOException {
 136         Path fileLocation = model.getInternalLocation();
 137         try (InputStream fileInputStream = createInputStream(fileLocation)) {
 138             put.setEntity(new ByteArrayEntity(IOUtils.toByteArray(fileInputStream)));
 139         }
 140     }
 141
 142     private URI getPublishUri(String fileName) {
 143         return dmaapProducerReactiveHttpClient.getBaseUri() //
 144                 .pathSegment(PUBLISH_TOPIC) //
 145                 .pathSegment(DEFAULT_FEED_ID) //
 146                 .pathSegment(fileName).build();
 147     }
 148
 149     private Mono<ConsumerDmaapModel> handleHttpResponse(HttpStatus response, ConsumerDmaapModel model,
 150             Map<String, String> contextMap) {
 151         MdcVariables.setMdcContextMap(contextMap);
 152         if (HttpUtils.isSuccessfulResponseCode(response.value())) {
 153             logger.trace("Publish to DR successful!");
 154             return Mono.just(model);
 155         } else {
 156             logger.warn("Publish to DR unsuccessful, response code: {}", response);
 157             return Mono.error(new Exception("Publish to DR unsuccessful, response code: " + response));
 158         }
 159     }
 160
 161     InputStream createInputStream(Path filePath) throws IOException {
 162         FileSystemResource realResource = new FileSystemResource(filePath);
 163         return realResource.getInputStream();
 164     }
 165
 166     DmaapPublisherConfiguration resolveConfiguration() {
 167         return datafileAppConfig.getDmaapPublisherConfiguration();
 168     }
 169
 170     DmaapProducerReactiveHttpClient resolveClient() {
 171         return new DmaapProducerReactiveHttpClient(resolveConfiguration());
 172     }
 173 }