gerrit.onap Code Review - dcaegen2/services.git/blob

   1 /*
   2 * ============LICENSE_START=======================================================
   3 * ONAP : DATALAKE
   4 * ================================================================================
   5 * Copyright 2018 China Mobile
   6 *=================================================================================
   7 * Licensed under the Apache License, Version 2.0 (the "License");
   8 * you may not use this file except in compliance with the License.
   9 * You may obtain a copy of the License at
  10 *
  11 *     http://www.apache.org/licenses/LICENSE-2.0
  12 *
  13 * Unless required by applicable law or agreed to in writing, software
  14 * distributed under the License is distributed on an "AS IS" BASIS,
  15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16 * See the License for the specific language governing permissions and
  17 * limitations under the License.
  18 * ============LICENSE_END=========================================================
  19 */
  20
  21 package org.onap.datalake.feeder.service.db;
  22
  23 import java.util.ArrayList;
  24 import java.util.HashMap;
  25 import java.util.List;
  26 import java.util.Map;
  27
  28 import javax.annotation.PostConstruct;
  29 import javax.annotation.PreDestroy;
  30
  31 import org.apache.commons.lang3.StringUtils;
  32 import org.bson.Document;
  33
  34 import org.json.JSONObject;
  35 import org.onap.datalake.feeder.config.ApplicationConfiguration;
  36 import org.onap.datalake.feeder.domain.Db;
  37 import org.onap.datalake.feeder.domain.EffectiveTopic;
  38 import org.slf4j.Logger;
  39 import org.slf4j.LoggerFactory;
  40
  41 import org.springframework.beans.factory.annotation.Autowired;
  42 import org.springframework.context.annotation.Scope;
  43 import org.springframework.stereotype.Service;
  44
  45 import com.mongodb.bulk.BulkWriteError;
  46 import com.mongodb.MongoBulkWriteException;
  47 import com.mongodb.MongoClient;
  48 import com.mongodb.MongoClientOptions;
  49 import com.mongodb.MongoClientOptions.Builder;
  50 import com.mongodb.MongoCredential;
  51 import com.mongodb.MongoTimeoutException;
  52 import com.mongodb.ServerAddress;
  53 import com.mongodb.client.MongoCollection;
  54 import com.mongodb.client.MongoDatabase;
  55 import com.mongodb.client.model.InsertManyOptions;
  56
  57 /**
  58  * Service for using MongoDB
  59  *
  60  * @author Guobiao Mo
  61  *
  62  */
  63 @Service
  64 @Scope("prototype")
  65 public class MongodbService implements DbStoreService {
  66
  67         private final Logger log = LoggerFactory.getLogger(this.getClass());
  68
  69         private Db mongodb;
  70
  71         @Autowired
  72         private ApplicationConfiguration config;
  73         private boolean dbReady = false;
  74
  75         //@Autowired
  76 //      private DbService dbService;
  77
  78         private MongoDatabase database;
  79         private MongoClient mongoClient;
  80         private Map<String, MongoCollection<Document>> mongoCollectionMap = new HashMap<>();
  81         private InsertManyOptions insertManyOptions;
  82
  83         public MongodbService(Db db) {
  84                 mongodb = db;
  85         }
  86
  87         @PostConstruct
  88         @Override
  89         public void init() {
  90                 String host = mongodb.getHost();
  91
  92                 Integer port = mongodb.getPort();
  93                 if (port == null || port == 0) {
  94                         port = 27017; //MongoDB default
  95                 }
  96
  97                 String databaseName = mongodb.getDatabase();
  98                 String userName = mongodb.getLogin();
  99                 String password = mongodb.getPass();
 100
 101                 MongoCredential credential = null;
 102                 if (StringUtils.isNoneBlank(userName) && StringUtils.isNoneBlank(password)) {
 103                         credential = MongoCredential.createCredential(userName, databaseName, password.toCharArray());
 104                 }
 105
 106                 Builder builder = MongoClientOptions.builder();
 107                 builder.serverSelectionTimeout(30000);//server selection timeout, in milliseconds
 108
 109                 //http://mongodb.github.io/mongo-java-driver/3.0/driver/reference/connecting/ssl/
 110                 if (config.isEnableSSL()) {
 111                         builder.sslEnabled(Boolean.TRUE.equals(mongodb.getEncrypt()));// getEncrypt() can be null
 112                 }
 113                 MongoClientOptions options = builder.build();
 114                 List<ServerAddress> addrs = new ArrayList<ServerAddress>();
 115
 116                 addrs.add(new ServerAddress(host, port)); // FIXME should be a list of address
 117
 118                 try {
 119                         if (StringUtils.isNoneBlank(userName) && StringUtils.isNoneBlank(password)) {
 120                                 credential = MongoCredential.createCredential(userName, databaseName, password.toCharArray());
 121                                 List<MongoCredential> credentialList = new ArrayList<MongoCredential>();
 122                                 credentialList.add(credential);
 123                                 mongoClient = new MongoClient(addrs, credentialList, options);
 124                         } else {
 125                                 mongoClient = new MongoClient(addrs, options);
 126                         }
 127                 } catch (Exception ex) {
 128                         dbReady = false;
 129                         log.error("Fail to initiate MongoDB" + mongodb.getHost());
 130                         return;
 131                 }
 132                 database = mongoClient.getDatabase(mongodb.getDatabase());
 133
 134                 insertManyOptions = new InsertManyOptions();
 135                 insertManyOptions.ordered(false);
 136
 137                 dbReady = true;
 138         }
 139
 140         @PreDestroy
 141         public void cleanUp() {
 142                 config.getShutdownLock().readLock().lock();
 143
 144                 try {
 145                         log.info("mongoClient.close() at cleanUp.");
 146                         mongoClient.close();
 147                 } finally {
 148                         config.getShutdownLock().readLock().unlock();
 149                 }
 150         }
 151
 152         public void saveJsons(EffectiveTopic effectiveTopic, List<JSONObject> jsons) {
 153                 if (dbReady == false)//TOD throw exception
 154                         return;
 155                 List<Document> documents = new ArrayList<>(jsons.size());
 156                 for (JSONObject json : jsons) {
 157                         //convert org.json JSONObject to MongoDB Document
 158                         Document doc = Document.parse(json.toString());
 159
 160                         String id = effectiveTopic.getTopic().getMessageId(json); //id can be null
 161                         if (id != null) {
 162                                 doc.put("_id", id);
 163                         }
 164                         documents.add(doc);
 165                 }
 166
 167                 String collectionName = effectiveTopic.getName().replaceAll("[^a-zA-Z0-9]", "");//remove - _ .
 168                 MongoCollection<Document> collection = mongoCollectionMap.computeIfAbsent(collectionName, k -> database.getCollection(k));
 169
 170                 try {
 171                         collection.insertMany(documents, insertManyOptions);
 172                 } catch (MongoBulkWriteException e) {
 173                         List<BulkWriteError> bulkWriteErrors = e.getWriteErrors();
 174                         for (BulkWriteError bulkWriteError : bulkWriteErrors) {
 175                                 log.error("Failed record: {}", bulkWriteError);
 176                         }
 177                 } catch (MongoTimeoutException e) {
 178                         log.error("saveJsons()", e);
 179                 }
 180
 181                 log.debug("saved text to effectiveTopic = {}, batch count = {} ", effectiveTopic, jsons.size());
 182         }
 183
 184 }