asdctool/src/main/java/org/openecomp/sdc/asdctool/impl/DataMigration.java

   1 /*-
   2  * ============LICENSE_START=======================================================
   3  * SDC
   4  * ================================================================================
   5  * Copyright (C) 2017 AT&T Intellectual Property. All rights reserved.
   6  * ================================================================================
   7  * Licensed under the Apache License, Version 2.0 (the "License");
   8  * you may not use this file except in compliance with the License.
   9  * You may obtain a copy of the License at
  10  *
  11  *      http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  * ============LICENSE_END=========================================================
  19  */
  20
  21 package org.openecomp.sdc.asdctool.impl;
  22
  23
  24 import com.carrotsearch.hppc.cursors.ObjectCursor;
  25 import com.fasterxml.jackson.core.type.TypeReference;
  26 import com.fasterxml.jackson.databind.ObjectMapper;
  27 import fj.data.Either;
  28 import org.apache.commons.lang.SystemUtils;
  29 import org.elasticsearch.action.search.SearchResponse;
  30 import org.elasticsearch.cluster.metadata.IndexMetaData;
  31 import org.elasticsearch.common.collect.ImmutableOpenMap;
  32 import org.elasticsearch.common.settings.Settings;
  33 import org.elasticsearch.common.unit.TimeValue;
  34 import org.elasticsearch.index.query.QueryBuilder;
  35 import org.elasticsearch.index.query.QueryBuilders;
  36 import org.elasticsearch.search.SearchHit;
  37 import org.openecomp.sdc.be.auditing.api.AuditEventFactory;
  38 import org.openecomp.sdc.be.auditing.impl.AuditAuthRequestEventFactory;
  39 import org.openecomp.sdc.be.auditing.impl.AuditConsumerEventFactory;
  40 import org.openecomp.sdc.be.auditing.impl.AuditGetUebClusterEventFactory;
  41 import org.openecomp.sdc.be.auditing.impl.category.AuditCategoryEventFactory;
  42 import org.openecomp.sdc.be.auditing.impl.category.AuditGetCategoryHierarchyEventFactory;
  43 import org.openecomp.sdc.be.auditing.impl.distribution.*;
  44 import org.openecomp.sdc.be.auditing.impl.resourceadmin.AuditResourceAdminEventMigrationFactory;
  45 import org.openecomp.sdc.be.auditing.impl.usersadmin.AuditGetUsersListEventFactory;
  46 import org.openecomp.sdc.be.auditing.impl.usersadmin.AuditUserAccessEventFactory;
  47 import org.openecomp.sdc.be.auditing.impl.usersadmin.AuditUserAdminEventFactory;
  48 import org.openecomp.sdc.be.dao.cassandra.ArtifactCassandraDao;
  49 import org.openecomp.sdc.be.dao.cassandra.AuditCassandraDao;
  50 import org.openecomp.sdc.be.dao.cassandra.CassandraOperationStatus;
  51 import org.openecomp.sdc.be.dao.cassandra.schema.Table;
  52 import org.openecomp.sdc.be.dao.es.ElasticSearchClient;
  53 import org.openecomp.sdc.be.resources.data.ESArtifactData;
  54 import org.openecomp.sdc.be.resources.data.auditing.AuditingActionEnum;
  55 import org.openecomp.sdc.be.resources.data.auditing.AuditingGenericEvent;
  56 import org.openecomp.sdc.be.resources.data.auditing.AuditingTypesConstants;
  57 import org.openecomp.sdc.be.resources.data.auditing.model.*;
  58 import org.openecomp.sdc.common.datastructure.AuditingFieldsKey;
  59 import org.openecomp.sdc.common.log.wrappers.Logger;
  60 import org.springframework.beans.factory.annotation.Autowired;
  61
  62 import java.io.*;
  63 import java.net.MalformedURLException;
  64 import java.net.URISyntaxException;
  65 import java.net.URL;
  66 import java.nio.file.Files;
  67 import java.nio.file.Paths;
  68 import java.util.EnumMap;
  69 import java.util.Map;
  70
  71 /**
  72  * Created by mlando on 5/16/2016.
  73  */
  74 public class DataMigration {
  75
  76         private ObjectMapper jsonMapper = new ObjectMapper();
  77
  78         private static Logger log = Logger.getLogger(DataMigration.class.getName());
  79
  80         private ElasticSearchClient elasticSearchClient;
  81         private AuditCassandraDao auditCassandraDao;
  82         private ArtifactCassandraDao artifactCassandraDao;
  83
  84         @Autowired
  85         public DataMigration(AuditCassandraDao auditCassandraDao,
  86                 ArtifactCassandraDao artifactCassandraDao) {
  87                 this.auditCassandraDao = auditCassandraDao;
  88                 this.artifactCassandraDao = artifactCassandraDao;
  89         }
  90
  91     /**
  92          * the method exports and imports the records from ES to cassandra the flow
  93          * will check to see if the files are not empty if the files are not empty
  94          * the export will be skiped and the flow will use the existing files. the
  95          * flow will check if the tables in cassandra are empty, if the tables are
  96          * not empty the proces will stop and exit. if the tables are empty the
  97          * method will import the records from the files. in case of a fail the flow
  98          * will exit and clear all the Cassandra tables.
  99          *
 100          * @param appConfigDir
 101          *            the location of the dir in wich the output files will be
 102          *            stored
 103          * @param exportFromEs
 104          *            should the es be exported again and overwrite the old export
 105          * @param importToCassandra
 106          *            should we import the data into cassandra
 107          * @return true in case the operation was successful.
 108          */
 109         public boolean migrateDataESToCassndra(String appConfigDir, boolean exportFromEs, boolean importToCassandra) {
 110                 if (!initEsClient()) {
 111                         return false;
 112                 }
 113                 Map<Table, File> files = createOutPutFiles(appConfigDir, exportFromEs);
 114                 if (files == null) {
 115                         return false;
 116                 }
 117                 if (exportFromEs && filesEmpty(files)) {
 118                         Map<Table, PrintWriter> printerWritersMap = createWriters(files);
 119                         if (printerWritersMap == null) {
 120                                 return false;
 121                         }
 122                         try {
 123                                 ImmutableOpenMap<String, IndexMetaData> indexData = getIndexData();
 124                                 for (ObjectCursor<String> key : indexData.keys()) {
 125                                         if (("resources".equalsIgnoreCase(key.value) || key.value.startsWith("auditingevents"))
 126                         && !exportArtifacts(key.value, printerWritersMap)) {
 127                         return false;
 128                     }
 129                                 }
 130                         } finally {
 131                                 if (elasticSearchClient != null) {
 132                                         elasticSearchClient.close();
 133                                 }
 134                                 for (PrintWriter writer : printerWritersMap.values()) {
 135                                         writer.close();
 136                                 }
 137                         }
 138                 }
 139
 140                 return !importToCassandra || importToCassndra(files);
 141         }
 142
 143         private boolean initEsClient() {
 144                 String configHome = System.getProperty("config.home");
 145                 URL url = null;
 146                 Settings settings = null;
 147                 try {
 148                         if (SystemUtils.IS_OS_WINDOWS) {
 149                                 url = new URL("file:///" + configHome + "/elasticsearch.yml");
 150                         } else {
 151                                 url = new URL("file:" + configHome + "/elasticsearch.yml");
 152                         }
 153                         log.debug("URL {}", url);
 154                         settings = Settings.settingsBuilder().loadFromPath(Paths.get(url.toURI())).build();
 155                 } catch (MalformedURLException | URISyntaxException e1) {
 156                         log.error("Failed to create URL in order to load elasticsearch yml", e1);
 157                         return true;
 158                 }
 159
 160                 this.elasticSearchClient = new ElasticSearchClient();
 161                 this.elasticSearchClient.setClusterName(settings.get("cluster.name"));
 162                 this.elasticSearchClient.setLocal(settings.get("elasticSearch.local"));
 163                 this.elasticSearchClient.setTransportClient(settings.get("elasticSearch.transportclient"));
 164                 try {
 165                         elasticSearchClient.initialize();
 166                 } catch (URISyntaxException e) {
 167                     log.error(e.getMessage());
 168                         return false;
 169                 }
 170                 return true;
 171         }
 172
 173         /**
 174          * the method clears all the cassandra tables
 175          */
 176         private void truncateCassandraTable() {
 177                 log.info("import failed. truncating Cassandra tables.");
 178                 artifactCassandraDao.deleteAllArtifacts();
 179                 auditCassandraDao.deleteAllAudit();
 180         }
 181
 182         /**
 183          * the method imports the records from the files into cassandra
 184          *
 185          * @param files
 186          *            a map of files holding
 187          * @return true if the operation was successful
 188          */
 189         private boolean importToCassndra(Map<Table, File> files) {
 190                 log.info("starting to import date into Cassandra.");
 191                 if (!validtaTablsNotEmpty(files))
 192                         return true;
 193                 for (Table table : files.keySet()) {
 194                         log.info("importing recordes into {}", table.getTableDescription().getTableName());
 195                         if (!handleImport(files, table)) {
 196                                 truncateCassandraTable();
 197                                 return false;
 198                         }
 199                 }
 200                 log.info("finished to import date into Cassandra.");
 201                 return true;
 202         }
 203
 204         private boolean validtaTablsNotEmpty(Map<Table, File> files) {
 205                 for (Table table : files.keySet()) {
 206                         Either<Boolean, CassandraOperationStatus> isTableEmptyRes = checkIfTableIsEmpty(table);
 207                         if (isTableEmptyRes.isRight() || !isTableEmptyRes.left().value()) {
 208                                 log.error("Cassandra table {} is not empty operation aborted.",
 209                                                 table.getTableDescription().getTableName());
 210                                 return false;
 211                         }
 212                 }
 213                 return true;
 214         }
 215
 216         /**
 217          * the method retrieves the fields from the given map and generates
 218      * corresponding audit event according to the table name
 219          *
 220          * @param map
 221          *            the map from which we will retrieve the fields enum values
 222          * @param table
 223          *            the table we are going to store the record in.
 224          * @return an AuditingGenericEvent event representing the audit record that is going to be
 225          *         created.
 226          */
 227         AuditingGenericEvent createAuditEvent(Map<AuditingFieldsKey, String> map, Table table) {
 228                 AuditEventFactory factory = null;
 229                 switch (table) {
 230                         case USER_ADMIN_EVENT:
 231                                 factory = getAuditUserAdminEventFactory(map);
 232                                 break;
 233                         case USER_ACCESS_EVENT:
 234                                 factory = getAuditUserAccessEventFactory(map);
 235                                 break;
 236                         case RESOURCE_ADMIN_EVENT:
 237                                 factory = getAuditResourceAdminEventMigrationFactory(map);
 238                                 break;
 239                         case DISTRIBUTION_DOWNLOAD_EVENT:
 240                                 factory = getAuditDistributionDownloadEventFactory(map);
 241                                 break;
 242                         case DISTRIBUTION_ENGINE_EVENT:
 243                                 factory = getAuditDistributionEngineEventMigrationFactory(map);
 244                                 break;
 245                         case DISTRIBUTION_NOTIFICATION_EVENT:
 246                                 factory = getAuditDistributionNotificationEventFactory(map);
 247                                 break;
 248                         case DISTRIBUTION_STATUS_EVENT:
 249                                 factory = getAuditDistributionStatusEventFactory(map);
 250                                 break;
 251                         case DISTRIBUTION_DEPLOY_EVENT:
 252                                 factory = getAuditDistributionDeployEventFactory(map);
 253                                 break;
 254                         case DISTRIBUTION_GET_UEB_CLUSTER_EVENT:
 255                                 factory = getAuditGetUebClusterEventFactory(map);
 256                                 break;
 257                         case AUTH_EVENT:
 258                                 factory = getAuditAuthRequestEventFactory(map);
 259                                 break;
 260                         case CONSUMER_EVENT:
 261                                 factory = getAuditConsumerEventFactory(map);
 262                                 break;
 263                         case CATEGORY_EVENT:
 264                                 factory = getAuditCategoryEventFactory(map);
 265                                 break;
 266                         case GET_USERS_LIST_EVENT:
 267                                 factory = getAuditGetUsersListEventFactory(map);
 268                                 break;
 269                         case GET_CATEGORY_HIERARCHY_EVENT:
 270                                 factory = getAuditGetCategoryHierarchyEventFactory(map);
 271                                 break;
 272                         default:
 273                                 break;
 274                 }
 275                 return factory != null ? factory.getDbEvent() : null;
 276         }
 277
 278         private AuditEventFactory getAuditGetCategoryHierarchyEventFactory(Map<AuditingFieldsKey, String> map) {
 279                 return new AuditGetCategoryHierarchyEventFactory(
 280                         CommonAuditData.newBuilder()
 281                                         .description(map.get(AuditingFieldsKey.AUDIT_DESC))
 282                                         .status(map.get(AuditingFieldsKey.AUDIT_STATUS))
 283                                         .requestId(map.get(AuditingFieldsKey.AUDIT_REQUEST_ID))
 284                                         .serviceInstanceId(map.get(AuditingFieldsKey.AUDIT_SERVICE_INSTANCE_ID))
 285                                         .build(),
 286                         map.get(AuditingFieldsKey.AUDIT_MODIFIER_UID),
 287                         map.get(AuditingFieldsKey.AUDIT_DETAILS),
 288                         map.get(AuditingFieldsKey.AUDIT_TIMESTAMP));
 289         }
 290
 291         private AuditEventFactory getAuditGetUsersListEventFactory(Map<AuditingFieldsKey, String> map) {
 292                 return new AuditGetUsersListEventFactory(
 293                         CommonAuditData.newBuilder()
 294                                         .description(map.get(AuditingFieldsKey.AUDIT_DESC))
 295                                         .status(map.get(AuditingFieldsKey.AUDIT_STATUS))
 296                                         .requestId(map.get(AuditingFieldsKey.AUDIT_REQUEST_ID))
 297                                         .serviceInstanceId(map.get(AuditingFieldsKey.AUDIT_SERVICE_INSTANCE_ID))
 298                                         .build(),
 299                         map.get(AuditingFieldsKey.AUDIT_MODIFIER_UID),
 300                         map.get(AuditingFieldsKey.AUDIT_USER_DETAILS),
 301                         map.get(AuditingFieldsKey.AUDIT_TIMESTAMP));
 302         }
 303
 304         private AuditEventFactory getAuditCategoryEventFactory(Map<AuditingFieldsKey, String> map) {
 305                 return new AuditCategoryEventFactory(
 306                         AuditingActionEnum.fromName(map.get(AuditingFieldsKey.AUDIT_ACTION)),
 307                         CommonAuditData.newBuilder()
 308                                         .description(map.get(AuditingFieldsKey.AUDIT_DESC))
 309                                         .status(map.get(AuditingFieldsKey.AUDIT_STATUS))
 310                                         .requestId(map.get(AuditingFieldsKey.AUDIT_REQUEST_ID))
 311                                         .serviceInstanceId(map.get(AuditingFieldsKey.AUDIT_SERVICE_INSTANCE_ID))
 312                                         .build(),
 313                         map.get(AuditingFieldsKey.AUDIT_MODIFIER_UID),
 314                         map.get(AuditingFieldsKey.AUDIT_CATEGORY_NAME),
 315                         map.get(AuditingFieldsKey.AUDIT_SUB_CATEGORY_NAME),
 316                         map.get(AuditingFieldsKey.AUDIT_GROUPING_NAME),
 317                         map.get(AuditingFieldsKey.AUDIT_RESOURCE_TYPE),
 318                         map.get(AuditingFieldsKey.AUDIT_TIMESTAMP));
 319         }
 320
 321         private AuditEventFactory getAuditUserAccessEventFactory(Map<AuditingFieldsKey, String> map) {
 322                 return new AuditUserAccessEventFactory(
 323                         CommonAuditData.newBuilder()
 324                                         .description(map.get(AuditingFieldsKey.AUDIT_DESC))
 325                                         .status(map.get(AuditingFieldsKey.AUDIT_STATUS))
 326                                         .requestId(map.get(AuditingFieldsKey.AUDIT_REQUEST_ID))
 327                                         .serviceInstanceId(map.get(AuditingFieldsKey.AUDIT_SERVICE_INSTANCE_ID))
 328                                         .build(),
 329                         map.get(AuditingFieldsKey.AUDIT_USER_UID),
 330                         map.get(AuditingFieldsKey.AUDIT_TIMESTAMP));
 331         }
 332
 333         private AuditEventFactory getAuditUserAdminEventFactory(Map<AuditingFieldsKey, String> map) {
 334                 return new AuditUserAdminEventFactory(
 335             AuditingActionEnum.fromName(map.get(AuditingFieldsKey.AUDIT_ACTION)),
 336                         CommonAuditData.newBuilder()
 337                                         .description(map.get(AuditingFieldsKey.AUDIT_DESC))
 338                                         .status(map.get(AuditingFieldsKey.AUDIT_STATUS))
 339                                         .requestId(map.get(AuditingFieldsKey.AUDIT_REQUEST_ID))
 340                                         .serviceInstanceId(map.get(AuditingFieldsKey.AUDIT_SERVICE_INSTANCE_ID))
 341                                         .build(),
 342                         map.get(AuditingFieldsKey.AUDIT_MODIFIER_UID),
 343                         map.get(AuditingFieldsKey.AUDIT_USER_BEFORE),
 344                         map.get(AuditingFieldsKey.AUDIT_USER_AFTER),
 345                         map.get(AuditingFieldsKey.AUDIT_TIMESTAMP));
 346         }
 347
 348         private AuditEventFactory getAuditConsumerEventFactory(Map<AuditingFieldsKey, String> map) {
 349                 return new AuditConsumerEventFactory(
 350                     AuditingActionEnum.fromName(map.get(AuditingFieldsKey.AUDIT_ACTION)),
 351                         CommonAuditData.newBuilder()
 352                                         .description(map.get(AuditingFieldsKey.AUDIT_DESC))
 353                                         .status(map.get(AuditingFieldsKey.AUDIT_STATUS))
 354                                         .requestId(map.get(AuditingFieldsKey.AUDIT_REQUEST_ID))
 355                                         .serviceInstanceId(map.get(AuditingFieldsKey.AUDIT_SERVICE_INSTANCE_ID))
 356                                         .build(),
 357                         map.get(AuditingFieldsKey.AUDIT_MODIFIER_UID),
 358                         map.get(AuditingFieldsKey.AUDIT_ECOMP_USER),
 359                         map.get(AuditingFieldsKey.AUDIT_TIMESTAMP));
 360         }
 361
 362         private AuditEventFactory getAuditAuthRequestEventFactory(Map<AuditingFieldsKey, String> map) {
 363                 return new AuditAuthRequestEventFactory(
 364                         CommonAuditData.newBuilder()
 365                                         .description(map.get(AuditingFieldsKey.AUDIT_DESC))
 366                                         .status(map.get(AuditingFieldsKey.AUDIT_STATUS))
 367                                         .requestId(map.get(AuditingFieldsKey.AUDIT_REQUEST_ID))
 368                                         .serviceInstanceId(map.get(AuditingFieldsKey.AUDIT_SERVICE_INSTANCE_ID))
 369                                         .build(),
 370                         map.get(AuditingFieldsKey.AUDIT_USER_UID),
 371                         map.get(AuditingFieldsKey.AUDIT_AUTH_URL),
 372                         map.get(AuditingFieldsKey.AUDIT_AUTH_REALM),
 373                         map.get(AuditingFieldsKey.AUDIT_AUTH_STATUS),
 374                         map.get(AuditingFieldsKey.AUDIT_TIMESTAMP));
 375         }
 376
 377         private AuditEventFactory getAuditGetUebClusterEventFactory(Map<AuditingFieldsKey, String> map) {
 378                 return new AuditGetUebClusterEventFactory(
 379                         CommonAuditData.newBuilder()
 380                                         .description(map.get(AuditingFieldsKey.AUDIT_DESC))
 381                                         .status(map.get(AuditingFieldsKey.AUDIT_STATUS))
 382                                         .requestId(map.get(AuditingFieldsKey.AUDIT_REQUEST_ID))
 383                                         .serviceInstanceId(map.get(AuditingFieldsKey.AUDIT_SERVICE_INSTANCE_ID))
 384                                         .build(),
 385                         map.get(AuditingFieldsKey.AUDIT_DISTRIBUTION_CONSUMER_ID),
 386                         map.get(AuditingFieldsKey.AUDIT_TIMESTAMP));
 387         }
 388
 389         private AuditEventFactory getAuditResourceAdminEventMigrationFactory(Map<AuditingFieldsKey, String> map) {
 390                 return new AuditResourceAdminEventMigrationFactory(
 391                 AuditingActionEnum.fromName(map.get(AuditingFieldsKey.AUDIT_ACTION)),
 392                         CommonAuditData.newBuilder()
 393                                         .description(map.get(AuditingFieldsKey.AUDIT_DESC))
 394                                         .status(map.get(AuditingFieldsKey.AUDIT_STATUS))
 395                                         .requestId(map.get(AuditingFieldsKey.AUDIT_REQUEST_ID))
 396                                         .serviceInstanceId(map.get(AuditingFieldsKey.AUDIT_SERVICE_INSTANCE_ID))
 397                                         .build(),
 398                         new ResourceCommonInfo(map.get(AuditingFieldsKey.AUDIT_RESOURCE_NAME),
 399                                         map.get(AuditingFieldsKey.AUDIT_RESOURCE_TYPE)),
 400                         ResourceVersionInfo.newBuilder()
 401                                         .artifactUuid(map.get(AuditingFieldsKey.AUDIT_PREV_ARTIFACT_UUID))
 402                                         .state(map.get(AuditingFieldsKey.AUDIT_RESOURCE_PREV_STATE))
 403                                         .version(map.get(AuditingFieldsKey.AUDIT_RESOURCE_PREV_VERSION))
 404                                         .distributionStatus(map.get(AuditingFieldsKey.AUDIT_RESOURCE_DPREV_STATUS))
 405                                         .build(),
 406                         ResourceVersionInfo.newBuilder()
 407                                         .artifactUuid(map.get(AuditingFieldsKey.AUDIT_CURR_ARTIFACT_UUID))
 408                                         .state(map.get(AuditingFieldsKey.AUDIT_RESOURCE_CURR_STATE))
 409                                         .version(map.get(AuditingFieldsKey.AUDIT_RESOURCE_CURR_VERSION))
 410                                         .distributionStatus(map.get(AuditingFieldsKey.AUDIT_RESOURCE_DCURR_STATUS))
 411                                         .build(),
 412                         map.get(AuditingFieldsKey.AUDIT_INVARIANT_UUID),
 413                         map.get(AuditingFieldsKey.AUDIT_MODIFIER_UID),
 414                         map.get(AuditingFieldsKey.AUDIT_ARTIFACT_DATA),
 415                         map.get(AuditingFieldsKey.AUDIT_RESOURCE_COMMENT),
 416                         map.get(AuditingFieldsKey.AUDIT_DISTRIBUTION_ID),
 417                         map.get(AuditingFieldsKey.AUDIT_RESOURCE_TOSCA_NODE_TYPE),
 418                         map.get(AuditingFieldsKey.AUDIT_TIMESTAMP));
 419         }
 420
 421         private AuditEventFactory getAuditDistributionDownloadEventFactory(Map<AuditingFieldsKey, String> map) {
 422                 return new AuditDistributionDownloadEventFactory(
 423                         CommonAuditData.newBuilder()
 424                                         .description(map.get(AuditingFieldsKey.AUDIT_DESC))
 425                                         .status(map.get(AuditingFieldsKey.AUDIT_STATUS))
 426                                         .requestId(map.get(AuditingFieldsKey.AUDIT_REQUEST_ID))
 427                                         .serviceInstanceId(map.get(AuditingFieldsKey.AUDIT_SERVICE_INSTANCE_ID))
 428                                         .build(),
 429                         new DistributionData(map.get(AuditingFieldsKey.AUDIT_DISTRIBUTION_CONSUMER_ID),
 430                                         map.get(AuditingFieldsKey.AUDIT_RESOURCE_URL)),
 431                                 map.get(AuditingFieldsKey.AUDIT_TIMESTAMP));
 432         }
 433
 434         private AuditEventFactory getAuditDistributionEngineEventMigrationFactory(Map<AuditingFieldsKey, String> map) {
 435                 return new AuditDistributionEngineEventMigrationFactory(
 436                     AuditingActionEnum.fromName(map.get(AuditingFieldsKey.AUDIT_ACTION)),
 437                         CommonAuditData.newBuilder()
 438                                         .description(map.get(AuditingFieldsKey.AUDIT_DESC))
 439                                         .status(map.get(AuditingFieldsKey.AUDIT_STATUS))
 440                                         .requestId(map.get(AuditingFieldsKey.AUDIT_REQUEST_ID))
 441                                         .serviceInstanceId(map.get(AuditingFieldsKey.AUDIT_SERVICE_INSTANCE_ID))
 442                                         .build(),
 443                         DistributionTopicData.newBuilder()
 444                                         .notificationTopic(map.get(AuditingFieldsKey.AUDIT_DISTRIBUTION_NOTIFICATION_TOPIC_NAME))
 445                                         .statusTopic(map.get(AuditingFieldsKey.AUDIT_DISTRIBUTION_STATUS_TOPIC_NAME))
 446                                         .build(),
 447                         map.get(AuditingFieldsKey.AUDIT_DISTRIBUTION_CONSUMER_ID),
 448                         map.get(AuditingFieldsKey.AUDIT_DISTRIBUTION_API_KEY),
 449                         map.get(AuditingFieldsKey.AUDIT_DISTRIBUTION_ENVRIONMENT_NAME),
 450                         map.get(AuditingFieldsKey.AUDIT_DISTRIBUTION_ROLE),
 451                         map.get(AuditingFieldsKey.AUDIT_TIMESTAMP));
 452         }
 453
 454         private AuditEventFactory getAuditDistributionDeployEventFactory(Map<AuditingFieldsKey, String> map) {
 455                 return new AuditDistributionDeployEventFactory(
 456                         CommonAuditData.newBuilder()
 457                                         .description(map.get(AuditingFieldsKey.AUDIT_DESC))
 458                                         .status(map.get(AuditingFieldsKey.AUDIT_STATUS))
 459                                         .requestId(map.get(AuditingFieldsKey.AUDIT_REQUEST_ID))
 460                                         .serviceInstanceId(map.get(AuditingFieldsKey.AUDIT_SERVICE_INSTANCE_ID))
 461                                         .build(),
 462                         new ResourceCommonInfo(map.get(AuditingFieldsKey.AUDIT_RESOURCE_NAME),
 463                                         map.get(AuditingFieldsKey.AUDIT_RESOURCE_TYPE)),
 464                         map.get(AuditingFieldsKey.AUDIT_RESOURCE_CURR_VERSION),
 465                         map.get(AuditingFieldsKey.AUDIT_DISTRIBUTION_ID),
 466                         map.get(AuditingFieldsKey.AUDIT_MODIFIER_UID),
 467                         map.get(AuditingFieldsKey.AUDIT_TIMESTAMP));
 468         }
 469
 470         private AuditEventFactory getAuditDistributionStatusEventFactory(Map<AuditingFieldsKey, String> map) {
 471                 return new AuditDistributionStatusEventFactory(
 472                         CommonAuditData.newBuilder()
 473                                         .description(map.get(AuditingFieldsKey.AUDIT_DESC))
 474                                         .status(map.get(AuditingFieldsKey.AUDIT_STATUS))
 475                                         .requestId(map.get(AuditingFieldsKey.AUDIT_REQUEST_ID))
 476                                         .serviceInstanceId(map.get(AuditingFieldsKey.AUDIT_SERVICE_INSTANCE_ID))
 477                                         .build(),
 478                         new DistributionData(map.get(AuditingFieldsKey.AUDIT_DISTRIBUTION_CONSUMER_ID),
 479                                         map.get(AuditingFieldsKey.AUDIT_RESOURCE_URL)),
 480                         map.get(AuditingFieldsKey.AUDIT_DISTRIBUTION_ID),
 481                         map.get(AuditingFieldsKey.AUDIT_DISTRIBUTION_TOPIC_NAME),
 482                         map.get(AuditingFieldsKey.AUDIT_DISTRIBUTION_STATUS_TIME),
 483                         map.get(AuditingFieldsKey.AUDIT_TIMESTAMP));
 484         }
 485
 486         private AuditEventFactory getAuditDistributionNotificationEventFactory(Map<AuditingFieldsKey, String> map) {
 487                 return new AuditDistributionNotificationEventFactory(
 488                         CommonAuditData.newBuilder()
 489                                         .description(map.get(AuditingFieldsKey.AUDIT_DESC))
 490                                         .status(map.get(AuditingFieldsKey.AUDIT_STATUS))
 491                                         .requestId(map.get(AuditingFieldsKey.AUDIT_REQUEST_ID))
 492                                         .serviceInstanceId(map.get(AuditingFieldsKey.AUDIT_SERVICE_INSTANCE_ID))
 493                                         .build(),
 494                         new ResourceCommonInfo(map.get(AuditingFieldsKey.AUDIT_RESOURCE_NAME),
 495                                         map.get(AuditingFieldsKey.AUDIT_RESOURCE_TYPE)),
 496                         ResourceVersionInfo.newBuilder()
 497                                         .state(map.get(AuditingFieldsKey.AUDIT_RESOURCE_CURR_STATE))
 498                                         .version(map.get(AuditingFieldsKey.AUDIT_RESOURCE_CURR_VERSION))
 499                                         .build(),
 500                         map.get(AuditingFieldsKey.AUDIT_DISTRIBUTION_ID),
 501                         map.get(AuditingFieldsKey.AUDIT_MODIFIER_UID),
 502                         map.get(AuditingFieldsKey.AUDIT_DISTRIBUTION_TOPIC_NAME),
 503                         new OperationalEnvAuditData(map.get(AuditingFieldsKey.AUDIT_DISTRIBUTION_ENVIRONMENT_ID),
 504                                         map.get(AuditingFieldsKey.AUDIT_DISTRIBUTION_VNF_WORKLOAD_CONTEXT),
 505                                         map.get(AuditingFieldsKey.AUDIT_DISTRIBUTION_TENANT)),
 506                         map.get(AuditingFieldsKey.AUDIT_TIMESTAMP));
 507         }
 508
 509
 510
 511         /**
 512          * the method reads the content of the file intended for a given table, and
 513          * sores them in cassandra
 514          *
 515          * @param files
 516          *            a map of files from which the recordes will be retrieved.
 517          * @param table
 518          *            the name of the table we want to look up in the files and sore
 519          *            in Cassandra // * @param store the function to call when
 520          *            storing recordes in cassndra
 521          * @return true if the operation was successful
 522          */
 523         private boolean handleImport(Map<Table, File> files, Table table) {
 524                 BufferedReader br = null;
 525                 try {
 526                         br = new BufferedReader(new FileReader(files.get(table)));
 527                         String line = null;
 528                         while ((line = br.readLine()) != null) {
 529                                 CassandraOperationStatus res = CassandraOperationStatus.GENERAL_ERROR;
 530                                 if (Table.ARTIFACT.equals(table)) {
 531                                         res = artifactCassandraDao.saveArtifact(jsonMapper.readValue(line, ESArtifactData.class));
 532                                 }
 533                                 else {
 534                     AuditingGenericEvent recordForCassandra = createAuditRecordForCassandra(line, table);
 535                                         if (recordForCassandra != null) {
 536                         res = auditCassandraDao.saveRecord(recordForCassandra);
 537                     }
 538                                 }
 539                                 if (!res.equals(CassandraOperationStatus.OK)) {
 540                                         log.error("save recored to cassndra {} failed with status {} aborting.",
 541                                                         table.getTableDescription().getTableName(), res);
 542                                         return false;
 543                                 }
 544                         }
 545                         return true;
 546                 } catch (IOException e) {
 547                         log.error("failed to read file", e);
 548                         return false;
 549                 } finally {
 550                         if (br != null) {
 551                                 try {
 552                                         br.close();
 553                                 } catch (IOException e) {
 554                                         log.error("failed to close file reader", e);
 555                                 }
 556                         }
 557                 }
 558         }
 559
 560     AuditingGenericEvent createAuditRecordForCassandra(String json, Table table) throws IOException{
 561         return createAuditEvent(parseToMap(json), table);
 562     }
 563
 564         private Map<AuditingFieldsKey, String> parseToMap(String json) throws IOException {
 565                 return jsonMapper.readValue(json, new TypeReference<Map<AuditingFieldsKey, String>>(){});
 566         }
 567
 568         /**
 569          * the method checks if the given table is empty
 570          *
 571          * @param table
 572          *            the name of the table we want to check
 573          * @return true if the table is empty
 574          */
 575         private Either<Boolean, CassandraOperationStatus> checkIfTableIsEmpty(Table table) {
 576                 if (Table.ARTIFACT.equals(table)) {
 577                         return artifactCassandraDao.isTableEmpty(table.getTableDescription().getTableName());
 578                 } else {
 579                         return auditCassandraDao.isTableEmpty(table.getTableDescription().getTableName());
 580                 }
 581         }
 582
 583         private boolean filesEmpty(Map<Table, File> files) {
 584                 for (Table table : files.keySet()) {
 585                         File file = files.get(table);
 586                         if (file.length() != 0) {
 587                                 log.info("file:{} is not empty skipping export", table.getTableDescription().getTableName());
 588                                 return false;
 589                         }
 590                 }
 591                 return true;
 592         }
 593
 594         /**
 595          * the method reads the records from es index of audit's into a file as
 596          * json's.
 597          *
 598          * @param value
 599          *            the name of the index we want
 600          * @param printerWritersMap
 601          *            a map of the writers we use to write to a file.
 602          * @return true in case the export was successful.
 603          */
 604         private boolean exportAudit(String value, Map<Table, PrintWriter> printerWritersMap) {
 605                 log.info("stratng to export audit data from es index{} to file.", value);
 606                 QueryBuilder queryBuilder = QueryBuilders.matchAllQuery();
 607                 SearchResponse scrollResp = elasticSearchClient.getClient().prepareSearch(value).setScroll(new TimeValue(60000))
 608                                 .setQuery(queryBuilder).setSize(100).execute().actionGet();
 609                 while (true) {
 610                         for (SearchHit hit : scrollResp.getHits().getHits()) {
 611                                 PrintWriter out = printerWritersMap.get(TypeToTableMapping.getTableByType(hit.getType()));
 612                                 out.println(hit.getSourceAsString());
 613                         }
 614                         scrollResp = elasticSearchClient.getClient().prepareSearchScroll(scrollResp.getScrollId())
 615                                         .setScroll(new TimeValue(60000)).execute().actionGet();
 616                         if (scrollResp.getHits().getHits().length == 0) {
 617                                 break;
 618
 619                         }
 620                 }
 621
 622                 log.info("export audit data from es to file. finished succsesfully");
 623                 return true;
 624         }
 625
 626         /**
 627          * the method reads the records from es index of resources into a file as
 628          * json's.
 629          *
 630          * @param index
 631          *            the name of the index we want to read
 632          * @param printerWritersMap
 633          *            a map of the writers we use to write to a file.
 634          * @return true in case the export was successful.
 635          */
 636         private boolean exportArtifacts(String index, Map<Table, PrintWriter> printerWritersMap) {
 637                 log.info("stratng to export artifact data from es to file.");
 638                 PrintWriter out = printerWritersMap.get(Table.ARTIFACT);
 639                 QueryBuilder queryBuilder = QueryBuilders.matchAllQuery();
 640                 SearchResponse scrollResp = elasticSearchClient.getClient().prepareSearch(index).setScroll(new TimeValue(60000))
 641                                 .setQuery(queryBuilder).setSize(100).execute().actionGet();
 642                 while (true) {
 643                         for (SearchHit hit : scrollResp.getHits().getHits()) {
 644                                 ;
 645                                 out.println(hit.getSourceAsString());
 646                         }
 647                         scrollResp = elasticSearchClient.getClient().prepareSearchScroll(scrollResp.getScrollId())
 648                                         .setScroll(new TimeValue(60000)).execute().actionGet();
 649                         if (scrollResp.getHits().getHits().length == 0) {
 650                                 break;
 651
 652                         }
 653                 }
 654
 655                 log.info("export artifact data from es to file. finished succsesfully");
 656                 return true;
 657         }
 658
 659         /**
 660          * the method retrieves all the indexes from elasticsearch
 661          *
 662          * @return a map of indexes and there metadata
 663          */
 664         private ImmutableOpenMap<String, IndexMetaData> getIndexData() {
 665                 return elasticSearchClient.getClient().admin().cluster().prepareState().get().getState().getMetaData()
 666                                 .getIndices();
 667         }
 668
 669         /**
 670          * the method creates all the files and dir which holds them. in case the
 671          * files exist they will not be created again.
 672          *
 673          * @param appConfigDir
 674          *            the base path under which the output dir will be created and
 675          *            the export result files the created filesa are named according
 676          *            to the name of the table into which it will be imported.
 677          * @param exportToEs
 678          *            if true all the export files will be recreated
 679          * @returnthe returns a map of tables and the files representing them them
 680          */
 681         private Map<Table, File> createOutPutFiles(String appConfigDir, boolean exportToEs) {
 682                 Map<Table, File> result = new EnumMap<Table, File>(Table.class);
 683                 File outputDir = new File(appConfigDir + "/output/");
 684                 if (!createOutPutFolder(outputDir)) {
 685                         return null;
 686                 }
 687                 for (Table table : Table.values()) {
 688                         File file = new File(outputDir + "/" + table.getTableDescription().getTableName());
 689                         if (exportToEs) {
 690                                 try {
 691                                         if (file.exists()) {
 692                                                 Files.delete(file.toPath());
 693                                         }
 694                                 } catch (IOException e) {
 695                                         log.error("failed to delete output file {}", file.getAbsolutePath(), e);
 696                                         return null;
 697                                 }
 698                                 file = new File(outputDir + "/" + table.getTableDescription().getTableName());
 699                         }
 700                         if (!file.exists()) {
 701                                 try {
 702                                         file.createNewFile();
 703                                 } catch (IOException e) {
 704                                         log.error("failed to create output file {}", file.getAbsolutePath(), e);
 705                                         return null;
 706                                 }
 707                         }
 708                         result.put(table, file);
 709
 710                 }
 711                 return result;
 712         }
 713
 714         /**
 715          * the method create the writers to each file
 716          *
 717          * @param files
 718          *            a map of the files according to table
 719          * @return returns a map of writers according to table.
 720          */
 721         private Map<Table, PrintWriter> createWriters(Map<Table, File> files) {
 722                 Map<Table, PrintWriter> printerWritersMap = new EnumMap<>(Table.class);
 723
 724                         for (Table table : files.keySet()) {
 725                                 log.info("creating writer for {}", table);
 726                                 File file = files.get(table);
 727                 try(PrintWriter out = new PrintWriter(new BufferedWriter(new FileWriter(file)) )){
 728                                 printerWritersMap.put(table, out);
 729                                 log.info("creating writer for {} was successful", table);
 730             } catch (IOException e) {
 731                 log.error("create writer to file failed",e);
 732                 return null;
 733                         }
 734         }
 735                 return printerWritersMap;
 736         }
 737
 738         /**
 739          * the method creates the output dir in case it does not exist
 740          *
 741          * @param outputDir
 742          *            the path under wich the directory will be created.
 743          * @return true in case the create was succsesful or the dir already exists
 744          */
 745         private boolean createOutPutFolder(File outputDir) {
 746                 if (!outputDir.exists()) {
 747                         log.info("creating output dir {}", outputDir.getAbsolutePath());
 748                         try {
 749                                 Files.createDirectories(outputDir.toPath());
 750                         } catch (IOException e) {
 751                                 log.error("failed to create output dir {}", outputDir.getAbsolutePath(), e);
 752                                 return false;
 753                         }
 754                 }
 755                 return true;
 756         }
 757
 758         public enum TypeToTableMapping {
 759                 USER_ADMIN_EVENT_TYPE(AuditingTypesConstants.USER_ADMIN_EVENT_TYPE,
 760                                 Table.USER_ADMIN_EVENT), USER_ACCESS_EVENT_TYPE(AuditingTypesConstants.USER_ACCESS_EVENT_TYPE,
 761                                                 Table.USER_ACCESS_EVENT), RESOURCE_ADMIN_EVENT_TYPE(
 762                                                                 AuditingTypesConstants.RESOURCE_ADMIN_EVENT_TYPE,
 763                                                                 Table.RESOURCE_ADMIN_EVENT), DISTRIBUTION_DOWNLOAD_EVENT_TYPE(
 764                                                                                 AuditingTypesConstants.DISTRIBUTION_DOWNLOAD_EVENT_TYPE,
 765                                                                                 Table.DISTRIBUTION_DOWNLOAD_EVENT), DISTRIBUTION_ENGINE_EVENT_TYPE(
 766                                                                                                 AuditingTypesConstants.DISTRIBUTION_ENGINE_EVENT_TYPE,
 767                                                                                                 Table.DISTRIBUTION_ENGINE_EVENT), DISTRIBUTION_NOTIFICATION_EVENT_TYPE(
 768                                                                                                                 AuditingTypesConstants.DISTRIBUTION_NOTIFICATION_EVENT_TYPE,
 769                                                                                                                 Table.DISTRIBUTION_NOTIFICATION_EVENT), DISTRIBUTION_STATUS_EVENT_TYPE(
 770                                                                                                                                 AuditingTypesConstants.DISTRIBUTION_STATUS_EVENT_TYPE,
 771                                                                                                                                 Table.DISTRIBUTION_STATUS_EVENT), DISTRIBUTION_DEPLOY_EVENT_TYPE(
 772                                                                                                                                                 AuditingTypesConstants.DISTRIBUTION_DEPLOY_EVENT_TYPE,
 773                                                                                                                                                 Table.DISTRIBUTION_DEPLOY_EVENT), DISTRIBUTION_GET_UEB_CLUSTER_EVENT_TYPE(
 774                                                                                                                                                                 AuditingTypesConstants.DISTRIBUTION_GET_UEB_CLUSTER_EVENT_TYPE,
 775                                                                                                                                                                 Table.DISTRIBUTION_GET_UEB_CLUSTER_EVENT), AUTH_EVENT_TYPE(
 776                                                                                                                                                                                 AuditingTypesConstants.AUTH_EVENT_TYPE,
 777                                                                                                                                                                                 Table.AUTH_EVENT), CONSUMER_EVENT_TYPE(
 778                                                                                                                                                                                                 AuditingTypesConstants.CONSUMER_EVENT_TYPE,
 779                                                                                                                                                                                                 Table.CONSUMER_EVENT), CATEGORY_EVENT_TYPE(
 780                                                                                                                                                                                                                 AuditingTypesConstants.CATEGORY_EVENT_TYPE,
 781                                                                                                                                                                                                                 Table.CATEGORY_EVENT), GET_USERS_LIST_EVENT_TYPE(
 782                                                                                                                                                                                                                                 AuditingTypesConstants.GET_USERS_LIST_EVENT_TYPE,
 783                                                                                                                                                                                                                                 Table.GET_USERS_LIST_EVENT), GET_CATEGORY_HIERARCHY_EVENT_TYPE(
 784                                                                                                                                                                                                                                                 AuditingTypesConstants.GET_CATEGORY_HIERARCHY_EVENT_TYPE,
 785                                                                                                                                                                                                                                                 Table.GET_CATEGORY_HIERARCHY_EVENT);
 786
 787                 String typeName;
 788                 Table table;
 789
 790                 TypeToTableMapping(String typeName, Table table) {
 791                         this.typeName = typeName;
 792                         this.table = table;
 793                 }
 794
 795                 public String getTypeName() {
 796                         return typeName;
 797                 }
 798
 799                 public Table getTable() {
 800                         return table;
 801                 }
 802
 803                 public static Table getTableByType(String type) {
 804                         for (TypeToTableMapping mapping : TypeToTableMapping.values()) {
 805                                 if (mapping.getTypeName().equalsIgnoreCase(type)) {
 806                                         return mapping.getTable();
 807                                 }
 808                         }
 809                         return null;
 810                 }
 811         }
 812
 813 }