asdctool/src/main/java/org/openecomp/sdc/asdctool/impl/DataMigration.java

   1 /*-
   2  * ============LICENSE_START=======================================================
   3  * SDC
   4  * ================================================================================
   5  * Copyright (C) 2017 AT&T Intellectual Property. All rights reserved.
   6  * ================================================================================
   7  * Licensed under the Apache License, Version 2.0 (the "License");
   8  * you may not use this file except in compliance with the License.
   9  * You may obtain a copy of the License at
  10  *
  11  *      http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  * ============LICENSE_END=========================================================
  19  */
  20
  21 package org.openecomp.sdc.asdctool.impl;
  22
  23
  24 import com.carrotsearch.hppc.cursors.ObjectCursor;
  25 import com.fasterxml.jackson.core.type.TypeReference;
  26 import com.fasterxml.jackson.databind.ObjectMapper;
  27 import fj.data.Either;
  28 import org.apache.commons.lang.SystemUtils;
  29 import org.elasticsearch.action.search.SearchResponse;
  30 import org.elasticsearch.cluster.metadata.IndexMetaData;
  31 import org.elasticsearch.common.collect.ImmutableOpenMap;
  32 import org.elasticsearch.common.settings.Settings;
  33 import org.elasticsearch.common.unit.TimeValue;
  34 import org.elasticsearch.index.query.QueryBuilder;
  35 import org.elasticsearch.index.query.QueryBuilders;
  36 import org.elasticsearch.search.SearchHit;
  37 import org.openecomp.sdc.be.auditing.api.AuditEventFactory;
  38 import org.openecomp.sdc.be.auditing.impl.AuditAuthRequestEventFactory;
  39 import org.openecomp.sdc.be.auditing.impl.AuditConsumerEventFactory;
  40 import org.openecomp.sdc.be.auditing.impl.AuditGetUebClusterEventFactory;
  41 import org.openecomp.sdc.be.auditing.impl.category.AuditCategoryEventFactory;
  42 import org.openecomp.sdc.be.auditing.impl.category.AuditGetCategoryHierarchyEventFactory;
  43 import org.openecomp.sdc.be.auditing.impl.distribution.*;
  44 import org.openecomp.sdc.be.auditing.impl.resourceadmin.AuditResourceAdminEventMigrationFactory;
  45 import org.openecomp.sdc.be.auditing.impl.usersadmin.AuditGetUsersListEventFactory;
  46 import org.openecomp.sdc.be.auditing.impl.usersadmin.AuditUserAccessEventFactory;
  47 import org.openecomp.sdc.be.auditing.impl.usersadmin.AuditUserAdminEventFactory;
  48 import org.openecomp.sdc.be.dao.cassandra.ArtifactCassandraDao;
  49 import org.openecomp.sdc.be.dao.cassandra.AuditCassandraDao;
  50 import org.openecomp.sdc.be.dao.cassandra.CassandraOperationStatus;
  51 import org.openecomp.sdc.be.dao.cassandra.schema.Table;
  52 import org.openecomp.sdc.be.dao.es.ElasticSearchClient;
  53 import org.openecomp.sdc.be.resources.data.ESArtifactData;
  54 import org.openecomp.sdc.be.resources.data.auditing.AuditingActionEnum;
  55 import org.openecomp.sdc.be.resources.data.auditing.AuditingGenericEvent;
  56 import org.openecomp.sdc.be.resources.data.auditing.AuditingTypesConstants;
  57 import org.openecomp.sdc.be.resources.data.auditing.model.*;
  58 import org.openecomp.sdc.common.datastructure.AuditingFieldsKey;
  59 import org.openecomp.sdc.common.log.wrappers.Logger;
  60 import org.springframework.beans.factory.annotation.Autowired;
  61
  62 import java.io.*;
  63 import java.net.MalformedURLException;
  64 import java.net.URISyntaxException;
  65 import java.net.URL;
  66 import java.nio.file.Files;
  67 import java.nio.file.Paths;
  68 import java.util.EnumMap;
  69 import java.util.Map;
  70
  71 /**
  72  * Created by mlando on 5/16/2016.
  73  */
  74 public class DataMigration {
  75
  76         private ObjectMapper jsonMapper = new ObjectMapper();
  77
  78         private static Logger log = Logger.getLogger(DataMigration.class.getName());
  79
  80         private ElasticSearchClient elasticSearchClient;
  81         @Autowired
  82         private AuditCassandraDao auditCassandraDao;
  83         @Autowired
  84         private ArtifactCassandraDao artifactCassandraDao;
  85
  86     /**
  87          * the method exports and imports the records from ES to cassandra the flow
  88          * will check to see if the files are not empty if the files are not empty
  89          * the export will be skiped and the flow will use the existing files. the
  90          * flow will check if the tables in cassandra are empty, if the tables are
  91          * not empty the proces will stop and exit. if the tables are empty the
  92          * method will import the records from the files. in case of a fail the flow
  93          * will exit and clear all the Cassandra tables.
  94          *
  95          * @param appConfigDir
  96          *            the location of the dir in wich the output files will be
  97          *            stored
  98          * @param exportFromEs
  99          *            should the es be exported again and overwrite the old export
 100          * @param importToCassandra
 101          *            should we import the data into cassandra
 102          * @return true in case the operation was successful.
 103          */
 104         public boolean migrateDataESToCassndra(String appConfigDir, boolean exportFromEs, boolean importToCassandra) {
 105                 if (!initEsClient()) {
 106                         return false;
 107                 }
 108                 Map<Table, File> files = createOutPutFiles(appConfigDir, exportFromEs);
 109                 if (files == null) {
 110                         return false;
 111                 }
 112                 if (exportFromEs && filesEmpty(files)) {
 113                         Map<Table, PrintWriter> printerWritersMap = createWriters(files);
 114                         if (printerWritersMap == null) {
 115                                 return false;
 116                         }
 117                         try {
 118                                 ImmutableOpenMap<String, IndexMetaData> indexData = getIndexData();
 119                                 for (ObjectCursor<String> key : indexData.keys()) {
 120                                         if (("resources".equalsIgnoreCase(key.value) || key.value.startsWith("auditingevents"))
 121                         && !exportArtifacts(key.value, printerWritersMap)) {
 122                         return false;
 123                     }
 124                                 }
 125                         } finally {
 126                                 if (elasticSearchClient != null) {
 127                                         elasticSearchClient.close();
 128                                 }
 129                                 for (PrintWriter writer : printerWritersMap.values()) {
 130                                         writer.close();
 131                                 }
 132                         }
 133                 }
 134
 135                 return !importToCassandra || importToCassndra(files);
 136         }
 137
 138         private boolean initEsClient() {
 139                 String configHome = System.getProperty("config.home");
 140                 URL url = null;
 141                 Settings settings = null;
 142                 try {
 143                         if (SystemUtils.IS_OS_WINDOWS) {
 144                                 url = new URL("file:///" + configHome + "/elasticsearch.yml");
 145                         } else {
 146                                 url = new URL("file:" + configHome + "/elasticsearch.yml");
 147                         }
 148                         log.debug("URL {}", url);
 149                         settings = Settings.settingsBuilder().loadFromPath(Paths.get(url.toURI())).build();
 150                 } catch (MalformedURLException | URISyntaxException e1) {
 151                         log.error("Failed to create URL in order to load elasticsearch yml", e1);
 152                         return true;
 153                 }
 154
 155                 this.elasticSearchClient = new ElasticSearchClient();
 156                 this.elasticSearchClient.setClusterName(settings.get("cluster.name"));
 157                 this.elasticSearchClient.setLocal(settings.get("elasticSearch.local"));
 158                 this.elasticSearchClient.setTransportClient(settings.get("elasticSearch.transportclient"));
 159                 try {
 160                         elasticSearchClient.initialize();
 161                 } catch (URISyntaxException e) {
 162                     log.error(e.getMessage());
 163                         return false;
 164                 }
 165                 return true;
 166         }
 167
 168         /**
 169          * the method clears all the cassandra tables
 170          */
 171         private void truncateCassandraTable() {
 172                 log.info("import failed. truncating Cassandra tables.");
 173                 artifactCassandraDao.deleteAllArtifacts();
 174                 auditCassandraDao.deleteAllAudit();
 175         }
 176
 177         /**
 178          * the method imports the records from the files into cassandra
 179          *
 180          * @param files
 181          *            a map of files holding
 182          * @return true if the operation was successful
 183          */
 184         private boolean importToCassndra(Map<Table, File> files) {
 185                 log.info("starting to import date into Cassandra.");
 186                 if (!validtaTablsNotEmpty(files))
 187                         return true;
 188                 for (Table table : files.keySet()) {
 189                         log.info("importing recordes into {}", table.getTableDescription().getTableName());
 190                         if (!handleImport(files, table)) {
 191                                 truncateCassandraTable();
 192                                 return false;
 193                         }
 194                 }
 195                 log.info("finished to import date into Cassandra.");
 196                 return true;
 197         }
 198
 199         private boolean validtaTablsNotEmpty(Map<Table, File> files) {
 200                 for (Table table : files.keySet()) {
 201                         Either<Boolean, CassandraOperationStatus> isTableEmptyRes = checkIfTableIsEmpty(table);
 202                         if (isTableEmptyRes.isRight() || !isTableEmptyRes.left().value()) {
 203                                 log.error("Cassandra table {} is not empty operation aborted.",
 204                                                 table.getTableDescription().getTableName());
 205                                 return false;
 206                         }
 207                 }
 208                 return true;
 209         }
 210
 211         /**
 212          * the method retrieves the fields from the given map and generates
 213      * corresponding audit event according to the table name
 214          *
 215          * @param map
 216          *            the map from which we will retrive the fields enum values
 217          * @param table
 218          *            the table we are going to store the record in.
 219          * @return an AuditingGenericEvent event representing the audit record that is going to be
 220          *         created.
 221          */
 222         AuditingGenericEvent createAuditEvent(Map<AuditingFieldsKey, String> map, Table table) {
 223                 AuditEventFactory factory = null;
 224                 switch (table) {
 225                         case USER_ADMIN_EVENT:
 226                                 factory = getAuditUserAdminEventFactory(map);
 227                                 break;
 228                         case USER_ACCESS_EVENT:
 229                                 factory = getAuditUserAccessEventFactory(map);
 230                                 break;
 231                         case RESOURCE_ADMIN_EVENT:
 232                                 factory = getAuditResourceAdminEventMigrationFactory(map);
 233                                 break;
 234                         case DISTRIBUTION_DOWNLOAD_EVENT:
 235                                 factory = getAuditDistributionDownloadEventFactory(map);
 236                                 break;
 237                         case DISTRIBUTION_ENGINE_EVENT:
 238                                 factory = getAuditDistributionEngineEventMigrationFactory(map);
 239                                 break;
 240                         case DISTRIBUTION_NOTIFICATION_EVENT:
 241                                 factory = getAuditDistributionNotificationEventFactory(map);
 242                                 break;
 243                         case DISTRIBUTION_STATUS_EVENT:
 244                                 factory = getAuditDistributionStatusEventFactory(map);
 245                                 break;
 246                         case DISTRIBUTION_DEPLOY_EVENT:
 247                                 factory = getAuditDistributionDeployEventFactory(map);
 248                                 break;
 249                         case DISTRIBUTION_GET_UEB_CLUSTER_EVENT:
 250                                 factory = getAuditGetUebClusterEventFactory(map);
 251                                 break;
 252                         case AUTH_EVENT:
 253                                 factory = getAuditAuthRequestEventFactory(map);
 254                                 break;
 255                         case CONSUMER_EVENT:
 256                                 factory = getAuditConsumerEventFactory(map);
 257                                 break;
 258                         case CATEGORY_EVENT:
 259                                 factory = getAuditCategoryEventFactory(map);
 260                                 break;
 261                         case GET_USERS_LIST_EVENT:
 262                                 factory = getAuditGetUsersListEventFactory(map);
 263                                 break;
 264                         case GET_CATEGORY_HIERARCHY_EVENT:
 265                                 factory = getAuditGetCategoryHierarchyEventFactory(map);
 266                                 break;
 267                         default:
 268                                 break;
 269                 }
 270                 return factory != null ? factory.getDbEvent() : null;
 271         }
 272
 273         private AuditEventFactory getAuditGetCategoryHierarchyEventFactory(Map<AuditingFieldsKey, String> map) {
 274                 return new AuditGetCategoryHierarchyEventFactory(
 275                         CommonAuditData.newBuilder()
 276                                         .description(map.get(AuditingFieldsKey.AUDIT_DESC))
 277                                         .status(map.get(AuditingFieldsKey.AUDIT_STATUS))
 278                                         .requestId(map.get(AuditingFieldsKey.AUDIT_REQUEST_ID))
 279                                         .serviceInstanceId(map.get(AuditingFieldsKey.AUDIT_SERVICE_INSTANCE_ID))
 280                                         .build(),
 281                         map.get(AuditingFieldsKey.AUDIT_MODIFIER_UID),
 282                         map.get(AuditingFieldsKey.AUDIT_DETAILS),
 283                         map.get(AuditingFieldsKey.AUDIT_TIMESTAMP));
 284         }
 285
 286         private AuditEventFactory getAuditGetUsersListEventFactory(Map<AuditingFieldsKey, String> map) {
 287                 return new AuditGetUsersListEventFactory(
 288                         CommonAuditData.newBuilder()
 289                                         .description(map.get(AuditingFieldsKey.AUDIT_DESC))
 290                                         .status(map.get(AuditingFieldsKey.AUDIT_STATUS))
 291                                         .requestId(map.get(AuditingFieldsKey.AUDIT_REQUEST_ID))
 292                                         .serviceInstanceId(map.get(AuditingFieldsKey.AUDIT_SERVICE_INSTANCE_ID))
 293                                         .build(),
 294                         map.get(AuditingFieldsKey.AUDIT_MODIFIER_UID),
 295                         map.get(AuditingFieldsKey.AUDIT_USER_DETAILS),
 296                         map.get(AuditingFieldsKey.AUDIT_TIMESTAMP));
 297         }
 298
 299         private AuditEventFactory getAuditCategoryEventFactory(Map<AuditingFieldsKey, String> map) {
 300                 return new AuditCategoryEventFactory(
 301                         AuditingActionEnum.fromName(map.get(AuditingFieldsKey.AUDIT_ACTION)),
 302                         CommonAuditData.newBuilder()
 303                                         .description(map.get(AuditingFieldsKey.AUDIT_DESC))
 304                                         .status(map.get(AuditingFieldsKey.AUDIT_STATUS))
 305                                         .requestId(map.get(AuditingFieldsKey.AUDIT_REQUEST_ID))
 306                                         .serviceInstanceId(map.get(AuditingFieldsKey.AUDIT_SERVICE_INSTANCE_ID))
 307                                         .build(),
 308                         map.get(AuditingFieldsKey.AUDIT_MODIFIER_UID),
 309                         map.get(AuditingFieldsKey.AUDIT_CATEGORY_NAME),
 310                         map.get(AuditingFieldsKey.AUDIT_SUB_CATEGORY_NAME),
 311                         map.get(AuditingFieldsKey.AUDIT_GROUPING_NAME),
 312                         map.get(AuditingFieldsKey.AUDIT_RESOURCE_TYPE),
 313                         map.get(AuditingFieldsKey.AUDIT_TIMESTAMP));
 314         }
 315
 316         private AuditEventFactory getAuditUserAccessEventFactory(Map<AuditingFieldsKey, String> map) {
 317                 return new AuditUserAccessEventFactory(
 318                         CommonAuditData.newBuilder()
 319                                         .description(map.get(AuditingFieldsKey.AUDIT_DESC))
 320                                         .status(map.get(AuditingFieldsKey.AUDIT_STATUS))
 321                                         .requestId(map.get(AuditingFieldsKey.AUDIT_REQUEST_ID))
 322                                         .serviceInstanceId(map.get(AuditingFieldsKey.AUDIT_SERVICE_INSTANCE_ID))
 323                                         .build(),
 324                         map.get(AuditingFieldsKey.AUDIT_USER_UID),
 325                         map.get(AuditingFieldsKey.AUDIT_TIMESTAMP));
 326         }
 327
 328         private AuditEventFactory getAuditUserAdminEventFactory(Map<AuditingFieldsKey, String> map) {
 329                 return new AuditUserAdminEventFactory(
 330             AuditingActionEnum.fromName(map.get(AuditingFieldsKey.AUDIT_ACTION)),
 331                         CommonAuditData.newBuilder()
 332                                         .description(map.get(AuditingFieldsKey.AUDIT_DESC))
 333                                         .status(map.get(AuditingFieldsKey.AUDIT_STATUS))
 334                                         .requestId(map.get(AuditingFieldsKey.AUDIT_REQUEST_ID))
 335                                         .serviceInstanceId(map.get(AuditingFieldsKey.AUDIT_SERVICE_INSTANCE_ID))
 336                                         .build(),
 337                         map.get(AuditingFieldsKey.AUDIT_MODIFIER_UID),
 338                         map.get(AuditingFieldsKey.AUDIT_USER_BEFORE),
 339                         map.get(AuditingFieldsKey.AUDIT_USER_AFTER),
 340                         map.get(AuditingFieldsKey.AUDIT_TIMESTAMP));
 341         }
 342
 343         private AuditEventFactory getAuditConsumerEventFactory(Map<AuditingFieldsKey, String> map) {
 344                 return new AuditConsumerEventFactory(
 345                     AuditingActionEnum.fromName(map.get(AuditingFieldsKey.AUDIT_ACTION)),
 346                         CommonAuditData.newBuilder()
 347                                         .description(map.get(AuditingFieldsKey.AUDIT_DESC))
 348                                         .status(map.get(AuditingFieldsKey.AUDIT_STATUS))
 349                                         .requestId(map.get(AuditingFieldsKey.AUDIT_REQUEST_ID))
 350                                         .serviceInstanceId(map.get(AuditingFieldsKey.AUDIT_SERVICE_INSTANCE_ID))
 351                                         .build(),
 352                         map.get(AuditingFieldsKey.AUDIT_MODIFIER_UID),
 353                         map.get(AuditingFieldsKey.AUDIT_ECOMP_USER),
 354                         map.get(AuditingFieldsKey.AUDIT_TIMESTAMP));
 355         }
 356
 357         private AuditEventFactory getAuditAuthRequestEventFactory(Map<AuditingFieldsKey, String> map) {
 358                 return new AuditAuthRequestEventFactory(
 359                         CommonAuditData.newBuilder()
 360                                         .description(map.get(AuditingFieldsKey.AUDIT_DESC))
 361                                         .status(map.get(AuditingFieldsKey.AUDIT_STATUS))
 362                                         .requestId(map.get(AuditingFieldsKey.AUDIT_REQUEST_ID))
 363                                         .serviceInstanceId(map.get(AuditingFieldsKey.AUDIT_SERVICE_INSTANCE_ID))
 364                                         .build(),
 365                         map.get(AuditingFieldsKey.AUDIT_USER_UID),
 366                         map.get(AuditingFieldsKey.AUDIT_AUTH_URL),
 367                         map.get(AuditingFieldsKey.AUDIT_AUTH_REALM),
 368                         map.get(AuditingFieldsKey.AUDIT_AUTH_STATUS),
 369                         map.get(AuditingFieldsKey.AUDIT_TIMESTAMP));
 370         }
 371
 372         private AuditEventFactory getAuditGetUebClusterEventFactory(Map<AuditingFieldsKey, String> map) {
 373                 return new AuditGetUebClusterEventFactory(
 374                         CommonAuditData.newBuilder()
 375                                         .description(map.get(AuditingFieldsKey.AUDIT_DESC))
 376                                         .status(map.get(AuditingFieldsKey.AUDIT_STATUS))
 377                                         .requestId(map.get(AuditingFieldsKey.AUDIT_REQUEST_ID))
 378                                         .serviceInstanceId(map.get(AuditingFieldsKey.AUDIT_SERVICE_INSTANCE_ID))
 379                                         .build(),
 380                         map.get(AuditingFieldsKey.AUDIT_DISTRIBUTION_CONSUMER_ID),
 381                         map.get(AuditingFieldsKey.AUDIT_TIMESTAMP));
 382         }
 383
 384         private AuditEventFactory getAuditResourceAdminEventMigrationFactory(Map<AuditingFieldsKey, String> map) {
 385                 return new AuditResourceAdminEventMigrationFactory(
 386                 AuditingActionEnum.fromName(map.get(AuditingFieldsKey.AUDIT_ACTION)),
 387                         CommonAuditData.newBuilder()
 388                                         .description(map.get(AuditingFieldsKey.AUDIT_DESC))
 389                                         .status(map.get(AuditingFieldsKey.AUDIT_STATUS))
 390                                         .requestId(map.get(AuditingFieldsKey.AUDIT_REQUEST_ID))
 391                                         .serviceInstanceId(map.get(AuditingFieldsKey.AUDIT_SERVICE_INSTANCE_ID))
 392                                         .build(),
 393                         new ResourceCommonInfo(map.get(AuditingFieldsKey.AUDIT_RESOURCE_NAME),
 394                                         map.get(AuditingFieldsKey.AUDIT_RESOURCE_TYPE)),
 395                         ResourceVersionInfo.newBuilder()
 396                                         .artifactUuid(map.get(AuditingFieldsKey.AUDIT_PREV_ARTIFACT_UUID))
 397                                         .state(map.get(AuditingFieldsKey.AUDIT_RESOURCE_PREV_STATE))
 398                                         .version(map.get(AuditingFieldsKey.AUDIT_RESOURCE_PREV_VERSION))
 399                                         .distributionStatus(map.get(AuditingFieldsKey.AUDIT_RESOURCE_DPREV_STATUS))
 400                                         .build(),
 401                         ResourceVersionInfo.newBuilder()
 402                                         .artifactUuid(map.get(AuditingFieldsKey.AUDIT_CURR_ARTIFACT_UUID))
 403                                         .state(map.get(AuditingFieldsKey.AUDIT_RESOURCE_CURR_STATE))
 404                                         .version(map.get(AuditingFieldsKey.AUDIT_RESOURCE_CURR_VERSION))
 405                                         .distributionStatus(map.get(AuditingFieldsKey.AUDIT_RESOURCE_DCURR_STATUS))
 406                                         .build(),
 407                         map.get(AuditingFieldsKey.AUDIT_INVARIANT_UUID),
 408                         map.get(AuditingFieldsKey.AUDIT_MODIFIER_UID),
 409                         map.get(AuditingFieldsKey.AUDIT_ARTIFACT_DATA),
 410                         map.get(AuditingFieldsKey.AUDIT_RESOURCE_COMMENT),
 411                         map.get(AuditingFieldsKey.AUDIT_DISTRIBUTION_ID),
 412                         map.get(AuditingFieldsKey.AUDIT_RESOURCE_TOSCA_NODE_TYPE),
 413                         map.get(AuditingFieldsKey.AUDIT_TIMESTAMP));
 414         }
 415
 416         private AuditEventFactory getAuditDistributionDownloadEventFactory(Map<AuditingFieldsKey, String> map) {
 417                 return new AuditDistributionDownloadEventFactory(
 418                         CommonAuditData.newBuilder()
 419                                         .description(map.get(AuditingFieldsKey.AUDIT_DESC))
 420                                         .status(map.get(AuditingFieldsKey.AUDIT_STATUS))
 421                                         .requestId(map.get(AuditingFieldsKey.AUDIT_REQUEST_ID))
 422                                         .serviceInstanceId(map.get(AuditingFieldsKey.AUDIT_SERVICE_INSTANCE_ID))
 423                                         .build(),
 424                         new DistributionData(map.get(AuditingFieldsKey.AUDIT_DISTRIBUTION_CONSUMER_ID),
 425                                         map.get(AuditingFieldsKey.AUDIT_RESOURCE_URL)),
 426                                 map.get(AuditingFieldsKey.AUDIT_TIMESTAMP));
 427         }
 428
 429         private AuditEventFactory getAuditDistributionEngineEventMigrationFactory(Map<AuditingFieldsKey, String> map) {
 430                 return new AuditDistributionEngineEventMigrationFactory(
 431                     AuditingActionEnum.fromName(map.get(AuditingFieldsKey.AUDIT_ACTION)),
 432                         CommonAuditData.newBuilder()
 433                                         .description(map.get(AuditingFieldsKey.AUDIT_DESC))
 434                                         .status(map.get(AuditingFieldsKey.AUDIT_STATUS))
 435                                         .requestId(map.get(AuditingFieldsKey.AUDIT_REQUEST_ID))
 436                                         .serviceInstanceId(map.get(AuditingFieldsKey.AUDIT_SERVICE_INSTANCE_ID))
 437                                         .build(),
 438                         DistributionTopicData.newBuilder()
 439                                         .notificationTopic(map.get(AuditingFieldsKey.AUDIT_DISTRIBUTION_NOTIFICATION_TOPIC_NAME))
 440                                         .statusTopic(map.get(AuditingFieldsKey.AUDIT_DISTRIBUTION_STATUS_TOPIC_NAME))
 441                                         .build(),
 442                         map.get(AuditingFieldsKey.AUDIT_DISTRIBUTION_CONSUMER_ID),
 443                         map.get(AuditingFieldsKey.AUDIT_DISTRIBUTION_API_KEY),
 444                         map.get(AuditingFieldsKey.AUDIT_DISTRIBUTION_ENVRIONMENT_NAME),
 445                         map.get(AuditingFieldsKey.AUDIT_DISTRIBUTION_ROLE),
 446                         map.get(AuditingFieldsKey.AUDIT_TIMESTAMP));
 447         }
 448
 449         private AuditEventFactory getAuditDistributionDeployEventFactory(Map<AuditingFieldsKey, String> map) {
 450                 return new AuditDistributionDeployEventFactory(
 451                         CommonAuditData.newBuilder()
 452                                         .description(map.get(AuditingFieldsKey.AUDIT_DESC))
 453                                         .status(map.get(AuditingFieldsKey.AUDIT_STATUS))
 454                                         .requestId(map.get(AuditingFieldsKey.AUDIT_REQUEST_ID))
 455                                         .serviceInstanceId(map.get(AuditingFieldsKey.AUDIT_SERVICE_INSTANCE_ID))
 456                                         .build(),
 457                         new ResourceCommonInfo(map.get(AuditingFieldsKey.AUDIT_RESOURCE_NAME),
 458                                         map.get(AuditingFieldsKey.AUDIT_RESOURCE_TYPE)),
 459                         map.get(AuditingFieldsKey.AUDIT_RESOURCE_CURR_VERSION),
 460                         map.get(AuditingFieldsKey.AUDIT_DISTRIBUTION_ID),
 461                         map.get(AuditingFieldsKey.AUDIT_MODIFIER_UID),
 462                         map.get(AuditingFieldsKey.AUDIT_TIMESTAMP));
 463         }
 464
 465         private AuditEventFactory getAuditDistributionStatusEventFactory(Map<AuditingFieldsKey, String> map) {
 466                 return new AuditDistributionStatusEventFactory(
 467                         CommonAuditData.newBuilder()
 468                                         .description(map.get(AuditingFieldsKey.AUDIT_DESC))
 469                                         .status(map.get(AuditingFieldsKey.AUDIT_STATUS))
 470                                         .requestId(map.get(AuditingFieldsKey.AUDIT_REQUEST_ID))
 471                                         .serviceInstanceId(map.get(AuditingFieldsKey.AUDIT_SERVICE_INSTANCE_ID))
 472                                         .build(),
 473                         new DistributionData(map.get(AuditingFieldsKey.AUDIT_DISTRIBUTION_CONSUMER_ID),
 474                                         map.get(AuditingFieldsKey.AUDIT_RESOURCE_URL)),
 475                         map.get(AuditingFieldsKey.AUDIT_DISTRIBUTION_ID),
 476                         map.get(AuditingFieldsKey.AUDIT_DISTRIBUTION_TOPIC_NAME),
 477                         map.get(AuditingFieldsKey.AUDIT_DISTRIBUTION_STATUS_TIME),
 478                         map.get(AuditingFieldsKey.AUDIT_TIMESTAMP));
 479         }
 480
 481         private AuditEventFactory getAuditDistributionNotificationEventFactory(Map<AuditingFieldsKey, String> map) {
 482                 return new AuditDistributionNotificationEventFactory(
 483                         CommonAuditData.newBuilder()
 484                                         .description(map.get(AuditingFieldsKey.AUDIT_DESC))
 485                                         .status(map.get(AuditingFieldsKey.AUDIT_STATUS))
 486                                         .requestId(map.get(AuditingFieldsKey.AUDIT_REQUEST_ID))
 487                                         .serviceInstanceId(map.get(AuditingFieldsKey.AUDIT_SERVICE_INSTANCE_ID))
 488                                         .build(),
 489                         new ResourceCommonInfo(map.get(AuditingFieldsKey.AUDIT_RESOURCE_NAME),
 490                                         map.get(AuditingFieldsKey.AUDIT_RESOURCE_TYPE)),
 491                         ResourceVersionInfo.newBuilder()
 492                                         .state(map.get(AuditingFieldsKey.AUDIT_RESOURCE_CURR_STATE))
 493                                         .version(map.get(AuditingFieldsKey.AUDIT_RESOURCE_CURR_VERSION))
 494                                         .build(),
 495                         map.get(AuditingFieldsKey.AUDIT_DISTRIBUTION_ID),
 496                         map.get(AuditingFieldsKey.AUDIT_MODIFIER_UID),
 497                         map.get(AuditingFieldsKey.AUDIT_DISTRIBUTION_TOPIC_NAME),
 498                         new OperationalEnvAuditData(map.get(AuditingFieldsKey.AUDIT_DISTRIBUTION_ENVIRONMENT_ID),
 499                                         map.get(AuditingFieldsKey.AUDIT_DISTRIBUTION_VNF_WORKLOAD_CONTEXT),
 500                                         map.get(AuditingFieldsKey.AUDIT_DISTRIBUTION_TENANT)),
 501                         map.get(AuditingFieldsKey.AUDIT_TIMESTAMP));
 502         }
 503
 504
 505
 506         /**
 507          * the method reads the content of the file intended for a given table, and
 508          * sores them in cassandra
 509          *
 510          * @param files
 511          *            a map of files from which the recordes will be retrieved.
 512          * @param table
 513          *            the name of the table we want to look up in the files and sore
 514          *            in Cassandra // * @param store the function to call when
 515          *            storing recordes in cassndra
 516          * @return true if the operation was successful
 517          */
 518         private boolean handleImport(Map<Table, File> files, Table table) {
 519                 BufferedReader br = null;
 520                 try {
 521                         br = new BufferedReader(new FileReader(files.get(table)));
 522                         String line = null;
 523                         while ((line = br.readLine()) != null) {
 524                                 CassandraOperationStatus res = CassandraOperationStatus.GENERAL_ERROR;
 525                                 if (Table.ARTIFACT.equals(table)) {
 526                                         res = artifactCassandraDao.saveArtifact(jsonMapper.readValue(line, ESArtifactData.class));
 527                                 }
 528                                 else {
 529                     AuditingGenericEvent recordForCassandra = createAuditRecordForCassandra(line, table);
 530                                         if (recordForCassandra != null) {
 531                         res = auditCassandraDao.saveRecord(recordForCassandra);
 532                     }
 533                                 }
 534                                 if (!res.equals(CassandraOperationStatus.OK)) {
 535                                         log.error("save recored to cassndra {} failed with status {} aborting.",
 536                                                         table.getTableDescription().getTableName(), res);
 537                                         return false;
 538                                 }
 539                         }
 540                         return true;
 541                 } catch (IOException e) {
 542                         log.error("failed to read file", e);
 543                         return false;
 544                 } finally {
 545                         if (br != null) {
 546                                 try {
 547                                         br.close();
 548                                 } catch (IOException e) {
 549                                         log.error("failed to close file reader", e);
 550                                 }
 551                         }
 552                 }
 553         }
 554
 555     AuditingGenericEvent createAuditRecordForCassandra(String json, Table table) throws IOException{
 556         return createAuditEvent(parseToMap(json), table);
 557     }
 558
 559         private Map<AuditingFieldsKey, String> parseToMap(String json) throws IOException {
 560                 return jsonMapper.readValue(json, new TypeReference<Map<AuditingFieldsKey, String>>(){});
 561         }
 562
 563         /**
 564          * the method checks if the given table is empty
 565          *
 566          * @param table
 567          *            the name of the table we want to check
 568          * @return true if the table is empty
 569          */
 570         private Either<Boolean, CassandraOperationStatus> checkIfTableIsEmpty(Table table) {
 571                 if (Table.ARTIFACT.equals(table)) {
 572                         return artifactCassandraDao.isTableEmpty(table.getTableDescription().getTableName());
 573                 } else {
 574                         return auditCassandraDao.isTableEmpty(table.getTableDescription().getTableName());
 575                 }
 576         }
 577
 578         private boolean filesEmpty(Map<Table, File> files) {
 579                 for (Table table : files.keySet()) {
 580                         File file = files.get(table);
 581                         if (file.length() != 0) {
 582                                 log.info("file:{} is not empty skipping export", table.getTableDescription().getTableName());
 583                                 return false;
 584                         }
 585                 }
 586                 return true;
 587         }
 588
 589         /**
 590          * the method reads the records from es index of audit's into a file as
 591          * json's.
 592          *
 593          * @param value
 594          *            the name of the index we want
 595          * @param printerWritersMap
 596          *            a map of the writers we use to write to a file.
 597          * @return true in case the export was successful.
 598          */
 599         private boolean exportAudit(String value, Map<Table, PrintWriter> printerWritersMap) {
 600                 log.info("stratng to export audit data from es index{} to file.", value);
 601                 QueryBuilder queryBuilder = QueryBuilders.matchAllQuery();
 602                 SearchResponse scrollResp = elasticSearchClient.getClient().prepareSearch(value).setScroll(new TimeValue(60000))
 603                                 .setQuery(queryBuilder).setSize(100).execute().actionGet();
 604                 while (true) {
 605                         for (SearchHit hit : scrollResp.getHits().getHits()) {
 606                                 PrintWriter out = printerWritersMap.get(TypeToTableMapping.getTableByType(hit.getType()));
 607                                 out.println(hit.getSourceAsString());
 608                         }
 609                         scrollResp = elasticSearchClient.getClient().prepareSearchScroll(scrollResp.getScrollId())
 610                                         .setScroll(new TimeValue(60000)).execute().actionGet();
 611                         if (scrollResp.getHits().getHits().length == 0) {
 612                                 break;
 613
 614                         }
 615                 }
 616
 617                 log.info("export audit data from es to file. finished succsesfully");
 618                 return true;
 619         }
 620
 621         /**
 622          * the method reads the records from es index of resources into a file as
 623          * json's.
 624          *
 625          * @param index
 626          *            the name of the index we want to read
 627          * @param printerWritersMap
 628          *            a map of the writers we use to write to a file.
 629          * @return true in case the export was successful.
 630          */
 631         private boolean exportArtifacts(String index, Map<Table, PrintWriter> printerWritersMap) {
 632                 log.info("stratng to export artifact data from es to file.");
 633                 PrintWriter out = printerWritersMap.get(Table.ARTIFACT);
 634                 QueryBuilder queryBuilder = QueryBuilders.matchAllQuery();
 635                 SearchResponse scrollResp = elasticSearchClient.getClient().prepareSearch(index).setScroll(new TimeValue(60000))
 636                                 .setQuery(queryBuilder).setSize(100).execute().actionGet();
 637                 while (true) {
 638                         for (SearchHit hit : scrollResp.getHits().getHits()) {
 639                                 ;
 640                                 out.println(hit.getSourceAsString());
 641                         }
 642                         scrollResp = elasticSearchClient.getClient().prepareSearchScroll(scrollResp.getScrollId())
 643                                         .setScroll(new TimeValue(60000)).execute().actionGet();
 644                         if (scrollResp.getHits().getHits().length == 0) {
 645                                 break;
 646
 647                         }
 648                 }
 649
 650                 log.info("export artifact data from es to file. finished succsesfully");
 651                 return true;
 652         }
 653
 654         /**
 655          * the method retrieves all the indexes from elasticsearch
 656          *
 657          * @return a map of indexes and there metadata
 658          */
 659         private ImmutableOpenMap<String, IndexMetaData> getIndexData() {
 660                 return elasticSearchClient.getClient().admin().cluster().prepareState().get().getState().getMetaData()
 661                                 .getIndices();
 662         }
 663
 664         /**
 665          * the method creates all the files and dir which holds them. in case the
 666          * files exist they will not be created again.
 667          *
 668          * @param appConfigDir
 669          *            the base path under which the output dir will be created and
 670          *            the export result files the created filesa are named according
 671          *            to the name of the table into which it will be imported.
 672          * @param exportToEs
 673          *            if true all the export files will be recreated
 674          * @returnthe returns a map of tables and the files representing them them
 675          */
 676         private Map<Table, File> createOutPutFiles(String appConfigDir, boolean exportToEs) {
 677                 Map<Table, File> result = new EnumMap<Table, File>(Table.class);
 678                 File outputDir = new File(appConfigDir + "/output/");
 679                 if (!createOutPutFolder(outputDir)) {
 680                         return null;
 681                 }
 682                 for (Table table : Table.values()) {
 683                         File file = new File(outputDir + "/" + table.getTableDescription().getTableName());
 684                         if (exportToEs) {
 685                                 try {
 686                                         if (file.exists()) {
 687                                                 Files.delete(file.toPath());
 688                                         }
 689                                 } catch (IOException e) {
 690                                         log.error("failed to delete output file {}", file.getAbsolutePath(), e);
 691                                         return null;
 692                                 }
 693                                 file = new File(outputDir + "/" + table.getTableDescription().getTableName());
 694                         }
 695                         if (!file.exists()) {
 696                                 try {
 697                                         file.createNewFile();
 698                                 } catch (IOException e) {
 699                                         log.error("failed to create output file {}", file.getAbsolutePath(), e);
 700                                         return null;
 701                                 }
 702                         }
 703                         result.put(table, file);
 704
 705                 }
 706                 return result;
 707         }
 708
 709         /**
 710          * the method create the writers to each file
 711          *
 712          * @param files
 713          *            a map of the files according to table
 714          * @return returns a map of writers according to table.
 715          */
 716         private Map<Table, PrintWriter> createWriters(Map<Table, File> files) {
 717                 Map<Table, PrintWriter> printerWritersMap = new EnumMap<>(Table.class);
 718
 719                         for (Table table : files.keySet()) {
 720                                 log.info("creating writer for {}", table);
 721                                 File file = files.get(table);
 722                 try(PrintWriter out = new PrintWriter(new BufferedWriter(new FileWriter(file)) )){
 723                                 printerWritersMap.put(table, out);
 724                                 log.info("creating writer for {} was successful", table);
 725             } catch (IOException e) {
 726                 log.error("create writer to file failed",e);
 727                 return null;
 728                         }
 729         }
 730                 return printerWritersMap;
 731         }
 732
 733         /**
 734          * the method creates the output dir in case it does not exist
 735          *
 736          * @param outputDir
 737          *            the path under wich the directory will be created.
 738          * @return true in case the create was succsesful or the dir already exists
 739          */
 740         private boolean createOutPutFolder(File outputDir) {
 741                 if (!outputDir.exists()) {
 742                         log.info("creating output dir {}", outputDir.getAbsolutePath());
 743                         try {
 744                                 Files.createDirectories(outputDir.toPath());
 745                         } catch (IOException e) {
 746                                 log.error("failed to create output dir {}", outputDir.getAbsolutePath(), e);
 747                                 return false;
 748                         }
 749                 }
 750                 return true;
 751         }
 752
 753         public enum TypeToTableMapping {
 754                 USER_ADMIN_EVENT_TYPE(AuditingTypesConstants.USER_ADMIN_EVENT_TYPE,
 755                                 Table.USER_ADMIN_EVENT), USER_ACCESS_EVENT_TYPE(AuditingTypesConstants.USER_ACCESS_EVENT_TYPE,
 756                                                 Table.USER_ACCESS_EVENT), RESOURCE_ADMIN_EVENT_TYPE(
 757                                                                 AuditingTypesConstants.RESOURCE_ADMIN_EVENT_TYPE,
 758                                                                 Table.RESOURCE_ADMIN_EVENT), DISTRIBUTION_DOWNLOAD_EVENT_TYPE(
 759                                                                                 AuditingTypesConstants.DISTRIBUTION_DOWNLOAD_EVENT_TYPE,
 760                                                                                 Table.DISTRIBUTION_DOWNLOAD_EVENT), DISTRIBUTION_ENGINE_EVENT_TYPE(
 761                                                                                                 AuditingTypesConstants.DISTRIBUTION_ENGINE_EVENT_TYPE,
 762                                                                                                 Table.DISTRIBUTION_ENGINE_EVENT), DISTRIBUTION_NOTIFICATION_EVENT_TYPE(
 763                                                                                                                 AuditingTypesConstants.DISTRIBUTION_NOTIFICATION_EVENT_TYPE,
 764                                                                                                                 Table.DISTRIBUTION_NOTIFICATION_EVENT), DISTRIBUTION_STATUS_EVENT_TYPE(
 765                                                                                                                                 AuditingTypesConstants.DISTRIBUTION_STATUS_EVENT_TYPE,
 766                                                                                                                                 Table.DISTRIBUTION_STATUS_EVENT), DISTRIBUTION_DEPLOY_EVENT_TYPE(
 767                                                                                                                                                 AuditingTypesConstants.DISTRIBUTION_DEPLOY_EVENT_TYPE,
 768                                                                                                                                                 Table.DISTRIBUTION_DEPLOY_EVENT), DISTRIBUTION_GET_UEB_CLUSTER_EVENT_TYPE(
 769                                                                                                                                                                 AuditingTypesConstants.DISTRIBUTION_GET_UEB_CLUSTER_EVENT_TYPE,
 770                                                                                                                                                                 Table.DISTRIBUTION_GET_UEB_CLUSTER_EVENT), AUTH_EVENT_TYPE(
 771                                                                                                                                                                                 AuditingTypesConstants.AUTH_EVENT_TYPE,
 772                                                                                                                                                                                 Table.AUTH_EVENT), CONSUMER_EVENT_TYPE(
 773                                                                                                                                                                                                 AuditingTypesConstants.CONSUMER_EVENT_TYPE,
 774                                                                                                                                                                                                 Table.CONSUMER_EVENT), CATEGORY_EVENT_TYPE(
 775                                                                                                                                                                                                                 AuditingTypesConstants.CATEGORY_EVENT_TYPE,
 776                                                                                                                                                                                                                 Table.CATEGORY_EVENT), GET_USERS_LIST_EVENT_TYPE(
 777                                                                                                                                                                                                                                 AuditingTypesConstants.GET_USERS_LIST_EVENT_TYPE,
 778                                                                                                                                                                                                                                 Table.GET_USERS_LIST_EVENT), GET_CATEGORY_HIERARCHY_EVENT_TYPE(
 779                                                                                                                                                                                                                                                 AuditingTypesConstants.GET_CATEGORY_HIERARCHY_EVENT_TYPE,
 780                                                                                                                                                                                                                                                 Table.GET_CATEGORY_HIERARCHY_EVENT);
 781
 782                 String typeName;
 783                 Table table;
 784
 785                 TypeToTableMapping(String typeName, Table table) {
 786                         this.typeName = typeName;
 787                         this.table = table;
 788                 }
 789
 790                 public String getTypeName() {
 791                         return typeName;
 792                 }
 793
 794                 public Table getTable() {
 795                         return table;
 796                 }
 797
 798                 public static Table getTableByType(String type) {
 799                         for (TypeToTableMapping mapping : TypeToTableMapping.values()) {
 800                                 if (mapping.getTypeName().equalsIgnoreCase(type)) {
 801                                         return mapping.getTable();
 802                                 }
 803                         }
 804                         return null;
 805                 }
 806         }
 807
 808 }