ONAP-PAP-REST/src/main/java/org/onap/policy/pap/xacml/rest/elk/client/ElasticSearchPolicyUpdate.java

   1 /*-
   2  * ============LICENSE_START=======================================================
   3  * ONAP Policy Engine
   4  * ================================================================================
   5  * Copyright (C) 2017-2018 AT&T Intellectual Property. All rights reserved.
   6  * ================================================================================
   7  * Licensed under the Apache License, Version 2.0 (the "License");
   8  * you may not use this file except in compliance with the License.
   9  * You may obtain a copy of the License at
  10  *
  11  *      http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  * ============LICENSE_END=========================================================
  19  */
  20 package org.onap.policy.pap.xacml.rest.elk.client;
  21
  22 import java.io.ByteArrayInputStream;
  23 import java.io.FileInputStream;
  24 import java.io.InputStream;
  25 import java.nio.charset.StandardCharsets;
  26 import java.nio.file.Path;
  27 import java.nio.file.Paths;
  28 import java.sql.Connection;
  29 import java.sql.DriverManager;
  30 import java.sql.PreparedStatement;
  31 import java.sql.ResultSet;
  32 import java.sql.Statement;
  33 import java.util.ArrayList;
  34 import java.util.Iterator;
  35 import java.util.List;
  36 import java.util.Properties;
  37
  38 import org.onap.policy.common.logging.flexlogger.FlexLogger;
  39 import org.onap.policy.common.logging.flexlogger.Logger;
  40 import org.onap.policy.utils.CryptoUtils;
  41 import org.onap.policy.xacml.util.XACMLPolicyScanner;
  42
  43 import com.google.gson.Gson;
  44
  45 import io.searchbox.client.JestClientFactory;
  46 import io.searchbox.client.config.HttpClientConfig;
  47 import io.searchbox.client.http.JestHttpClient;
  48 import io.searchbox.core.Bulk;
  49 import io.searchbox.core.Bulk.Builder;
  50 import io.searchbox.core.BulkResult;
  51 import io.searchbox.core.Index;
  52 import oasis.names.tc.xacml._3_0.core.schema.wd_17.AllOfType;
  53 import oasis.names.tc.xacml._3_0.core.schema.wd_17.AnyOfType;
  54 import oasis.names.tc.xacml._3_0.core.schema.wd_17.AttributeDesignatorType;
  55 import oasis.names.tc.xacml._3_0.core.schema.wd_17.AttributeValueType;
  56 import oasis.names.tc.xacml._3_0.core.schema.wd_17.MatchType;
  57 import oasis.names.tc.xacml._3_0.core.schema.wd_17.PolicyType;
  58 import oasis.names.tc.xacml._3_0.core.schema.wd_17.TargetType;
  59
  60
  61
  62 /**
  63  * This code will deals with parsing the XACML content on reading from
  64  * database(PolicyEntity, ConfigurationDataEntity and ActionBodyEntity tables)
  65  * and convert the data into json to do bulk operation on putting to elastic search database.
  66  * Which is used to support Elastic Search in Policy Application GUI to search policies.
  67  *
  68  *
  69  *
  70  * properties should be configured in policyelk.properties
  71  *
  72  */
  73 public class ElasticSearchPolicyUpdate {
  74
  75         private static final Logger LOGGER = FlexLogger.getLogger(ElasticSearchPolicyUpdate.class);
  76         protected final static JestClientFactory jestFactory = new JestClientFactory();
  77
  78         public static void main(String[] args) {
  79
  80                 String elkURL = null;
  81                 String databseUrl = null;
  82                 String userName = null;
  83                 String txt = null;
  84                 String databaseDriver = null;
  85
  86                 String propertyFile = System.getProperty("PROPERTY_FILE");
  87                 Properties config = new Properties();
  88                 Path file = Paths.get(propertyFile);
  89                 if(!file.toFile().exists()){
  90                         LOGGER.error("Config File doesn't Exist in the specified Path " + file.toString());
  91                 }else{
  92                         if(file.toString().endsWith(".properties")){
  93                                 try {
  94                                         InputStream in = new FileInputStream(file.toFile());
  95                                         config.load(in);
  96                                         elkURL = config.getProperty("policy.elk.url");
  97                                         databseUrl = config.getProperty("policy.database.url");
  98                                         userName = config.getProperty("policy.database.username");
  99                                         txt = CryptoUtils.decryptTxtNoExStr(config.getProperty("policy.database.password"));
 100                                         databaseDriver = config.getProperty("policy.database.driver");
 101                                         if(elkURL == null || databseUrl == null || userName == null || txt == null || databaseDriver == null){
 102                                                 LOGGER.error("please check the elk configuration");
 103                                         }
 104                                 } catch (Exception e) {
 105                                         LOGGER.error("Config File doesn't Exist in the specified Path " + file.toString(),e);
 106                                 }
 107                         }
 108                 }
 109
 110                 Builder bulk = null;
 111
 112                 HttpClientConfig httpClientConfig = new HttpClientConfig.Builder(elkURL).multiThreaded(true).build();
 113                 jestFactory.setHttpClientConfig(httpClientConfig);
 114             JestHttpClient client = (JestHttpClient) jestFactory.getObject();
 115
 116                 Connection conn = null;
 117                 Statement stmt = null;
 118                 ResultSet result = null;
 119
 120                 List<Index> listIndex = new ArrayList<>();
 121
 122                 try {
 123                         Class.forName(databaseDriver);
 124                         conn = DriverManager.getConnection(databseUrl, userName, txt);
 125                         stmt = conn.createStatement();
 126
 127                         String policyEntityQuery = "Select * from PolicyEntity";
 128                         result = stmt.executeQuery(policyEntityQuery);
 129
 130                         while(result.next()){
 131                                 StringBuilder policyDataString = new StringBuilder("{");
 132                                 String scope = result.getString("scope");
 133                                 String policyName = result.getString("policyName");
 134                                 if(policyName != null){
 135                                         policyDataString.append("\"policyName\":\""+scope+"."+policyName+"\",");
 136                                 }
 137                                 String description = result.getString("description");
 138                                 if(description != null){
 139                                         policyDataString.append("\"policyDescription\":\""+description+"\",");
 140                                 }
 141                                 Object policyData = result.getString("policydata");
 142
 143                                 if(scope != null){
 144                                         policyDataString.append("\"scope\":\""+scope+"\",");
 145                                 }
 146                                 String actionbodyid = result.getString("actionbodyid");
 147                                 String configurationdataid = result.getString("configurationdataid");
 148
 149
 150                                 String policyWithScopeName = scope + "." + policyName;
 151                                 String _type = null;
 152
 153                                 if(policyWithScopeName.contains(".Config_")){
 154                                         policyDataString.append("\"policyType\":\"Config\",");
 155                                         if(policyWithScopeName.contains(".Config_Fault_")){
 156                                                 _type = "closedloop";
 157                                                 policyDataString.append("\"configPolicyType\":\"ClosedLoop_Fault\",");
 158                                         }else if(policyWithScopeName.contains(".Config_PM_")){
 159                                                 _type = "closedloop";
 160                                                 policyDataString.append("\"configPolicyType\":\"ClosedLoop_PM\",");
 161                                         }else{
 162                                                 _type = "config";
 163                                                 policyDataString.append("\"configPolicyType\":\"Base\",");
 164                                         }
 165                                 }else if(policyWithScopeName.contains(".Action_")){
 166                                         _type = "action";
 167                                         policyDataString.append("\"policyType\":\"Action\",");
 168                                 }else if(policyWithScopeName.contains(".Decision_")){
 169                                         _type = "decision";
 170                                         policyDataString.append("\"policyType\":\"Decision\",");
 171                                 }
 172
 173                                 if(!"decision".equals(_type)){
 174                                         if(configurationdataid != null){
 175                                             updateConfigData(conn, configurationdataid, policyDataString);
 176                                         }
 177                                         if(actionbodyid != null){
 178                                             updateActionData(conn, actionbodyid, policyDataString);
 179                                         }
 180                                 }
 181
 182                                 String _id = policyWithScopeName;
 183
 184                                 String dataString = constructPolicyData(policyData, policyDataString);
 185                                 dataString = dataString.substring(0, dataString.length()-1);
 186                                 dataString = dataString.trim().replace(System.getProperty("line.separator"), "") + "}";
 187                                 dataString = dataString.replace("null", "\"\"");
 188                                 dataString = dataString.replaceAll("\n", "");
 189
 190                                 try{
 191                                         Gson gson = new Gson();
 192                                         gson.fromJson(dataString, Object.class);
 193                                 }catch(Exception e){
 194                                         LOGGER.error(e);
 195                                         continue;
 196                                 }
 197
 198                                 if("config".equals(_type)){
 199                                         listIndex.add(new Index.Builder(dataString).index("policy").type("config").id(_id).build());
 200                                 }else if("closedloop".equals(_type)){
 201                                         listIndex.add(new Index.Builder(dataString).index("policy").type("closedloop").id(_id).build());
 202                                 }else if("action".equals(_type)){
 203                                         listIndex.add(new Index.Builder(dataString).index("policy").type("action").id(_id).build());
 204                                 }else if("decision".equals(_type)){
 205                                         listIndex.add(new Index.Builder(dataString).index("policy").type("decision").id(_id).build());
 206                                 }
 207                         }
 208
 209                         result.close();
 210                         bulk = new Bulk.Builder();
 211                         for(int i =0; i < listIndex.size(); i++){
 212                                 bulk.addAction(listIndex.get(i));
 213                         }
 214                         BulkResult searchResult = client.execute(bulk.build());
 215                         if(searchResult.isSucceeded()){
 216                                 LOGGER.debug("Success");
 217                         }else{
 218                                 LOGGER.error("Failure");
 219                         }
 220                 } catch (Exception e) {
 221                         LOGGER.error("Exception Occured while performing database Operation for Elastic Search Policy Upgrade"+e);
 222                 }finally{
 223                         if(result != null){
 224                             try {
 225                                 result.close();
 226                             } catch (Exception e) {
 227                                 LOGGER.error("Exception Occured while closing the resultset"+e);
 228                             }
 229                         }
 230                         if(stmt != null){
 231                             try {
 232                                 stmt.close();
 233                             } catch (Exception e) {
 234                                 LOGGER.error("Exception Occured while closing the statement"+e);
 235                             }
 236                         }
 237                         if(conn != null){
 238                                 try {
 239                                         conn.close();
 240                                 } catch (Exception e) {
 241                                         LOGGER.error("Exception Occured while closing the connection"+e);
 242                                 }
 243                         }
 244                 }
 245         }
 246
 247         public static String constructPolicyData(Object policyContent, StringBuilder policyDataString){
 248                 InputStream stream = new ByteArrayInputStream(policyContent.toString().getBytes(StandardCharsets.UTF_8));
 249                 Object policyData = XACMLPolicyScanner.readPolicy(stream);
 250                 if(policyData instanceof PolicyType){
 251                         PolicyType policy = (PolicyType) policyData;
 252                         TargetType target = policy.getTarget();
 253                         if (target != null) {
 254                                 // Under target we have AnyOFType
 255                                 List<AnyOfType> anyOfList = target.getAnyOf();
 256                                 if (anyOfList != null) {
 257                                         Iterator<AnyOfType> iterAnyOf = anyOfList.iterator();
 258                                         while (iterAnyOf.hasNext()) {
 259                                                 AnyOfType anyOf = iterAnyOf.next();
 260                                                 // Under AnyOFType we have AllOFType
 261                                                 List<AllOfType> allOfList = anyOf.getAllOf();
 262                                                 if (allOfList != null) {
 263                                                         Iterator<AllOfType> iterAllOf = allOfList.iterator();
 264                                                         while (iterAllOf.hasNext()) {
 265                                                                 AllOfType allOf = iterAllOf.next();
 266                                                                 // Under AllOFType we have Match
 267                                                                 List<MatchType> matchList = allOf.getMatch();
 268                                                                 if (matchList != null) {
 269                                                                         Iterator<MatchType> iterMatch = matchList.iterator();
 270                                                                         while (iterMatch.hasNext()) {
 271                                                                                 MatchType match = iterMatch.next();
 272                                                                                 //
 273                                                                                 // Under the match we have attribute value and
 274                                                                                 // attributeDesignator. So,finally down to the actual attribute.
 275                                                                                 //
 276                                                                                 AttributeValueType attributeValue = match.getAttributeValue();
 277                                                                                 String value = (String) attributeValue.getContent().get(0);
 278                                                                                 AttributeDesignatorType designator = match.getAttributeDesignator();
 279                                                                                 String attributeId = designator.getAttributeId();
 280                                                                                 // First match in the target is OnapName, so set that value.
 281                                                                                 if ("ONAPName".equals(attributeId)) {
 282                                                                                         policyDataString.append("\"onapName\":\""+value+"\",");
 283                                                                                 }
 284                                                                                 if ("RiskType".equals(attributeId)){
 285                                                                                         policyDataString.append("\"riskType\":\""+value+"\",");
 286                                                                                 }
 287                                                                                 if ("RiskLevel".equals(attributeId)){
 288                                                                                         policyDataString.append("\"riskLevel\":\""+value+"\",");
 289                                                                                 }
 290                                                                                 if ("guard".equals(attributeId)){
 291                                                                                         policyDataString.append("\"guard\":\""+value+"\",");
 292                                                                                 }
 293                                                                                 if ("ConfigName".equals(attributeId)){
 294                                                                                         policyDataString.append("\"configName\":\""+value+"\",");
 295                                                                                 }
 296                                                                         }
 297                                                                 }
 298                                                         }
 299                                                 }
 300                                         }
 301                                 }
 302                         }
 303                 }
 304                 return policyDataString.toString();
 305         }
 306
 307         private static void updateConfigData(Connection conn, String configurationdataid, StringBuilder policyDataString) throws Exception {
 308
 309                 PreparedStatement pstmt = null;
 310                 ResultSet configResult = null;
 311                 try {
 312                     String configEntityQuery = "Select * from ConfigurationDataEntity where configurationDataId = ?";
 313                     pstmt = null;
 314                     pstmt = conn.prepareStatement(configEntityQuery);
 315                     pstmt.setString(1, configurationdataid);
 316                     configResult = pstmt.executeQuery();
 317                     while(configResult.next()){
 318                         String configBody = configResult.getString("configbody");
 319                         String configType = configResult.getString("configtype");
 320                         if(configBody!=null){
 321                             configBody = configBody.replace("null", "\"\"");
 322                             configBody= configBody.replace("\"", "\\\"");
 323                             policyDataString.append("\"jsonBodyData\":\""+configBody+"\",\"configType\":\""+configType+"\",");
 324                         }
 325                     }
 326                 } catch(Exception e) {
 327                     LOGGER.error("Exception Occured while updating configData"+e);
 328                     throw(e);
 329                 } finally {
 330                     if(configResult != null){
 331                         try {
 332                             configResult.close();
 333                         } catch (Exception e) {
 334                             LOGGER.error("Exception Occured while closing the ResultSet"+e);
 335                         }
 336                     }
 337                     if(pstmt != null){
 338                         try {
 339                             pstmt.close();
 340                         } catch (Exception e) {
 341                             LOGGER.error("Exception Occured while closing the PreparedStatement"+e);
 342                         }
 343                     }
 344                }
 345         }
 346
 347         private static void updateActionData(Connection conn, String actionbodyid, StringBuilder policyDataString) throws Exception {
 348
 349             PreparedStatement pstmt = null;
 350             ResultSet actionResult = null;
 351             try {
 352                 String actionEntityQuery = "Select * from ActionBodyEntity where actionBodyId = ?";
 353                 pstmt = conn.prepareStatement(actionEntityQuery);
 354                 pstmt.setString(1, actionbodyid);
 355                 actionResult = pstmt.executeQuery();
 356                 while(actionResult.next()){
 357                     String actionBody = actionResult.getString("actionbody");
 358                     actionBody = actionBody.replace("null", "\"\"");
 359                     actionBody = actionBody.replace("\"", "\\\"");
 360                     policyDataString.append("\"jsonBodyData\":\""+actionBody+"\",");
 361                 }
 362             } catch(Exception e) {
 363                 LOGGER.error("Exception Occured while updating actionData"+e);
 364                 throw(e);
 365             } finally {
 366                 if(actionResult != null){
 367                     try {
 368                         actionResult.close();
 369                     } catch (Exception e) {
 370                         LOGGER.error("Exception Occured while closing the ResultSet"+e);
 371                     }
 372                 }
 373                 if(pstmt != null){
 374                     try {
 375                         pstmt.close();
 376                     } catch (Exception e) {
 377                         LOGGER.error("Exception Occured while closing the PreparedStatement"+e);
 378                     }
 379                 }
 380            }
 381         }
 382 }