ems/boco/src/main/java/org/onap/vfc/nfvo/emsdriver/collector/TaskThread.java

   1 /**
   2  * Copyright 2017 BOCO Corporation.  CMCC Technologies Co., Ltd
   3  *
   4  * Licensed under the Apache License, Version 2.0 (the "License");
   5  * you may not use this file except in compliance with the License.
   6  * You may obtain a copy of the License at
   7  *
   8  *     http://www.apache.org/licenses/LICENSE-2.0
   9  *
  10  * Unless required by applicable law or agreed to in writing, software
  11  * distributed under the License is distributed on an "AS IS" BASIS,
  12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  * See the License for the specific language governing permissions and
  14  * limitations under the License.
  15  */
  16 package org.onap.vfc.nfvo.emsdriver.collector;
  17
  18 import java.io.BufferedOutputStream;
  19 import java.io.BufferedReader;
  20 import java.io.File;
  21 import java.io.FileInputStream;
  22 import java.io.FileNotFoundException;
  23 import java.io.FileOutputStream;
  24 import java.io.IOException;
  25 import java.io.InputStreamReader;
  26 import java.io.OutputStream;
  27 import java.io.OutputStreamWriter;
  28 import java.text.SimpleDateFormat;
  29 import java.util.ArrayList;
  30 import java.util.Date;
  31 import java.util.HashMap;
  32 import java.util.Hashtable;
  33 import java.util.LinkedHashMap;
  34 import java.util.List;
  35 import java.util.Properties;
  36 import java.util.regex.Matcher;
  37 import java.util.regex.Pattern;
  38
  39 import javax.xml.stream.XMLInputFactory;
  40 import javax.xml.stream.XMLStreamConstants;
  41 import javax.xml.stream.XMLStreamReader;
  42
  43 import org.apache.commons.io.FileUtils;
  44 import org.apache.commons.logging.Log;
  45 import org.apache.commons.logging.LogFactory;
  46 import org.onap.vfc.nfvo.emsdriver.commons.constant.Constant;
  47 import org.onap.vfc.nfvo.emsdriver.commons.ftp.AFtpRemoteFile;
  48 import org.onap.vfc.nfvo.emsdriver.commons.ftp.FTPInterface;
  49 import org.onap.vfc.nfvo.emsdriver.commons.ftp.FTPSrv;
  50 import org.onap.vfc.nfvo.emsdriver.commons.model.CollectMsg;
  51 import org.onap.vfc.nfvo.emsdriver.commons.model.CollectVo;
  52 import org.onap.vfc.nfvo.emsdriver.commons.utils.DateUtil;
  53 import org.onap.vfc.nfvo.emsdriver.commons.utils.Gunzip;
  54 import org.onap.vfc.nfvo.emsdriver.commons.utils.StringUtil;
  55 import org.onap.vfc.nfvo.emsdriver.commons.utils.UnZip;
  56 import org.onap.vfc.nfvo.emsdriver.commons.utils.VarExprParser;
  57 import org.onap.vfc.nfvo.emsdriver.commons.utils.Zip;
  58 import org.onap.vfc.nfvo.emsdriver.configmgr.ConfigurationImp;
  59 import org.onap.vfc.nfvo.emsdriver.configmgr.ConfigurationInterface;
  60 import org.onap.vfc.nfvo.emsdriver.messagemgr.MessageChannel;
  61 import org.onap.vfc.nfvo.emsdriver.messagemgr.MessageChannelFactory;
  62
  63
  64 public class TaskThread implements Runnable{
  65
  66         public  Log log = LogFactory.getLog(TaskThread.class);
  67
  68         private MessageChannel cmResultChannel;
  69         public MessageChannel pmResultChannel;
  70
  71         private CollectMsg data;
  72
  73         private ConfigurationInterface configurationInterface = new ConfigurationImp();
  74
  75         private String localPath = Constant.SYS_DATA_TEMP;
  76         private String resultPath = Constant.SYS_DATA_RESULT;
  77
  78         private SimpleDateFormat dateFormat = new SimpleDateFormat("yyyyMMddHHmmss");
  79
  80         private SimpleDateFormat dateFormat2 = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
  81
  82 //      private String csvpathAndFileName;
  83 //      private String xmlPathAndFileName;
  84 //      private int countNum = 0 ;
  85
  86         public TaskThread(CollectMsg data) {
  87                 this.data = data;
  88         }
  89         public TaskThread() {
  90                 super();
  91         }
  92
  93         @Override
  94         public void run(){
  95
  96                 cmResultChannel = MessageChannelFactory.getMessageChannel(Constant.COLLECT_RESULT_CHANNEL_KEY);
  97                 pmResultChannel = MessageChannelFactory.getMessageChannel(Constant.COLLECT_CHANNEL_KEY);
  98                 try {
  99                         collectMsgHandle(data);
 100                 } catch (Exception e) {
 101                         log.error("",e);
 102                 }
 103         }
 104
 105         private void collectMsgHandle(CollectMsg collectMsg) {
 106                 String emsName = collectMsg.getEmsName();
 107                 String type = collectMsg.getType();
 108                 CollectVo collectVo = configurationInterface.getCollectVoByEmsNameAndType(emsName, type);
 109
 110                 //ftp download
 111                 List<String> downloadfiles = this.ftpDownload(collectVo);
 112                 //paser ftp update message send
 113                 for(String fileName :downloadfiles){
 114                         this.parseFtpAndSendMessage(fileName,collectVo);
 115                 }
 116         }
 117
 118         public void parseFtpAndSendMessage(String fileName, CollectVo collectVo) {
 119                 //
 120                 List<File> filelist = decompressed(fileName);
 121
 122                 for (File tempfile : filelist) {
 123
 124                         String unfileName = tempfile.getName();
 125
 126                         Pattern pa = Pattern.compile(".*-(.*)-\\w{2}-");
 127                         Matcher ma = pa.matcher(unfileName);
 128                         if (!ma.find())
 129                           continue;
 130                         String nename = ma.group(1);
 131                         boolean parseResult = false;
 132                         if("ems-resource".equalsIgnoreCase(collectVo.getType())){
 133                                 parseResult = processCMXml(tempfile, nename,"CM");
 134                         }else{
 135                                 if(unfileName.indexOf(".csv") > 0){
 136                                         parseResult = processPMCsv(tempfile);
 137                                 }else{
 138                                         parseResult = processPMCsv(tempfile);
 139                                 }
 140                         }
 141
 142                         if (parseResult){
 143                                 log.info("parser "+tempfile+" sucess");
 144                         }else {
 145                                 log.info("parser "+tempfile+" fail");
 146                         }
 147
 148                 }
 149         }
 150
 151         public boolean processPMCsv(File tempfile) {
 152
 153                 FileInputStream brs = null;
 154                 InputStreamReader isr = null;
 155                 BufferedReader br = null;
 156
 157                 List<String> columnNames = new ArrayList<String>();
 158                 List<String> commonValues = new ArrayList<String>();
 159                 try {
 160
 161                         brs = new FileInputStream(tempfile);
 162                         isr = new InputStreamReader(brs, Constant.ENCODING_UTF8);
 163                         br = new BufferedReader(isr);
 164                         //common field
 165                         String commonField = br.readLine();
 166                         String[] fields = commonField.split("\\|",-1);
 167                         for(String com : fields){
 168                                 String[] comNameAndValue = com.split("=",2);
 169                                 columnNames.add(comNameAndValue[0].trim());
 170                                 commonValues.add(comNameAndValue[1]);
 171                         }
 172                         //column names
 173                         String columnName = br.readLine();
 174                         String[] names = columnName.split("\\|",-1);
 175                         for(String name : names){
 176                                 columnNames.add(name);
 177                         }
 178
 179 //                      xmlPathAndFileName = this.setColumnNames(nename, columnNames,type);
 180
 181                         String valueLine = "";
 182                         List<String> valuelist = new ArrayList<String>();
 183
 184                         while ((valueLine = br.readLine()) != null) {
 185                                 if (valueLine.trim().equals("")) {
 186                                         continue;
 187                                 }
 188 //                              countNum ++;
 189                                 String [] values = valueLine.split("\\|",-1);
 190
 191                                 valuelist.addAll(commonValues);
 192                                 for(String value : values){
 193                                         valuelist.add(value);
 194                                 }
 195 //                              this.appendLine(valuelist, bos);
 196                                 //resultMap
 197                                 HashMap<String,String> resultMap = this.resultMap(columnNames,valuelist);
 198                                 try {
 199                                         pmResultChannel.put(resultMap);
 200                                 } catch (InterruptedException e) {
 201                                         log.error("collectResultChannel.put(resultMap) error ",e);
 202                                 }
 203                                 valuelist.clear();
 204                         }
 205                 } catch (IOException e) {
 206                         log.error("processPMCsv is fail ",e);
 207                         return false;
 208                 }finally{
 209                         try{
 210                                 if (br != null)
 211                                         br.close();
 212                                 if (isr != null)
 213                                         isr.close();
 214                                 if (brs != null)
 215                                         brs.close();
 216
 217                         } catch (Exception e){
 218                                 log.error(e);
 219                         }
 220                 }
 221                 return true;
 222
 223         }
 224
 225         private HashMap<String,String> resultMap(List<String> columnNames, List<String> valuelist) {
 226
 227                 HashMap<String,String>  resultMap = new HashMap<String,String>();
 228                 if(columnNames.size() == valuelist.size()){
 229                         for(int i =0;i<columnNames.size();i++){
 230                                 resultMap.put(columnNames.get(i), valuelist.get(i));
 231                         }
 232                 }
 233
 234                 return resultMap;
 235
 236         }
 237         private boolean processCMXml(File tempfile, String nename, String type) {
 238
 239                 String csvpath = localPath+nename+"/"+type+"/";
 240                 File csvpathfile = new File(csvpath);
 241                 if(!csvpathfile.exists()){
 242                         csvpathfile.mkdirs();
 243                 }
 244                 String csvFileName = nename +dateFormat.format(new Date())+  System.nanoTime();
 245                 String csvpathAndFileName = csvpath+csvFileName+".csv";
 246                 BufferedOutputStream  bos = null;
 247                 FileOutputStream fos = null;
 248                 try {
 249                         fos = new FileOutputStream(csvpathAndFileName,false);
 250                         bos = new BufferedOutputStream(fos, 10240);
 251                 } catch (FileNotFoundException e1) {
 252                         log.error("FileNotFoundException "+StringUtil.getStackTrace(e1));
 253                 }
 254
 255                 boolean FieldNameFlag = false;
 256                 boolean FieldValueFlag = false;
 257                 //line num
 258                 int countNum = 0;
 259                 String xmlPathAndFileName = null;
 260                 String localName = null;
 261                 String endLocalName = null;
 262                 String rmUID = null;
 263                 int index = -1;
 264                 ArrayList<String> names = new ArrayList<String>();// colname
 265                 LinkedHashMap<String, String> nameAndValue = new LinkedHashMap<String, String>();
 266
 267
 268                 FileInputStream fis = null;
 269                 InputStreamReader isr = null;
 270                 XMLStreamReader reader = null;
 271                 try{
 272                         fis = new FileInputStream(tempfile);
 273                         isr = new InputStreamReader(fis, Constant.ENCODING_UTF8);
 274                         XMLInputFactory fac = XMLInputFactory.newInstance();
 275                         reader = fac.createXMLStreamReader(isr);
 276                         int event = -1;
 277                         boolean setcolum = true;
 278                         while (reader.hasNext()){
 279                                 try{
 280                                         event = reader.next();
 281                                         switch (event){
 282                                         case XMLStreamConstants.START_ELEMENT:
 283                                                 localName = reader.getLocalName();
 284                                                 if ("FieldName".equalsIgnoreCase(localName)){
 285                                                         FieldNameFlag = true;
 286                                                 }
 287                                                 if (FieldNameFlag){
 288                                                         if ("N".equalsIgnoreCase(localName)){
 289                                                                 String colName = reader.getElementText().trim();
 290                                                                 names.add(colName);
 291                                                         }
 292                                                 }
 293                                                 if ("FieldValue".equalsIgnoreCase(localName)){
 294                                                         FieldValueFlag = true;
 295
 296                                                 }
 297                                                 if (FieldValueFlag){
 298                                                         if(setcolum){
 299                                                                 xmlPathAndFileName = this.setColumnNames(nename, names,type);
 300                                                                 setcolum = false;
 301                                                         }
 302
 303                                                         if ("Object".equalsIgnoreCase(localName)){
 304                                                                 int ac = reader.getAttributeCount();
 305                                                                 for (int i = 0; i < ac; i++){
 306                                                                         if ("rmUID".equalsIgnoreCase(reader.getAttributeLocalName(i))){
 307                                                                                 rmUID = reader.getAttributeValue(i).trim();
 308                                                                         }
 309                                                                 }
 310                                                                 nameAndValue.put("rmUID", rmUID);
 311                                                         }
 312                                                         if ("V".equalsIgnoreCase(localName)) {
 313                                                                 index = Integer.parseInt(reader
 314                                                                                 .getAttributeValue(0)) - 1;
 315                                                                 String currentName = names.get(index);
 316                                                                 String v = reader.getElementText().trim();
 317                                                                 nameAndValue.put(currentName, v);
 318                                                         }
 319                                                 }
 320                                                 break;
 321                                         case XMLStreamConstants.CHARACTERS:
 322                                                 break;
 323                                         case XMLStreamConstants.END_ELEMENT:
 324                                                 endLocalName = reader.getLocalName();
 325
 326                                                 if ("FieldName".equalsIgnoreCase(endLocalName)){
 327                                                         FieldNameFlag = false;
 328                                                 }
 329                                                 if ("FieldValue".equalsIgnoreCase(endLocalName)){
 330                                                         FieldValueFlag = false;
 331                                                 }
 332                                                 if ("Object".equalsIgnoreCase(endLocalName)){
 333                                                         countNum ++;
 334                                                         this.appendLine(nameAndValue,bos);
 335                                                         nameAndValue.clear();
 336                                                 }
 337                                                 break;
 338                                         }
 339                                 } catch (Exception e)
 340                                 {
 341                                         log.error(""+StringUtil.getStackTrace(e));
 342                                         event = reader.next();
 343                                 }
 344                         }
 345
 346
 347                         if(bos != null){
 348                                 bos.close();
 349                                 bos = null;
 350                         }
 351                         if(fos != null){
 352                                 fos.close();
 353                                 fos = null;
 354                         }
 355
 356                         String[] fileKeys = this.createZipFile(csvpathAndFileName,xmlPathAndFileName,nename);
 357                         //ftp store
 358                         Properties ftpPro = configurationInterface.getProperties();
 359                         String ip = ftpPro.getProperty("ftp_ip");
 360                         String port = ftpPro.getProperty("ftp_port");
 361                         String ftp_user = ftpPro.getProperty("ftp_user");
 362                         String ftp_password = ftpPro.getProperty("ftp_password");
 363
 364                         String ftp_passive = ftpPro.getProperty("ftp_passive");
 365                         String ftp_type = ftpPro.getProperty("ftp_type");
 366                         String remoteFile = ftpPro.getProperty("ftp_remote_path");
 367                         this.ftpStore(fileKeys,ip,port,ftp_user,ftp_password,ftp_passive,ftp_type,remoteFile);
 368                         //create Message
 369                         String message = this.createMessage(fileKeys[1], ftp_user, ftp_password, ip,  port, countNum,nename);
 370
 371                         //set message
 372                         this.setMessage(message);
 373                 } catch (Exception e){
 374                         log.error(""+StringUtil.getStackTrace(e));
 375                         return false;
 376                 } finally{
 377                         try{
 378                                 if (reader != null){
 379                                         reader.close();
 380                                 }
 381                                 if (isr != null){
 382                                         isr.close();
 383                                 }
 384                                 if (fis != null){
 385                                         fis.close();
 386                                 }
 387                                 if(bos != null){
 388                                         bos.close();
 389                                 }
 390
 391                                 if(fos != null){
 392                                         fos.close();
 393                                 }
 394                         } catch (Exception e){
 395                                 log.error(e);
 396                         }
 397                 }
 398                 return true;
 399         }
 400
 401         private void setMessage(String message) {
 402
 403                 try {
 404                         cmResultChannel.put(message);
 405                 } catch (Exception e) {
 406                         log.error("collectResultChannel.put(message) is error "+StringUtil.getStackTrace(e));
 407                 }
 408         }
 409
 410         public String createMessage(String zipName,String user,String pwd,String ip, String port,int countNum, String nename) {
 411
 412                 StringBuffer strBuffer = new StringBuffer();
 413                 strBuffer
 414                                 .append("<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
 415                                                 + "<FILE_DATA_READY_UL xmlns:xsi=\" http://www.w3.org/2001/XMLSchema-instance\">"
 416                                                 + "<Header SessionID=\"");
 417                 strBuffer.append("");
 418                 strBuffer.append("\" LicenceID=\"");
 419                 strBuffer.append("");
 420                 strBuffer.append("\" SystemID=\"");
 421                 strBuffer.append("");
 422                 strBuffer.append("\" Time=\"");
 423                 strBuffer.append( dateFormat2.format(new Date()));
 424                 strBuffer.append("\" PolicyID=\"");
 425                 strBuffer.append("");
 426                 strBuffer.append("\"/><Body>");
 427                 strBuffer.append("<DataCatalog>");
 428                 strBuffer.append("");
 429                 strBuffer.append("</DataCatalog><GroupID>");
 430                 strBuffer.append(nename);
 431                 strBuffer.append("</GroupID><DataSourceName>");
 432                 strBuffer.append("");
 433                 strBuffer.append("</DataSourceName><InstanceID>");
 434                 strBuffer.append("");
 435                 strBuffer.append("</InstanceID><FileFormat>");
 436                 strBuffer.append("csv");
 437                 strBuffer.append("</FileFormat><CharSet>");
 438                 strBuffer.append("gbk");
 439                 strBuffer.append("</CharSet><FieldSeparator>");
 440                 strBuffer.append("|");
 441                 strBuffer.append("</FieldSeparator><IsCompressed>");
 442                 strBuffer.append("true");
 443                 strBuffer.append("</IsCompressed><StartTime>");
 444                 strBuffer.append(dateFormat2.format(new Date()));
 445                 strBuffer.append("</StartTime><EndTime>");
 446                 strBuffer.append("");
 447                 strBuffer.append("</EndTime><FileList>");
 448                 strBuffer.append(zipName);
 449                 strBuffer.append("</FileList><ConnectionString>");
 450                 strBuffer.append("ftp://" + user + ":" + pwd + "@" + ip + ":" + port);
 451                 strBuffer.append("</ConnectionString>");
 452                 strBuffer.append("<DataCount>");
 453                 strBuffer.append(countNum);
 454                 strBuffer.append("</DataCount>");
 455
 456                 strBuffer.append("<FileSize>").append("").append("</FileSize>");
 457                 strBuffer.append("<DataGranularity>").append("").append("</DataGranularity>");
 458
 459
 460                 strBuffer.append("</Body></FILE_DATA_READY_UL>");
 461                 return strBuffer.toString();
 462
 463         }
 464
 465         private void ftpStore(String[] fileKeys, String ip, String port, String ftp_user, String ftp_password,
 466                         String ftp_passive, String ftp_type, String remoteFile) {
 467                 String zipFilePath = fileKeys[0];
 468
 469
 470                 FTPInterface ftpClient;
 471                 ftpClient = new FTPSrv();
 472                 //login
 473                 try {
 474                         ftpClient.login(ip, Integer.parseInt(port), ftp_user, ftp_password, "GBK", Boolean.parseBoolean(ftp_passive), 5*60*1000);
 475                 } catch (Exception e) {
 476                         log.error("login fail,ip=["+ip+"] port=["+port+"] user=["+ftp_user+"]pwd=["+ftp_password+"]"+StringUtil.getStackTrace(e));
 477                     return;
 478                 }
 479 //              ftpClient.store(zipFilePath, remoteFile);
 480                 log.debug("store  ["+zipFilePath+"]to["+remoteFile+"]");
 481
 482                 FileUtils.deleteQuietly(new File(zipFilePath));
 483
 484
 485         }
 486
 487         private String[] createZipFile(String csvpathAndFileName,String xmlPathAndFileName,String nename) {
 488
 489                 String zipPath = resultPath+nename +dateFormat.format(new Date())+"_"+System.nanoTime();
 490
 491                 File destDir = new File(zipPath);
 492                 destDir.mkdirs();
 493
 494                 try {
 495                         FileUtils.copyFileToDirectory(new File(csvpathAndFileName), destDir);
 496                         FileUtils.copyFileToDirectory(new File(xmlPathAndFileName), destDir);
 497                 } catch (IOException e) {
 498
 499                 }
 500
 501                 String destFilePath = zipPath + ".zip";
 502                 try {
 503                         Zip zip = new Zip(destDir.getAbsolutePath(), destFilePath);
 504                         zip.setCompressLevel(9);
 505                         zip.compress();
 506
 507                         FileUtils.deleteDirectory(destDir);
 508                 } catch (IOException e) {
 509                         log.error("zip.compress() is fail "+StringUtil.getStackTrace(e));
 510                 }
 511                 return new String[] { destFilePath, zipPath + ".zip"};
 512         }
 513
 514
 515         private String setColumnNames(String nename, List<String> names,String type) {
 516                 //write xml
 517                 String xmlpath = localPath+nename +"/"+type+"/";
 518                 File xmlpathfile = new File(xmlpath);
 519                 if(!xmlpathfile.exists()){
 520                         xmlpathfile.mkdirs();
 521                 }
 522                 String xmlFileName = nename +dateFormat.format(new Date())+ System.nanoTime();
 523                 String fieldLine = "";
 524                 for (int i = 0; i < names.size(); i++) {
 525                         String field = "\t<Field>\r\n" + "\t\t<FieldNo>" + i
 526                                         + "</FieldNo>\r\n" + "\t\t<FieldName>"
 527                                         + names.get(i) + "</FieldName>\r\n"
 528                                         + "\t\t<FieldType>" + names.get(i)
 529                                         + "</FieldType>\r\n"
 530                                         + "\t\t<FieldNameOther>" + names.get(i)
 531                                         + "</FieldNameOther>\r\n" +
 532                                         "\t</Field>\r\n";
 533                         fieldLine = fieldLine + field;
 534                 }
 535
 536                 String str = "<?xml version=\"1.0\" encoding=\"gbk\"?>\r\n"
 537                                 + "<xml>\r\n" + "<FILE_STRUCTURE>\r\n" + fieldLine
 538                                 + "</FILE_STRUCTURE>\r\n" + "</xml>\r\n";
 539                 String xmlPathAndFileName = xmlpath+xmlFileName+".xml";
 540                 try {
 541                         this.writeDetail(xmlPathAndFileName,str);
 542                 } catch (Exception e) {
 543                         log.error("writeDetail is fail ,xmlFileName="+xmlFileName +StringUtil.getStackTrace(e));
 544                 }
 545
 546                 return xmlPathAndFileName;
 547         }
 548
 549         private void writeDetail(String detailFileName,String str) throws Exception {
 550                 OutputStreamWriter writer = null;
 551                 OutputStream readOut = null;
 552                 try {
 553                         readOut = new FileOutputStream(new File(detailFileName), false);
 554                         writer = new OutputStreamWriter(readOut);
 555                         writer.write(str);
 556                         writer.flush();
 557                 } finally {
 558
 559                         if(null != writer){
 560                                 writer.close();
 561                         }
 562                         if(readOut != null){
 563                                 readOut.close();
 564                         }
 565
 566                 }
 567
 568         }
 569
 570
 571         private void appendLine(LinkedHashMap<String, String> nameAndValue,BufferedOutputStream  bos) {
 572                 StringBuilder lineDatas =  new StringBuilder();
 573
 574                 for (String key : nameAndValue.keySet()) {
 575                         lineDatas.append(nameAndValue.get(key)).append("|");
 576                 }
 577                 try {
 578                         bos.write(lineDatas.toString().getBytes());
 579                         bos.write("\n".getBytes());
 580                 } catch (IOException e) {
 581                         log.error("appendLine error "+StringUtil.getStackTrace(e));
 582                 }
 583         }
 584
 585 //      private void appendLine(List<String> values,BufferedOutputStream  bos) {
 586 //              StringBuilder lineDatas =  new StringBuilder();
 587 //
 588 //              for (String value : values) {
 589 //                      lineDatas.append(value).append("|");
 590 //              }
 591 //              try {
 592 //                      bos.write(lineDatas.toString().getBytes());
 593 //                      bos.write("\n".getBytes());
 594 //              } catch (IOException e) {
 595 //                      log.error("appendLine error "+StringUtil.getStackTrace(e));
 596 //              }
 597 //      }
 598
 599         public List<File> decompressed(String fileName){
 600             List<File> filelist = new ArrayList<File>();
 601
 602             if (fileName.indexOf(".gz") > 1)
 603             {
 604                 try {
 605                                 File decompressFile = deGz(fileName);
 606                                 filelist.add(decompressFile);
 607                         } catch (IOException e) {
 608                                 log.error("decompressed is fail "+StringUtil.getStackTrace(e));
 609                         }
 610             } else if (fileName.indexOf(".zip") > 1)
 611             {
 612                 try {
 613                                 File[] files = deZip(new File(fileName));
 614                                 for(File temp :files){
 615                                         filelist.add(temp);
 616                                 }
 617                         } catch (Exception e) {
 618                                 log.error("decompressed is fail "+StringUtil.getStackTrace(e));
 619                         }
 620             }
 621             else {
 622                 filelist.add(new File(fileName));
 623             }
 624
 625             return filelist;
 626         }
 627
 628         private File deGz(String gzFileName) throws IOException {
 629                 Gunzip gunzip = new Gunzip();
 630                 String orgFile = gzFileName.replace(".gz", "");
 631                 gunzip.unCompress(gzFileName, orgFile);
 632                 return new File(orgFile);
 633         }
 634
 635         public File[] deZip(File file) throws Exception{
 636
 637                 String regx = "(.*).zip";
 638               Pattern p = Pattern.compile(regx);
 639               Matcher m = p.matcher(file.getName());
 640               if (m.find())
 641               {
 642                 String orgFile = localPath + m.group(1) + "/";
 643                 UnZip unzip = new UnZip(file.getAbsolutePath(), orgFile);
 644             unzip.deCompress();
 645                 file = new File(orgFile);
 646               }
 647               File[] files = file.listFiles();
 648
 649               return files;
 650
 651         }
 652
 653         private List<String> ftpDownload(CollectVo collectVo) {
 654
 655                 List<String> fileList = new ArrayList<String>();
 656                 //IP
 657                 String ip = collectVo.getIP();
 658                 //port
 659                 String port = collectVo.getPort();
 660                 //user
 661                 String user = collectVo.getUser();
 662                 //password
 663                 String password = collectVo.getPassword();
 664                 //isPassiveMode
 665                 String passivemode = collectVo.getPassive();
 666
 667                 FTPInterface ftpClient = new FTPSrv();
 668
 669                 //login
 670                 try {
 671                         ftpClient.login(ip, Integer.parseInt(port), user, password, "GBK", Boolean.parseBoolean(passivemode), 5*60*1000);
 672                 } catch (Exception e) {
 673                         log.error("login fail,ip=["+ip+"] port=["+port+"] user=["+user+"]password=["+password+"]"+StringUtil.getStackTrace(e));
 674                     return fileList;
 675                 }
 676
 677                 //download
 678                 String dir = collectVo.getRemotepath();
 679                 List<String> searchExprList = new ArrayList<String>();
 680                 String []FPath = dir.split(";");
 681                 for(int i=0;i<FPath.length;i++){
 682                         int oldSize = searchExprList.size();
 683                         String conpath = FPath[i];
 684                         Hashtable<String,String> varMap = new Hashtable<String,String>();
 685                         long[] d = DateUtil.getScanScope(new Date(), 900);
 686                         searchExprList.add(VarExprParser.replaceVar(conpath,d[0],d[1]));
 687
 688                         varMap.clear();
 689                         varMap = null;
 690                         log.info("["+conpath+"]，result["+(searchExprList.size()-oldSize)+"] path");
 691                         conpath = null;
 692                 }
 693                 searchExprList =getLocalPathNoRegular(searchExprList);
 694                 List<AFtpRemoteFile> remoteFiles = new ArrayList<AFtpRemoteFile>();
 695                 for(String expr :searchExprList){
 696                         String keys[] = parseExprKeys(expr);
 697                         String ftpRegular = keys[1];
 698                         String ftpDir = keys[0];
 699
 700                         boolean cdsucess = ftpClient.chdir(expr);
 701                         if(cdsucess){
 702                                 AFtpRemoteFile[] arf = (AFtpRemoteFile[]) ftpClient.list();
 703                                 log.info(" list ["+ftpDir+"]，result["+(arf==null?"null":arf.length)+"] files");
 704                                 //filter
 705
 706                                 rfileFilter(remoteFiles,arf,ftpRegular);
 707
 708                                 keys = null;
 709                                 ftpRegular=ftpDir = null;
 710
 711                                 for(AFtpRemoteFile ftpRemoteFile: remoteFiles){
 712                                         if(!new File(localPath).exists()){
 713                                                 try {
 714                                                         new File(localPath).mkdir();
 715                                                 } catch (Exception e) {
 716                                                         log.error("create localPath is fail localPath="+localPath+" "+StringUtil.getStackTrace(e));
 717                                                 }
 718                                         }
 719
 720                                         if(!new File(localPath).exists()){
 721                                                 new File(localPath).mkdirs();
 722                                         }
 723
 724                                         String localFileName = localPath + ftpRemoteFile.getFileName();
 725                                         File loaclFile = new File(localFileName);
 726                                         if (loaclFile.exists()) {
 727                                                 loaclFile.delete();
 728                                         }
 729
 730                                         boolean flag = ftpClient.downloadFile(ftpRemoteFile.getAbsFileName(), localFileName);
 731
 732                                         if(flag){
 733                                                 fileList.add(localFileName);
 734                                         }else{
 735                                                 log.error("download file fail fileName="+ftpRemoteFile.getAbsFileName());
 736                                         }
 737                                 }
 738
 739                         }else{
 740                                 log.error("chdir is faill dir =["+dir+"]");
 741                         }
 742                 }
 743
 744
 745
 746                 return fileList;
 747         }
 748
 749         private void rfileFilter(List<AFtpRemoteFile> fileContainer, AFtpRemoteFile[] arfs, String ftpRegular) {
 750                 if (ftpRegular!=null && ftpRegular.length()>0) {
 751                         Pattern pattern = null;
 752                         try {
 753                                 pattern = Pattern.compile(ftpRegular, Pattern.CASE_INSENSITIVE);
 754                         } catch (Exception e) {
 755                                 log.info("["+ftpRegular+"]Pattern.compile exception:"+e.getMessage());
 756                         }
 757                         int hisSize = fileContainer.size();
 758                         for (int j=0; arfs!=null&&j<arfs.length; j++) {
 759                                 String fileName = parseFileName(arfs[j].getFileName());
 760                                 Matcher matcher = pattern.matcher(fileName);
 761                                 if (matcher.find())
 762                                         fileContainer.add(arfs[j]);
 763                         }
 764                         log.info("["+ftpRegular+"]filter["+(fileContainer.size()-hisSize)+"]filse");
 765                         pattern = null;
 766                 }else {
 767                         for (int j=0; arfs!=null&&j<arfs.length; j++)
 768                                 fileContainer.add(arfs[j]);
 769                 }
 770
 771         }
 772
 773         private String parseFileName(String fileName) {
 774                 int idx = fileName.lastIndexOf("/");
 775                 if (idx == -1)
 776                         return fileName;
 777                 return fileName.substring(idx+1, fileName.length());
 778         }
 779
 780         private String[] parseExprKeys(String source) {
 781
 782                 if(source.indexOf(";") > -1){
 783                         source = source.substring(0, source.indexOf(";"));
 784                 }
 785                 if (source.endsWith("/"))
 786                         return new String[]{source,""};
 787
 788                 int idx = source.lastIndexOf("/");
 789                 String[] dirkeys = new String[2];
 790                 dirkeys[0] = source.substring(0, idx+1);
 791                 dirkeys[1] = source.substring(idx+1, source.length());
 792                 return dirkeys;
 793         }
 794
 795         public  List<String> getLocalPathNoRegular(List<String> searchExprList){
 796                 boolean isregular = false;
 797                 List<String> regularList = new ArrayList<String>();
 798                 for(String regular : searchExprList){
 799                         Pattern lpattern = null;
 800                         try{
 801                                 lpattern = Pattern.compile("(.*/)<([^/]+)>(/.*)");
 802                         }catch (Exception e) {
 803                                 log.info("["+regular+"]compile fails:"+e.getMessage());
 804                         }
 805
 806                         Matcher  matcher = lpattern.matcher(regular);
 807                         if(matcher.find()){
 808                                 isregular = true;
 809                                 String parpath  = matcher.group(1);
 810                                 File[] arryFile = new File(parpath).listFiles();
 811                                 for(File file :arryFile){
 812                                         if(file.isDirectory()&&file.getName().matches(matcher.group(2))){
 813                                                 regularList.add(matcher.group(1)+file.getName()+matcher.group(3));
 814                                         }
 815                                 }
 816                         }else{
 817                                 regularList.add(regular);
 818                         }
 819
 820                 }
 821                 if(isregular==true){
 822                         getLocalPathNoRegular(regularList);
 823                 }
 824                 return regularList;
 825         }
 826
 827 }