ems/boco/src/main/java/org/onap/nfvo/emsdriver/collector/TaskThread.java

   1 /**
   2  * Copyright 2017 BOCO Corporation.  CMCC Technologies Co., Ltd
   3  *
   4  * Licensed under the Apache License, Version 2.0 (the "License");
   5  * you may not use this file except in compliance with the License.
   6  * You may obtain a copy of the License at
   7  *
   8  *     http://www.apache.org/licenses/LICENSE-2.0
   9  *
  10  * Unless required by applicable law or agreed to in writing, software
  11  * distributed under the License is distributed on an "AS IS" BASIS,
  12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  * See the License for the specific language governing permissions and
  14  * limitations under the License.
  15  */
  16 package org.onap.nfvo.emsdriver.collector;
  17
  18 import java.io.BufferedOutputStream;
  19 import java.io.BufferedReader;
  20 import java.io.File;
  21 import java.io.FileInputStream;
  22 import java.io.FileNotFoundException;
  23 import java.io.FileOutputStream;
  24 import java.io.IOException;
  25 import java.io.InputStreamReader;
  26 import java.io.OutputStream;
  27 import java.io.OutputStreamWriter;
  28 import java.text.SimpleDateFormat;
  29 import java.util.ArrayList;
  30 import java.util.Date;
  31 import java.util.LinkedHashMap;
  32 import java.util.List;
  33 import java.util.Properties;
  34 import java.util.regex.Matcher;
  35 import java.util.regex.Pattern;
  36
  37 import javax.xml.stream.XMLInputFactory;
  38 import javax.xml.stream.XMLStreamConstants;
  39 import javax.xml.stream.XMLStreamReader;
  40
  41 import org.apache.commons.io.FileUtils;
  42 import org.apache.commons.logging.Log;
  43 import org.apache.commons.logging.LogFactory;
  44 import org.onap.nfvo.emsdriver.commons.constant.Constant;
  45 import org.onap.nfvo.emsdriver.commons.ftp.AFtpRemoteFile;
  46 import org.onap.nfvo.emsdriver.commons.ftp.FTPInterface;
  47 import org.onap.nfvo.emsdriver.commons.ftp.FTPSrv;
  48 import org.onap.nfvo.emsdriver.commons.ftp.SFTPSrv;
  49 import org.onap.nfvo.emsdriver.commons.model.CollectMsg;
  50 import org.onap.nfvo.emsdriver.commons.model.CollectVo;
  51 import org.onap.nfvo.emsdriver.commons.utils.StringUtil;
  52 import org.onap.nfvo.emsdriver.commons.utils.UnZip;
  53 import org.onap.nfvo.emsdriver.commons.utils.Zip;
  54 import org.onap.nfvo.emsdriver.configmgr.ConfigurationImp;
  55 import org.onap.nfvo.emsdriver.configmgr.ConfigurationInterface;
  56 import org.onap.nfvo.emsdriver.messagemgr.MessageChannel;
  57 import org.onap.nfvo.emsdriver.messagemgr.MessageChannelFactory;
  58
  59
  60 public class TaskThread implements Runnable{
  61
  62         public  Log log = LogFactory.getLog(TaskThread.class);
  63
  64         private MessageChannel collectResultChannel;
  65
  66         private CollectMsg data;
  67
  68         private ConfigurationInterface configurationInterface = new ConfigurationImp();
  69
  70         private String localPath = Constant.SYS_DATA_TEMP;
  71         private String resultPath = Constant.SYS_DATA_RESULT;
  72
  73         private SimpleDateFormat dateFormat = new SimpleDateFormat("yyyyMMddHHmmss");
  74
  75         private SimpleDateFormat dateFormat2 = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
  76
  77         public TaskThread(CollectMsg data) {
  78                 this.data = data;
  79         }
  80
  81         @Override
  82         public void run(){
  83
  84                 collectResultChannel = MessageChannelFactory.getMessageChannel(Constant.COLLECT_RESULT_CHANNEL_KEY);
  85
  86                 try {
  87                         collectMsgHandle(data);
  88                 } catch (Exception e) {
  89                         log.error("",e);
  90                 }
  91         }
  92
  93         private void collectMsgHandle(CollectMsg collectMsg) {
  94                 String emsName = collectMsg.getEmsName();
  95                 String type = collectMsg.getType();
  96                 CollectVo collectVo = configurationInterface.getCollectVoByEmsNameAndType(emsName, type);
  97
  98                 //ftp download
  99                 List<String> downloadfiles = this.ftpDownload(collectVo);
 100                 //paser ftp update message send
 101                 for(String fileName :downloadfiles){
 102                         this.parseFtpAndSendMessage(fileName,collectVo);
 103                 }
 104         }
 105
 106         private void parseFtpAndSendMessage(String fileName, CollectVo collectVo) {
 107                 //
 108                 List<File> filelist = decompressed(fileName);
 109
 110                 for (File tempfile : filelist) {
 111
 112                         String unfileName = tempfile.getName();
 113
 114                         Pattern pa = Pattern.compile(".*-(.*)-\\w{2}-");
 115                         Matcher ma = pa.matcher(unfileName);
 116                         if (!ma.find())
 117                           continue;
 118                         String nename = ma.group(1);
 119                         boolean parseResult = false;
 120                         if("CM".equalsIgnoreCase(collectVo.getType())){
 121                                 parseResult = processCMXml(tempfile, nename,"CM");
 122                         }else{
 123                                 parseResult = processPMCsv(tempfile, nename,"PM");
 124                         }
 125
 126                         if (parseResult){
 127                                 log.info("parser "+tempfile+" sucess");
 128                         }else {
 129                                 log.info("parser "+tempfile+" fail");
 130                         }
 131
 132                 }
 133         }
 134
 135         private boolean processPMCsv(File tempfile, String nename,String type) {
 136
 137                 String csvpath = localPath+nename+"/"+type+"/";
 138                 File csvpathfile = new File(csvpath);
 139                 if(!csvpathfile.exists()){
 140                         csvpathfile.mkdirs();
 141                 }
 142                 String csvFileName = nename +dateFormat.format(new Date())+  System.nanoTime();
 143                 String csvpathAndFileName = csvpath+csvFileName;
 144                 BufferedOutputStream  bos = null;
 145                 FileOutputStream fos = null;
 146                 try {
 147                         fos = new FileOutputStream(csvpathAndFileName,false);
 148                         bos = new BufferedOutputStream(fos, 10240);
 149                 } catch (FileNotFoundException e1) {
 150                         log.error("FileNotFoundException "+StringUtil.getStackTrace(e1));
 151                 }
 152
 153                 FileInputStream brs = null;
 154                 InputStreamReader isr = null;
 155                 BufferedReader br = null;
 156
 157                 List<String> columnNames = new ArrayList<String>();
 158                 List<String> commonValues = new ArrayList<String>();
 159                 try {
 160
 161                         brs = new FileInputStream(tempfile);
 162                         isr = new InputStreamReader(brs, Constant.ENCODING_UTF8);
 163                         br = new BufferedReader(isr);
 164                         //common field
 165                         String commonField = br.readLine();
 166                         String[] fields = commonField.split("|",-1);
 167                         for(String com : fields){
 168                                 String[] comNameAndValue = com.split("=",2);
 169                                 columnNames.add(comNameAndValue[0].trim());
 170                                 commonValues.add(comNameAndValue[1]);
 171                         }
 172                         //column names
 173                         String columnName = br.readLine();
 174                         String[] names = columnName.split("|",-1);
 175                         for(String name : names){
 176                                 columnNames.add(name);
 177                         }
 178
 179                         String xmlPathAndFileName = this.setColumnNames(nename, columnNames,type);
 180
 181                         String valueLine = "";
 182                         List<String> valuelist = new ArrayList<String>();
 183                         int countNum = 0 ;
 184                         while (br.readLine() != null) {
 185
 186                                 if (valueLine.trim().equals("")) {
 187                                         continue;
 188                                 }
 189                                 countNum ++;
 190                                 String [] values = valueLine.split("|",-1);
 191
 192                                 valuelist.addAll(commonValues);
 193                                 for(String value : values){
 194                                         valuelist.add(value);
 195                                 }
 196                                 this.appendLine(valuelist, bos);
 197
 198                                 valuelist.clear();
 199                         }
 200
 201                         if(bos != null){
 202                                 bos.close();
 203                                 bos = null;
 204                         }
 205                         if(fos != null){
 206                                 fos.close();
 207                                 fos = null;
 208                         }
 209
 210                         String[] fileKeys = this.createZipFile(csvpathAndFileName,xmlPathAndFileName,nename);
 211                         //ftp store
 212                         Properties ftpPro = configurationInterface.getProperties();
 213                         String ip = ftpPro.getProperty("ftp_ip");
 214                         String port = ftpPro.getProperty("ftp_port");
 215                         String ftp_user = ftpPro.getProperty("ftp_user");
 216                         String ftp_password = ftpPro.getProperty("ftp_password");
 217
 218                         String ftp_passive = ftpPro.getProperty("ftp_passive");
 219                         String ftp_type = ftpPro.getProperty("ftp_type");
 220                         String remoteFile = ftpPro.getProperty("ftp_remote_path");
 221                         this.ftpStore(fileKeys,ip,port,ftp_user,ftp_password,ftp_passive,ftp_type,remoteFile);
 222                         //create Message
 223                         String message = this.createMessage(fileKeys[1], ftp_user, ftp_password, ip,  port, countNum,nename);
 224
 225                         //set message
 226                         this.setMessage(message);
 227                 } catch (IOException e) {
 228                         log.error("processPMCsv is fail ",e);
 229                         return false;
 230                 }finally{
 231                         try{
 232                                 if (br != null)
 233                                         br.close();
 234                                 if (isr != null)
 235                                         isr.close();
 236                                 if (brs != null)
 237                                         brs.close();
 238                                 if(bos != null){
 239                                         bos.close();
 240                                 }
 241
 242                                 if(fos != null){
 243                                         fos.close();
 244                                 }
 245                         } catch (Exception e){
 246                                 log.error(e);
 247                         }
 248                 }
 249                 return true;
 250
 251         }
 252
 253         private boolean processCMXml(File tempfile, String nename, String type) {
 254
 255                 String csvpath = localPath+nename+"/"+type+"/";
 256                 File csvpathfile = new File(csvpath);
 257                 if(!csvpathfile.exists()){
 258                         csvpathfile.mkdirs();
 259                 }
 260                 String csvFileName = nename +dateFormat.format(new Date())+  System.nanoTime();
 261                 String csvpathAndFileName = csvpath+csvFileName+".csv";
 262                 BufferedOutputStream  bos = null;
 263                 FileOutputStream fos = null;
 264                 try {
 265                         fos = new FileOutputStream(csvpathAndFileName,false);
 266                         bos = new BufferedOutputStream(fos, 10240);
 267                 } catch (FileNotFoundException e1) {
 268                         log.error("FileNotFoundException "+StringUtil.getStackTrace(e1));
 269                 }
 270
 271                 boolean FieldNameFlag = false;
 272                 boolean FieldValueFlag = false;
 273                 //line num
 274                 int countNum = 0;
 275                 String xmlPathAndFileName = null;
 276                 String localName = null;
 277                 String endLocalName = null;
 278                 String rmUID = null;
 279                 int index = -1;
 280                 ArrayList<String> names = new ArrayList<String>();// colname
 281                 LinkedHashMap<String, String> nameAndValue = new LinkedHashMap<String, String>();
 282
 283
 284                 FileInputStream fis = null;
 285                 InputStreamReader isr = null;
 286                 XMLStreamReader reader = null;
 287                 try{
 288                         fis = new FileInputStream(tempfile);
 289                         isr = new InputStreamReader(fis, Constant.ENCODING_UTF8);
 290                         XMLInputFactory fac = XMLInputFactory.newInstance();
 291                         reader = fac.createXMLStreamReader(isr);
 292                         int event = -1;
 293                         boolean setcolum = true;
 294                         while (reader.hasNext()){
 295                                 try{
 296                                         event = reader.next();
 297                                         switch (event){
 298                                         case XMLStreamConstants.START_ELEMENT:
 299                                                 localName = reader.getLocalName();
 300                                                 if ("FieldName".equalsIgnoreCase(localName)){
 301                                                         FieldNameFlag = true;
 302                                                 }
 303                                                 if (FieldNameFlag){
 304                                                         if ("N".equalsIgnoreCase(localName)){
 305                                                                 String colName = reader.getElementText().trim();
 306                                                                 names.add(colName);
 307                                                         }
 308                                                 }
 309                                                 if ("FieldValue".equalsIgnoreCase(localName)){
 310                                                         FieldValueFlag = true;
 311
 312                                                 }
 313                                                 if (FieldValueFlag){
 314                                                         if(setcolum){
 315                                                                 xmlPathAndFileName = this.setColumnNames(nename, names,type);
 316                                                                 setcolum = false;
 317                                                         }
 318
 319                                                         if ("Object".equalsIgnoreCase(localName)){
 320                                                                 int ac = reader.getAttributeCount();
 321                                                                 for (int i = 0; i < ac; i++){
 322                                                                         if ("rmUID".equalsIgnoreCase(reader.getAttributeLocalName(i))){
 323                                                                                 rmUID = reader.getAttributeValue(i).trim();
 324                                                                         }
 325                                                                 }
 326                                                                 nameAndValue.put("rmUID", rmUID);
 327                                                         }
 328                                                         if ("V".equalsIgnoreCase(localName)) {
 329                                                                 index = Integer.parseInt(reader
 330                                                                                 .getAttributeValue(0)) - 1;
 331                                                                 String currentName = names.get(index);
 332                                                                 String v = reader.getElementText().trim();
 333                                                                 nameAndValue.put(currentName, v);
 334                                                         }
 335                                                 }
 336                                                 break;
 337                                         case XMLStreamConstants.CHARACTERS:
 338                                                 break;
 339                                         case XMLStreamConstants.END_ELEMENT:
 340                                                 endLocalName = reader.getLocalName();
 341
 342                                                 if ("FieldName".equalsIgnoreCase(endLocalName)){
 343                                                         FieldNameFlag = false;
 344                                                 }
 345                                                 if ("FieldValue".equalsIgnoreCase(endLocalName)){
 346                                                         FieldValueFlag = false;
 347                                                 }
 348                                                 if ("Object".equalsIgnoreCase(endLocalName)){
 349                                                         countNum ++;
 350                                                         this.appendLine(nameAndValue,bos);
 351                                                         nameAndValue.clear();
 352                                                 }
 353                                                 break;
 354                                         }
 355                                 } catch (Exception e)
 356                                 {
 357                                         log.error(""+StringUtil.getStackTrace(e));
 358                                         event = reader.next();
 359                                 }
 360                         }
 361
 362
 363                         if(bos != null){
 364                                 bos.close();
 365                                 bos = null;
 366                         }
 367                         if(fos != null){
 368                                 fos.close();
 369                                 fos = null;
 370                         }
 371
 372                         String[] fileKeys = this.createZipFile(csvpathAndFileName,xmlPathAndFileName,nename);
 373                         //ftp store
 374                         Properties ftpPro = configurationInterface.getProperties();
 375                         String ip = ftpPro.getProperty("ftp_ip");
 376                         String port = ftpPro.getProperty("ftp_port");
 377                         String ftp_user = ftpPro.getProperty("ftp_user");
 378                         String ftp_password = ftpPro.getProperty("ftp_password");
 379
 380                         String ftp_passive = ftpPro.getProperty("ftp_passive");
 381                         String ftp_type = ftpPro.getProperty("ftp_type");
 382                         String remoteFile = ftpPro.getProperty("ftp_remote_path");
 383                         this.ftpStore(fileKeys,ip,port,ftp_user,ftp_password,ftp_passive,ftp_type,remoteFile);
 384                         //create Message
 385                         String message = this.createMessage(fileKeys[1], ftp_user, ftp_password, ip,  port, countNum,nename);
 386
 387                         //set message
 388                         this.setMessage(message);
 389                 } catch (Exception e){
 390                         log.error(""+StringUtil.getStackTrace(e));
 391                         return false;
 392                 } finally{
 393                         try{
 394                                 if (reader != null){
 395                                         reader.close();
 396                                 }
 397                                 if (isr != null){
 398                                         isr.close();
 399                                 }
 400                                 if (fis != null){
 401                                         fis.close();
 402                                 }
 403                                 if(bos != null){
 404                                         bos.close();
 405                                 }
 406
 407                                 if(fos != null){
 408                                         fos.close();
 409                                 }
 410                         } catch (Exception e){
 411                                 log.error(e);
 412                         }
 413                 }
 414                 return true;
 415         }
 416
 417         private void setMessage(String message) {
 418
 419                 try {
 420                         collectResultChannel.put(message);
 421                 } catch (Exception e) {
 422                         log.error("collectResultChannel.put(message) is error "+StringUtil.getStackTrace(e));
 423                 }
 424         }
 425
 426         private String createMessage(String zipName,String user,String pwd,String ip, String port,int countNum, String nename) {
 427
 428                 StringBuffer strBuffer = new StringBuffer();
 429                 strBuffer
 430                                 .append("<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
 431                                                 + "<FILE_DATA_READY_UL xmlns:xsi=\" http://www.w3.org/2001/XMLSchema-instance\">"
 432                                                 + "<Header SessionID=\"");
 433                 strBuffer.append("");
 434                 strBuffer.append("\" LicenceID=\"");
 435                 strBuffer.append("");
 436                 strBuffer.append("\" SystemID=\"");
 437                 strBuffer.append("");
 438                 strBuffer.append("\" Time=\"");
 439                 strBuffer.append( dateFormat2.format(new Date()));
 440                 strBuffer.append("\" PolicyID=\"");
 441                 strBuffer.append("");
 442                 strBuffer.append("\"/><Body>");
 443                 strBuffer.append("<DataCatalog>");
 444                 strBuffer.append("");
 445                 strBuffer.append("</DataCatalog><GroupID>");
 446                 strBuffer.append(nename);
 447                 strBuffer.append("</GroupID><DataSourceName>");
 448                 strBuffer.append("");
 449                 strBuffer.append("</DataSourceName><InstanceID>");
 450                 strBuffer.append("");
 451                 strBuffer.append("</InstanceID><FileFormat>");
 452                 strBuffer.append("csv");
 453                 strBuffer.append("</FileFormat><CharSet>");
 454                 strBuffer.append("gbk");
 455                 strBuffer.append("</CharSet><FieldSeparator>");
 456                 strBuffer.append("|");
 457                 strBuffer.append("</FieldSeparator><IsCompressed>");
 458                 strBuffer.append("true");
 459                 strBuffer.append("</IsCompressed><StartTime>");
 460                 strBuffer.append(dateFormat2.format(new Date()));
 461                 strBuffer.append("</StartTime><EndTime>");
 462                 strBuffer.append("");
 463                 strBuffer.append("</EndTime><FileList>");
 464                 strBuffer.append(zipName);
 465                 strBuffer.append("</FileList><ConnectionString>");
 466                 strBuffer.append("ftp://" + user + ":" + pwd + "@" + ip + ":" + port);
 467                 strBuffer.append("</ConnectionString>");
 468                 strBuffer.append("<DataCount>");
 469                 strBuffer.append(countNum);
 470                 strBuffer.append("</DataCount>");
 471
 472                 strBuffer.append("<FileSize>").append("").append("</FileSize>");
 473                 strBuffer.append("<DataGranularity>").append("").append("</DataGranularity>");
 474
 475
 476                 strBuffer.append("</Body></FILE_DATA_READY_UL>");
 477                 return strBuffer.toString();
 478
 479         }
 480
 481         private void ftpStore(String[] fileKeys, String ip, String port, String ftp_user, String ftp_password,
 482                         String ftp_passive, String ftp_type, String remoteFile) {
 483                 String zipFilePath = fileKeys[0];
 484
 485
 486                 FTPInterface ftpClient;
 487                 if("ftp".equalsIgnoreCase(ftp_type)){
 488                          ftpClient = new FTPSrv();
 489                 }else{
 490                          ftpClient = new SFTPSrv();
 491                 }
 492
 493                 //login
 494                 try {
 495                         ftpClient.login(ip, Integer.parseInt(port), ftp_user, ftp_password, "GBK", Boolean.parseBoolean(ftp_passive), 5*60*1000);
 496                 } catch (Exception e) {
 497                         log.error("login fail,ip=["+ip+"] port=["+port+"] user=["+ftp_user+"]pwd=["+ftp_password+"]"+StringUtil.getStackTrace(e));
 498                     return;
 499                 }
 500                 ftpClient.store(zipFilePath, remoteFile);
 501                 log.debug("store  ["+zipFilePath+"]to["+remoteFile+"]");
 502
 503                 FileUtils.deleteQuietly(new File(zipFilePath));
 504
 505
 506         }
 507
 508         private String[] createZipFile(String csvpathAndFileName,String xmlPathAndFileName,String nename) {
 509
 510                 String zipPath = resultPath+nename +dateFormat.format(new Date())+"_"+System.nanoTime();
 511
 512                 File destDir = new File(zipPath);
 513                 destDir.mkdirs();
 514
 515                 try {
 516                         FileUtils.copyFileToDirectory(new File(csvpathAndFileName), destDir);
 517                         FileUtils.copyFileToDirectory(new File(xmlPathAndFileName), destDir);
 518                 } catch (IOException e) {
 519
 520                 }
 521
 522                 String destFilePath = zipPath + ".zip";
 523                 try {
 524                         Zip zip = new Zip(destDir.getAbsolutePath(), destFilePath);
 525                         zip.setCompressLevel(9);
 526                         zip.compress();
 527
 528                         FileUtils.deleteDirectory(destDir);
 529                 } catch (IOException e) {
 530                         log.error("zip.compress() is fail "+StringUtil.getStackTrace(e));
 531                 }
 532                 return new String[] { destFilePath, zipPath + ".zip"};
 533         }
 534
 535
 536         private String setColumnNames(String nename, List<String> names,String type) {
 537                 //write xml
 538                 String xmlpath = localPath+nename +"/"+type+"/";
 539                 File xmlpathfile = new File(xmlpath);
 540                 if(!xmlpathfile.exists()){
 541                         xmlpathfile.mkdirs();
 542                 }
 543                 String xmlFileName = nename +dateFormat.format(new Date())+ System.nanoTime();
 544                 String fieldLine = "";
 545                 for (int i = 0; i < names.size(); i++) {
 546                         String field = "\t<Field>\r\n" + "\t\t<FieldNo>" + i
 547                                         + "</FieldNo>\r\n" + "\t\t<FieldName>"
 548                                         + names.get(i) + "</FieldName>\r\n"
 549                                         + "\t\t<FieldType>" + names.get(i)
 550                                         + "</FieldType>\r\n"
 551                                         + "\t\t<FieldNameOther>" + names.get(i)
 552                                         + "</FieldNameOther>\r\n" +
 553                                         "\t</Field>\r\n";
 554                         fieldLine = fieldLine + field;
 555                 }
 556
 557                 String str = "<?xml version=\"1.0\" encoding=\"gbk\"?>\r\n"
 558                                 + "<xml>\r\n" + "<FILE_STRUCTURE>\r\n" + fieldLine
 559                                 + "</FILE_STRUCTURE>\r\n" + "</xml>\r\n";
 560                 String xmlPathAndFileName = xmlpath+xmlFileName+".xml";
 561                 try {
 562                         this.writeDetail(xmlPathAndFileName,str);
 563                 } catch (Exception e) {
 564                         log.error("writeDetail is fail ,xmlFileName="+xmlFileName +StringUtil.getStackTrace(e));
 565                 }
 566
 567                 return xmlPathAndFileName;
 568         }
 569
 570         private void writeDetail(String detailFileName,String str) throws Exception {
 571                 OutputStreamWriter writer = null;
 572                 OutputStream readOut = null;
 573                 try {
 574                         readOut = new FileOutputStream(new File(detailFileName), false);
 575                         writer = new OutputStreamWriter(readOut);
 576                         writer.write(str);
 577                         writer.flush();
 578                 } finally {
 579
 580                         if(null != writer){
 581                                 writer.close();
 582                         }
 583                         if(readOut != null){
 584                                 readOut.close();
 585                         }
 586
 587                 }
 588
 589         }
 590
 591
 592         private void appendLine(LinkedHashMap<String, String> nameAndValue,BufferedOutputStream  bos) {
 593                 StringBuilder lineDatas =  new StringBuilder();
 594
 595                 for (String key : nameAndValue.keySet()) {
 596                         lineDatas.append(nameAndValue.get(key)).append("|");
 597                 }
 598                 try {
 599                         bos.write(lineDatas.toString().getBytes());
 600                         bos.write("\n".getBytes());
 601                 } catch (IOException e) {
 602                         log.error("appendLine error "+StringUtil.getStackTrace(e));
 603                 }
 604         }
 605
 606         private void appendLine(List<String> values,BufferedOutputStream  bos) {
 607                 StringBuilder lineDatas =  new StringBuilder();
 608
 609                 for (String value : values) {
 610                         lineDatas.append(value).append("|");
 611                 }
 612                 try {
 613                         bos.write(lineDatas.toString().getBytes());
 614                         bos.write("\n".getBytes());
 615                 } catch (IOException e) {
 616                         log.error("appendLine error "+StringUtil.getStackTrace(e));
 617                 }
 618         }
 619
 620         public List<File> decompressed(String fileName){
 621             List<File> filelist = new ArrayList<File>();
 622
 623             if (fileName.indexOf(".gz") > 1)
 624             {
 625 //            decompressFile = deGz(file);
 626             } else if (fileName.indexOf(".zip") > 1)
 627             {
 628                 try {
 629                                 File[] files = deZip(new File(fileName));
 630                                 for(File temp :files){
 631                                         filelist.add(temp);
 632                                 }
 633                         } catch (Exception e) {
 634                                 log.error("decompressed is fail "+StringUtil.getStackTrace(e));
 635                         }
 636             }
 637             else {
 638                 filelist.add(new File(fileName));
 639             }
 640
 641             return filelist;
 642         }
 643
 644         public File[] deZip(File file) throws Exception{
 645
 646                 String regx = "(.*).zip";
 647               Pattern p = Pattern.compile(regx);
 648               Matcher m = p.matcher(file.getName());
 649               if (m.find())
 650               {
 651                 String orgFile = localPath + m.group(1) + "/";
 652                 UnZip unzip = new UnZip(file.getAbsolutePath(), orgFile);
 653             unzip.deCompress();
 654                 file = new File(orgFile);
 655               }
 656               File[] files = file.listFiles();
 657
 658               return files;
 659
 660         }
 661
 662         private List<String> ftpDownload(CollectVo collectVo) {
 663
 664                 List<String> fileList = new ArrayList<String>();
 665                 //IP
 666                 String ip = collectVo.getIP();
 667                 //port
 668                 String port = collectVo.getPort();
 669                 //user
 670                 String user = collectVo.getUser();
 671                 //password
 672                 String password = collectVo.getPassword();
 673                 //isPassiveMode
 674                 String passivemode = collectVo.getPassive();
 675
 676                 String ftpType = collectVo.getFtptype();
 677                 FTPInterface ftpClient = null;
 678                 if("ftp".equalsIgnoreCase(ftpType)){
 679                          ftpClient = new FTPSrv();
 680                 }else{
 681                          ftpClient = new SFTPSrv();
 682                 }
 683
 684                 //login
 685                 try {
 686                         ftpClient.login(ip, Integer.parseInt(port), user, password, "GBK", Boolean.parseBoolean(passivemode), 5*60*1000);
 687                 } catch (Exception e) {
 688                         log.error("login fail,ip=["+ip+"] port=["+port+"] user=["+user+"]password=["+password+"]"+StringUtil.getStackTrace(e));
 689                     return fileList;
 690                 }
 691
 692                 //download
 693                 String dir = collectVo.getRemotepath();
 694                 boolean cdsucess = ftpClient.chdir(dir);
 695                 if(cdsucess){
 696                         AFtpRemoteFile[] remoteFiles = (AFtpRemoteFile[]) ftpClient.list();
 697
 698                         for(AFtpRemoteFile ftpRemoteFile: remoteFiles){
 699                                 if(!new File(localPath).exists()){
 700                                         try {
 701                                                 new File(localPath).mkdir();
 702                                         } catch (Exception e) {
 703                                                 log.error("create localPath is fail localPath="+localPath+" "+StringUtil.getStackTrace(e));
 704                                         }
 705                                 }
 706
 707                                 if(!new File(localPath).exists()){
 708                                         new File(localPath).mkdirs();
 709                                 }
 710
 711                                 String localFileName = localPath + ftpRemoteFile.getFileName();
 712                                 File loaclFile = new File(localFileName);
 713                                 if (loaclFile.exists()) {
 714                                         loaclFile.delete();
 715                                 }
 716
 717                                 boolean flag = ftpClient.downloadFile(ftpRemoteFile.getAbsFileName(), localFileName);
 718
 719                                 if(flag){
 720                                         fileList.add(localFileName);
 721                                 }else{
 722                                         log.error("download file fail fileName="+ftpRemoteFile.getAbsFileName());
 723                                 }
 724                         }
 725
 726                 }else{
 727                         log.error("chdir is faill dir =["+dir+"]");
 728                 }
 729
 730                 return fileList;
 731         }
 732
 733
 734 }