ems/boco/src/main/java/org/onap/vfc/nfvo/emsdriver/collector/TaskThread.java

   1 /**
   2  * Copyright 2017 BOCO Corporation.  CMCC Technologies Co., Ltd
   3  *
   4  * Licensed under the Apache License, Version 2.0 (the "License");
   5  * you may not use this file except in compliance with the License.
   6  * You may obtain a copy of the License at
   7  *
   8  *     http://www.apache.org/licenses/LICENSE-2.0
   9  *
  10  * Unless required by applicable law or agreed to in writing, software
  11  * distributed under the License is distributed on an "AS IS" BASIS,
  12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  * See the License for the specific language governing permissions and
  14  * limitations under the License.
  15  */
  16 package org.onap.vfc.nfvo.emsdriver.collector;
  17
  18 import java.io.BufferedOutputStream;
  19 import java.io.BufferedReader;
  20 import java.io.File;
  21 import java.io.FileInputStream;
  22 import java.io.FileNotFoundException;
  23 import java.io.FileOutputStream;
  24 import java.io.IOException;
  25 import java.io.InputStreamReader;
  26 import java.io.OutputStream;
  27 import java.io.OutputStreamWriter;
  28 import java.text.SimpleDateFormat;
  29 import java.util.ArrayList;
  30 import java.util.Date;
  31 import java.util.LinkedHashMap;
  32 import java.util.List;
  33 import java.util.Properties;
  34 import java.util.regex.Matcher;
  35 import java.util.regex.Pattern;
  36
  37 import javax.xml.stream.XMLInputFactory;
  38 import javax.xml.stream.XMLStreamConstants;
  39 import javax.xml.stream.XMLStreamReader;
  40
  41 import org.apache.commons.io.FileUtils;
  42 import org.apache.commons.logging.Log;
  43 import org.apache.commons.logging.LogFactory;
  44 import org.onap.vfc.nfvo.emsdriver.commons.constant.Constant;
  45 import org.onap.vfc.nfvo.emsdriver.commons.ftp.AFtpRemoteFile;
  46 import org.onap.vfc.nfvo.emsdriver.commons.ftp.FTPInterface;
  47 import org.onap.vfc.nfvo.emsdriver.commons.ftp.FTPSrv;
  48 import org.onap.vfc.nfvo.emsdriver.commons.ftp.SFTPSrv;
  49 import org.onap.vfc.nfvo.emsdriver.commons.model.CollectMsg;
  50 import org.onap.vfc.nfvo.emsdriver.commons.model.CollectVo;
  51 import org.onap.vfc.nfvo.emsdriver.commons.utils.Gunzip;
  52 import org.onap.vfc.nfvo.emsdriver.commons.utils.StringUtil;
  53 import org.onap.vfc.nfvo.emsdriver.commons.utils.UnZip;
  54 import org.onap.vfc.nfvo.emsdriver.commons.utils.Zip;
  55 import org.onap.vfc.nfvo.emsdriver.configmgr.ConfigurationImp;
  56 import org.onap.vfc.nfvo.emsdriver.configmgr.ConfigurationInterface;
  57 import org.onap.vfc.nfvo.emsdriver.messagemgr.MessageChannel;
  58 import org.onap.vfc.nfvo.emsdriver.messagemgr.MessageChannelFactory;
  59
  60
  61 public class TaskThread implements Runnable{
  62
  63         public  Log log = LogFactory.getLog(TaskThread.class);
  64
  65         private MessageChannel collectResultChannel;
  66
  67         private CollectMsg data;
  68
  69         private ConfigurationInterface configurationInterface = new ConfigurationImp();
  70
  71         private String localPath = Constant.SYS_DATA_TEMP;
  72         private String resultPath = Constant.SYS_DATA_RESULT;
  73
  74         private SimpleDateFormat dateFormat = new SimpleDateFormat("yyyyMMddHHmmss");
  75
  76         private SimpleDateFormat dateFormat2 = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
  77
  78         private String csvpathAndFileName;
  79         private String xmlPathAndFileName;
  80         private int countNum = 0 ;
  81
  82         public TaskThread(CollectMsg data) {
  83                 this.data = data;
  84         }
  85         public TaskThread() {
  86                 super();
  87         }
  88
  89         @Override
  90         public void run(){
  91
  92                 collectResultChannel = MessageChannelFactory.getMessageChannel(Constant.COLLECT_RESULT_CHANNEL_KEY);
  93
  94                 try {
  95                         collectMsgHandle(data);
  96                 } catch (Exception e) {
  97                         log.error("",e);
  98                 }
  99         }
 100
 101         private void collectMsgHandle(CollectMsg collectMsg) {
 102                 String emsName = collectMsg.getEmsName();
 103                 String type = collectMsg.getType();
 104                 CollectVo collectVo = configurationInterface.getCollectVoByEmsNameAndType(emsName, type);
 105
 106                 //ftp download
 107                 List<String> downloadfiles = this.ftpDownload(collectVo);
 108                 //paser ftp update message send
 109                 for(String fileName :downloadfiles){
 110                         this.parseFtpAndSendMessage(fileName,collectVo);
 111                 }
 112         }
 113
 114         private void parseFtpAndSendMessage(String fileName, CollectVo collectVo) {
 115                 //
 116                 List<File> filelist = decompressed(fileName);
 117
 118                 for (File tempfile : filelist) {
 119
 120                         String unfileName = tempfile.getName();
 121
 122                         Pattern pa = Pattern.compile(".*-(.*)-\\w{2}-");
 123                         Matcher ma = pa.matcher(unfileName);
 124                         if (!ma.find())
 125                           continue;
 126                         String nename = ma.group(1);
 127                         boolean parseResult = false;
 128                         if("CM".equalsIgnoreCase(collectVo.getType())){
 129                                 parseResult = processCMXml(tempfile, nename,"CM");
 130                         }else{
 131                                 parseResult = processPMCsv(tempfile, nename,"PM");
 132
 133                                 //createzipFile
 134                                 String[] fileKeys = this.createZipFile(csvpathAndFileName,xmlPathAndFileName,nename);
 135                                 //ftp store
 136                                 Properties ftpPro = configurationInterface.getProperties();
 137                                 String ip = ftpPro.getProperty("ftp_ip");
 138                                 String port = ftpPro.getProperty("ftp_port");
 139                                 String ftp_user = ftpPro.getProperty("ftp_user");
 140                                 String ftp_password = ftpPro.getProperty("ftp_password");
 141
 142                                 String ftp_passive = ftpPro.getProperty("ftp_passive");
 143                                 String ftp_type = ftpPro.getProperty("ftp_type");
 144                                 String remoteFile = ftpPro.getProperty("ftp_remote_path");
 145                                 this.ftpStore(fileKeys,ip,port,ftp_user,ftp_password,ftp_passive,ftp_type,remoteFile);
 146                                 //create Message
 147                                 String message = this.createMessage(fileKeys[1], ftp_user, ftp_password, ip,  port, countNum,nename);
 148
 149                                 //set message
 150                                 this.setMessage(message);
 151                         }
 152
 153                         if (parseResult){
 154                                 log.info("parser "+tempfile+" sucess");
 155                         }else {
 156                                 log.info("parser "+tempfile+" fail");
 157                         }
 158
 159                 }
 160         }
 161
 162         public boolean processPMCsv(File tempfile, String nename,String type) {
 163
 164                 String csvpath = localPath+nename+"/"+type+"/";
 165                 File csvpathfile = new File(csvpath);
 166                 if(!csvpathfile.exists()){
 167                         csvpathfile.mkdirs();
 168                 }
 169                 String csvFileName = nename +dateFormat.format(new Date())+  System.nanoTime();
 170                 csvpathAndFileName = csvpath+csvFileName+".csv";
 171                 BufferedOutputStream  bos = null;
 172                 FileOutputStream fos = null;
 173                 try {
 174                         fos = new FileOutputStream(csvpathAndFileName,false);
 175                         bos = new BufferedOutputStream(fos, 10240);
 176                 } catch (FileNotFoundException e1) {
 177                         log.error("FileNotFoundException "+StringUtil.getStackTrace(e1));
 178                 }
 179
 180                 FileInputStream brs = null;
 181                 InputStreamReader isr = null;
 182                 BufferedReader br = null;
 183
 184                 List<String> columnNames = new ArrayList<String>();
 185                 List<String> commonValues = new ArrayList<String>();
 186                 try {
 187
 188                         brs = new FileInputStream(tempfile);
 189                         isr = new InputStreamReader(brs, Constant.ENCODING_UTF8);
 190                         br = new BufferedReader(isr);
 191                         //common field
 192                         String commonField = br.readLine();
 193                         String[] fields = commonField.split("\\|",-1);
 194                         for(String com : fields){
 195                                 String[] comNameAndValue = com.split("=",2);
 196                                 columnNames.add(comNameAndValue[0].trim());
 197                                 commonValues.add(comNameAndValue[1]);
 198                         }
 199                         //column names
 200                         String columnName = br.readLine();
 201                         String[] names = columnName.split("\\|",-1);
 202                         for(String name : names){
 203                                 columnNames.add(name);
 204                         }
 205
 206                         xmlPathAndFileName = this.setColumnNames(nename, columnNames,type);
 207
 208                         String valueLine = "";
 209                         List<String> valuelist = new ArrayList<String>();
 210
 211                         while (br.readLine() != null) {
 212                                 if (valueLine.trim().equals("")) {
 213                                         continue;
 214                                 }
 215                                 countNum ++;
 216                                 String [] values = valueLine.split("\\|",-1);
 217
 218                                 valuelist.addAll(commonValues);
 219                                 for(String value : values){
 220                                         valuelist.add(value);
 221                                 }
 222                                 this.appendLine(valuelist, bos);
 223
 224                                 valuelist.clear();
 225                         }
 226                 } catch (IOException e) {
 227                         log.error("processPMCsv is fail ",e);
 228                         return false;
 229                 }finally{
 230                         try{
 231                                 if (br != null)
 232                                         br.close();
 233                                 if (isr != null)
 234                                         isr.close();
 235                                 if (brs != null)
 236                                         brs.close();
 237                                 if(bos != null){
 238                                         bos.close();
 239                                 }
 240                                 if(fos != null){
 241                                         fos.close();
 242                                 }
 243                         } catch (Exception e){
 244                                 log.error(e);
 245                         }
 246                 }
 247                 return true;
 248
 249         }
 250
 251         private boolean processCMXml(File tempfile, String nename, String type) {
 252
 253                 String csvpath = localPath+nename+"/"+type+"/";
 254                 File csvpathfile = new File(csvpath);
 255                 if(!csvpathfile.exists()){
 256                         csvpathfile.mkdirs();
 257                 }
 258                 String csvFileName = nename +dateFormat.format(new Date())+  System.nanoTime();
 259                 String csvpathAndFileName = csvpath+csvFileName+".csv";
 260                 BufferedOutputStream  bos = null;
 261                 FileOutputStream fos = null;
 262                 try {
 263                         fos = new FileOutputStream(csvpathAndFileName,false);
 264                         bos = new BufferedOutputStream(fos, 10240);
 265                 } catch (FileNotFoundException e1) {
 266                         log.error("FileNotFoundException "+StringUtil.getStackTrace(e1));
 267                 }
 268
 269                 boolean FieldNameFlag = false;
 270                 boolean FieldValueFlag = false;
 271                 //line num
 272                 int countNum = 0;
 273                 String xmlPathAndFileName = null;
 274                 String localName = null;
 275                 String endLocalName = null;
 276                 String rmUID = null;
 277                 int index = -1;
 278                 ArrayList<String> names = new ArrayList<String>();// colname
 279                 LinkedHashMap<String, String> nameAndValue = new LinkedHashMap<String, String>();
 280
 281
 282                 FileInputStream fis = null;
 283                 InputStreamReader isr = null;
 284                 XMLStreamReader reader = null;
 285                 try{
 286                         fis = new FileInputStream(tempfile);
 287                         isr = new InputStreamReader(fis, Constant.ENCODING_UTF8);
 288                         XMLInputFactory fac = XMLInputFactory.newInstance();
 289                         reader = fac.createXMLStreamReader(isr);
 290                         int event = -1;
 291                         boolean setcolum = true;
 292                         while (reader.hasNext()){
 293                                 try{
 294                                         event = reader.next();
 295                                         switch (event){
 296                                         case XMLStreamConstants.START_ELEMENT:
 297                                                 localName = reader.getLocalName();
 298                                                 if ("FieldName".equalsIgnoreCase(localName)){
 299                                                         FieldNameFlag = true;
 300                                                 }
 301                                                 if (FieldNameFlag){
 302                                                         if ("N".equalsIgnoreCase(localName)){
 303                                                                 String colName = reader.getElementText().trim();
 304                                                                 names.add(colName);
 305                                                         }
 306                                                 }
 307                                                 if ("FieldValue".equalsIgnoreCase(localName)){
 308                                                         FieldValueFlag = true;
 309
 310                                                 }
 311                                                 if (FieldValueFlag){
 312                                                         if(setcolum){
 313                                                                 xmlPathAndFileName = this.setColumnNames(nename, names,type);
 314                                                                 setcolum = false;
 315                                                         }
 316
 317                                                         if ("Object".equalsIgnoreCase(localName)){
 318                                                                 int ac = reader.getAttributeCount();
 319                                                                 for (int i = 0; i < ac; i++){
 320                                                                         if ("rmUID".equalsIgnoreCase(reader.getAttributeLocalName(i))){
 321                                                                                 rmUID = reader.getAttributeValue(i).trim();
 322                                                                         }
 323                                                                 }
 324                                                                 nameAndValue.put("rmUID", rmUID);
 325                                                         }
 326                                                         if ("V".equalsIgnoreCase(localName)) {
 327                                                                 index = Integer.parseInt(reader
 328                                                                                 .getAttributeValue(0)) - 1;
 329                                                                 String currentName = names.get(index);
 330                                                                 String v = reader.getElementText().trim();
 331                                                                 nameAndValue.put(currentName, v);
 332                                                         }
 333                                                 }
 334                                                 break;
 335                                         case XMLStreamConstants.CHARACTERS:
 336                                                 break;
 337                                         case XMLStreamConstants.END_ELEMENT:
 338                                                 endLocalName = reader.getLocalName();
 339
 340                                                 if ("FieldName".equalsIgnoreCase(endLocalName)){
 341                                                         FieldNameFlag = false;
 342                                                 }
 343                                                 if ("FieldValue".equalsIgnoreCase(endLocalName)){
 344                                                         FieldValueFlag = false;
 345                                                 }
 346                                                 if ("Object".equalsIgnoreCase(endLocalName)){
 347                                                         countNum ++;
 348                                                         this.appendLine(nameAndValue,bos);
 349                                                         nameAndValue.clear();
 350                                                 }
 351                                                 break;
 352                                         }
 353                                 } catch (Exception e)
 354                                 {
 355                                         log.error(""+StringUtil.getStackTrace(e));
 356                                         event = reader.next();
 357                                 }
 358                         }
 359
 360
 361                         if(bos != null){
 362                                 bos.close();
 363                                 bos = null;
 364                         }
 365                         if(fos != null){
 366                                 fos.close();
 367                                 fos = null;
 368                         }
 369
 370                         String[] fileKeys = this.createZipFile(csvpathAndFileName,xmlPathAndFileName,nename);
 371                         //ftp store
 372                         Properties ftpPro = configurationInterface.getProperties();
 373                         String ip = ftpPro.getProperty("ftp_ip");
 374                         String port = ftpPro.getProperty("ftp_port");
 375                         String ftp_user = ftpPro.getProperty("ftp_user");
 376                         String ftp_password = ftpPro.getProperty("ftp_password");
 377
 378                         String ftp_passive = ftpPro.getProperty("ftp_passive");
 379                         String ftp_type = ftpPro.getProperty("ftp_type");
 380                         String remoteFile = ftpPro.getProperty("ftp_remote_path");
 381                         this.ftpStore(fileKeys,ip,port,ftp_user,ftp_password,ftp_passive,ftp_type,remoteFile);
 382                         //create Message
 383                         String message = this.createMessage(fileKeys[1], ftp_user, ftp_password, ip,  port, countNum,nename);
 384
 385                         //set message
 386                         this.setMessage(message);
 387                 } catch (Exception e){
 388                         log.error(""+StringUtil.getStackTrace(e));
 389                         return false;
 390                 } finally{
 391                         try{
 392                                 if (reader != null){
 393                                         reader.close();
 394                                 }
 395                                 if (isr != null){
 396                                         isr.close();
 397                                 }
 398                                 if (fis != null){
 399                                         fis.close();
 400                                 }
 401                                 if(bos != null){
 402                                         bos.close();
 403                                 }
 404
 405                                 if(fos != null){
 406                                         fos.close();
 407                                 }
 408                         } catch (Exception e){
 409                                 log.error(e);
 410                         }
 411                 }
 412                 return true;
 413         }
 414
 415         private void setMessage(String message) {
 416
 417                 try {
 418                         collectResultChannel.put(message);
 419                 } catch (Exception e) {
 420                         log.error("collectResultChannel.put(message) is error "+StringUtil.getStackTrace(e));
 421                 }
 422         }
 423
 424         private String createMessage(String zipName,String user,String pwd,String ip, String port,int countNum, String nename) {
 425
 426                 StringBuffer strBuffer = new StringBuffer();
 427                 strBuffer
 428                                 .append("<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
 429                                                 + "<FILE_DATA_READY_UL xmlns:xsi=\" http://www.w3.org/2001/XMLSchema-instance\">"
 430                                                 + "<Header SessionID=\"");
 431                 strBuffer.append("");
 432                 strBuffer.append("\" LicenceID=\"");
 433                 strBuffer.append("");
 434                 strBuffer.append("\" SystemID=\"");
 435                 strBuffer.append("");
 436                 strBuffer.append("\" Time=\"");
 437                 strBuffer.append( dateFormat2.format(new Date()));
 438                 strBuffer.append("\" PolicyID=\"");
 439                 strBuffer.append("");
 440                 strBuffer.append("\"/><Body>");
 441                 strBuffer.append("<DataCatalog>");
 442                 strBuffer.append("");
 443                 strBuffer.append("</DataCatalog><GroupID>");
 444                 strBuffer.append(nename);
 445                 strBuffer.append("</GroupID><DataSourceName>");
 446                 strBuffer.append("");
 447                 strBuffer.append("</DataSourceName><InstanceID>");
 448                 strBuffer.append("");
 449                 strBuffer.append("</InstanceID><FileFormat>");
 450                 strBuffer.append("csv");
 451                 strBuffer.append("</FileFormat><CharSet>");
 452                 strBuffer.append("gbk");
 453                 strBuffer.append("</CharSet><FieldSeparator>");
 454                 strBuffer.append("|");
 455                 strBuffer.append("</FieldSeparator><IsCompressed>");
 456                 strBuffer.append("true");
 457                 strBuffer.append("</IsCompressed><StartTime>");
 458                 strBuffer.append(dateFormat2.format(new Date()));
 459                 strBuffer.append("</StartTime><EndTime>");
 460                 strBuffer.append("");
 461                 strBuffer.append("</EndTime><FileList>");
 462                 strBuffer.append(zipName);
 463                 strBuffer.append("</FileList><ConnectionString>");
 464                 strBuffer.append("ftp://" + user + ":" + pwd + "@" + ip + ":" + port);
 465                 strBuffer.append("</ConnectionString>");
 466                 strBuffer.append("<DataCount>");
 467                 strBuffer.append(countNum);
 468                 strBuffer.append("</DataCount>");
 469
 470                 strBuffer.append("<FileSize>").append("").append("</FileSize>");
 471                 strBuffer.append("<DataGranularity>").append("").append("</DataGranularity>");
 472
 473
 474                 strBuffer.append("</Body></FILE_DATA_READY_UL>");
 475                 return strBuffer.toString();
 476
 477         }
 478
 479         private void ftpStore(String[] fileKeys, String ip, String port, String ftp_user, String ftp_password,
 480                         String ftp_passive, String ftp_type, String remoteFile) {
 481                 String zipFilePath = fileKeys[0];
 482
 483
 484                 FTPInterface ftpClient;
 485                 if("ftp".equalsIgnoreCase(ftp_type)){
 486                          ftpClient = new FTPSrv();
 487                 }else{
 488                          ftpClient = new SFTPSrv();
 489                 }
 490
 491                 //login
 492                 try {
 493                         ftpClient.login(ip, Integer.parseInt(port), ftp_user, ftp_password, "GBK", Boolean.parseBoolean(ftp_passive), 5*60*1000);
 494                 } catch (Exception e) {
 495                         log.error("login fail,ip=["+ip+"] port=["+port+"] user=["+ftp_user+"]pwd=["+ftp_password+"]"+StringUtil.getStackTrace(e));
 496                     return;
 497                 }
 498                 ftpClient.store(zipFilePath, remoteFile);
 499                 log.debug("store  ["+zipFilePath+"]to["+remoteFile+"]");
 500
 501                 FileUtils.deleteQuietly(new File(zipFilePath));
 502
 503
 504         }
 505
 506         private String[] createZipFile(String csvpathAndFileName,String xmlPathAndFileName,String nename) {
 507
 508                 String zipPath = resultPath+nename +dateFormat.format(new Date())+"_"+System.nanoTime();
 509
 510                 File destDir = new File(zipPath);
 511                 destDir.mkdirs();
 512
 513                 try {
 514                         FileUtils.copyFileToDirectory(new File(csvpathAndFileName), destDir);
 515                         FileUtils.copyFileToDirectory(new File(xmlPathAndFileName), destDir);
 516                 } catch (IOException e) {
 517
 518                 }
 519
 520                 String destFilePath = zipPath + ".zip";
 521                 try {
 522                         Zip zip = new Zip(destDir.getAbsolutePath(), destFilePath);
 523                         zip.setCompressLevel(9);
 524                         zip.compress();
 525
 526                         FileUtils.deleteDirectory(destDir);
 527                 } catch (IOException e) {
 528                         log.error("zip.compress() is fail "+StringUtil.getStackTrace(e));
 529                 }
 530                 return new String[] { destFilePath, zipPath + ".zip"};
 531         }
 532
 533
 534         private String setColumnNames(String nename, List<String> names,String type) {
 535                 //write xml
 536                 String xmlpath = localPath+nename +"/"+type+"/";
 537                 File xmlpathfile = new File(xmlpath);
 538                 if(!xmlpathfile.exists()){
 539                         xmlpathfile.mkdirs();
 540                 }
 541                 String xmlFileName = nename +dateFormat.format(new Date())+ System.nanoTime();
 542                 String fieldLine = "";
 543                 for (int i = 0; i < names.size(); i++) {
 544                         String field = "\t<Field>\r\n" + "\t\t<FieldNo>" + i
 545                                         + "</FieldNo>\r\n" + "\t\t<FieldName>"
 546                                         + names.get(i) + "</FieldName>\r\n"
 547                                         + "\t\t<FieldType>" + names.get(i)
 548                                         + "</FieldType>\r\n"
 549                                         + "\t\t<FieldNameOther>" + names.get(i)
 550                                         + "</FieldNameOther>\r\n" +
 551                                         "\t</Field>\r\n";
 552                         fieldLine = fieldLine + field;
 553                 }
 554
 555                 String str = "<?xml version=\"1.0\" encoding=\"gbk\"?>\r\n"
 556                                 + "<xml>\r\n" + "<FILE_STRUCTURE>\r\n" + fieldLine
 557                                 + "</FILE_STRUCTURE>\r\n" + "</xml>\r\n";
 558                 String xmlPathAndFileName = xmlpath+xmlFileName+".xml";
 559                 try {
 560                         this.writeDetail(xmlPathAndFileName,str);
 561                 } catch (Exception e) {
 562                         log.error("writeDetail is fail ,xmlFileName="+xmlFileName +StringUtil.getStackTrace(e));
 563                 }
 564
 565                 return xmlPathAndFileName;
 566         }
 567
 568         private void writeDetail(String detailFileName,String str) throws Exception {
 569                 OutputStreamWriter writer = null;
 570                 OutputStream readOut = null;
 571                 try {
 572                         readOut = new FileOutputStream(new File(detailFileName), false);
 573                         writer = new OutputStreamWriter(readOut);
 574                         writer.write(str);
 575                         writer.flush();
 576                 } finally {
 577
 578                         if(null != writer){
 579                                 writer.close();
 580                         }
 581                         if(readOut != null){
 582                                 readOut.close();
 583                         }
 584
 585                 }
 586
 587         }
 588
 589
 590         private void appendLine(LinkedHashMap<String, String> nameAndValue,BufferedOutputStream  bos) {
 591                 StringBuilder lineDatas =  new StringBuilder();
 592
 593                 for (String key : nameAndValue.keySet()) {
 594                         lineDatas.append(nameAndValue.get(key)).append("|");
 595                 }
 596                 try {
 597                         bos.write(lineDatas.toString().getBytes());
 598                         bos.write("\n".getBytes());
 599                 } catch (IOException e) {
 600                         log.error("appendLine error "+StringUtil.getStackTrace(e));
 601                 }
 602         }
 603
 604         private void appendLine(List<String> values,BufferedOutputStream  bos) {
 605                 StringBuilder lineDatas =  new StringBuilder();
 606
 607                 for (String value : values) {
 608                         lineDatas.append(value).append("|");
 609                 }
 610                 try {
 611                         bos.write(lineDatas.toString().getBytes());
 612                         bos.write("\n".getBytes());
 613                 } catch (IOException e) {
 614                         log.error("appendLine error "+StringUtil.getStackTrace(e));
 615                 }
 616         }
 617
 618         public List<File> decompressed(String fileName){
 619             List<File> filelist = new ArrayList<File>();
 620
 621             if (fileName.indexOf(".gz") > 1)
 622             {
 623                 try {
 624                                 File decompressFile = deGz(fileName);
 625                                 filelist.add(decompressFile);
 626                         } catch (IOException e) {
 627                                 log.error("decompressed is fail "+StringUtil.getStackTrace(e));
 628                         }
 629             } else if (fileName.indexOf(".zip") > 1)
 630             {
 631                 try {
 632                                 File[] files = deZip(new File(fileName));
 633                                 for(File temp :files){
 634                                         filelist.add(temp);
 635                                 }
 636                         } catch (Exception e) {
 637                                 log.error("decompressed is fail "+StringUtil.getStackTrace(e));
 638                         }
 639             }
 640             else {
 641                 filelist.add(new File(fileName));
 642             }
 643
 644             return filelist;
 645         }
 646
 647         private File deGz(String gzFileName) throws IOException {
 648                 Gunzip gunzip = new Gunzip();
 649                 String orgFile = gzFileName.replace(".gz", "");
 650                 gunzip.unCompress(gzFileName, orgFile);
 651                 return new File(orgFile);
 652         }
 653
 654         public File[] deZip(File file) throws Exception{
 655
 656                 String regx = "(.*).zip";
 657               Pattern p = Pattern.compile(regx);
 658               Matcher m = p.matcher(file.getName());
 659               if (m.find())
 660               {
 661                 String orgFile = localPath + m.group(1) + "/";
 662                 UnZip unzip = new UnZip(file.getAbsolutePath(), orgFile);
 663             unzip.deCompress();
 664                 file = new File(orgFile);
 665               }
 666               File[] files = file.listFiles();
 667
 668               return files;
 669
 670         }
 671
 672         private List<String> ftpDownload(CollectVo collectVo) {
 673
 674                 List<String> fileList = new ArrayList<String>();
 675                 //IP
 676                 String ip = collectVo.getIP();
 677                 //port
 678                 String port = collectVo.getPort();
 679                 //user
 680                 String user = collectVo.getUser();
 681                 //password
 682                 String password = collectVo.getPassword();
 683                 //isPassiveMode
 684                 String passivemode = collectVo.getPassive();
 685
 686                 String ftpType = collectVo.getFtptype();
 687                 FTPInterface ftpClient = null;
 688                 if("ftp".equalsIgnoreCase(ftpType)){
 689                          ftpClient = new FTPSrv();
 690                 }else{
 691                          ftpClient = new SFTPSrv();
 692                 }
 693
 694                 //login
 695                 try {
 696                         ftpClient.login(ip, Integer.parseInt(port), user, password, "GBK", Boolean.parseBoolean(passivemode), 5*60*1000);
 697                 } catch (Exception e) {
 698                         log.error("login fail,ip=["+ip+"] port=["+port+"] user=["+user+"]password=["+password+"]"+StringUtil.getStackTrace(e));
 699                     return fileList;
 700                 }
 701
 702                 //download
 703                 String dir = collectVo.getRemotepath();
 704                 boolean cdsucess = ftpClient.chdir(dir);
 705                 if(cdsucess){
 706                         AFtpRemoteFile[] remoteFiles = (AFtpRemoteFile[]) ftpClient.list();
 707
 708                         for(AFtpRemoteFile ftpRemoteFile: remoteFiles){
 709                                 if(!new File(localPath).exists()){
 710                                         try {
 711                                                 new File(localPath).mkdir();
 712                                         } catch (Exception e) {
 713                                                 log.error("create localPath is fail localPath="+localPath+" "+StringUtil.getStackTrace(e));
 714                                         }
 715                                 }
 716
 717                                 if(!new File(localPath).exists()){
 718                                         new File(localPath).mkdirs();
 719                                 }
 720
 721                                 String localFileName = localPath + ftpRemoteFile.getFileName();
 722                                 File loaclFile = new File(localFileName);
 723                                 if (loaclFile.exists()) {
 724                                         loaclFile.delete();
 725                                 }
 726
 727                                 boolean flag = ftpClient.downloadFile(ftpRemoteFile.getAbsFileName(), localFileName);
 728
 729                                 if(flag){
 730                                         fileList.add(localFileName);
 731                                 }else{
 732                                         log.error("download file fail fileName="+ftpRemoteFile.getAbsFileName());
 733                                 }
 734                         }
 735
 736                 }else{
 737                         log.error("chdir is faill dir =["+dir+"]");
 738                 }
 739
 740                 return fileList;
 741         }
 742
 743
 744 }