gerrit.onap Code Review - sdc/sdc-distribution-client.git/blob

   1 ######################## BEGIN LICENSE BLOCK ########################
   2 # The Original Code is mozilla.org code.
   3 #
   4 # The Initial Developer of the Original Code is
   5 # Netscape Communications Corporation.
   6 # Portions created by the Initial Developer are Copyright (C) 1998
   7 # the Initial Developer. All Rights Reserved.
   8 #
   9 # Contributor(s):
  10 #   Mark Pilgrim - port to Python
  11 #
  12 # This library is free software; you can redistribute it and/or
  13 # modify it under the terms of the GNU Lesser General Public
  14 # License as published by the Free Software Foundation; either
  15 # version 2.1 of the License, or (at your option) any later version.
  16 #
  17 # This library is distributed in the hope that it will be useful,
  18 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  19 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  20 # Lesser General Public License for more details.
  21 #
  22 # You should have received a copy of the GNU Lesser General Public
  23 # License along with this library; if not, write to the Free Software
  24 # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
  25 # 02110-1301  USA
  26 ######################### END LICENSE BLOCK #########################
  27
  28 import sys
  29 from .mbcharsetprober import MultiByteCharSetProber
  30 from .codingstatemachine import CodingStateMachine
  31 from .chardistribution import SJISDistributionAnalysis
  32 from .jpcntx import SJISContextAnalysis
  33 from .mbcssm import SJISSMModel
  34 from . import constants
  35
  36
  37 class SJISProber(MultiByteCharSetProber):
  38     def __init__(self):
  39         MultiByteCharSetProber.__init__(self)
  40         self._mCodingSM = CodingStateMachine(SJISSMModel)
  41         self._mDistributionAnalyzer = SJISDistributionAnalysis()
  42         self._mContextAnalyzer = SJISContextAnalysis()
  43         self.reset()
  44
  45     def reset(self):
  46         MultiByteCharSetProber.reset(self)
  47         self._mContextAnalyzer.reset()
  48
  49     def get_charset_name(self):
  50         return "SHIFT_JIS"
  51
  52     def feed(self, aBuf):
  53         aLen = len(aBuf)
  54         for i in range(0, aLen):
  55             codingState = self._mCodingSM.next_state(aBuf[i])
  56             if codingState == constants.eError:
  57                 if constants._debug:
  58                     sys.stderr.write(self.get_charset_name()
  59                                      + ' prober hit error at byte ' + str(i)
  60                                      + '\n')
  61                 self._mState = constants.eNotMe
  62                 break
  63             elif codingState == constants.eItsMe:
  64                 self._mState = constants.eFoundIt
  65                 break
  66             elif codingState == constants.eStart:
  67                 charLen = self._mCodingSM.get_current_charlen()
  68                 if i == 0:
  69                     self._mLastChar[1] = aBuf[0]
  70                     self._mContextAnalyzer.feed(self._mLastChar[2 - charLen:],
  71                                                 charLen)
  72                     self._mDistributionAnalyzer.feed(self._mLastChar, charLen)
  73                 else:
  74                     self._mContextAnalyzer.feed(aBuf[i + 1 - charLen:i + 3
  75                                                      - charLen], charLen)
  76                     self._mDistributionAnalyzer.feed(aBuf[i - 1:i + 1],
  77                                                      charLen)
  78
  79         self._mLastChar[0] = aBuf[aLen - 1]
  80
  81         if self.get_state() == constants.eDetecting:
  82             if (self._mContextAnalyzer.got_enough_data() and
  83                (self.get_confidence() > constants.SHORTCUT_THRESHOLD)):
  84                 self._mState = constants.eFoundIt
  85
  86         return self.get_state()
  87
  88     def get_confidence(self):
  89         contxtCf = self._mContextAnalyzer.get_confidence()
  90         distribCf = self._mDistributionAnalyzer.get_confidence()
  91         return max(contxtCf, distribCf)