XMAN Replicator

Path: /opt/alt/python312/lib/python3.12/site-packages/pip/_vendor/chardet/

Name	Size	Rights	Date	Actions
[.]	<DIR>	drwxr-xr-x	2026-02-10 06:06	🔄 🗑️ 🗑️📁 ⬇️ 🔒
[..]	<DIR>	drwxr-xr-x	2026-02-10 06:06	🔄 🗑️ 🗑️📁 ⬇️ 🔒
__init__.py	4,797 B	-rw-r--r--	2026-01-08 19:19	✏️ 🔄 🗑️ ⬇️ 🔒
[__pycache__]	<DIR>	drwxr-xr-x	2026-02-10 06:06	🔄 🗑️ 🗑️📁 ⬇️ 🔒
big5freq.py	31,274 B	-rw-r--r--	2026-01-08 19:19	✏️ 🔄 🗑️ ⬇️ 🔒
big5prober.py	1,763 B	-rw-r--r--	2026-01-08 19:19	✏️ 🔄 🗑️ ⬇️ 🔒
chardistribution.py	10,032 B	-rw-r--r--	2026-01-08 19:19	✏️ 🔄 🗑️ ⬇️ 🔒
charsetgroupprober.py	3,915 B	-rw-r--r--	2026-01-08 19:19	✏️ 🔄 🗑️ ⬇️ 🔒
charsetprober.py	5,420 B	-rw-r--r--	2026-01-08 19:19	✏️ 🔄 🗑️ ⬇️ 🔒
[cli]	<DIR>	drwxr-xr-x	2026-02-10 06:06	🔄 🗑️ 🗑️📁 ⬇️ 🔒
codingstatemachine.py	3,732 B	-rw-r--r--	2026-01-08 19:19	✏️ 🔄 🗑️ ⬇️ 🔒
codingstatemachinedict.py	542 B	-rw-r--r--	2026-01-08 19:19	✏️ 🔄 🗑️ ⬇️ 🔒
cp949prober.py	1,860 B	-rw-r--r--	2026-01-08 19:19	✏️ 🔄 🗑️ ⬇️ 🔒
enums.py	1,683 B	-rw-r--r--	2026-01-08 19:19	✏️ 🔄 🗑️ ⬇️ 🔒
escprober.py	4,006 B	-rw-r--r--	2026-01-08 19:19	✏️ 🔄 🗑️ ⬇️ 🔒
escsm.py	12,176 B	-rw-r--r--	2026-01-08 19:19	✏️ 🔄 🗑️ ⬇️ 🔒
eucjpprober.py	3,934 B	-rw-r--r--	2026-01-08 19:19	✏️ 🔄 🗑️ ⬇️ 🔒
euckrfreq.py	13,566 B	-rw-r--r--	2026-01-08 19:19	✏️ 🔄 🗑️ ⬇️ 🔒
euckrprober.py	1,753 B	-rw-r--r--	2026-01-08 19:19	✏️ 🔄 🗑️ ⬇️ 🔒
euctwfreq.py	36,913 B	-rw-r--r--	2026-01-08 19:19	✏️ 🔄 🗑️ ⬇️ 🔒
euctwprober.py	1,753 B	-rw-r--r--	2026-01-08 19:19	✏️ 🔄 🗑️ ⬇️ 🔒
gb2312freq.py	20,735 B	-rw-r--r--	2026-01-08 19:19	✏️ 🔄 🗑️ ⬇️ 🔒
gb2312prober.py	1,759 B	-rw-r--r--	2026-01-08 19:19	✏️ 🔄 🗑️ ⬇️ 🔒
hebrewprober.py	14,537 B	-rw-r--r--	2026-01-08 19:19	✏️ 🔄 🗑️ ⬇️ 🔒
jisfreq.py	25,796 B	-rw-r--r--	2026-01-08 19:19	✏️ 🔄 🗑️ ⬇️ 🔒
johabfreq.py	42,498 B	-rw-r--r--	2026-01-08 19:19	✏️ 🔄 🗑️ ⬇️ 🔒
johabprober.py	1,752 B	-rw-r--r--	2026-01-08 19:19	✏️ 🔄 🗑️ ⬇️ 🔒
jpcntx.py	27,055 B	-rw-r--r--	2026-01-08 19:19	✏️ 🔄 🗑️ ⬇️ 🔒
langbulgarianmodel.py	104,562 B	-rw-r--r--	2026-01-08 19:19	✏️ 🔄 🗑️ ⬇️ 🔒
langgreekmodel.py	98,484 B	-rw-r--r--	2026-01-08 19:19	✏️ 🔄 🗑️ ⬇️ 🔒
langhebrewmodel.py	98,196 B	-rw-r--r--	2026-01-08 19:19	✏️ 🔄 🗑️ ⬇️ 🔒
langhungarianmodel.py	101,363 B	-rw-r--r--	2026-01-08 19:19	✏️ 🔄 🗑️ ⬇️ 🔒
langrussianmodel.py	128,035 B	-rw-r--r--	2026-01-08 19:19	✏️ 🔄 🗑️ ⬇️ 🔒
langthaimodel.py	102,774 B	-rw-r--r--	2026-01-08 19:19	✏️ 🔄 🗑️ ⬇️ 🔒
langturkishmodel.py	95,372 B	-rw-r--r--	2026-01-08 19:19	✏️ 🔄 🗑️ ⬇️ 🔒
latin1prober.py	5,380 B	-rw-r--r--	2026-01-08 19:19	✏️ 🔄 🗑️ ⬇️ 🔒
macromanprober.py	6,077 B	-rw-r--r--	2026-01-08 19:19	✏️ 🔄 🗑️ ⬇️ 🔒
mbcharsetprober.py	3,715 B	-rw-r--r--	2026-01-08 19:19	✏️ 🔄 🗑️ ⬇️ 🔒
mbcsgroupprober.py	2,131 B	-rw-r--r--	2026-01-08 19:19	✏️ 🔄 🗑️ ⬇️ 🔒
mbcssm.py	30,391 B	-rw-r--r--	2026-01-08 19:19	✏️ 🔄 🗑️ ⬇️ 🔒
[metadata]	<DIR>	drwxr-xr-x	2026-02-10 06:06	🔄 🗑️ 🗑️📁 ⬇️ 🔒
py.typed	0 B	-rw-r--r--	2026-01-08 19:19	✏️ 🔄 🗑️ ⬇️ 🔒
resultdict.py	402 B	-rw-r--r--	2026-01-08 19:19	✏️ 🔄 🗑️ ⬇️ 🔒
sbcharsetprober.py	6,400 B	-rw-r--r--	2026-01-08 19:19	✏️ 🔄 🗑️ ⬇️ 🔒
sbcsgroupprober.py	4,137 B	-rw-r--r--	2026-01-08 19:19	✏️ 🔄 🗑️ ⬇️ 🔒
sjisprober.py	4,007 B	-rw-r--r--	2026-01-08 19:19	✏️ 🔄 🗑️ ⬇️ 🔒
universaldetector.py	14,848 B	-rw-r--r--	2026-01-08 19:19	✏️ 🔄 🗑️ ⬇️ 🔒
utf8prober.py	2,812 B	-rw-r--r--	2026-01-08 19:19	✏️ 🔄 🗑️ ⬇️ 🔒
utf1632prober.py	8,505 B	-rw-r--r--	2026-01-08 19:19	✏️ 🔄 🗑️ ⬇️ 🔒
version.py	244 B	-rw-r--r--	2026-01-08 19:19	✏️ 🔄 🗑️ ⬇️ 🔒

Edit: utf1632prober.py

######################## BEGIN LICENSE BLOCK ########################
#
# Contributor(s):
#   Jason Zavaglia
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
# 02110-1301  USA
######################### END LICENSE BLOCK #########################
from typing import List, Union

from .charsetprober import CharSetProber
from .enums import ProbingState

class UTF1632Prober(CharSetProber):
    """
    This class simply looks for occurrences of zero bytes, and infers
    whether the file is UTF16 or UTF32 (low-endian or big-endian)
    For instance, files looking like ( \0 \0 \0 [nonzero] )+
    have a good probability to be UTF32BE.  Files looking like ( \0 [nonzero] )+
    may be guessed to be UTF16BE, and inversely for little-endian varieties.
    """

# how many logical characters to scan before feeling confident of prediction
    MIN_CHARS_FOR_DETECTION = 20
    # a fixed constant ratio of expected zeros or non-zeros in modulo-position.
    EXPECTED_RATIO = 0.94

def __init__(self) -> None:
        super().__init__()
        self.position = 0
        self.zeros_at_mod = [0] * 4
        self.nonzeros_at_mod = [0] * 4
        self._state = ProbingState.DETECTING
        self.quad = [0, 0, 0, 0]
        self.invalid_utf16be = False
        self.invalid_utf16le = False
        self.invalid_utf32be = False
        self.invalid_utf32le = False
        self.first_half_surrogate_pair_detected_16be = False
        self.first_half_surrogate_pair_detected_16le = False
        self.reset()

def reset(self) -> None:
        super().reset()
        self.position = 0
        self.zeros_at_mod = [0] * 4
        self.nonzeros_at_mod = [0] * 4
        self._state = ProbingState.DETECTING
        self.invalid_utf16be = False
        self.invalid_utf16le = False
        self.invalid_utf32be = False
        self.invalid_utf32le = False
        self.first_half_surrogate_pair_detected_16be = False
        self.first_half_surrogate_pair_detected_16le = False
        self.quad = [0, 0, 0, 0]

@property
    def charset_name(self) -> str:
        if self.is_likely_utf32be():
            return "utf-32be"
        if self.is_likely_utf32le():
            return "utf-32le"
        if self.is_likely_utf16be():
            return "utf-16be"
        if self.is_likely_utf16le():
            return "utf-16le"
        # default to something valid
        return "utf-16"

@property
    def language(self) -> str:
        return ""

def approx_32bit_chars(self) -> float:
        return max(1.0, self.position / 4.0)

def approx_16bit_chars(self) -> float:
        return max(1.0, self.position / 2.0)

def is_likely_utf32be(self) -> bool:
        approx_chars = self.approx_32bit_chars()
        return approx_chars >= self.MIN_CHARS_FOR_DETECTION and (
            self.zeros_at_mod[0] / approx_chars > self.EXPECTED_RATIO
            and self.zeros_at_mod[1] / approx_chars > self.EXPECTED_RATIO
            and self.zeros_at_mod[2] / approx_chars > self.EXPECTED_RATIO
            and self.nonzeros_at_mod[3] / approx_chars > self.EXPECTED_RATIO
            and not self.invalid_utf32be
        )

def is_likely_utf32le(self) -> bool:
        approx_chars = self.approx_32bit_chars()
        return approx_chars >= self.MIN_CHARS_FOR_DETECTION and (
            self.nonzeros_at_mod[0] / approx_chars > self.EXPECTED_RATIO
            and self.zeros_at_mod[1] / approx_chars > self.EXPECTED_RATIO
            and self.zeros_at_mod[2] / approx_chars > self.EXPECTED_RATIO
            and self.zeros_at_mod[3] / approx_chars > self.EXPECTED_RATIO
            and not self.invalid_utf32le
        )

def is_likely_utf16be(self) -> bool:
        approx_chars = self.approx_16bit_chars()
        return approx_chars >= self.MIN_CHARS_FOR_DETECTION and (
            (self.nonzeros_at_mod[1] + self.nonzeros_at_mod[3]) / approx_chars
            > self.EXPECTED_RATIO
            and (self.zeros_at_mod[0] + self.zeros_at_mod[2]) / approx_chars
            > self.EXPECTED_RATIO
            and not self.invalid_utf16be
        )

def is_likely_utf16le(self) -> bool:
        approx_chars = self.approx_16bit_chars()
        return approx_chars >= self.MIN_CHARS_FOR_DETECTION and (
            (self.nonzeros_at_mod[0] + self.nonzeros_at_mod[2]) / approx_chars
            > self.EXPECTED_RATIO
            and (self.zeros_at_mod[1] + self.zeros_at_mod[3]) / approx_chars
            > self.EXPECTED_RATIO
            and not self.invalid_utf16le
        )

def validate_utf32_characters(self, quad: List[int]) -> None:
        """
        Validate if the quad of bytes is valid UTF-32.

UTF-32 is valid in the range 0x00000000 - 0x0010FFFF
        excluding 0x0000D800 - 0x0000DFFF

https://en.wikipedia.org/wiki/UTF-32
        """
        if (
            quad[0] != 0
            or quad[1] > 0x10
            or (quad[0] == 0 and quad[1] == 0 and 0xD8 <= quad[2] <= 0xDF)
        ):
            self.invalid_utf32be = True
        if (
            quad[3] != 0
            or quad[2] > 0x10
            or (quad[3] == 0 and quad[2] == 0 and 0xD8 <= quad[1] <= 0xDF)
        ):
            self.invalid_utf32le = True

def validate_utf16_characters(self, pair: List[int]) -> None:
        """
        Validate if the pair of bytes is  valid UTF-16.

UTF-16 is valid in the range 0x0000 - 0xFFFF excluding 0xD800 - 0xFFFF
        with an exception for surrogate pairs, which must be in the range
        0xD800-0xDBFF followed by 0xDC00-0xDFFF

https://en.wikipedia.org/wiki/UTF-16
        """
        if not self.first_half_surrogate_pair_detected_16be:
            if 0xD8 <= pair[0] <= 0xDB:
                self.first_half_surrogate_pair_detected_16be = True
            elif 0xDC <= pair[0] <= 0xDF:
                self.invalid_utf16be = True
        else:
            if 0xDC <= pair[0] <= 0xDF:
                self.first_half_surrogate_pair_detected_16be = False
            else:
                self.invalid_utf16be = True

if not self.first_half_surrogate_pair_detected_16le:
            if 0xD8 <= pair[1] <= 0xDB:
                self.first_half_surrogate_pair_detected_16le = True
            elif 0xDC <= pair[1] <= 0xDF:
                self.invalid_utf16le = True
        else:
            if 0xDC <= pair[1] <= 0xDF:
                self.first_half_surrogate_pair_detected_16le = False
            else:
                self.invalid_utf16le = True

def feed(self, byte_str: Union[bytes, bytearray]) -> ProbingState:
        for c in byte_str:
            mod4 = self.position % 4
            self.quad[mod4] = c
            if mod4 == 3:
                self.validate_utf32_characters(self.quad)
                self.validate_utf16_characters(self.quad[0:2])
                self.validate_utf16_characters(self.quad[2:4])
            if c == 0:
                self.zeros_at_mod[mod4] += 1
            else:
                self.nonzeros_at_mod[mod4] += 1
            self.position += 1
        return self.state

@property
    def state(self) -> ProbingState:
        if self._state in {ProbingState.NOT_ME, ProbingState.FOUND_IT}:
            # terminal, decided states
            return self._state
        if self.get_confidence() > 0.80:
            self._state = ProbingState.FOUND_IT
        elif self.position > 4 * 1024:
            # if we get to 4kb into the file, and we can't conclude it's UTF,
            # let's give up
            self._state = ProbingState.NOT_ME
        return self._state

def get_confidence(self) -> float:
        return (
            0.85
            if (
                self.is_likely_utf16le()
                or self.is_likely_utf16be()
                or self.is_likely_utf32le()
                or self.is_likely_utf32be()
            )
            else 0.00
        )

🛸 XMAN Replicator

Edit: utf1632prober.py

PHP Console

SQL Console