I wanted to get the same results we got by using TextEncoder and TextDecoder on javascript but couldn't find real code solution the soluction I found doesn't give me the real same results.
exp:
const textencoder = new TextEncoder();
console.log(textencoder.encode('$'));
//[36]
const textdecoder = TextDecoder();
console.log(textdecoder.decode(new Uint8Array([36]));
// $
after a many tries and search, I got the solution form a friend and I decided to share it with you, may someone need it some day;
class TextEncoder():
def __init__(self):
pass
def encode(self, text):
"""
exp:
>>> textencoder = TextEncoder()
>>> textencoder.encode('$')
>>> [36]
"""
if isinstance(text, str):
encoded_text = text.encode('utf-8')
byte_array = bytearray(encoded_text)
return list(byte_array)
else:
raise TypeError(f'Expecting a str but got {type(text)}')
class TextDecoder():
def __init__(self):
pass
def decode(self, array):
"""
exp:
>>> textdecoder = TextDecoder()
>>> textdecoder.decode([36])
>>> $
"""
if isinstance(array, list):
return bytearray(array).decode('utf-8')
elif isinstance(array, bytearray):
return array.decode('utf-8')
else:
raise TypeError(f'expecting a list or bytearray got: {type(array)}')
codecs.getincrementaldecoder() is the equivalent of TextDecoder in JavaScript.
import codecs
# 世 界 \n
# e4 b8 96 e7 95 8c 0a
decoder = incremental_ansi_strip_decoder('utf-8')()
# s = decoder.decode(b'\xe4\xb8', final=True) # UnicodeDecodeError: 'utf-8' codec can't decode bytes in position 0-1: unexpected end of data
decoder = incremental_ansi_strip_decoder('utf-8')(errors='replace')
assert decoder.decode(b'\xe4\xb8', final=True) == '�'
assert decoder.decode(b'\x96\xe7\x95', final=True) == '��'
assert decoder.decode(b'\x8c\x0a', final=True) == '�\n'
# final=False
assert decoder.decode(b'\xe4\xb8') == ''
assert decoder.decode(b'\x96\xe7\x95') == '世'
assert decoder.decode(b'\x8c\x0a') == '界\n'
# invalid utf-8
assert decoder.decode(b'\xff', final=True) == '�'
assert decoder.decode(b'\xff') == '�'
assert decoder.decode(b'\xe4\xb8') == ''
assert decoder.decode(b'\xff') == '��'
is equivalent to:
// 世 界 \n
// e4 b8 96 e7 95 8c 0a
const decoder = new TextDecoder();
let s;
s = decoder.decode(Uint8Array.from([0xe4, 0xb8])); // "�"
s = decoder.decode(Uint8Array.from([0x96, 0xe7, 0x95])); // "��"
s = decoder.decode(Uint8Array.from([0x8c, 0x0a])); // "�\n"
// stream: true
s = decoder.decode(Uint8Array.from([0xe4, 0xb8]), { stream: true }); // ""
s = decoder.decode(Uint8Array.from([0x96, 0xe7, 0x95]), { stream: true }); // "世"
s = decoder.decode(Uint8Array.from([0x8c, 0x0a]), { stream: true }); // "界\n"
// invalid utf-8
s = decoder.decode(Uint8Array.from([0xff])); // "�"
s = decoder.decode(Uint8Array.from([0xff]), { stream: true }); // "�"
s = decoder.decode(Uint8Array.from([0xe4, 0xb8]), { stream: true }); // ""
s = decoder.decode(Uint8Array.from([0xff]), { stream: true }); // "��"
I guess codecs.getincrementalencoder() is the one equivalent to TextEncoder.
