2
0
mirror of https://github.com/xcat2/confluent.git synced 2026-09-28 16:20:54 +00:00

Rework the normalize

Seemingly, decode then encode can mess up some utf-8.
If possible, leave it untouched, apart from binary replacement.
translate won't work on a byte string, so use replace.
This commit is contained in:
Jarrod Johnson
2017-03-15 10:07:18 -04:00
parent a517436f71
commit 1dc8fc973d
+8 -6
View File
@@ -56,19 +56,21 @@ pytecolors2ansi = {
# in the same way that Screen's draw method would do
# for now at least get some of the arrows in there (note ESC is one
# of those arrows... so skip it...
ansichars = dict(zip((24, 25, 26), u'\u2191\u2193\u2192'))
ansichars = dict(zip(('\x18', '\x19'), u'\u2191\u2193'))
def _utf8_normalize(data, shiftin):
try:
data = data.decode('utf-8')
data.decode('utf-8')
except UnicodeDecodeError:
try:
data = data.decode('cp437')
data = data.decode('cp437').encode('utf-8')
except UnicodeDecodeError:
data = data.decode('utf-8', 'replace')
data = data.decode('utf-8', 'replace').encode('utf-8')
if shiftin is None:
data = data.translate(ansichars)
return data.encode('utf-8')
for d in ansichars:
data = data.replace(d, ansichars[d].encode('utf-8'))
return data
def pytechars2line(chars, maxlen=None):