From d2b8697ea2c8f77214608eafe57f887b24e527bc Mon Sep 17 00:00:00 2001 From: Igor Chubin Date: Sun, 29 Nov 2020 21:37:30 +0000 Subject: [PATCH] Hotfix unicode processing for Python 3 (fixes #265) This fix is temprorary, and must be implemented properly, after Python 2 support will be fully dropped. --- lib/fmt/comments.py | 5 +++-- lib/postprocessing.py | 2 ++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/fmt/comments.py b/lib/fmt/comments.py index 5b7625b..8bd122d 100644 --- a/lib/fmt/comments.py +++ b/lib/fmt/comments.py @@ -230,7 +230,7 @@ def _beautify(text, filetype, add_comments=False, remove_text=False): # or remove the text completely. Otherwise the code has to remain aligned unindent_code = add_comments or remove_text - lines = [x.rstrip('\n') for x in text.splitlines()] + lines = [x.decode("utf-8").rstrip('\n') for x in text.splitlines()] lines = _cleanup_lines(lines) lines_classes = zip(_classify_lines(lines), lines) lines_classes = _wrap_lines(lines_classes, unindent_code=unindent_code) @@ -292,7 +292,8 @@ def beautify(text, lang, options): # if mode is unknown, just don't transform the text at all return text - text = text.encode('utf-8') + if isinstance(text, str): + text = text.encode('utf-8') digest = "t:%s:%s:%s" % (hashlib.md5(text).hexdigest(), lang, mode) # temporary added line that removes invalid cache entries diff --git a/lib/postprocessing.py b/lib/postprocessing.py index bc82c1e..649c4a6 100644 --- a/lib/postprocessing.py +++ b/lib/postprocessing.py @@ -40,6 +40,8 @@ def _filter_by_keyword(answer, keyword, options): def _split_paragraphs(text): answer = [] paragraph = "" + if isinstance(text, bytes): + text = text.decode("utf-8") for line in text.splitlines(): if line == "": answer.append(paragraph)