diff --git a/lib/cheat_wrapper.py b/lib/cheat_wrapper.py index 8b9545b..f51fb43 100644 --- a/lib/cheat_wrapper.py +++ b/lib/cheat_wrapper.py @@ -1,10 +1,14 @@ -import gevent -from gevent.wsgi import WSGIServer -from gevent.queue import Queue +""" +Main cheat.sh wrapper. +Gets answer from the getters, add syntax highlighting or html markup and returns it. +At the moment, it contains the getters that should be moved out to a separate file. +""" + from gevent.monkey import patch_all -from gevent.subprocess import Popen, PIPE, STDOUT +from gevent.subprocess import Popen, PIPE patch_all() +# pylint: disable=wrong-import-position,wrong-import-order import sys import os import glob @@ -13,30 +17,42 @@ import random import string import collections +import colored +import redis from fuzzywuzzy import process, fuzz -import redis -import colored - from pygments import highlight as pygments_highlight -import pygments.lexers # from pygments.lexers import BashLexer, GoLexer, ScalaLexer, RustLexer, PythonLexer, PhpLexer, PerlLexer -from pygments.formatters import TerminalFormatter, Terminal256Formatter - +import pygments.lexers +from pygments.formatters import Terminal256Formatter # pylint: disable=no-name-in-module from pygments.styles import get_all_styles -COLOR_STYLES = sorted(list(get_all_styles())) MYDIR = os.path.abspath(os.path.dirname(os.path.dirname('__file__'))) sys.path.append("%s/lib/" % MYDIR) from globals import error, ANSI2HTML, \ PATH_TLDR_PAGES, PATH_CHEAT_PAGES, \ PATH_CHEAT_SHEETS, PATH_CHEAT_SHEETS_SPOOL - -from buttons import TWITTER_BUTTON, GITHUB_BUTTON, GITHUB_BUTTON_2, GITHUB_BUTTON_FOOTER - +from buttons import TWITTER_BUTTON, GITHUB_BUTTON, GITHUB_BUTTON_FOOTER from adapter_learnxiny import get_learnxiny, get_learnxiny_list, is_valid_learnxy - +# pylint: disable=wrong-import-position,wrong-import-order + +COLOR_STYLES = sorted(list(get_all_styles())) + # globals -INTERNAL_TOPICS = [":list", ":firstpage", ':post', ':bash_completion', ':help', ':styles', ':styles-demo', ':emacs', ':emacs-ivy', ':fish', ':bash', ':zsh'] +INTERNAL_TOPICS = [ + ":list", + ":firstpage", + ':post', + ':bash_completion', + ':help', + ':styles', + ':styles-demo', + ':emacs', + ':emacs-ivy', + ':fish', + ':bash', + ':zsh' + ] + LEXER = { "clojure": pygments.lexers.ClojureLexer, "c++" : pygments.lexers.CppLexer, @@ -54,30 +70,32 @@ LEXER = { "perl" : pygments.lexers.PerlLexer, "python": pygments.lexers.PythonLexer, "php" : pygments.lexers.PhpLexer, + "psql" : pygments.lexers.PostgresLexer, "ruby" : pygments.lexers.RubyLexer, "rust" : pygments.lexers.RustLexer, "scala" : pygments.lexers.ScalaLexer, } REDIS = redis.StrictRedis(host='localhost', port=6379, db=0) +MAX_SEARCH_LEN = 20 -def update_tldr_topics(): +def _update_tldr_topics(): answer = [] for topic in glob.glob(PATH_TLDR_PAGES): _, filename = os.path.split(topic) if filename.endswith('.md'): answer.append(filename[:-3]) return answer -TLDR_TOPICS = update_tldr_topics() +TLDR_TOPICS = _update_tldr_topics() -def update_cheat_topics(): +def _update_cheat_topics(): answer = [] for topic in glob.glob(PATH_CHEAT_PAGES): _, filename = os.path.split(topic) answer.append(filename) return answer -CHEAT_TOPICS = update_cheat_topics() +CHEAT_TOPICS = _update_cheat_topics() -def update_cheat_sheets_topics(): +def _update_cheat_sheets_topics(): answer = [] answer_dirs = [] @@ -87,7 +105,7 @@ def update_cheat_sheets_topics(): if dirname.startswith('_'): dirname = dirname[1:] answer.append("%s/%s" % (dirname, filename)) - + for topic in glob.glob(PATH_CHEAT_SHEETS + "*"): _, filename = os.path.split(topic) if os.path.isdir(topic): @@ -97,17 +115,25 @@ def update_cheat_sheets_topics(): else: answer.append(filename) return answer, answer_dirs -CHEAT_SHEETS_TOPICS, CHEAT_SHEETS_DIRS = update_cheat_sheets_topics() +CHEAT_SHEETS_TOPICS, CHEAT_SHEETS_DIRS = _update_cheat_sheets_topics() ANSI_ESCAPE = re.compile(r'(\x9B|\x1B\[)[0-?]*[ -\/]*[@-~]') def remove_ansi(sometext): + """ + Remove ANSI sequences from `sometext` and convert it into plaintext. + """ return ANSI_ESCAPE.sub('', sometext) def html_wrapper(data): - p = Popen([ "bash", ANSI2HTML, "--palette=solarized", "--bg=dark" ], stdin=PIPE, stdout=PIPE, stderr=PIPE) + """ + Convert ANSI text `data` to HTML + """ + proc = Popen( + ["bash", ANSI2HTML, "--palette=solarized", "--bg=dark"], + stdin=PIPE, stdout=PIPE, stderr=PIPE) data = data.encode('utf-8') - stdout, stderr = p.communicate(data) - if p.returncode != 0: + stdout, stderr = proc.communicate(data) + if proc.returncode != 0: error(stdout + stderr) return stdout.decode('utf-8') @@ -115,14 +141,14 @@ def html_wrapper(data): # # -cached_topics_list = [[]] +CACHED_TOPICS_LIST = [[]] def get_topics_list(skip_dirs=False, skip_internal=False): """ List of topics returned on /:list """ - - if cached_topics_list[0] != []: - return cached_topics_list[0] + + if CACHED_TOPICS_LIST[0] != []: + return CACHED_TOPICS_LIST[0] answer = CHEAT_TOPICS + TLDR_TOPICS + CHEAT_SHEETS_TOPICS answer = sorted(set(answer)) @@ -137,59 +163,70 @@ def get_topics_list(skip_dirs=False, skip_internal=False): if not skip_internal: answer += INTERNAL_TOPICS - cached_topics_list[0] = answer + CACHED_TOPICS_LIST[0] = answer return answer -def get_topics_dirs(): +def _get_topics_dirs(): return set([x.split('/', 1)[0] for x in get_topics_list() if '/' in x]) -def get_stat(): +def _get_stat(): stat = collections.Counter([ get_topic_type(topic) for topic in get_topics_list() ]) answer = "" - for k,v in stat.items(): - answer += "%s %s\n" % (k,v) + for key, val in stat.items(): + answer += "%s %s\n" % (key, val) return answer # # # def get_topic_type(topic): + """ + Return topic type for `topic` or "unknown" if topic can't be determined. + """ + result = '' if topic == "": - return "search" - - if topic.startswith(":"): - return "internal" - if '/' in topic: + result = "search" + elif topic.startswith(":"): + result = "internal" + elif '/' in topic and not ' ' in topic: topic_type, topic_name = topic.split('/', 1) - if topic_type in get_topics_dirs() and topic_name in [':list']: - return "internal" + if topic_type in _get_topics_dirs() and topic_name in [':list']: + result = "internal" if is_valid_learnxy(topic): - return 'learnxiny' - if topic in CHEAT_SHEETS_TOPICS: - return "cheat.sheets" - if topic.rstrip('/') in CHEAT_SHEETS_DIRS and topic.endswith('/'): - return "cheat.sheets dir" - if topic in CHEAT_TOPICS: - return "cheat" - if topic in TLDR_TOPICS: - return "tldr" - return "unknown" + result = 'learnxiny' + elif topic in CHEAT_SHEETS_TOPICS: + result = "cheat.sheets" + elif topic.rstrip('/') in CHEAT_SHEETS_DIRS and topic.endswith('/'): + result = "cheat.sheets dir" + elif topic in CHEAT_TOPICS: + result = "cheat" + elif topic in TLDR_TOPICS: + result = "tldr" + elif ' ' in topic: + result = "question" + else: + result = 'unknown' + return result # # Various cheat sheets getters # +# +#def registered_answer_getter(func): +# REGISTERED_ANSWER_GETTERS.append(funct) +# return cls -def get_internal(topic): +def _get_internal(topic): if '/' in topic: topic_type, topic_name = topic.split('/', 1) if topic_name == ":list": - topic_list = [x[len(topic_type)+1:] - for x in get_topics_list() - if x.startswith(topic_type + "/")] + topic_list = [x[len(topic_type)+1:] + for x in get_topics_list() + if x.startswith(topic_type + "/")] return "\n".join(topic_list)+"\n" if topic == ":list": @@ -199,17 +236,17 @@ def get_internal(topic): return "\n".join(COLOR_STYLES) + "\n" if topic == ":stat": - return get_stat()+"\n" + return _get_stat()+"\n" if topic in INTERNAL_TOPICS: return open(os.path.join(MYDIR, "share", topic[1:]+".txt"), "r").read() return "" -def get_tldr(topic): +def _get_tldr(topic): cmd = ["tldr", topic] - p = Popen(cmd, stdout=PIPE, stderr=PIPE) - answer = p.communicate()[0] + proc = Popen(cmd, stdout=PIPE, stderr=PIPE) + answer = proc.communicate()[0] fixed_answer = [] for line in answer.splitlines(): @@ -226,13 +263,13 @@ def get_tldr(topic): answer = "\n".join(fixed_answer) + "\n" return answer.decode('utf-8') -def get_cheat(topic): +def _get_cheat(topic): cmd = ["cheat", topic] - p = Popen(cmd, stdout=PIPE, stderr=PIPE) - answer = p.communicate()[0].decode('utf-8') + proc = Popen(cmd, stdout=PIPE, stderr=PIPE) + answer = proc.communicate()[0].decode('utf-8') return answer -def get_cheat_sheets(topic): +def _get_cheat_sheets(topic): """ Get the cheat sheet topic from the own repository (cheat.sheets). It's possible that topic directory starts with omited underscore @@ -242,14 +279,23 @@ def get_cheat_sheets(topic): filename = PATH_CHEAT_SHEETS + "_%s" % topic return open(filename, "r").read().decode('utf-8') -def get_cheat_sheets_dir(topic): +def _get_cheat_sheets_dir(topic): answer = [] for f_name in glob.glob(PATH_CHEAT_SHEETS + "%s/*" % topic.rstrip('/')): answer.append(os.path.basename(f_name)) topics = sorted(answer) return "\n".join(topics) + "\n" -def get_unknown(topic): +def _get_answer_for_question(topic): + """ + Find answer for the `topic` question. + """ + cmd = ["/home/igor/cheat.sh/bin/get-answer-for-question", topic] + proc = Popen(cmd, stdout=PIPE, stderr=PIPE) + answer = proc.communicate()[0].decode('utf-8') + return answer + +def _get_unknown(topic): topics_list = get_topics_list() if topic.startswith(':'): topics_list = [x for x in topics_list if x.startswith(':')] @@ -266,78 +312,81 @@ Do you mean one of these topics may be? """ % possible_topics_text TOPIC_GETTERS = ( - ("cheat.sheets" , get_cheat_sheets), - ("cheat.sheets dir" , get_cheat_sheets_dir), - ("tldr" , get_tldr), - ("internal" , get_internal), - ("cheat" , get_cheat), - ("learnxiny" , get_learnxiny), - ("unknown" , get_unknown), + ("cheat.sheets", _get_cheat_sheets), + ("cheat.sheets dir", _get_cheat_sheets_dir), + ("tldr", _get_tldr), + ("internal", _get_internal), + ("cheat", _get_cheat), + ("learnxiny", get_learnxiny), + ("question", _get_answer_for_question), + ("unknown", _get_unknown), ) -# -# -# - -def split_paragraphs(text): - answer = [] - paragraph = "" - for line in text.splitlines(): - if line == "": - answer.append(paragraph) - paragraph = "" - else: - paragraph += line+"\n" - answer.append(paragraph) - - return answer - -def paragraph_contains(paragraph, keyword, insensitive=False, word_boundaries=True): - """ - Several keywords can be joined together using ~ - For example: ~ssh~passphrase - """ - answer = True - - if '~' in keyword: - keywords = keyword.split('~') - else: - keywords = [keyword] - - for keyword in keywords: - regex = re.escape(keyword) - if not word_boundaries: - regex = r"\b%s\b" % keyword - - if insensitive: - answer = answer and bool(re.search(regex, paragraph, re.IGNORECASE)) - else: - answer = answer and bool(re.search(regex, paragraph)) - - return answer - -def join_paragraphs(paragraphs): - answer = "\n".join(paragraphs) - return answer - def get_answer(topic, keyword, options=""): """ Find cheat sheet for the topic. - If not keyword, return answer. - Otherwise cut the paragraphs cotaining keywords. + If `keyword` is None or rempty, return the whole answer. + Otherwise cut the paragraphs containing keywords. Args: topic (str): the name of the topic of the cheat sheet keyword (str): the name of the keywords to search in the cheat sheets - + Returns: string: the cheat sheet """ + def _join_paragraphs(paragraphs): + answer = "\n".join(paragraphs) + return answer + + def _split_paragraphs(text): + answer = [] + paragraph = "" + for line in text.splitlines(): + if line == "": + answer.append(paragraph) + paragraph = "" + else: + paragraph += line+"\n" + answer.append(paragraph) + return answer + + def _paragraph_contains(paragraph, keyword, insensitive=False, word_boundaries=True): + """ + Check if `paragraph` contains `keyword`. + Several keywords can be joined together using ~ + For example: ~ssh~passphrase + """ + answer = True + + if '~' in keyword: + keywords = keyword.split('~') + else: + keywords = [keyword] + + for kwrd in keywords: + regex = re.escape(kwrd) + if not word_boundaries: + regex = r"\b%s\b" % kwrd + + if insensitive: + answer = answer and bool(re.search(regex, paragraph, re.IGNORECASE)) + else: + answer = answer and bool(re.search(regex, paragraph)) + + return answer + answer = None # checking if the answer is in the cache if topic != "": + # temporary hack for "questions": + # the topic name has to be prefixed with q: + # so we can later delete them from redis + if '/' in topic and ' ' in topic: + topic = "q:" + topic + answer = REDIS.get(topic) if answer: answer = answer.decode('utf-8') @@ -351,7 +400,7 @@ def get_answer(topic, keyword, options=""): answer = topic_getter(topic) break if not answer: - answer = get_unknown(topic) + answer = _get_unknown(topic) # saving answers in the cache if topic_type not in ["search", "internal", "unknown"]: @@ -360,22 +409,29 @@ def get_answer(topic, keyword, options=""): if not keyword: return answer + # + # shorten the answer, because keyword is specified + # insensitive = 'i' in options word_boundaries = 'b' in options - paragraphs = split_paragraphs(answer) - paragraphs = [ p for p in paragraphs - if paragraph_contains( - p, keyword, - insensitive=insensitive, - word_boundaries=word_boundaries) ] - if len(paragraphs) == 0: + paragraphs = _split_paragraphs(answer) + paragraphs = [p for p in paragraphs + if _paragraph_contains(p, keyword, + insensitive=insensitive, + word_boundaries=word_boundaries)] + if paragraphs == []: return "" - answer = join_paragraphs(paragraphs) + answer = _join_paragraphs(paragraphs) return answer def find_answer_by_keyword(directory, keyword, options=""): + """ + Search in the whole tree of all cheatsheets or in its subtree `directory` + by `keyword` + """ + recursive = 'r' in options answer_paragraphs = [] @@ -386,16 +442,15 @@ def find_answer_by_keyword(directory, keyword, options=""): if not topic.startswith(directory): continue - + subtopic = topic[len(directory):] if not recursive and '/' in subtopic: - continue + continue answer = get_answer(topic, keyword, options=options) if answer: answer_paragraphs.append((topic, answer)) - MAX_SEARCH_LEN = 20 if len(answer_paragraphs) > MAX_SEARCH_LEN: answer_paragraphs.append(("LIMITED", "LIMITED TO %s ANSWERS" % MAX_SEARCH_LEN)) break @@ -403,8 +458,14 @@ def find_answer_by_keyword(directory, keyword, options=""): return answer_paragraphs # +# +# def save_cheatsheet(topic_name, cheatsheet): + """ + Save posted cheat sheet `cheatsheet` with `topic_name` + in the spool directory + """ nonce = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(9)) filename = topic_name.replace('/', '.') + "." + nonce @@ -415,18 +476,28 @@ def save_cheatsheet(topic_name, cheatsheet): # # -def colorize_internal(topic, answer, html_needed): +def _colorize_internal(topic, answer, html_needed): - def colorize_line(line): + def _colorize_line(line): if line.startswith('T'): line = colored.fg("grey_62") + line + colored.attr('reset') - line = re.sub(r"\{(.*?)\}", colored.fg("orange_3") + r"\1"+colored.fg('grey_35'), line) + line = re.sub(r"\{(.*?)\}", colored.fg("orange_3") + r"\1"+colored.fg('grey_35'), line) return line - line = re.sub(r"\[(F.*?)\]", colored.bg("black") + colored.fg("cyan") + r"[\1]"+colored.attr('reset'), line) - line = re.sub(r"\[(g.*?)\]", colored.bg("dark_gray") + colored.fg("grey_0") + r"[\1]"+colored.attr('reset'), line) - line = re.sub(r"\{(.*?)\}", colored.fg("orange_3") + r"\1"+colored.attr('reset'), line) - line = re.sub(r"<(.*?)>", colored.fg("cyan") + r"\1"+colored.attr('reset'), line) + line = re.sub(r"\[(F.*?)\]", + colored.bg("black") + colored.fg("cyan") + r"[\1]"+colored.attr('reset'), + line) + line = re.sub(r"\[(g.*?)\]", + colored.bg("dark_gray") \ + + colored.fg("grey_0") \ + + r"[\1]"+colored.attr('reset'), + line) + line = re.sub(r"\{(.*?)\}", + colored.fg("orange_3") + r"\1"+colored.attr('reset'), + line) + line = re.sub(r"<(.*?)>", + colored.fg("cyan") + r"\1"+colored.attr('reset'), + line) return line if topic in [':list', ':bash_completion']: @@ -437,14 +508,15 @@ def colorize_internal(topic, answer, html_needed): answer_lines = lines[:9] answer_lines.append(colored.fg('grey_35')+lines[9]+colored.attr('reset')) for line in lines[10:]: - answer_lines.append(colorize_line(line)) + answer_lines.append(_colorize_line(line)) if html_needed: answer_lines = answer_lines[:-2] answer = "\n".join(answer_lines) + "\n" return answer -def github_button(topic_type): +def _github_button(topic_type): + repository = { "cheat.sheets" : 'chubin/cheat.sheets', "cheat.sheets dir" : 'chubin/cheat.sheets', @@ -460,13 +532,15 @@ def github_button(topic_type): if not full_name: return '' - short_name = full_name.split('/',1)[1] + short_name = full_name.split('/', 1)[1] # pylint: disable=unused-variable + button = ( "" - '%(short_name)s' ) % locals() @@ -474,20 +548,29 @@ def github_button(topic_type): # -def rewrite_aliases(word): +def _rewrite_aliases(word): if word == ':bash.completion': return ':bash_completion' return word -def cheat_wrapper(query, request_options=None, html=False): +def cheat_wrapper(query, request_options=None, html=False): # pylint: disable=too-many-locals,too-many-branches,too-many-statements + """ + Giant megafunction that delivers cheat sheet for `query`. + If `html` is True, the answer is formated as HTML. + Additional request options specified in `request_options`. - # + This function is really really bad, and should be rewritten + as soon as possible. + """ + + # # at the moment, we just remove trailing slashes # so queries python/ and python are equal # query = query.rstrip('/') - query = rewrite_aliases(query) + query = _rewrite_aliases(query) + query = query.replace('+', ' ') highlight = not bool(request_options and request_options.get('no-terminal')) color_style = request_options.get('style', '') @@ -517,10 +600,13 @@ def cheat_wrapper(query, request_options=None, html=False): found = True # if the page was found in the database editable = False # can generated page be edited on github (only cheat.sheets pages can) result = "" - for topic, answer in answers: - + for topic, answer in answers: # pylint: disable=too-many-nested-blocks + if topic == 'LIMITED': - result += colored.bg('dark_goldenrod') + colored.fg('yellow_1') + ' ' + answer + ' ' + colored.attr('reset') + "\n" + result += colored.bg('dark_goldenrod') \ + + colored.fg('yellow_1') \ + + ' ' + answer + ' ' \ + + colored.attr('reset') + "\n" break if topic in [":list", ":bash_completion"]: @@ -533,9 +619,9 @@ def cheat_wrapper(query, request_options=None, html=False): if highlight: #if topic_type.endswith(" dir"): # pass - + if topic_type == "internal": - answer = colorize_internal(topic, answer, html) + answer = _colorize_internal(topic, answer, html) else: color_style = color_style or "native" lexer = pygments.lexers.BashLexer @@ -555,8 +641,12 @@ def cheat_wrapper(query, request_options=None, html=False): if search_mode: if highlight: - result += "\n%s%s %s %s%s\n" % (colored.bg('dark_gray'), colored.attr("res_underlined"), topic, colored.attr("res_underlined"), colored.attr('reset')) - else: + result += "\n%s%s %s %s%s\n" % (colored.bg('dark_gray'), + colored.attr("res_underlined"), + topic, + colored.attr("res_underlined"), + colored.attr('reset')) + else: result += "\n[%s]\n" % topic result += answer @@ -566,22 +656,31 @@ def cheat_wrapper(query, request_options=None, html=False): editable = False repository_button = '' else: - repository_button = github_button(topic_type) + repository_button = _github_button(topic_type) if html: result = result + "\n$" result = html_wrapper(result) title = "cheat.sh/%s" % topic - # title += '\nscript src="/files/awesomplete.min.js" async>' + # title += ('\nscript' + # ' src="/files/awesomplete.min.js" async>') # submit button: thanks to http://stackoverflow.com/questions/477691/ - submit_button = '' + submit_button = ('') topic_list = ('%s' - % ("\n".join("" % x for x in get_topics_list()))) + % ("\n".join("" % x for x in get_topics_list()))) curl_line = "$ curl cheat.sh/" if query == ':firstpage': query = "" - form_html = '
%s%s%s
' % (submit_button, curl_line, query, topic_list) + form_html = ('
' + '%s%s' + '' + '%s' + '') \ + % (submit_button, curl_line, query, topic_list) edit_button = '' if editable: @@ -589,11 +688,18 @@ def cheat_wrapper(query, request_options=None, html=False): if '/' in topic: topic = '_' + topic edit_page_link = 'https://github.com/chubin/cheat.sheets/edit/master/sheets/' + topic - edit_button = '
[edit]
' % edit_page_link + edit_button = ( + '
'
+                '[edit]'
+                '
') % edit_page_link result = re.sub("
", edit_button + form_html + "
", result)
         result = re.sub("", "" + title, result)
         if not request_options.get('quiet'):
-            result = result.replace('', TWITTER_BUTTON + GITHUB_BUTTON + repository_button + GITHUB_BUTTON_FOOTER + '')
+            result = result.replace('',
+                                    TWITTER_BUTTON \
+                                    + GITHUB_BUTTON \
+                                    + repository_button \
+                                    + GITHUB_BUTTON_FOOTER \
+                                    + '')
 
     return result, found
-