From 2333ed21bcf81f0f64eab0550bf6feb6c1a088f0 Mon Sep 17 00:00:00 2001 From: Igor Chubin Date: Sat, 5 May 2018 20:38:31 +0000 Subject: [PATCH] refactored lib/cheat_wrapper.py (splitted it into lib/cheat_wrapper.py and lib/get_answer.py) --- lib/cheat_wrapper.py | 664 ++++++++----------------------------------- lib/get_answer.py | 446 +++++++++++++++++++++++++++++ lib/globals.py | 3 + 3 files changed, 568 insertions(+), 545 deletions(-) create mode 100644 lib/get_answer.py diff --git a/lib/cheat_wrapper.py b/lib/cheat_wrapper.py index 44d423d..b130620 100644 --- a/lib/cheat_wrapper.py +++ b/lib/cheat_wrapper.py @@ -1,7 +1,7 @@ """ Main cheat.sh wrapper. -Gets answer from the getters, add syntax highlighting or html markup and returns it. -At the moment, it contains the getters that should be moved out to a separate file. +Get answers from getters (in get_answer), adds syntax highlighting +or html markup and returns the result. """ from gevent.monkey import patch_all @@ -11,90 +11,21 @@ patch_all() # pylint: disable=wrong-import-position,wrong-import-order import sys import os -import glob import re -import collections import colored -import redis -from fuzzywuzzy import process, fuzz - from pygments import highlight as pygments_highlight from pygments.formatters import Terminal256Formatter # pylint: disable=no-name-in-module -from pygments.styles import get_all_styles MYDIR = os.path.abspath(os.path.dirname(os.path.dirname('__file__'))) sys.path.append("%s/lib/" % MYDIR) -from globals import error, ANSI2HTML, \ - PATH_TLDR_PAGES, PATH_CHEAT_PAGES, \ - PATH_CHEAT_SHEETS +from globals import error, ANSI2HTML, COLOR_STYLES from buttons import TWITTER_BUTTON, GITHUB_BUTTON, GITHUB_BUTTON_FOOTER -from adapter_learnxiny import get_learnxiny, get_learnxiny_list, is_valid_learnxy -from languages_data import LEXER, LANGUAGE_ALIAS -import beautifier +from languages_data import LEXER +from get_answer import get_topic_type, get_topics_list, get_answer, find_answer_by_keyword +# import beautifier # pylint: disable=wrong-import-position,wrong-import-order -COLOR_STYLES = sorted(list(get_all_styles())) - -# globals -INTERNAL_TOPICS = [ - ":list", - ":firstpage", - ':post', - ':bash_completion', - ':help', - ':styles', - ':styles-demo', - ':emacs', - ':emacs-ivy', - ':fish', - ':bash', - ':zsh' - ] - - -REDIS = redis.StrictRedis(host='localhost', port=6379, db=0) -MAX_SEARCH_LEN = 20 - -def _update_tldr_topics(): - answer = [] - for topic in glob.glob(PATH_TLDR_PAGES): - _, filename = os.path.split(topic) - if filename.endswith('.md'): - answer.append(filename[:-3]) - return answer -TLDR_TOPICS = _update_tldr_topics() - -def _update_cheat_topics(): - answer = [] - for topic in glob.glob(PATH_CHEAT_PAGES): - _, filename = os.path.split(topic) - answer.append(filename) - return answer -CHEAT_TOPICS = _update_cheat_topics() - -def _update_cheat_sheets_topics(): - answer = [] - answer_dirs = [] - - for topic in glob.glob(PATH_CHEAT_SHEETS + "*/*"): - dirname, filename = os.path.split(topic) - dirname = os.path.basename(dirname) - if dirname.startswith('_'): - dirname = dirname[1:] - answer.append("%s/%s" % (dirname, filename)) - - for topic in glob.glob(PATH_CHEAT_SHEETS + "*"): - _, filename = os.path.split(topic) - if os.path.isdir(topic): - if filename.startswith('_'): - filename = filename[1:] - answer_dirs.append(filename+'/') - else: - answer.append(filename) - return answer, answer_dirs -CHEAT_SHEETS_TOPICS, CHEAT_SHEETS_DIRS = _update_cheat_sheets_topics() - ANSI_ESCAPE = re.compile(r'(\x9B|\x1B\[)[0-?]*[ -\/]*[@-~]') def remove_ansi(sometext): """ @@ -115,357 +46,6 @@ def html_wrapper(data): error(stdout + stderr) return stdout.decode('utf-8') -# -# -# - -CACHED_TOPICS_LIST = [[]] -def get_topics_list(skip_dirs=False, skip_internal=False): - """ - List of topics returned on /:list - """ - - if CACHED_TOPICS_LIST[0] != []: - return CACHED_TOPICS_LIST[0] - - answer = CHEAT_TOPICS + TLDR_TOPICS + CHEAT_SHEETS_TOPICS - answer = sorted(set(answer)) - - # doing it in this strange way to save the order of the topics - for topic in get_learnxiny_list(): - if topic not in answer: - answer.append(topic) - - if not skip_dirs: - answer += CHEAT_SHEETS_DIRS - if not skip_internal: - answer += INTERNAL_TOPICS - - CACHED_TOPICS_LIST[0] = answer - return answer - -def _get_topics_dirs(): - return set([x.split('/', 1)[0] for x in get_topics_list() if '/' in x]) - - -def _get_stat(): - stat = collections.Counter([ - get_topic_type(topic) for topic in get_topics_list() - ]) - - answer = "" - for key, val in stat.items(): - answer += "%s %s\n" % (key, val) - return answer -# -# -# - -def get_topic_type(topic): # pylint: disable=too-many-locals,too-many-branches,too-many-statements - """ - Return topic type for `topic` or "unknown" if topic can't be determined. - """ - result = '' - if topic == "": - result = "search" - elif topic.startswith(":"): - result = "internal" - elif '/' in topic: - topic_type, topic_name = topic.split('/', 1) - if '+' in topic_name: - result = 'question' - else: - if topic_type in _get_topics_dirs() and topic_name in [':list']: - result = "internal" - elif is_valid_learnxy(topic): - result = 'learnxiny' - else: - result = 'question' - - elif topic in CHEAT_SHEETS_TOPICS: - result = "cheat.sheets" - elif topic.rstrip('/') in CHEAT_SHEETS_DIRS and topic.endswith('/'): - result = "cheat.sheets dir" - elif topic in CHEAT_TOPICS: - result = "cheat" - elif topic in TLDR_TOPICS: - result = "tldr" - elif '+' in topic: - result = "question" - else: - result = 'unknown' - print topic, " ", result - return result - -# -# Various cheat sheets getters -# -# -#def registered_answer_getter(func): -# REGISTERED_ANSWER_GETTERS.append(funct) -# return cls - -def _get_internal(topic): - if '/' in topic: - topic_type, topic_name = topic.split('/', 1) - if topic_name == ":list": - topic_list = [x[len(topic_type)+1:] - for x in get_topics_list() - if x.startswith(topic_type + "/")] - return "\n".join(topic_list)+"\n" - - if topic == ":list": - return "\n".join(x for x in get_topics_list()) + "\n" - - if topic == ':styles': - return "\n".join(COLOR_STYLES) + "\n" - - if topic == ":stat": - return _get_stat()+"\n" - - if topic in INTERNAL_TOPICS: - return open(os.path.join(MYDIR, "share", topic[1:]+".txt"), "r").read() - - return "" - -def _get_tldr(topic): - cmd = ["tldr", topic] - proc = Popen(cmd, stdout=PIPE, stderr=PIPE) - answer = proc.communicate()[0] - - fixed_answer = [] - for line in answer.splitlines(): - line = line[2:] - if line.startswith('-'): - line = '# '+line[2:] - elif line == "": - pass - elif not line.startswith(' '): - line = "# "+line - - fixed_answer.append(line) - - answer = "\n".join(fixed_answer) + "\n" - return answer.decode('utf-8') - -def _get_cheat(topic): - cmd = ["cheat", topic] - proc = Popen(cmd, stdout=PIPE, stderr=PIPE) - answer = proc.communicate()[0].decode('utf-8') - return answer - -def _get_cheat_sheets(topic): - """ - Get the cheat sheet topic from the own repository (cheat.sheets). - It's possible that topic directory starts with omited underscore - """ - filename = PATH_CHEAT_SHEETS + "%s" % topic - if not os.path.exists(filename): - filename = PATH_CHEAT_SHEETS + "_%s" % topic - return open(filename, "r").read().decode('utf-8') - -def _get_cheat_sheets_dir(topic): - answer = [] - for f_name in glob.glob(PATH_CHEAT_SHEETS + "%s/*" % topic.rstrip('/')): - answer.append(os.path.basename(f_name)) - topics = sorted(answer) - return "\n".join(topics) + "\n" - -def _get_answer_for_question(topic): - """ - Find answer for the `topic` question. - """ - topic = " ".join(topic.replace('+', ' ').strip().split()) - cmd = ["/home/igor/cheat.sh/bin/get-answer-for-question", topic] - proc = Popen(cmd, stdout=PIPE, stderr=PIPE) - answer = proc.communicate()[0].decode('utf-8') - return answer - -def _get_unknown(topic): - topics_list = get_topics_list() - if topic.startswith(':'): - topics_list = [x for x in topics_list if x.startswith(':')] - else: - topics_list = [x for x in topics_list if not x.startswith(':')] - - possible_topics = process.extract(topic, topics_list, scorer=fuzz.ratio)[:3] - possible_topics_text = "\n".join([(" * %s %s" % x) for x in possible_topics]) - return """ -Unknown topic. -Do you mean one of these topics may be? - -%s - """ % possible_topics_text - -# pylint: disable=bad-whitespace -# -# topic_type, function_getter -# should be replaced with a decorator -TOPIC_GETTERS = ( - ("cheat.sheets", _get_cheat_sheets), - ("cheat.sheets dir", _get_cheat_sheets_dir), - ("tldr", _get_tldr), - ("internal", _get_internal), - ("cheat", _get_cheat), - ("learnxiny", get_learnxiny), - ("question", _get_answer_for_question), - ("unknown", _get_unknown), -) -# pylint: enable=bad-whitespace - -def get_answer(topic, keyword, options="", request_options=None): # pylint: disable=too-many-locals,too-many-branches,too-many-statements - """ - Find cheat sheet for the topic. - If `keyword` is None or rempty, return the whole answer. - Otherwise cut the paragraphs containing keywords. - - Args: - topic (str): the name of the topic of the cheat sheet - keyword (str): the name of the keywords to search in the cheat sheets - - Returns: - string: the cheat sheet - """ - - def _join_paragraphs(paragraphs): - answer = "\n".join(paragraphs) - return answer - - def _split_paragraphs(text): - answer = [] - paragraph = "" - for line in text.splitlines(): - if line == "": - answer.append(paragraph) - paragraph = "" - else: - paragraph += line+"\n" - answer.append(paragraph) - return answer - - def _paragraph_contains(paragraph, keyword, insensitive=False, word_boundaries=True): - """ - Check if `paragraph` contains `keyword`. - Several keywords can be joined together using ~ - For example: ~ssh~passphrase - """ - answer = True - - if '~' in keyword: - keywords = keyword.split('~') - else: - keywords = [keyword] - - for kwrd in keywords: - regex = re.escape(kwrd) - if not word_boundaries: - regex = r"\b%s\b" % kwrd - - if insensitive: - answer = answer and bool(re.search(regex, paragraph, re.IGNORECASE)) - else: - answer = answer and bool(re.search(regex, paragraph)) - - return answer - - answer = None - needs_beautification = False - - # checking if the answer is in the cache - if topic != "": - # temporary hack for "questions": - # the topic name has to be prefixed with q: - # so we can later delete them from redis - # and we known that they need beautification - if '/' in topic and '+' in topic: - topic = "q:" + topic - needs_beautification = True - - answer = REDIS.get(topic) - if answer: - answer = answer.decode('utf-8') - - # if answer was not found in the cache - # try to find it in one of the repositories - if not answer: - topic_type = get_topic_type(topic) - - for topic_getter_type, topic_getter in TOPIC_GETTERS: - if topic_type == topic_getter_type: - answer = topic_getter(topic) - break - if not answer: - topic_type = "unknown" - answer = _get_unknown(topic) - - # saving answers in the cache - if topic_type not in ["search", "internal", "unknown"]: - REDIS.set(topic, answer) - - if needs_beautification: - filetype = 'bash' - if '/' in topic: - filetype = topic.split('/', 1)[0] - if filetype.startswith('q:'): - filetype = filetype[2:] - - answer = beautifier.beautify(answer.encode('utf-8'), filetype, request_options) - - if not keyword: - return answer - - # - # shorten the answer, because keyword is specified - # - insensitive = 'i' in options - word_boundaries = 'b' in options - - paragraphs = _split_paragraphs(answer) - paragraphs = [p for p in paragraphs - if _paragraph_contains(p, keyword, - insensitive=insensitive, - word_boundaries=word_boundaries)] - if paragraphs == []: - return "" - - answer = _join_paragraphs(paragraphs) - return answer - -def find_answer_by_keyword(directory, keyword, options="", request_options=None): - """ - Search in the whole tree of all cheatsheets or in its subtree `directory` - by `keyword` - """ - - recursive = 'r' in options - - answer_paragraphs = [] - for topic in get_topics_list(skip_internal=True, skip_dirs=True): - # skip the internal pages, don't show them in search - if topic in INTERNAL_TOPICS: - continue - - if not topic.startswith(directory): - continue - - subtopic = topic[len(directory):] - if not recursive and '/' in subtopic: - continue - - answer = get_answer(topic, keyword, options=options, request_options=request_options) - if answer: - answer_paragraphs.append((topic, answer)) - - if len(answer_paragraphs) > MAX_SEARCH_LEN: - answer_paragraphs.append(("LIMITED", "LIMITED TO %s ANSWERS" % MAX_SEARCH_LEN)) - break - - return answer_paragraphs - -# -#========================>8 cut here 8<=================================================== -# - def _colorize_internal(topic, answer, html_needed): def _colorize_line(line): @@ -505,6 +85,21 @@ def _colorize_internal(topic, answer, html_needed): return answer +def _colorize_ansi_answer(topic, answer, color_style): + + color_style = color_style or "native" + lexer = LEXER['bash'] + for lexer_name, lexer_value in LEXER.items(): + if topic.startswith("%s/" % lexer_name): + color_style = color_style or "monokai" + if lexer_name == 'php': + answer = "\n" % answer + lexer = lexer_value + break + + formatter = Terminal256Formatter(style=color_style) + return pygments_highlight(answer, lexer(), formatter).lstrip('\n') + def _github_button(topic_type): repository = { @@ -536,68 +131,61 @@ def _github_button(topic_type): ) % locals() return button -# +def _render_html(query, result, editable, repository_button, request_options): -def _rewrite_aliases(word): - if word == ':bash.completion': - return ':bash_completion' - return word + result = result + "\n$" + result = html_wrapper(result) + title = "cheat.sh/%s" % query + # title += ('\nscript' + # ' src="/files/awesomplete.min.js" async>') + # submit button: thanks to http://stackoverflow.com/questions/477691/ + submit_button = ('') + topic_list = ('%s' + % ("\n".join("" % x for x in get_topics_list()))) -def _rewrite_section_name(query): - """ - """ - if '/' not in query: - return query + curl_line = "$ curl cheat.sh/" + if query == ':firstpage': + query = "" + form_html = ('
' + '%s%s' + '' + '%s' + '') \ + % (submit_button, curl_line, query, topic_list) - section_name, rest = query.split('/', 1) - section_name = LANGUAGE_ALIAS.get(section_name, section_name) - return "%s/%s" % (section_name, rest) + edit_button = '' + if editable: + # It's possible that topic directory starts with omited underscore + if '/' in query: + query = '_' + query + edit_page_link = 'https://github.com/chubin/cheat.sheets/edit/master/sheets/' + query + edit_button = ( + '
'
+            '[edit]'
+            '
') % edit_page_link + result = re.sub("
", edit_button + form_html + "
", result)
+    result = re.sub("", "" + title, result)
+    if not request_options.get('quiet'):
+        result = result.replace('',
+                                TWITTER_BUTTON \
+                                + GITHUB_BUTTON \
+                                + repository_button \
+                                + GITHUB_BUTTON_FOOTER \
+                                + '')
+    return result
 
-def cheat_wrapper(query, request_options=None, html=False): # pylint: disable=too-many-locals,too-many-branches,too-many-statements
-    """
-    Giant megafunction that delivers cheat sheet for `query`.
-    If `html` is True, the answer is formated as HTML.
-    Additional request options specified in `request_options`.
+def _visualize(query, keyword, answers, request_options, html=None):
 
-    This function is really really bad, and should be rewritten
-    as soon as possible.
-    """
-
-    #
-    # at the moment, we just remove trailing slashes
-    # so queries python/ and python are equal
-    #
-    query = query.rstrip('/')
-
-    query = _rewrite_aliases(query)
-    query = _rewrite_section_name(query)
+    search_mode = bool(keyword)
 
     highlight = not bool(request_options and request_options.get('no-terminal'))
     color_style = request_options.get('style', '')
     if color_style not in COLOR_STYLES:
         color_style = ''
 
-    keyword = None
-    if '~' in query:
-        topic = query
-        pos = topic.index('~')
-        keyword = topic[pos+1:]
-        topic = topic[:pos]
-
-        options = ""
-        if '/' in keyword:
-            options = keyword[::-1]
-            options = options[:options.index('/')]
-            keyword = keyword[:-len(options)-1]
-
-        answers = find_answer_by_keyword(
-            topic, keyword, options=options, request_options=request_options)
-        search_mode = True
-    else:
-        answers = [(query, get_answer(query, keyword, request_options=request_options))]
-        search_mode = False
-
-
     found = True            # if the page was found in the database
     editable = False        # can generated page be edited on github (only cheat.sheets pages can)
     result = ""
@@ -617,93 +205,79 @@ def cheat_wrapper(query, request_options=None, html=False): # pylint: disable=to
         if topic_type == 'unknown':
             found = False
 
-        if highlight:
-            #if topic_type.endswith(" dir"):
-            #    pass
+        if topic_type == "cheat.sheets":
+            editable = True
+
+        if not highlight:
+            if search_mode:
+                result += "\n[%s]\n" % topic
+        else:
 
             if topic_type == "internal":
                 answer = _colorize_internal(topic, answer, html)
             else:
-                color_style = color_style or "native"
-                lexer = LEXER['bash']
-                for lexer_name, lexer_value in LEXER.items():
-                    if topic.startswith('cpp'):
-                        topic = 'c++' + topic[3:]
+                answer = _colorize_ansi_answer(topic, answer, color_style)
 
-                    if topic.startswith("%s/" % lexer_name):
-                        color_style = color_style or "monokai"
-                        if lexer_name == 'php':
-                            answer = "\n" % answer
-                        lexer = lexer_value
-                        break
-
-                formatter = Terminal256Formatter(style=color_style)
-                answer = pygments_highlight(answer, lexer(), formatter).lstrip('\n')
-
-        if topic_type == "cheat.sheets":
-            editable = True
-
-        if search_mode:
-            if highlight:
+            if search_mode:
                 result += "\n%s%s %s %s%s\n" % (colored.bg('dark_gray'),
                                                 colored.attr("res_underlined"),
                                                 topic,
                                                 colored.attr("res_underlined"),
                                                 colored.attr('reset'))
-            else:
-                result += "\n[%s]\n" % topic
-
         result += answer
 
     if search_mode:
-        result = result[1:]
+        result = result.lstrip('\n')
         editable = False
         repository_button = ''
     else:
         repository_button = _github_button(topic_type)
 
     if html:
-        result = result + "\n$"
-        result = html_wrapper(result)
-        title = "cheat.sh/%s" % topic
-        # title += ('\nscript'
-        #           ' src="/files/awesomplete.min.js" async>')
-        # submit button: thanks to http://stackoverflow.com/questions/477691/
-        submit_button = ('')
-        topic_list = ('%s'
-                      % ("\n".join("" % x for x in get_topics_list())))
+        result = _render_html(
+            query, result, editable, repository_button, request_options)
 
-        curl_line = "$ curl cheat.sh/"
-        if query == ':firstpage':
-            query = ""
-        form_html = ('
'
-                     '%s%s'
-                     ''
-                     '%s'
-                     '') \
-                     % (submit_button, curl_line, query, topic_list)
-
-        edit_button = ''
-        if editable:
-            # It's possible that topic directory starts with omited underscore
-            if '/' in topic:
-                topic = '_' + topic
-            edit_page_link = 'https://github.com/chubin/cheat.sheets/edit/master/sheets/' + topic
-            edit_button = (
-                '
'
-                '[edit]'
-                '
') % edit_page_link
-        result = re.sub("
", edit_button + form_html + "
", result)
-        result = re.sub("", "" + title, result)
-        if not request_options.get('quiet'):
-            result = result.replace('',
-                                    TWITTER_BUTTON \
-                                    + GITHUB_BUTTON \
-                                    + repository_button \
-                                    + GITHUB_BUTTON_FOOTER \
-                                    + '')
 
     return result, found
+
+def cheat_wrapper(query, request_options=None, html=False):
+    """
+    Giant megafunction that delivers cheat sheet for `query`.
+    If `html` is True, the answer is formated as HTML.
+    Additional request options specified in `request_options`.
+
+    This function is really really bad, and should be rewritten
+    as soon as possible.
+    """
+
+    def _parse_query(query):
+        topic = query
+        keyword = None
+        search_options = ""
+
+        keyword = None
+        if '~' in query:
+            topic = query
+            pos = topic.index('~')
+            keyword = topic[pos+1:]
+            topic = topic[:pos]
+
+            if '/' in keyword:
+                search_options = keyword[::-1]
+                search_options = search_options[:search_options.index('/')]
+                keyword = keyword[:-len(search_options)-1]
+
+        return topic, keyword, search_options
+
+    # at the moment, we just remove trailing slashes
+    # so queries python/ and python are equal
+    query = query.rstrip('/')
+    topic, keyword, search_options = _parse_query(query)
+
+    if keyword:
+        answers = find_answer_by_keyword(
+            topic, keyword, options=search_options, request_options=request_options)
+    else:
+        answers = [(topic, get_answer(topic, keyword, request_options=request_options))]
+
+    return _visualize(query, keyword, answers, request_options, html=html)
diff --git a/lib/get_answer.py b/lib/get_answer.py
new file mode 100644
index 0000000..6d3f6ad
--- /dev/null
+++ b/lib/get_answer.py
@@ -0,0 +1,446 @@
+"""
+Main module, answers hub.
+
+Exports:
+
+    get_topics_list()
+    get_topic_type()
+    get_answer()
+"""
+
+from gevent.monkey import patch_all
+from gevent.subprocess import Popen, PIPE
+patch_all()
+
+# pylint: disable=wrong-import-position,wrong-import-order
+import collections
+import glob
+import os
+import re
+import redis
+from fuzzywuzzy import process, fuzz
+
+import beautifier
+from globals import MYDIR, PATH_TLDR_PAGES, PATH_CHEAT_PAGES, PATH_CHEAT_SHEETS, COLOR_STYLES
+from adapter_learnxiny import get_learnxiny, get_learnxiny_list, is_valid_learnxy
+from languages_data import LANGUAGE_ALIAS
+# pylint: enable=wrong-import-position,wrong-import-order
+
+REDIS = redis.StrictRedis(host='localhost', port=6379, db=0)
+
+MAX_SEARCH_LEN = 20
+
+INTERNAL_TOPICS = [
+    ":list",
+    ":firstpage",
+    ':post',
+    ':bash_completion',
+    ':help',
+    ':styles',
+    ':styles-demo',
+    ':emacs',
+    ':emacs-ivy',
+    ':fish',
+    ':bash',
+    ':zsh'
+    ]
+
+def _update_tldr_topics():
+    answer = []
+    for topic in glob.glob(PATH_TLDR_PAGES):
+        _, filename = os.path.split(topic)
+        if filename.endswith('.md'):
+            answer.append(filename[:-3])
+    return answer
+TLDR_TOPICS = _update_tldr_topics()
+
+def _update_cheat_topics():
+    answer = []
+    for topic in glob.glob(PATH_CHEAT_PAGES):
+        _, filename = os.path.split(topic)
+        answer.append(filename)
+    return answer
+CHEAT_TOPICS = _update_cheat_topics()
+
+def _update_cheat_sheets_topics():
+    answer = []
+    answer_dirs = []
+
+    for topic in glob.glob(PATH_CHEAT_SHEETS + "*/*"):
+        dirname, filename = os.path.split(topic)
+        dirname = os.path.basename(dirname)
+        if dirname.startswith('_'):
+            dirname = dirname[1:]
+        answer.append("%s/%s" % (dirname, filename))
+
+    for topic in glob.glob(PATH_CHEAT_SHEETS + "*"):
+        _, filename = os.path.split(topic)
+        if os.path.isdir(topic):
+            if filename.startswith('_'):
+                filename = filename[1:]
+            answer_dirs.append(filename+'/')
+        else:
+            answer.append(filename)
+    return answer, answer_dirs
+CHEAT_SHEETS_TOPICS, CHEAT_SHEETS_DIRS = _update_cheat_sheets_topics()
+
+CACHED_TOPICS_LIST = [[]]
+def get_topics_list(skip_dirs=False, skip_internal=False):
+    """
+    List of topics returned on /:list
+    """
+
+    if CACHED_TOPICS_LIST[0] != []:
+        return CACHED_TOPICS_LIST[0]
+
+    answer = CHEAT_TOPICS + TLDR_TOPICS + CHEAT_SHEETS_TOPICS
+    answer = sorted(set(answer))
+
+    # doing it in this strange way to save the order of the topics
+    for topic in get_learnxiny_list():
+        if topic not in answer:
+            answer.append(topic)
+
+    if not skip_dirs:
+        answer += CHEAT_SHEETS_DIRS
+    if not skip_internal:
+        answer += INTERNAL_TOPICS
+
+    CACHED_TOPICS_LIST[0] = answer
+    return answer
+
+def _get_topics_dirs():
+    return set([x.split('/', 1)[0] for x in get_topics_list() if '/' in x])
+
+
+def _get_stat():
+    stat = collections.Counter([
+        get_topic_type(topic) for topic in get_topics_list()
+    ])
+
+    answer = ""
+    for key, val in stat.items():
+        answer += "%s %s\n" % (key, val)
+    return answer
+#
+#
+#
+
+def get_topic_type(topic): # pylint: disable=too-many-locals,too-many-branches,too-many-statements
+    """
+    Return topic type for `topic` or "unknown" if topic can't be determined.
+    """
+    result = ''
+    if topic == "":
+        result = "search"
+    elif topic.startswith(":"):
+        result = "internal"
+    elif '/' in topic:
+        topic_type, topic_name = topic.split('/', 1)
+        if '+' in topic_name:
+            result = 'question'
+        else:
+            if topic_type in _get_topics_dirs() and topic_name in [':list']:
+                result = "internal"
+            elif is_valid_learnxy(topic):
+                result = 'learnxiny'
+            else:
+                result = 'question'
+
+    elif topic in CHEAT_SHEETS_TOPICS:
+        result = "cheat.sheets"
+    elif topic.rstrip('/') in CHEAT_SHEETS_DIRS and topic.endswith('/'):
+        result = "cheat.sheets dir"
+    elif topic in CHEAT_TOPICS:
+        result = "cheat"
+    elif topic in TLDR_TOPICS:
+        result = "tldr"
+    elif '+' in topic:
+        result = "question"
+    else:
+        result = 'unknown'
+    print topic, " ", result
+    return result
+
+#
+#   Various cheat sheets getters
+#
+#
+#def registered_answer_getter(func):
+#    REGISTERED_ANSWER_GETTERS.append(funct)
+#    return cls
+
+def _get_internal(topic):
+    if '/' in topic:
+        topic_type, topic_name = topic.split('/', 1)
+        if topic_name == ":list":
+            topic_list = [x[len(topic_type)+1:]
+                          for x in get_topics_list()
+                          if x.startswith(topic_type + "/")]
+            return "\n".join(topic_list)+"\n"
+
+    if topic == ":list":
+        return "\n".join(x for x in get_topics_list()) + "\n"
+
+    if topic == ':styles':
+        return "\n".join(COLOR_STYLES) + "\n"
+
+    if topic == ":stat":
+        return _get_stat()+"\n"
+
+    if topic in INTERNAL_TOPICS:
+        return open(os.path.join(MYDIR, "share", topic[1:]+".txt"), "r").read()
+
+    return ""
+
+def _get_tldr(topic):
+    cmd = ["tldr", topic]
+    proc = Popen(cmd, stdout=PIPE, stderr=PIPE)
+    answer = proc.communicate()[0]
+
+    fixed_answer = []
+    for line in answer.splitlines():
+        line = line[2:]
+        if line.startswith('-'):
+            line = '# '+line[2:]
+        elif line == "":
+            pass
+        elif not line.startswith(' '):
+            line = "# "+line
+
+        fixed_answer.append(line)
+
+    answer = "\n".join(fixed_answer) + "\n"
+    return answer.decode('utf-8')
+
+def _get_cheat(topic):
+    cmd = ["cheat", topic]
+    proc = Popen(cmd, stdout=PIPE, stderr=PIPE)
+    answer = proc.communicate()[0].decode('utf-8')
+    return answer
+
+def _get_cheat_sheets(topic):
+    """
+    Get the cheat sheet topic from the own repository (cheat.sheets).
+    It's possible that topic directory starts with omited underscore
+    """
+    filename = PATH_CHEAT_SHEETS + "%s" % topic
+    if not os.path.exists(filename):
+        filename = PATH_CHEAT_SHEETS + "_%s" % topic
+    return open(filename, "r").read().decode('utf-8')
+
+def _get_cheat_sheets_dir(topic):
+    answer = []
+    for f_name in glob.glob(PATH_CHEAT_SHEETS + "%s/*" % topic.rstrip('/')):
+        answer.append(os.path.basename(f_name))
+    topics = sorted(answer)
+    return "\n".join(topics) + "\n"
+
+def _get_answer_for_question(topic):
+    """
+    Find answer for the `topic` question.
+    """
+    topic = " ".join(topic.replace('+', ' ').strip().split())
+    cmd = ["/home/igor/cheat.sh/bin/get-answer-for-question", topic]
+    proc = Popen(cmd, stdout=PIPE, stderr=PIPE)
+    answer = proc.communicate()[0].decode('utf-8')
+    return answer
+
+def _get_unknown(topic):
+    topics_list = get_topics_list()
+    if topic.startswith(':'):
+        topics_list = [x for x in topics_list if x.startswith(':')]
+    else:
+        topics_list = [x for x in topics_list if not x.startswith(':')]
+
+    possible_topics = process.extract(topic, topics_list, scorer=fuzz.ratio)[:3]
+    possible_topics_text = "\n".join([("    * %s %s" % x) for x in possible_topics])
+    return """
+Unknown topic.
+Do you mean one of these topics may be?
+
+%s
+    """ % possible_topics_text
+
+# pylint: disable=bad-whitespace
+#
+# topic_type, function_getter
+# should be replaced with a decorator
+TOPIC_GETTERS = (
+    ("cheat.sheets",        _get_cheat_sheets),
+    ("cheat.sheets dir",    _get_cheat_sheets_dir),
+    ("tldr",                _get_tldr),
+    ("internal",            _get_internal),
+    ("cheat",               _get_cheat),
+    ("learnxiny",           get_learnxiny),
+    ("question",            _get_answer_for_question),
+    ("unknown",             _get_unknown),
+)
+# pylint: enable=bad-whitespace
+
+def get_answer(topic, keyword, options="", request_options=None): # pylint: disable=too-many-locals,too-many-branches,too-many-statements
+    """
+    Find cheat sheet for the topic.
+    If `keyword` is None or rempty, return the whole answer.
+    Otherwise cut the paragraphs containing keywords.
+
+    Args:
+        topic (str):    the name of the topic of the cheat sheet
+        keyword (str):  the name of the keywords to search in the cheat sheets
+
+    Returns:
+        string:         the cheat sheet
+    """
+
+    def _join_paragraphs(paragraphs):
+        answer = "\n".join(paragraphs)
+        return answer
+
+    def _split_paragraphs(text):
+        answer = []
+        paragraph = ""
+        for line in text.splitlines():
+            if line == "":
+                answer.append(paragraph)
+                paragraph = ""
+            else:
+                paragraph += line+"\n"
+        answer.append(paragraph)
+        return answer
+
+    def _paragraph_contains(paragraph, keyword, insensitive=False, word_boundaries=True):
+        """
+        Check if `paragraph` contains `keyword`.
+        Several keywords can be joined together using ~
+        For example: ~ssh~passphrase
+        """
+        answer = True
+
+        if '~' in keyword:
+            keywords = keyword.split('~')
+        else:
+            keywords = [keyword]
+
+        for kwrd in keywords:
+            regex = re.escape(kwrd)
+            if not word_boundaries:
+                regex = r"\b%s\b" % kwrd
+
+            if insensitive:
+                answer = answer and bool(re.search(regex, paragraph, re.IGNORECASE))
+            else:
+                answer = answer and bool(re.search(regex, paragraph))
+
+        return answer
+
+    def _rewrite_aliases(word):
+        if word == ':bash.completion':
+            return ':bash_completion'
+        return word
+
+    def _rewrite_section_name(query):
+        """
+        """
+        if '/' not in query:
+            return query
+
+        section_name, rest = query.split('/', 1)
+        section_name = LANGUAGE_ALIAS.get(section_name, section_name)
+        return "%s/%s" % (section_name, rest)
+
+    answer = None
+    needs_beautification = False
+
+    topic = _rewrite_aliases(topic)
+    topic = _rewrite_section_name(topic)
+
+    # checking if the answer is in the cache
+    if topic != "":
+        # temporary hack for "questions":
+        # the topic name has to be prefixed with q:
+        # so we can later delete them from redis
+        # and we known that they need beautification
+        if '/' in topic and '+' in topic:
+            topic = "q:" + topic
+            needs_beautification = True
+
+        answer = REDIS.get(topic)
+        if answer:
+            answer = answer.decode('utf-8')
+
+    # if answer was not found in the cache
+    # try to find it in one of the repositories
+    if not answer:
+        topic_type = get_topic_type(topic)
+
+        for topic_getter_type, topic_getter in TOPIC_GETTERS:
+            if topic_type == topic_getter_type:
+                answer = topic_getter(topic)
+                break
+        if not answer:
+            topic_type = "unknown"
+            answer = _get_unknown(topic)
+
+        # saving answers in the cache
+        if topic_type not in ["search", "internal", "unknown"]:
+            REDIS.set(topic, answer)
+
+    if needs_beautification:
+        filetype = 'bash'
+        if '/' in topic:
+            filetype = topic.split('/', 1)[0]
+            if filetype.startswith('q:'):
+                filetype = filetype[2:]
+
+        answer = beautifier.beautify(answer.encode('utf-8'), filetype, request_options)
+
+    if not keyword:
+        return answer
+
+    #
+    # shorten the answer, because keyword is specified
+    #
+    insensitive = 'i' in options
+    word_boundaries = 'b' in options
+
+    paragraphs = _split_paragraphs(answer)
+    paragraphs = [p for p in paragraphs
+                  if _paragraph_contains(p, keyword,
+                                         insensitive=insensitive,
+                                         word_boundaries=word_boundaries)]
+    if paragraphs == []:
+        return ""
+
+    answer = _join_paragraphs(paragraphs)
+    return answer
+
+def find_answer_by_keyword(directory, keyword, options="", request_options=None):
+    """
+    Search in the whole tree of all cheatsheets or in its subtree `directory`
+    by `keyword`
+    """
+
+    recursive = 'r' in options
+
+    answer_paragraphs = []
+    for topic in get_topics_list(skip_internal=True, skip_dirs=True):
+        # skip the internal pages, don't show them in search
+        if topic in INTERNAL_TOPICS:
+            continue
+
+        if not topic.startswith(directory):
+            continue
+
+        subtopic = topic[len(directory):]
+        if not recursive and '/' in subtopic:
+            continue
+
+        answer = get_answer(topic, keyword, options=options, request_options=request_options)
+        if answer:
+            answer_paragraphs.append((topic, answer))
+
+        if len(answer_paragraphs) > MAX_SEARCH_LEN:
+            answer_paragraphs.append(("LIMITED", "LIMITED TO %s ANSWERS" % MAX_SEARCH_LEN))
+            break
+
+    return answer_paragraphs
diff --git a/lib/globals.py b/lib/globals.py
index 98e6b54..d00939d 100644
--- a/lib/globals.py
+++ b/lib/globals.py
@@ -5,6 +5,7 @@ All hardcoded pathes should be (theoretically) here.
 
 import logging
 import os
+from pygments.styles import get_all_styles
 
 MYDIR = os.path.abspath(os.path.dirname(os.path.dirname('__file__')))
 
@@ -21,6 +22,8 @@ PATH_CHEAT_PAGES = "/usr/local/lib/python2.7/dist-packages/cheat/cheatsheets/*"
 PATH_CHEAT_SHEETS = "/home/igor/cheat.sheets/sheets/"
 PATH_CHEAT_SHEETS_SPOOL = "/home/igor/cheat.sheets/spool/"
 
+COLOR_STYLES = sorted(list(get_all_styles()))
+
 def error(text):
     """
     Log error `text` and produce a RuntimeError exception