diff --git a/lib/cheat_wrapper.py b/lib/cheat_wrapper.py
index 44d423d..b130620 100644
--- a/lib/cheat_wrapper.py
+++ b/lib/cheat_wrapper.py
@@ -1,7 +1,7 @@
"""
Main cheat.sh wrapper.
-Gets answer from the getters, add syntax highlighting or html markup and returns it.
-At the moment, it contains the getters that should be moved out to a separate file.
+Get answers from getters (in get_answer), adds syntax highlighting
+or html markup and returns the result.
"""
from gevent.monkey import patch_all
@@ -11,90 +11,21 @@ patch_all()
# pylint: disable=wrong-import-position,wrong-import-order
import sys
import os
-import glob
import re
-import collections
import colored
-import redis
-from fuzzywuzzy import process, fuzz
-
from pygments import highlight as pygments_highlight
from pygments.formatters import Terminal256Formatter # pylint: disable=no-name-in-module
-from pygments.styles import get_all_styles
MYDIR = os.path.abspath(os.path.dirname(os.path.dirname('__file__')))
sys.path.append("%s/lib/" % MYDIR)
-from globals import error, ANSI2HTML, \
- PATH_TLDR_PAGES, PATH_CHEAT_PAGES, \
- PATH_CHEAT_SHEETS
+from globals import error, ANSI2HTML, COLOR_STYLES
from buttons import TWITTER_BUTTON, GITHUB_BUTTON, GITHUB_BUTTON_FOOTER
-from adapter_learnxiny import get_learnxiny, get_learnxiny_list, is_valid_learnxy
-from languages_data import LEXER, LANGUAGE_ALIAS
-import beautifier
+from languages_data import LEXER
+from get_answer import get_topic_type, get_topics_list, get_answer, find_answer_by_keyword
+# import beautifier
# pylint: disable=wrong-import-position,wrong-import-order
-COLOR_STYLES = sorted(list(get_all_styles()))
-
-# globals
-INTERNAL_TOPICS = [
- ":list",
- ":firstpage",
- ':post',
- ':bash_completion',
- ':help',
- ':styles',
- ':styles-demo',
- ':emacs',
- ':emacs-ivy',
- ':fish',
- ':bash',
- ':zsh'
- ]
-
-
-REDIS = redis.StrictRedis(host='localhost', port=6379, db=0)
-MAX_SEARCH_LEN = 20
-
-def _update_tldr_topics():
- answer = []
- for topic in glob.glob(PATH_TLDR_PAGES):
- _, filename = os.path.split(topic)
- if filename.endswith('.md'):
- answer.append(filename[:-3])
- return answer
-TLDR_TOPICS = _update_tldr_topics()
-
-def _update_cheat_topics():
- answer = []
- for topic in glob.glob(PATH_CHEAT_PAGES):
- _, filename = os.path.split(topic)
- answer.append(filename)
- return answer
-CHEAT_TOPICS = _update_cheat_topics()
-
-def _update_cheat_sheets_topics():
- answer = []
- answer_dirs = []
-
- for topic in glob.glob(PATH_CHEAT_SHEETS + "*/*"):
- dirname, filename = os.path.split(topic)
- dirname = os.path.basename(dirname)
- if dirname.startswith('_'):
- dirname = dirname[1:]
- answer.append("%s/%s" % (dirname, filename))
-
- for topic in glob.glob(PATH_CHEAT_SHEETS + "*"):
- _, filename = os.path.split(topic)
- if os.path.isdir(topic):
- if filename.startswith('_'):
- filename = filename[1:]
- answer_dirs.append(filename+'/')
- else:
- answer.append(filename)
- return answer, answer_dirs
-CHEAT_SHEETS_TOPICS, CHEAT_SHEETS_DIRS = _update_cheat_sheets_topics()
-
ANSI_ESCAPE = re.compile(r'(\x9B|\x1B\[)[0-?]*[ -\/]*[@-~]')
def remove_ansi(sometext):
"""
@@ -115,357 +46,6 @@ def html_wrapper(data):
error(stdout + stderr)
return stdout.decode('utf-8')
-#
-#
-#
-
-CACHED_TOPICS_LIST = [[]]
-def get_topics_list(skip_dirs=False, skip_internal=False):
- """
- List of topics returned on /:list
- """
-
- if CACHED_TOPICS_LIST[0] != []:
- return CACHED_TOPICS_LIST[0]
-
- answer = CHEAT_TOPICS + TLDR_TOPICS + CHEAT_SHEETS_TOPICS
- answer = sorted(set(answer))
-
- # doing it in this strange way to save the order of the topics
- for topic in get_learnxiny_list():
- if topic not in answer:
- answer.append(topic)
-
- if not skip_dirs:
- answer += CHEAT_SHEETS_DIRS
- if not skip_internal:
- answer += INTERNAL_TOPICS
-
- CACHED_TOPICS_LIST[0] = answer
- return answer
-
-def _get_topics_dirs():
- return set([x.split('/', 1)[0] for x in get_topics_list() if '/' in x])
-
-
-def _get_stat():
- stat = collections.Counter([
- get_topic_type(topic) for topic in get_topics_list()
- ])
-
- answer = ""
- for key, val in stat.items():
- answer += "%s %s\n" % (key, val)
- return answer
-#
-#
-#
-
-def get_topic_type(topic): # pylint: disable=too-many-locals,too-many-branches,too-many-statements
- """
- Return topic type for `topic` or "unknown" if topic can't be determined.
- """
- result = ''
- if topic == "":
- result = "search"
- elif topic.startswith(":"):
- result = "internal"
- elif '/' in topic:
- topic_type, topic_name = topic.split('/', 1)
- if '+' in topic_name:
- result = 'question'
- else:
- if topic_type in _get_topics_dirs() and topic_name in [':list']:
- result = "internal"
- elif is_valid_learnxy(topic):
- result = 'learnxiny'
- else:
- result = 'question'
-
- elif topic in CHEAT_SHEETS_TOPICS:
- result = "cheat.sheets"
- elif topic.rstrip('/') in CHEAT_SHEETS_DIRS and topic.endswith('/'):
- result = "cheat.sheets dir"
- elif topic in CHEAT_TOPICS:
- result = "cheat"
- elif topic in TLDR_TOPICS:
- result = "tldr"
- elif '+' in topic:
- result = "question"
- else:
- result = 'unknown'
- print topic, " ", result
- return result
-
-#
-# Various cheat sheets getters
-#
-#
-#def registered_answer_getter(func):
-# REGISTERED_ANSWER_GETTERS.append(funct)
-# return cls
-
-def _get_internal(topic):
- if '/' in topic:
- topic_type, topic_name = topic.split('/', 1)
- if topic_name == ":list":
- topic_list = [x[len(topic_type)+1:]
- for x in get_topics_list()
- if x.startswith(topic_type + "/")]
- return "\n".join(topic_list)+"\n"
-
- if topic == ":list":
- return "\n".join(x for x in get_topics_list()) + "\n"
-
- if topic == ':styles':
- return "\n".join(COLOR_STYLES) + "\n"
-
- if topic == ":stat":
- return _get_stat()+"\n"
-
- if topic in INTERNAL_TOPICS:
- return open(os.path.join(MYDIR, "share", topic[1:]+".txt"), "r").read()
-
- return ""
-
-def _get_tldr(topic):
- cmd = ["tldr", topic]
- proc = Popen(cmd, stdout=PIPE, stderr=PIPE)
- answer = proc.communicate()[0]
-
- fixed_answer = []
- for line in answer.splitlines():
- line = line[2:]
- if line.startswith('-'):
- line = '# '+line[2:]
- elif line == "":
- pass
- elif not line.startswith(' '):
- line = "# "+line
-
- fixed_answer.append(line)
-
- answer = "\n".join(fixed_answer) + "\n"
- return answer.decode('utf-8')
-
-def _get_cheat(topic):
- cmd = ["cheat", topic]
- proc = Popen(cmd, stdout=PIPE, stderr=PIPE)
- answer = proc.communicate()[0].decode('utf-8')
- return answer
-
-def _get_cheat_sheets(topic):
- """
- Get the cheat sheet topic from the own repository (cheat.sheets).
- It's possible that topic directory starts with omited underscore
- """
- filename = PATH_CHEAT_SHEETS + "%s" % topic
- if not os.path.exists(filename):
- filename = PATH_CHEAT_SHEETS + "_%s" % topic
- return open(filename, "r").read().decode('utf-8')
-
-def _get_cheat_sheets_dir(topic):
- answer = []
- for f_name in glob.glob(PATH_CHEAT_SHEETS + "%s/*" % topic.rstrip('/')):
- answer.append(os.path.basename(f_name))
- topics = sorted(answer)
- return "\n".join(topics) + "\n"
-
-def _get_answer_for_question(topic):
- """
- Find answer for the `topic` question.
- """
- topic = " ".join(topic.replace('+', ' ').strip().split())
- cmd = ["/home/igor/cheat.sh/bin/get-answer-for-question", topic]
- proc = Popen(cmd, stdout=PIPE, stderr=PIPE)
- answer = proc.communicate()[0].decode('utf-8')
- return answer
-
-def _get_unknown(topic):
- topics_list = get_topics_list()
- if topic.startswith(':'):
- topics_list = [x for x in topics_list if x.startswith(':')]
- else:
- topics_list = [x for x in topics_list if not x.startswith(':')]
-
- possible_topics = process.extract(topic, topics_list, scorer=fuzz.ratio)[:3]
- possible_topics_text = "\n".join([(" * %s %s" % x) for x in possible_topics])
- return """
-Unknown topic.
-Do you mean one of these topics may be?
-
-%s
- """ % possible_topics_text
-
-# pylint: disable=bad-whitespace
-#
-# topic_type, function_getter
-# should be replaced with a decorator
-TOPIC_GETTERS = (
- ("cheat.sheets", _get_cheat_sheets),
- ("cheat.sheets dir", _get_cheat_sheets_dir),
- ("tldr", _get_tldr),
- ("internal", _get_internal),
- ("cheat", _get_cheat),
- ("learnxiny", get_learnxiny),
- ("question", _get_answer_for_question),
- ("unknown", _get_unknown),
-)
-# pylint: enable=bad-whitespace
-
-def get_answer(topic, keyword, options="", request_options=None): # pylint: disable=too-many-locals,too-many-branches,too-many-statements
- """
- Find cheat sheet for the topic.
- If `keyword` is None or rempty, return the whole answer.
- Otherwise cut the paragraphs containing keywords.
-
- Args:
- topic (str): the name of the topic of the cheat sheet
- keyword (str): the name of the keywords to search in the cheat sheets
-
- Returns:
- string: the cheat sheet
- """
-
- def _join_paragraphs(paragraphs):
- answer = "\n".join(paragraphs)
- return answer
-
- def _split_paragraphs(text):
- answer = []
- paragraph = ""
- for line in text.splitlines():
- if line == "":
- answer.append(paragraph)
- paragraph = ""
- else:
- paragraph += line+"\n"
- answer.append(paragraph)
- return answer
-
- def _paragraph_contains(paragraph, keyword, insensitive=False, word_boundaries=True):
- """
- Check if `paragraph` contains `keyword`.
- Several keywords can be joined together using ~
- For example: ~ssh~passphrase
- """
- answer = True
-
- if '~' in keyword:
- keywords = keyword.split('~')
- else:
- keywords = [keyword]
-
- for kwrd in keywords:
- regex = re.escape(kwrd)
- if not word_boundaries:
- regex = r"\b%s\b" % kwrd
-
- if insensitive:
- answer = answer and bool(re.search(regex, paragraph, re.IGNORECASE))
- else:
- answer = answer and bool(re.search(regex, paragraph))
-
- return answer
-
- answer = None
- needs_beautification = False
-
- # checking if the answer is in the cache
- if topic != "":
- # temporary hack for "questions":
- # the topic name has to be prefixed with q:
- # so we can later delete them from redis
- # and we known that they need beautification
- if '/' in topic and '+' in topic:
- topic = "q:" + topic
- needs_beautification = True
-
- answer = REDIS.get(topic)
- if answer:
- answer = answer.decode('utf-8')
-
- # if answer was not found in the cache
- # try to find it in one of the repositories
- if not answer:
- topic_type = get_topic_type(topic)
-
- for topic_getter_type, topic_getter in TOPIC_GETTERS:
- if topic_type == topic_getter_type:
- answer = topic_getter(topic)
- break
- if not answer:
- topic_type = "unknown"
- answer = _get_unknown(topic)
-
- # saving answers in the cache
- if topic_type not in ["search", "internal", "unknown"]:
- REDIS.set(topic, answer)
-
- if needs_beautification:
- filetype = 'bash'
- if '/' in topic:
- filetype = topic.split('/', 1)[0]
- if filetype.startswith('q:'):
- filetype = filetype[2:]
-
- answer = beautifier.beautify(answer.encode('utf-8'), filetype, request_options)
-
- if not keyword:
- return answer
-
- #
- # shorten the answer, because keyword is specified
- #
- insensitive = 'i' in options
- word_boundaries = 'b' in options
-
- paragraphs = _split_paragraphs(answer)
- paragraphs = [p for p in paragraphs
- if _paragraph_contains(p, keyword,
- insensitive=insensitive,
- word_boundaries=word_boundaries)]
- if paragraphs == []:
- return ""
-
- answer = _join_paragraphs(paragraphs)
- return answer
-
-def find_answer_by_keyword(directory, keyword, options="", request_options=None):
- """
- Search in the whole tree of all cheatsheets or in its subtree `directory`
- by `keyword`
- """
-
- recursive = 'r' in options
-
- answer_paragraphs = []
- for topic in get_topics_list(skip_internal=True, skip_dirs=True):
- # skip the internal pages, don't show them in search
- if topic in INTERNAL_TOPICS:
- continue
-
- if not topic.startswith(directory):
- continue
-
- subtopic = topic[len(directory):]
- if not recursive and '/' in subtopic:
- continue
-
- answer = get_answer(topic, keyword, options=options, request_options=request_options)
- if answer:
- answer_paragraphs.append((topic, answer))
-
- if len(answer_paragraphs) > MAX_SEARCH_LEN:
- answer_paragraphs.append(("LIMITED", "LIMITED TO %s ANSWERS" % MAX_SEARCH_LEN))
- break
-
- return answer_paragraphs
-
-#
-#========================>8 cut here 8<===================================================
-#
-
def _colorize_internal(topic, answer, html_needed):
def _colorize_line(line):
@@ -505,6 +85,21 @@ def _colorize_internal(topic, answer, html_needed):
return answer
+def _colorize_ansi_answer(topic, answer, color_style):
+
+ color_style = color_style or "native"
+ lexer = LEXER['bash']
+ for lexer_name, lexer_value in LEXER.items():
+ if topic.startswith("%s/" % lexer_name):
+ color_style = color_style or "monokai"
+ if lexer_name == 'php':
+ answer = "\n" % answer
+ lexer = lexer_value
+ break
+
+ formatter = Terminal256Formatter(style=color_style)
+ return pygments_highlight(answer, lexer(), formatter).lstrip('\n')
+
def _github_button(topic_type):
repository = {
@@ -536,68 +131,61 @@ def _github_button(topic_type):
) % locals()
return button
-#
+def _render_html(query, result, editable, repository_button, request_options):
-def _rewrite_aliases(word):
- if word == ':bash.completion':
- return ':bash_completion'
- return word
+ result = result + "\n$"
+ result = html_wrapper(result)
+ title = "cheat.sh/%s" % query
+ # title += ('\nscript'
+ # ' src="/files/awesomplete.min.js" async>')
+ # submit button: thanks to http://stackoverflow.com/questions/477691/
+ submit_button = ('')
+ topic_list = (''
+ % ("\n".join("" % x for x in get_topics_list())))
-def _rewrite_section_name(query):
- """
- """
- if '/' not in query:
- return query
+ curl_line = "$ curl cheat.sh/"
+ if query == ':firstpage':
+ query = ""
+ form_html = (''
+ '%s%s'
+ ''
+ '%s'
+ '') \
+ % (submit_button, curl_line, query, topic_list)
- section_name, rest = query.split('/', 1)
- section_name = LANGUAGE_ALIAS.get(section_name, section_name)
- return "%s/%s" % (section_name, rest)
+ edit_button = ''
+ if editable:
+ # It's possible that topic directory starts with omited underscore
+ if '/' in query:
+ query = '_' + query
+ edit_page_link = 'https://github.com/chubin/cheat.sheets/edit/master/sheets/' + query
+ edit_button = (
+ ''
+ '[edit]'
+ '
') % edit_page_link
+ result = re.sub("", edit_button + form_html + "", result)
+ result = re.sub("", "" + title, result)
+ if not request_options.get('quiet'):
+ result = result.replace('',
+ TWITTER_BUTTON \
+ + GITHUB_BUTTON \
+ + repository_button \
+ + GITHUB_BUTTON_FOOTER \
+ + '')
+ return result
-def cheat_wrapper(query, request_options=None, html=False): # pylint: disable=too-many-locals,too-many-branches,too-many-statements
- """
- Giant megafunction that delivers cheat sheet for `query`.
- If `html` is True, the answer is formated as HTML.
- Additional request options specified in `request_options`.
+def _visualize(query, keyword, answers, request_options, html=None):
- This function is really really bad, and should be rewritten
- as soon as possible.
- """
-
- #
- # at the moment, we just remove trailing slashes
- # so queries python/ and python are equal
- #
- query = query.rstrip('/')
-
- query = _rewrite_aliases(query)
- query = _rewrite_section_name(query)
+ search_mode = bool(keyword)
highlight = not bool(request_options and request_options.get('no-terminal'))
color_style = request_options.get('style', '')
if color_style not in COLOR_STYLES:
color_style = ''
- keyword = None
- if '~' in query:
- topic = query
- pos = topic.index('~')
- keyword = topic[pos+1:]
- topic = topic[:pos]
-
- options = ""
- if '/' in keyword:
- options = keyword[::-1]
- options = options[:options.index('/')]
- keyword = keyword[:-len(options)-1]
-
- answers = find_answer_by_keyword(
- topic, keyword, options=options, request_options=request_options)
- search_mode = True
- else:
- answers = [(query, get_answer(query, keyword, request_options=request_options))]
- search_mode = False
-
-
found = True # if the page was found in the database
editable = False # can generated page be edited on github (only cheat.sheets pages can)
result = ""
@@ -617,93 +205,79 @@ def cheat_wrapper(query, request_options=None, html=False): # pylint: disable=to
if topic_type == 'unknown':
found = False
- if highlight:
- #if topic_type.endswith(" dir"):
- # pass
+ if topic_type == "cheat.sheets":
+ editable = True
+
+ if not highlight:
+ if search_mode:
+ result += "\n[%s]\n" % topic
+ else:
if topic_type == "internal":
answer = _colorize_internal(topic, answer, html)
else:
- color_style = color_style or "native"
- lexer = LEXER['bash']
- for lexer_name, lexer_value in LEXER.items():
- if topic.startswith('cpp'):
- topic = 'c++' + topic[3:]
+ answer = _colorize_ansi_answer(topic, answer, color_style)
- if topic.startswith("%s/" % lexer_name):
- color_style = color_style or "monokai"
- if lexer_name == 'php':
- answer = "\n" % answer
- lexer = lexer_value
- break
-
- formatter = Terminal256Formatter(style=color_style)
- answer = pygments_highlight(answer, lexer(), formatter).lstrip('\n')
-
- if topic_type == "cheat.sheets":
- editable = True
-
- if search_mode:
- if highlight:
+ if search_mode:
result += "\n%s%s %s %s%s\n" % (colored.bg('dark_gray'),
colored.attr("res_underlined"),
topic,
colored.attr("res_underlined"),
colored.attr('reset'))
- else:
- result += "\n[%s]\n" % topic
-
result += answer
if search_mode:
- result = result[1:]
+ result = result.lstrip('\n')
editable = False
repository_button = ''
else:
repository_button = _github_button(topic_type)
if html:
- result = result + "\n$"
- result = html_wrapper(result)
- title = "cheat.sh/%s" % topic
- # title += ('\nscript'
- # ' src="/files/awesomplete.min.js" async>')
- # submit button: thanks to http://stackoverflow.com/questions/477691/
- submit_button = ('')
- topic_list = (''
- % ("\n".join("" % x for x in get_topics_list())))
+ result = _render_html(
+ query, result, editable, repository_button, request_options)
- curl_line = "$ curl cheat.sh/"
- if query == ':firstpage':
- query = ""
- form_html = ('
'
- '%s%s'
- ''
- '%s'
- '') \
- % (submit_button, curl_line, query, topic_list)
-
- edit_button = ''
- if editable:
- # It's possible that topic directory starts with omited underscore
- if '/' in topic:
- topic = '_' + topic
- edit_page_link = 'https://github.com/chubin/cheat.sheets/edit/master/sheets/' + topic
- edit_button = (
- ''
- '[edit]'
- '
') % edit_page_link
- result = re.sub("", edit_button + form_html + "", result)
- result = re.sub("", "" + title, result)
- if not request_options.get('quiet'):
- result = result.replace('',
- TWITTER_BUTTON \
- + GITHUB_BUTTON \
- + repository_button \
- + GITHUB_BUTTON_FOOTER \
- + '')
return result, found
+
+def cheat_wrapper(query, request_options=None, html=False):
+ """
+ Giant megafunction that delivers cheat sheet for `query`.
+ If `html` is True, the answer is formated as HTML.
+ Additional request options specified in `request_options`.
+
+ This function is really really bad, and should be rewritten
+ as soon as possible.
+ """
+
+ def _parse_query(query):
+ topic = query
+ keyword = None
+ search_options = ""
+
+ keyword = None
+ if '~' in query:
+ topic = query
+ pos = topic.index('~')
+ keyword = topic[pos+1:]
+ topic = topic[:pos]
+
+ if '/' in keyword:
+ search_options = keyword[::-1]
+ search_options = search_options[:search_options.index('/')]
+ keyword = keyword[:-len(search_options)-1]
+
+ return topic, keyword, search_options
+
+ # at the moment, we just remove trailing slashes
+ # so queries python/ and python are equal
+ query = query.rstrip('/')
+ topic, keyword, search_options = _parse_query(query)
+
+ if keyword:
+ answers = find_answer_by_keyword(
+ topic, keyword, options=search_options, request_options=request_options)
+ else:
+ answers = [(topic, get_answer(topic, keyword, request_options=request_options))]
+
+ return _visualize(query, keyword, answers, request_options, html=html)
diff --git a/lib/get_answer.py b/lib/get_answer.py
new file mode 100644
index 0000000..6d3f6ad
--- /dev/null
+++ b/lib/get_answer.py
@@ -0,0 +1,446 @@
+"""
+Main module, answers hub.
+
+Exports:
+
+ get_topics_list()
+ get_topic_type()
+ get_answer()
+"""
+
+from gevent.monkey import patch_all
+from gevent.subprocess import Popen, PIPE
+patch_all()
+
+# pylint: disable=wrong-import-position,wrong-import-order
+import collections
+import glob
+import os
+import re
+import redis
+from fuzzywuzzy import process, fuzz
+
+import beautifier
+from globals import MYDIR, PATH_TLDR_PAGES, PATH_CHEAT_PAGES, PATH_CHEAT_SHEETS, COLOR_STYLES
+from adapter_learnxiny import get_learnxiny, get_learnxiny_list, is_valid_learnxy
+from languages_data import LANGUAGE_ALIAS
+# pylint: enable=wrong-import-position,wrong-import-order
+
+REDIS = redis.StrictRedis(host='localhost', port=6379, db=0)
+
+MAX_SEARCH_LEN = 20
+
+INTERNAL_TOPICS = [
+ ":list",
+ ":firstpage",
+ ':post',
+ ':bash_completion',
+ ':help',
+ ':styles',
+ ':styles-demo',
+ ':emacs',
+ ':emacs-ivy',
+ ':fish',
+ ':bash',
+ ':zsh'
+ ]
+
+def _update_tldr_topics():
+ answer = []
+ for topic in glob.glob(PATH_TLDR_PAGES):
+ _, filename = os.path.split(topic)
+ if filename.endswith('.md'):
+ answer.append(filename[:-3])
+ return answer
+TLDR_TOPICS = _update_tldr_topics()
+
+def _update_cheat_topics():
+ answer = []
+ for topic in glob.glob(PATH_CHEAT_PAGES):
+ _, filename = os.path.split(topic)
+ answer.append(filename)
+ return answer
+CHEAT_TOPICS = _update_cheat_topics()
+
+def _update_cheat_sheets_topics():
+ answer = []
+ answer_dirs = []
+
+ for topic in glob.glob(PATH_CHEAT_SHEETS + "*/*"):
+ dirname, filename = os.path.split(topic)
+ dirname = os.path.basename(dirname)
+ if dirname.startswith('_'):
+ dirname = dirname[1:]
+ answer.append("%s/%s" % (dirname, filename))
+
+ for topic in glob.glob(PATH_CHEAT_SHEETS + "*"):
+ _, filename = os.path.split(topic)
+ if os.path.isdir(topic):
+ if filename.startswith('_'):
+ filename = filename[1:]
+ answer_dirs.append(filename+'/')
+ else:
+ answer.append(filename)
+ return answer, answer_dirs
+CHEAT_SHEETS_TOPICS, CHEAT_SHEETS_DIRS = _update_cheat_sheets_topics()
+
+CACHED_TOPICS_LIST = [[]]
+def get_topics_list(skip_dirs=False, skip_internal=False):
+ """
+ List of topics returned on /:list
+ """
+
+ if CACHED_TOPICS_LIST[0] != []:
+ return CACHED_TOPICS_LIST[0]
+
+ answer = CHEAT_TOPICS + TLDR_TOPICS + CHEAT_SHEETS_TOPICS
+ answer = sorted(set(answer))
+
+ # doing it in this strange way to save the order of the topics
+ for topic in get_learnxiny_list():
+ if topic not in answer:
+ answer.append(topic)
+
+ if not skip_dirs:
+ answer += CHEAT_SHEETS_DIRS
+ if not skip_internal:
+ answer += INTERNAL_TOPICS
+
+ CACHED_TOPICS_LIST[0] = answer
+ return answer
+
+def _get_topics_dirs():
+ return set([x.split('/', 1)[0] for x in get_topics_list() if '/' in x])
+
+
+def _get_stat():
+ stat = collections.Counter([
+ get_topic_type(topic) for topic in get_topics_list()
+ ])
+
+ answer = ""
+ for key, val in stat.items():
+ answer += "%s %s\n" % (key, val)
+ return answer
+#
+#
+#
+
+def get_topic_type(topic): # pylint: disable=too-many-locals,too-many-branches,too-many-statements
+ """
+ Return topic type for `topic` or "unknown" if topic can't be determined.
+ """
+ result = ''
+ if topic == "":
+ result = "search"
+ elif topic.startswith(":"):
+ result = "internal"
+ elif '/' in topic:
+ topic_type, topic_name = topic.split('/', 1)
+ if '+' in topic_name:
+ result = 'question'
+ else:
+ if topic_type in _get_topics_dirs() and topic_name in [':list']:
+ result = "internal"
+ elif is_valid_learnxy(topic):
+ result = 'learnxiny'
+ else:
+ result = 'question'
+
+ elif topic in CHEAT_SHEETS_TOPICS:
+ result = "cheat.sheets"
+ elif topic.rstrip('/') in CHEAT_SHEETS_DIRS and topic.endswith('/'):
+ result = "cheat.sheets dir"
+ elif topic in CHEAT_TOPICS:
+ result = "cheat"
+ elif topic in TLDR_TOPICS:
+ result = "tldr"
+ elif '+' in topic:
+ result = "question"
+ else:
+ result = 'unknown'
+ print topic, " ", result
+ return result
+
+#
+# Various cheat sheets getters
+#
+#
+#def registered_answer_getter(func):
+# REGISTERED_ANSWER_GETTERS.append(funct)
+# return cls
+
+def _get_internal(topic):
+ if '/' in topic:
+ topic_type, topic_name = topic.split('/', 1)
+ if topic_name == ":list":
+ topic_list = [x[len(topic_type)+1:]
+ for x in get_topics_list()
+ if x.startswith(topic_type + "/")]
+ return "\n".join(topic_list)+"\n"
+
+ if topic == ":list":
+ return "\n".join(x for x in get_topics_list()) + "\n"
+
+ if topic == ':styles':
+ return "\n".join(COLOR_STYLES) + "\n"
+
+ if topic == ":stat":
+ return _get_stat()+"\n"
+
+ if topic in INTERNAL_TOPICS:
+ return open(os.path.join(MYDIR, "share", topic[1:]+".txt"), "r").read()
+
+ return ""
+
+def _get_tldr(topic):
+ cmd = ["tldr", topic]
+ proc = Popen(cmd, stdout=PIPE, stderr=PIPE)
+ answer = proc.communicate()[0]
+
+ fixed_answer = []
+ for line in answer.splitlines():
+ line = line[2:]
+ if line.startswith('-'):
+ line = '# '+line[2:]
+ elif line == "":
+ pass
+ elif not line.startswith(' '):
+ line = "# "+line
+
+ fixed_answer.append(line)
+
+ answer = "\n".join(fixed_answer) + "\n"
+ return answer.decode('utf-8')
+
+def _get_cheat(topic):
+ cmd = ["cheat", topic]
+ proc = Popen(cmd, stdout=PIPE, stderr=PIPE)
+ answer = proc.communicate()[0].decode('utf-8')
+ return answer
+
+def _get_cheat_sheets(topic):
+ """
+ Get the cheat sheet topic from the own repository (cheat.sheets).
+ It's possible that topic directory starts with omited underscore
+ """
+ filename = PATH_CHEAT_SHEETS + "%s" % topic
+ if not os.path.exists(filename):
+ filename = PATH_CHEAT_SHEETS + "_%s" % topic
+ return open(filename, "r").read().decode('utf-8')
+
+def _get_cheat_sheets_dir(topic):
+ answer = []
+ for f_name in glob.glob(PATH_CHEAT_SHEETS + "%s/*" % topic.rstrip('/')):
+ answer.append(os.path.basename(f_name))
+ topics = sorted(answer)
+ return "\n".join(topics) + "\n"
+
+def _get_answer_for_question(topic):
+ """
+ Find answer for the `topic` question.
+ """
+ topic = " ".join(topic.replace('+', ' ').strip().split())
+ cmd = ["/home/igor/cheat.sh/bin/get-answer-for-question", topic]
+ proc = Popen(cmd, stdout=PIPE, stderr=PIPE)
+ answer = proc.communicate()[0].decode('utf-8')
+ return answer
+
+def _get_unknown(topic):
+ topics_list = get_topics_list()
+ if topic.startswith(':'):
+ topics_list = [x for x in topics_list if x.startswith(':')]
+ else:
+ topics_list = [x for x in topics_list if not x.startswith(':')]
+
+ possible_topics = process.extract(topic, topics_list, scorer=fuzz.ratio)[:3]
+ possible_topics_text = "\n".join([(" * %s %s" % x) for x in possible_topics])
+ return """
+Unknown topic.
+Do you mean one of these topics may be?
+
+%s
+ """ % possible_topics_text
+
+# pylint: disable=bad-whitespace
+#
+# topic_type, function_getter
+# should be replaced with a decorator
+TOPIC_GETTERS = (
+ ("cheat.sheets", _get_cheat_sheets),
+ ("cheat.sheets dir", _get_cheat_sheets_dir),
+ ("tldr", _get_tldr),
+ ("internal", _get_internal),
+ ("cheat", _get_cheat),
+ ("learnxiny", get_learnxiny),
+ ("question", _get_answer_for_question),
+ ("unknown", _get_unknown),
+)
+# pylint: enable=bad-whitespace
+
+def get_answer(topic, keyword, options="", request_options=None): # pylint: disable=too-many-locals,too-many-branches,too-many-statements
+ """
+ Find cheat sheet for the topic.
+ If `keyword` is None or rempty, return the whole answer.
+ Otherwise cut the paragraphs containing keywords.
+
+ Args:
+ topic (str): the name of the topic of the cheat sheet
+ keyword (str): the name of the keywords to search in the cheat sheets
+
+ Returns:
+ string: the cheat sheet
+ """
+
+ def _join_paragraphs(paragraphs):
+ answer = "\n".join(paragraphs)
+ return answer
+
+ def _split_paragraphs(text):
+ answer = []
+ paragraph = ""
+ for line in text.splitlines():
+ if line == "":
+ answer.append(paragraph)
+ paragraph = ""
+ else:
+ paragraph += line+"\n"
+ answer.append(paragraph)
+ return answer
+
+ def _paragraph_contains(paragraph, keyword, insensitive=False, word_boundaries=True):
+ """
+ Check if `paragraph` contains `keyword`.
+ Several keywords can be joined together using ~
+ For example: ~ssh~passphrase
+ """
+ answer = True
+
+ if '~' in keyword:
+ keywords = keyword.split('~')
+ else:
+ keywords = [keyword]
+
+ for kwrd in keywords:
+ regex = re.escape(kwrd)
+ if not word_boundaries:
+ regex = r"\b%s\b" % kwrd
+
+ if insensitive:
+ answer = answer and bool(re.search(regex, paragraph, re.IGNORECASE))
+ else:
+ answer = answer and bool(re.search(regex, paragraph))
+
+ return answer
+
+ def _rewrite_aliases(word):
+ if word == ':bash.completion':
+ return ':bash_completion'
+ return word
+
+ def _rewrite_section_name(query):
+ """
+ """
+ if '/' not in query:
+ return query
+
+ section_name, rest = query.split('/', 1)
+ section_name = LANGUAGE_ALIAS.get(section_name, section_name)
+ return "%s/%s" % (section_name, rest)
+
+ answer = None
+ needs_beautification = False
+
+ topic = _rewrite_aliases(topic)
+ topic = _rewrite_section_name(topic)
+
+ # checking if the answer is in the cache
+ if topic != "":
+ # temporary hack for "questions":
+ # the topic name has to be prefixed with q:
+ # so we can later delete them from redis
+ # and we known that they need beautification
+ if '/' in topic and '+' in topic:
+ topic = "q:" + topic
+ needs_beautification = True
+
+ answer = REDIS.get(topic)
+ if answer:
+ answer = answer.decode('utf-8')
+
+ # if answer was not found in the cache
+ # try to find it in one of the repositories
+ if not answer:
+ topic_type = get_topic_type(topic)
+
+ for topic_getter_type, topic_getter in TOPIC_GETTERS:
+ if topic_type == topic_getter_type:
+ answer = topic_getter(topic)
+ break
+ if not answer:
+ topic_type = "unknown"
+ answer = _get_unknown(topic)
+
+ # saving answers in the cache
+ if topic_type not in ["search", "internal", "unknown"]:
+ REDIS.set(topic, answer)
+
+ if needs_beautification:
+ filetype = 'bash'
+ if '/' in topic:
+ filetype = topic.split('/', 1)[0]
+ if filetype.startswith('q:'):
+ filetype = filetype[2:]
+
+ answer = beautifier.beautify(answer.encode('utf-8'), filetype, request_options)
+
+ if not keyword:
+ return answer
+
+ #
+ # shorten the answer, because keyword is specified
+ #
+ insensitive = 'i' in options
+ word_boundaries = 'b' in options
+
+ paragraphs = _split_paragraphs(answer)
+ paragraphs = [p for p in paragraphs
+ if _paragraph_contains(p, keyword,
+ insensitive=insensitive,
+ word_boundaries=word_boundaries)]
+ if paragraphs == []:
+ return ""
+
+ answer = _join_paragraphs(paragraphs)
+ return answer
+
+def find_answer_by_keyword(directory, keyword, options="", request_options=None):
+ """
+ Search in the whole tree of all cheatsheets or in its subtree `directory`
+ by `keyword`
+ """
+
+ recursive = 'r' in options
+
+ answer_paragraphs = []
+ for topic in get_topics_list(skip_internal=True, skip_dirs=True):
+ # skip the internal pages, don't show them in search
+ if topic in INTERNAL_TOPICS:
+ continue
+
+ if not topic.startswith(directory):
+ continue
+
+ subtopic = topic[len(directory):]
+ if not recursive and '/' in subtopic:
+ continue
+
+ answer = get_answer(topic, keyword, options=options, request_options=request_options)
+ if answer:
+ answer_paragraphs.append((topic, answer))
+
+ if len(answer_paragraphs) > MAX_SEARCH_LEN:
+ answer_paragraphs.append(("LIMITED", "LIMITED TO %s ANSWERS" % MAX_SEARCH_LEN))
+ break
+
+ return answer_paragraphs
diff --git a/lib/globals.py b/lib/globals.py
index 98e6b54..d00939d 100644
--- a/lib/globals.py
+++ b/lib/globals.py
@@ -5,6 +5,7 @@ All hardcoded pathes should be (theoretically) here.
import logging
import os
+from pygments.styles import get_all_styles
MYDIR = os.path.abspath(os.path.dirname(os.path.dirname('__file__')))
@@ -21,6 +22,8 @@ PATH_CHEAT_PAGES = "/usr/local/lib/python2.7/dist-packages/cheat/cheatsheets/*"
PATH_CHEAT_SHEETS = "/home/igor/cheat.sheets/sheets/"
PATH_CHEAT_SHEETS_SPOOL = "/home/igor/cheat.sheets/spool/"
+COLOR_STYLES = sorted(list(get_all_styles()))
+
def error(text):
"""
Log error `text` and produce a RuntimeError exception