From 981aa35c367a7ad2f02b651581857e2b7ec476f7 Mon Sep 17 00:00:00 2001 From: Anatoli Babenia Date: Mon, 27 Jul 2020 09:14:22 +0300 Subject: [PATCH 01/13] Fix Dockerfile (#172, #156) This still doesn't work as expected, because `cht.sh.txt` runs in interactive mode and gives the prompt. Where should cheat.sh be installed [/root/.cheat.sh]? --- Dockerfile | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 95c26a2..b74ca1b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,13 +1,15 @@ -FROM alpine:latest +FROM alpine:3.10 WORKDIR /app -COPY . /app +COPY requirements.txt /app/ RUN apk add --update --no-cache python2 py2-pip py2-gevent \ py2-flask py2-requests py2-pygments py2-redis \ py2-cffi py2-icu bash vim gawk sed \ && apk add --no-cache --virtual build-deps python2-dev \ build-base git \ && pip install -r requirements.txt \ - && sh share/scripts/get-sheets.sh \ && apk del build-deps +COPY . /app +RUN apk add --update --no-cache curl git py2-virtualenv \ + && ./share/cht.sh.txt --standalone-install ENTRYPOINT ["python2"] CMD ["bin/srv.py"] From 20e371c5756ce56c5c7bf5cbf005603665fe1bea Mon Sep 17 00:00:00 2001 From: Anatoli Babenia Date: Mon, 27 Jul 2020 11:03:46 +0300 Subject: [PATCH 02/13] Fetch cheat sheets without installing `cht.sh.txt` --- Dockerfile | 6 ++++-- lib/fetch.py | 6 +++++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index b74ca1b..347b515 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,7 +9,9 @@ RUN apk add --update --no-cache python2 py2-pip py2-gevent \ && pip install -r requirements.txt \ && apk del build-deps COPY . /app -RUN apk add --update --no-cache curl git py2-virtualenv \ - && ./share/cht.sh.txt --standalone-install +# fetching cheat sheets +RUN apk add --update --no-cache git \ + && mkdir -p /root/.cheat.sh/log/ \ + && python2 lib/fetch.py fetch-all ENTRYPOINT ["python2"] CMD ["bin/srv.py"] diff --git a/lib/fetch.py b/lib/fetch.py index fb8c951..75a0304 100644 --- a/lib/fetch.py +++ b/lib/fetch.py @@ -56,7 +56,11 @@ def fetch_all(skip_existing=True): sys.stdout.write("Fetching %s..." % (adptr)) sys.stdout.flush() - process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + try: + process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + except OSError: + print("\nERROR: %s" % cmd) + raise output = process.communicate()[0] if process.returncode != 0: sys.stdout.write("\nERROR:\n---\n" + output) From c3d5473123787f19869fb16b6278e38426fa6333 Mon Sep 17 00:00:00 2001 From: Anatoli Babenia Date: Mon, 27 Jul 2020 21:06:14 +0300 Subject: [PATCH 03/13] Use Python3 to fetch cheat sheets --- Dockerfile | 37 ++++++++++++++++++++++--------------- 1 file changed, 22 insertions(+), 15 deletions(-) diff --git a/Dockerfile b/Dockerfile index 347b515..05c7dd8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,17 +1,24 @@ -FROM alpine:3.10 -WORKDIR /app -COPY requirements.txt /app/ -RUN apk add --update --no-cache python2 py2-pip py2-gevent \ - py2-flask py2-requests py2-pygments py2-redis \ - py2-cffi py2-icu bash vim gawk sed \ - && apk add --no-cache --virtual build-deps python2-dev \ - build-base git \ - && pip install -r requirements.txt \ - && apk del build-deps -COPY . /app +FROM alpine:latest # fetching cheat sheets -RUN apk add --update --no-cache git \ - && mkdir -p /root/.cheat.sh/log/ \ - && python2 lib/fetch.py fetch-all -ENTRYPOINT ["python2"] +## installing dependencies +RUN apk add --update --no-cache git py3-six py3-pygments py3-yaml py3-gevent \ + libstdc++ py3-colorama py3-requests py3-icu py3-redis +## building missing python packages +RUN apk add --no-cache --virtual build-deps py3-pip g++ python3-dev \ + && pip3 install rapidfuzz colored polyglot pycld2 \ + && apk del build-deps +## copying +WORKDIR /app +COPY . /app +RUN mkdir -p /root/.cheat.sh/log/ \ + && python3 lib/fetch.py fetch-all + +# installing server dependencies +#RUN apk add --update --no-cache py3-cffi py2-pip py2-gevent \ +# py2-flask py2-requests py2-pygments py2-redis \ +# py2-cffi py2-icu bash vim gawk sed \ +# && apk add --no-cache --virtual build-deps python3-dev build-base \ +# && pip3 install -r requirements.txt \ +# && apk del build-deps +ENTRYPOINT ["python3"] CMD ["bin/srv.py"] From f727f2ceb47ff0580f21d0a952565442f1f830c1 Mon Sep 17 00:00:00 2001 From: Anatoli Babenia Date: Tue, 28 Jul 2020 11:33:28 +0300 Subject: [PATCH 04/13] Do not store `pip` cache -42MB --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 05c7dd8..c1d6b48 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,7 +5,7 @@ RUN apk add --update --no-cache git py3-six py3-pygments py3-yaml py3-gevent \ libstdc++ py3-colorama py3-requests py3-icu py3-redis ## building missing python packages RUN apk add --no-cache --virtual build-deps py3-pip g++ python3-dev \ - && pip3 install rapidfuzz colored polyglot pycld2 \ + && pip3 install --no-cache-dir rapidfuzz colored polyglot pycld2 \ && apk del build-deps ## copying WORKDIR /app From c0226133134fdc0518dd97a7d1fa53f19480b55e Mon Sep 17 00:00:00 2001 From: Anatoli Babenia Date: Tue, 28 Jul 2020 11:47:59 +0300 Subject: [PATCH 05/13] Use shallow copy for sheat clones -34MB --- lib/adapter/git_adapter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/adapter/git_adapter.py b/lib/adapter/git_adapter.py index e91a0d7..b8a8d29 100644 --- a/lib/adapter/git_adapter.py +++ b/lib/adapter/git_adapter.py @@ -79,7 +79,7 @@ class GitRepositoryAdapter(RepositoryAdapter): #pylint: disable=abstract-meth if not local_repository_dir: return None - return ['git', 'clone', cls._repository_url, local_repository_dir] + return ['git', 'clone', '--depth=1', cls._repository_url, local_repository_dir] @classmethod def update_command(cls): From f024e4d561d0610060859e358903d878d95b78b3 Mon Sep 17 00:00:00 2001 From: Anatoli Babenia Date: Tue, 28 Jul 2020 15:23:51 +0300 Subject: [PATCH 06/13] Run server --- Dockerfile | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index c1d6b48..66ff967 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,11 +14,6 @@ RUN mkdir -p /root/.cheat.sh/log/ \ && python3 lib/fetch.py fetch-all # installing server dependencies -#RUN apk add --update --no-cache py3-cffi py2-pip py2-gevent \ -# py2-flask py2-requests py2-pygments py2-redis \ -# py2-cffi py2-icu bash vim gawk sed \ -# && apk add --no-cache --virtual build-deps python3-dev build-base \ -# && pip3 install -r requirements.txt \ -# && apk del build-deps +RUN apk add --update --no-cache py3-jinja2 py3-flask ENTRYPOINT ["python3"] CMD ["bin/srv.py"] From b89e7196f46ce2e44a9e57a6d24f7d35528ea7de Mon Sep 17 00:00:00 2001 From: Anatoli Babenia Date: Tue, 28 Jul 2020 18:32:32 +0300 Subject: [PATCH 07/13] Show server port on startup --- bin/srv.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/bin/srv.py b/bin/srv.py index 8f623a8..cbedae0 100644 --- a/bin/srv.py +++ b/bin/srv.py @@ -275,8 +275,9 @@ def answer(topic=None): return Response(result, mimetype='text/plain') if 'CHEATSH_PORT' in os.environ: - SRV = WSGIServer((CONFIG['server.bind'], int(os.environ.get('CHEATSH_PORT'))), app) # log=None) - SRV.serve_forever() + PORT = int(os.environ.get('CHEATSH_PORT')) else: - SRV = WSGIServer((CONFIG['server.bind'], CONFIG['server.port']), app) # log=None) - SRV.serve_forever() + PORT = CONFIG['server.port'] +SRV = WSGIServer((CONFIG['server.bind'], PORT), app) # log=None) +print("Starting server on {}:{}".format(SRV.address[0], SRV.address[1])) +SRV.serve_forever() From b2114a43fac6526c2e3154ef9b276ea993fedb7b Mon Sep 17 00:00:00 2001 From: Anatoli Babenia Date: Tue, 28 Jul 2020 20:25:30 +0300 Subject: [PATCH 08/13] Pin Alpine to 3.12 to avoid surprises --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 66ff967..6b1c35f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM alpine:latest +FROM alpine:3.12 # fetching cheat sheets ## installing dependencies RUN apk add --update --no-cache git py3-six py3-pygments py3-yaml py3-gevent \ From c9a959f0872dda4f55d7aefc32442464102f5be1 Mon Sep 17 00:00:00 2001 From: Anatoli Babenia Date: Tue, 28 Jul 2020 23:08:04 +0300 Subject: [PATCH 09/13] Add --debug option to server to log tracebacks on failures podman build . -t cheat.sh podman run -it --rm -p 8002 cheat.sh bin/srv.py --debug https://stackoverflow.com/questions/26026148/how-to-run-wsgiserver-in-verbose-or-debug-mode --- bin/srv.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bin/srv.py b/bin/srv.py index cbedae0..77c3286 100644 --- a/bin/srv.py +++ b/bin/srv.py @@ -274,6 +274,9 @@ def answer(topic=None): return result return Response(result, mimetype='text/plain') + +if '--debug' in sys.argv: + app.debug = True if 'CHEATSH_PORT' in os.environ: PORT = int(os.environ.get('CHEATSH_PORT')) else: From dd00fa4188cc2cd9c271e71c4310542739cdb199 Mon Sep 17 00:00:00 2001 From: Anatoli Babenia Date: Wed, 29 Jul 2020 10:49:21 +0300 Subject: [PATCH 10/13] Print full cmd when `bash` is not found --- lib/frontend/html.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/frontend/html.py b/lib/frontend/html.py index c639e04..9c63120 100644 --- a/lib/frontend/html.py +++ b/lib/frontend/html.py @@ -75,9 +75,12 @@ def _render_html(query, result, editable, repository_button, topics_list, reques """ Convert ANSI text `data` to HTML """ - proc = Popen( - ["bash", CONFIG['path.internal.ansi2html'], "--palette=solarized", "--bg=dark"], - stdin=PIPE, stdout=PIPE, stderr=PIPE) + cmd = ["bash", CONFIG['path.internal.ansi2html'], "--palette=solarized", "--bg=dark"] + try: + proc = Popen(cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE) + except FileNotFoundError: + print("ERROR: %s" % cmd) + raise data = data.encode('utf-8') stdout, stderr = proc.communicate(data) if proc.returncode != 0: From 96cc3f64bd173aa6336e05d7fb12db6977b4a201 Mon Sep 17 00:00:00 2001 From: Anatoli Babenia Date: Wed, 29 Jul 2020 12:02:02 +0300 Subject: [PATCH 11/13] Install `bash` and fix Python3 subprocess bytes File "/app/lib/globals.py", line 25, in error if not text.startswith("Too many queries"): TypeError: startswith first arg must be bytes or a tuple of bytes, not str --- Dockerfile | 2 +- lib/frontend/html.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 6b1c35f..c0a461b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,6 +14,6 @@ RUN mkdir -p /root/.cheat.sh/log/ \ && python3 lib/fetch.py fetch-all # installing server dependencies -RUN apk add --update --no-cache py3-jinja2 py3-flask +RUN apk add --update --no-cache py3-jinja2 py3-flask bash ENTRYPOINT ["python3"] CMD ["bin/srv.py"] diff --git a/lib/frontend/html.py b/lib/frontend/html.py index 9c63120..c73e96c 100644 --- a/lib/frontend/html.py +++ b/lib/frontend/html.py @@ -84,7 +84,7 @@ def _render_html(query, result, editable, repository_button, topics_list, reques data = data.encode('utf-8') stdout, stderr = proc.communicate(data) if proc.returncode != 0: - error(stdout + stderr) + error((stdout + stderr).decode('utf-8')) return stdout.decode('utf-8') From aae40ae3a80670a469562edf09b4ca889aa2bb48 Mon Sep 17 00:00:00 2001 From: Anatoli Babenia Date: Wed, 29 Jul 2020 14:15:36 +0300 Subject: [PATCH 12/13] Install missing `gawk` needed by ansi2html.sh (#222) RuntimeError: /app/share/ansi2html.sh: line 38: gawk: command not found --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index c0a461b..e78be40 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,6 +14,6 @@ RUN mkdir -p /root/.cheat.sh/log/ \ && python3 lib/fetch.py fetch-all # installing server dependencies -RUN apk add --update --no-cache py3-jinja2 py3-flask bash +RUN apk add --update --no-cache py3-jinja2 py3-flask bash gawk ENTRYPOINT ["python3"] CMD ["bin/srv.py"] From fbe5414e9c3c449d0b154b3b6bd1e9ab7fb4f0f9 Mon Sep 17 00:00:00 2001 From: Anatoli Babenia Date: Wed, 29 Jul 2020 14:55:08 +0300 Subject: [PATCH 13/13] Write log as binary file in Python 3 Fixes File "bin/srv.py", line 101, in log_query my_file.write(log_entry.encode('utf-8')+"\n") TypeError: can't concat str to bytes --- bin/srv.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bin/srv.py b/bin/srv.py index 77c3286..52aaff1 100644 --- a/bin/srv.py +++ b/bin/srv.py @@ -96,9 +96,9 @@ def log_query(ip_addr, found, topic, user_agent): """ Log processed query and some internal data """ - log_entry = "%s %s %s %s" % (ip_addr, found, topic, user_agent) - with open(CONFIG["path.log.queries"], 'a') as my_file: - my_file.write(log_entry.encode('utf-8')+"\n") + log_entry = "%s %s %s %s\n" % (ip_addr, found, topic, user_agent) + with open(CONFIG["path.log.queries"], 'ab') as my_file: + my_file.write(log_entry.encode('utf-8')) def get_request_ip(req): """