diff --git a/scripts/autoserve.py b/scripts/autoserve.py new file mode 100644 index 0000000..6e69025 --- /dev/null +++ b/scripts/autoserve.py @@ -0,0 +1,122 @@ +#!/usr/bin/env python3 + +import sys +import http.server +import socketserver +import threading +import os +import time +import select + +import inotify.adapters + +PORT = 8000 + +THIS_DIR = os.path.dirname(os.path.abspath(__file__)) + +MONITORED_EVENT_TYPES = ( + 'IN_CREATE', + # 'IN_MODIFY', + 'IN_CLOSE_WRITE', + 'IN_DELETE', + 'IN_MOVED_FROM', + 'IN_MOVED_TO', + 'IN_DELETE_SELF', + 'IN_MOVE_SELF', +) + +WAITING_RESPONSES = [] +SLEEP_TIME = 0.5 +COUNTER = 0 +MAX_WAITS = 100 + +class Server(http.server.SimpleHTTPRequestHandler): + def do_GET(self): + if self.path.strip('/') == '__wait_for_changes': + WAITING_RESPONSES.append(self) + print(len(WAITING_RESPONSES), "waiting responses") + global COUNTER + ticket, COUNTER = COUNTER, COUNTER + 1 + + while self in WAITING_RESPONSES: + # This is an horribe way to wait! ... but it may work for quick tests 🤷 + + if COUNTER - ticket > MAX_WAITS: + # Connection closed by the other side + print("Manually closed for cleanup") + WAITING_RESPONSES.remove(self) + # send 502 response, timeout + self.send_response(502) + # send response headers + self.end_headers() + + return + + time.sleep(SLEEP_TIME) + return + + # send 200 response + self.send_response(200) + # send response headers + self.end_headers() + + with open(self.path.strip('/'), 'rb') as f: + # send the body of the response + self.wfile.write(f.read()) + + if not self.path.endswith('.html'): + return + else: + # Append update waiter + with open(os.path.join(THIS_DIR, 'wait_for_update.js'), 'rb') as f: + new_data = b'' + self.wfile.write(new_data) + new_data_len = len(new_data) + + return + +def notify_reloads(): + while len(WAITING_RESPONSES) > 0: + # Close opened connections + res = WAITING_RESPONSES.pop(0) + + try: + # send 200 response + res.send_response(200) + # send response headers + res.end_headers() + except Exception as e: + print("ERROR:", e) + + global COUNTER + COUNTER = 0 + + +def start_notifier(): + notifier = inotify.adapters.InotifyTree(os.getcwd()) + should_reload = False + for event in notifier.event_gen(yield_nones=True): + if event is None: + if should_reload: + print("Reloading!") + should_reload = False + notify_reloads() + continue + + (ev, types, directory, file) = event + if any([type in MONITORED_EVENT_TYPES for type in types]): + print("Detected change!", types, directory, file) + should_reload = True + +def serve(): + Handler = Server + notifier_thread = threading.Thread(target=start_notifier) + + with http.server.ThreadingHTTPServer(("127.0.0.1", PORT), Handler) as httpd: + print("serving at port", PORT) + notifier_thread.start() + httpd.serve_forever() + + +if __name__ == '__main__': + serve() diff --git a/scripts/blog.py b/scripts/blog.py new file mode 100644 index 0000000..74a5800 --- /dev/null +++ b/scripts/blog.py @@ -0,0 +1,345 @@ +#!/usr/bin/env python3 + +MARKDOWN_EXTENSION = '.md' +EXTENSIONS = [ + MARKDOWN_EXTENSION, +] + +MARKDOWN_EXTRA_FEATURES = [ + # See more in: https://python-markdown.github.io/extensions/ + 'markdown.extensions.fenced_code', + 'markdown.extensions.codehilite', + 'markdown.extensions.extra', +] + +import json +import logging +import sys +import os +import datetime +import shutil +import traceback +import time +import re +from typing import List + +from bs4 import BeautifulSoup as bs4 +import jinja2 +import inotify.adapters +import yaml +import markdown +from unidecode import unidecode + +NIKOLA_DATE_RE = re.compile(r'^([0-2]\d|30|31)\.(0\d|1[012])\.(\d{4}), (\d{1,2}):(\d{2})$') + +COMPLETE_DATE_RE = re.compile(r'^(\d{4})-(0\d|1[012])-([0-2]\d|30|31) ' + + r'(\d{2}):(\d{2})(:\d{2})( .+)?$') +SLUG_HYPHENATE_RE = re.compile(r'[\s\-]+') +SLUG_REMOVE_RE = re.compile(r'[^\s\-a-zA-Z0-9]*') + +ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + +STATIC_PATH = os.path.join(ROOT_DIR, 'static') +ARTICLE_TEMPLATE_NAME = 'article.tmpl.html' +BLOG_INDEX_TEMPLATE_NAME = 'blog_index.tmpl.html' +BLOG_INDEX_PAGE_SIZE = 10 + +STATIC_RESOURCES = ( + ('style.css', 'css/style.css'), + ('light-syntax.css', 'css/light-syntax.css'), + ('dark-syntax.css', 'css/dark-syntax.css', ('@media (prefers-color-scheme: dark) {\n', '\n}')), +) + +JINJA_ENV = jinja2.Environment( + loader=jinja2.FileSystemLoader(STATIC_PATH), + autoescape=jinja2.select_autoescape() +) + +def update_statics(): + global ARTICLE_TEMPLATE + ARTICLE_TEMPLATE = JINJA_ENV.get_template(ARTICLE_TEMPLATE_NAME) + global BLOG_INDEX_TEMPLATE + BLOG_INDEX_TEMPLATE = JINJA_ENV.get_template(BLOG_INDEX_TEMPLATE_NAME) + +update_statics() + +MONITORED_EVENT_TYPES = ( + 'IN_CREATE', + # 'IN_MODIFY', + 'IN_CLOSE_WRITE', + 'IN_DELETE', + 'IN_MOVED_FROM', + 'IN_MOVED_TO', + 'IN_DELETE_SELF', + 'IN_MOVE_SELF', +) + + +def parse_nikola_date(match): + return datetime.datetime(year=int(match.group(3)), + month=int(match.group(2)), + day=int(match.group(1)), + hour=int(match.group(4)), + minute=int(match.group(5)), + # Note this final assumption is not good + # and might get you in trouble if trying + # to sort closely-published posts + # when others are in complete-date format + tzinfo=datetime.timezone.utc, + ) + + +def parse_complete_date(match): + return datetime.datetime.strptime(match.group(0), '%Y-%m-%d %H:%M:%S %Z%z') + +def split_tags(tags: str) -> List[str]: + if isinstance(tags, str): + return [tag.strip() for tag in tags.split(',')] + elif isinstance(tags, list): + return tags + else: + raise NotImplementedError("Unknown tag type: {}".format(type(tags))) + +def slugify(title): + """ + Made for compatibility with Nikola's slugify within CodigoParaLlevar blog. + """ + slug = unidecode(title).lower() + slug = SLUG_REMOVE_RE.sub('', slug) + slug = SLUG_HYPHENATE_RE.sub('-', slug) + + return slug.strip() + + +def read_markdown(path): + with open(path, 'rt') as f: + data = f.read() + if data.startswith('---'): + start = data.index('\n') + if '---\n' not in data[start:]: + raise Exception('Front matter not finished on: {}'.format(path)) + front_matter_str, content = data[start:].split('---\n', 1) + front_matter = yaml.load(front_matter_str, Loader=yaml.SafeLoader) + else: + raise Exception('Front matter is needed for proper rendering. Not found on: {}'.format( + path + )) + doc = markdown.markdown(content, extensions=MARKDOWN_EXTRA_FEATURES) + return doc, front_matter + + +def get_out_path(front_matter): + if 'date' in front_matter: + if m := NIKOLA_DATE_RE.match(front_matter['date']): + front_matter['date'] = parse_nikola_date(m) + elif m := COMPLETE_DATE_RE.match(front_matter['date']): + front_matter['date'] = parse_complete_date(m) + else: + raise NotImplementedError('Unknown date format: {}'.format( + front_matter['date'])) + else: + raise Exception('No date found on: {}'.format( + path + )) + + if 'slug' not in front_matter: + if 'title' not in front_matter: + raise Exception('No title found on: {}'.format( + path + )) + + front_matter['slug'] = slugify(front_matter['title']) + + out_path = os.path.join(str(front_matter['date'].year), front_matter['slug']) + return out_path + + +def load_all(top_dir_relative): + top = os.path.abspath(top_dir_relative) + + docs = {} + + for root, dirs, files in os.walk(top): + for name in files: + if all([not name.endswith(ext) for ext in EXTENSIONS]): + # The logic is negative... but it works + continue + + if name.endswith(MARKDOWN_EXTENSION): + path = os.path.join(root, name) + doc, front_matter = read_markdown(path) + out_path = get_out_path(front_matter) + docs[path] = (doc, front_matter, out_path) + else: + raise NotImplementedError('Unknown filetype: {}'.format(name)) + + return docs + + +def load_doc(filepath): + doc, front_matter = read_markdown(filepath) + out_path = get_out_path(front_matter) + return (doc, front_matter, out_path) + + +def render_article(doc, front_matter, f): + result = ARTICLE_TEMPLATE.render( + content=doc, + title=front_matter['title'], + post_publication_date=front_matter['date'], + post_tags=split_tags(front_matter['tags']), + ) + f.write(result) + +def summarize(doc): + return bs4(doc, features='lxml').text[:1000] + +def render_index(docs, dest_top): + docs = sorted(docs.values(), key=lambda x: x[1]['date'], reverse=True) + + for off in range(0, len(docs), BLOG_INDEX_PAGE_SIZE): + page = docs[off: off + BLOG_INDEX_PAGE_SIZE] + + posts = [ + { + "doc": doc, + "title": front_matter['title'], + "post_publication_date": front_matter['date'], + "post_tags": split_tags(front_matter['tags']), + "summary": summarize(doc), + } + for (doc, front_matter, out_path) in page + ] + + result = BLOG_INDEX_TEMPLATE.render( + posts=posts, + ) + + if off == 0: + fname = 'index.html' + else: + fname = 'index-{}.html'.format(off // BLOG_INDEX_PAGE_SIZE) + with open(os.path.join(dest_top, fname), 'wt') as f: + f.write(result) + + +def regen_all(source_top, dest_top, docs=None): + if docs is None: + docs = load_all(source_top) + + # Render posts + for (doc, front_matter, out_path) in docs.values(): + doc_full_path = os.path.join(dest_top, out_path) + os.makedirs(os.path.dirname(doc_full_path), exist_ok=True) + # print("==", doc_full_path) + with open(doc_full_path + '.html', 'wt') as f: + try: + render_article(doc, front_matter, f) + except: + logging.error(traceback.format_exc()) + logging.error("Rendering failed 😿") + continue + + # Render statics + for static in STATIC_RESOURCES: + src_path = static[0] + dest_path = static[1] + + if len(static) > 2: + before, after = static[2] + else: + before, after = '', '' + target_dest = os.path.join(dest_top, dest_path) + os.makedirs(os.path.dirname(target_dest), exist_ok=True) + with open(os.path.join(STATIC_PATH, src_path), 'rt') as src: + data = before + src.read() + after + + with open(target_dest, 'wt') as f: + f.write(data) + + # Render index + render_index(docs, dest_top) + + return docs + + +def main(source_top, dest_top): + notifier = inotify.adapters.InotifyTrees([source_top, STATIC_PATH]) + + ## Initial load + t0 = time.time() + logging.info("Initial load...") + docs = regen_all(source_top, dest_top) + logging.info("Initial load completed in {:.2f}s".format(time.time() - t0)) + + ## Updating + for event in notifier.event_gen(yield_nones=False): + (ev, types, directory, file) = event + if not any([type in MONITORED_EVENT_TYPES for type in types]): + continue + filepath = os.path.join(directory, file) + if filepath.startswith(STATIC_PATH): + t0 = time.time() + try: + update_statics() + except: + logging.error(traceback.format_exc()) + logging.error("Loading new templates failed 😿") + continue + + is_static_resource = False + for static in STATIC_RESOURCES: + src_path = static[0] + dest_path = static[1] + if file == os.path.basename(src_path): + is_static_resource = True + + if len(static) > 2: + before, after = static[2] + else: + before, after = '', '' + target_dest = os.path.join(dest_top, dest_path) + os.makedirs(os.path.dirname(target_dest), exist_ok=True) + with open(os.path.join(STATIC_PATH, src_path), 'rt') as src: + data = before + src.read() + after + + with open(target_dest, 'wt') as f: + f.write(data) + + if is_static_resource: + logging.info("Updated static resources in {:.2f}s".format(time.time() - t0)) + else: + docs = regen_all(source_top, dest_top, docs) + logging.info("Updated all in {:.2f}s".format(time.time() - t0)) + + else: + try: + (doc, front_matter, out_path) = load_doc(filepath) + except: + logging.error(traceback.format_exc()) + logging.error("Skipping update 😿") + continue + + t0 = time.time() + docs[filepath] = (doc, front_matter, out_path) + doc_full_path = os.path.join(dest_top, out_path) + os.makedirs(os.path.dirname(doc_full_path), exist_ok=True) + # print("==", doc_full_path) + with open(doc_full_path + '.html', 'wt') as f: + try: + render_article(doc, front_matter, f) + except: + logging.error(traceback.format_exc()) + logging.error("Rendering failed 😿") + continue + + logging.info("Updated all in {:.2f}s".format(time.time() - t0)) + + +if __name__ == "__main__": + if len(sys.argv) != 3: + print("Usage: {} SOURCE_TOP DEST_TOP".format(sys.argv[0])) + exit(0) + + logging.basicConfig(level=logging.INFO, format="%(levelname)-8s %(message)s") + main(sys.argv[1], sys.argv[2]) diff --git a/scripts/generate.py b/scripts/generate.py index 05c3f63..3a6d2f0 100644 --- a/scripts/generate.py +++ b/scripts/generate.py @@ -138,7 +138,7 @@ def main(src_top, dest_top): elif l.value.startswith('./'): pass # TODO: Properly handle else: - raise NotImplementedError('On document {}, link to {}'.format(doc.path, l.value)) + logging.warning('On document {}, unknown link to {}'.format(doc.path, l.value)) if headline.parent: if isinstance(headline.parent, org_rw.Headline): @@ -169,7 +169,7 @@ def main(src_top, dest_top): json.dump(obj=graph, fp=f, indent=2) graph_explorer_path = os.path.join(dest_top, "graph.html") with open(graph_explorer_path, 'wt') as f: - with open(os.path.join(os.path.dirname(os.path.abspath(dest_top)), 'static', 'graph_explorer.html'), 'rt') as template: + with open(os.path.join(os.path.dirname(os.path.abspath(dest_top)), '..', 'static', 'graph_explorer.html'), 'rt') as template: source = template.read() f.write(source.replace('', json.dumps(graph))) diff --git a/scripts/requirements.txt b/scripts/requirements.txt new file mode 100644 index 0000000..9878f29 --- /dev/null +++ b/scripts/requirements.txt @@ -0,0 +1,2 @@ +Markdown +Jinja2 diff --git a/scripts/wait_for_update.js b/scripts/wait_for_update.js new file mode 100644 index 0000000..d53b9fd --- /dev/null +++ b/scripts/wait_for_update.js @@ -0,0 +1,19 @@ +(function (){ + var wait_for_update = function() { + console.debug("Waiting for changes..."); + + fetch('/__wait_for_changes').then(r => { + if (r.status !== 200) { + setTimeout( + wait_for_update, + 1000, + ); + } + else { + // Reload + window.location = window.location; + } + }); + }; + wait_for_update(); +})(); diff --git a/static/article.tmpl.html b/static/article.tmpl.html new file mode 100644 index 0000000..d2750ea --- /dev/null +++ b/static/article.tmpl.html @@ -0,0 +1,48 @@ + + + + + {{ title }} @ Código para llevar + + + + + + +
+

Codigo para llevar

+ +
+
+
+

{{ title }}

+
+ +
    + {% for post_tag in post_tags %} +
  • {{ post_tag }}
    • + {% endfor %} +
+
+
+ {{ content | safe }} +
+
+
+ + diff --git a/static/blog.css b/static/blog.css new file mode 100644 index 0000000..eb9a2ef --- /dev/null +++ b/static/blog.css @@ -0,0 +1,6 @@ +body { + margin: 0 auto; + width: fit-content; + max-width: 100ex; + padding: 0 1ex; +} diff --git a/static/blog_index.tmpl.html b/static/blog_index.tmpl.html new file mode 100644 index 0000000..4be44b9 --- /dev/null +++ b/static/blog_index.tmpl.html @@ -0,0 +1,52 @@ + + + + + Código para llevar + + + + + + +
+

Codigo para llevar

+ +
+
+ {% for post in posts %} +
+
+

{{ post.title }}

+
+ +
    + {% for post_tag in post.post_tags %} +
  • {{ post_tag }}
    • + {% endfor %} +
+
+
+ {{ post.summary | safe }} +
+
+
+ {% endfor %} +
+ + diff --git a/static/dark-syntax.css b/static/dark-syntax.css new file mode 100644 index 0000000..2938240 --- /dev/null +++ b/static/dark-syntax.css @@ -0,0 +1,82 @@ +pre { line-height: 125%; } +td.linenos .normal { color: inherit; background-color: transparent; padding-left: 5px; padding-right: 5px; } +span.linenos { color: inherit; background-color: transparent; padding-left: 5px; padding-right: 5px; } +td.linenos .special { color: #000000; background-color: #ffffc0; padding-left: 5px; padding-right: 5px; } +span.linenos.special { color: #000000; background-color: #ffffc0; padding-left: 5px; padding-right: 5px; } +.hll { background-color: #49483e } +.c { color: #75715e } /* Comment */ +.err { color: #960050; background-color: #1e0010 } /* Error */ +.esc { color: #f8f8f2 } /* Escape */ +.g { color: #f8f8f2 } /* Generic */ +.k { color: #66d9ef } /* Keyword */ +.l { color: #ae81ff } /* Literal */ +.n { color: #f8f8f2 } /* Name */ +.o { color: #f92672 } /* Operator */ +.x { color: #f8f8f2 } /* Other */ +.p { color: #f8f8f2 } /* Punctuation */ +.ch { color: #75715e } /* Comment.Hashbang */ +.cm { color: #75715e } /* Comment.Multiline */ +.cp { color: #75715e } /* Comment.Preproc */ +.cpf { color: #75715e } /* Comment.PreprocFile */ +.c1 { color: #75715e } /* Comment.Single */ +.cs { color: #75715e } /* Comment.Special */ +.gd { color: #f92672 } /* Generic.Deleted */ +.ge { color: #f8f8f2; font-style: italic } /* Generic.Emph */ +.gr { color: #f8f8f2 } /* Generic.Error */ +.gh { color: #f8f8f2 } /* Generic.Heading */ +.gi { color: #a6e22e } /* Generic.Inserted */ +.go { color: #66d9ef } /* Generic.Output */ +.gp { color: #f92672; font-weight: bold } /* Generic.Prompt */ +.gs { color: #f8f8f2; font-weight: bold } /* Generic.Strong */ +.gu { color: #75715e } /* Generic.Subheading */ +.gt { color: #f8f8f2 } /* Generic.Traceback */ +.kc { color: #66d9ef } /* Keyword.Constant */ +.kd { color: #66d9ef } /* Keyword.Declaration */ +.kn { color: #f92672 } /* Keyword.Namespace */ +.kp { color: #66d9ef } /* Keyword.Pseudo */ +.kr { color: #66d9ef } /* Keyword.Reserved */ +.kt { color: #66d9ef } /* Keyword.Type */ +.ld { color: #e6db74 } /* Literal.Date */ +.m { color: #ae81ff } /* Literal.Number */ +.s { color: #e6db74 } /* Literal.String */ +.na { color: #a6e22e } /* Name.Attribute */ +.nb { color: #f8f8f2 } /* Name.Builtin */ +.nc { color: #a6e22e } /* Name.Class */ +.no { color: #66d9ef } /* Name.Constant */ +.nd { color: #a6e22e } /* Name.Decorator */ +.ni { color: #f8f8f2 } /* Name.Entity */ +.ne { color: #a6e22e } /* Name.Exception */ +.nf { color: #a6e22e } /* Name.Function */ +.nl { color: #f8f8f2 } /* Name.Label */ +.nn { color: #f8f8f2 } /* Name.Namespace */ +.nx { color: #a6e22e } /* Name.Other */ +.py { color: #f8f8f2 } /* Name.Property */ +.nt { color: #f92672 } /* Name.Tag */ +.nv { color: #f8f8f2 } /* Name.Variable */ +.ow { color: #f92672 } /* Operator.Word */ +.w { color: #f8f8f2 } /* Text.Whitespace */ +.mb { color: #ae81ff } /* Literal.Number.Bin */ +.mf { color: #ae81ff } /* Literal.Number.Float */ +.mh { color: #ae81ff } /* Literal.Number.Hex */ +.mi { color: #ae81ff } /* Literal.Number.Integer */ +.mo { color: #ae81ff } /* Literal.Number.Oct */ +.sa { color: #e6db74 } /* Literal.String.Affix */ +.sb { color: #e6db74 } /* Literal.String.Backtick */ +.sc { color: #e6db74 } /* Literal.String.Char */ +.dl { color: #e6db74 } /* Literal.String.Delimiter */ +.sd { color: #e6db74 } /* Literal.String.Doc */ +.s2 { color: #e6db74 } /* Literal.String.Double */ +.se { color: #ae81ff } /* Literal.String.Escape */ +.sh { color: #e6db74 } /* Literal.String.Heredoc */ +.si { color: #e6db74 } /* Literal.String.Interpol */ +.sx { color: #e6db74 } /* Literal.String.Other */ +.sr { color: #e6db74 } /* Literal.String.Regex */ +.s1 { color: #e6db74 } /* Literal.String.Single */ +.ss { color: #e6db74 } /* Literal.String.Symbol */ +.bp { color: #f8f8f2 } /* Name.Builtin.Pseudo */ +.fm { color: #a6e22e } /* Name.Function.Magic */ +.vc { color: #f8f8f2 } /* Name.Variable.Class */ +.vg { color: #f8f8f2 } /* Name.Variable.Global */ +.vi { color: #f8f8f2 } /* Name.Variable.Instance */ +.vm { color: #f8f8f2 } /* Name.Variable.Magic */ +.il { color: #ae81ff } /* Literal.Number.Integer.Long */ diff --git a/static/homepage.html b/static/homepage.html index c4f1ebf..01bb87e 100644 --- a/static/homepage.html +++ b/static/homepage.html @@ -5,12 +5,16 @@ Código para llevar