From 6d621ffc3cdf12b55769e195b39c1132ba02c09a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Mart=C3=ADnez=20Portela?= Date: Wed, 4 Oct 2023 00:19:39 +0200 Subject: [PATCH 01/20] WIP: Save blog & notes data on FTS search DB. --- scripts/blog.py | 44 +++++++++++++++++++++++++++++++++++++++++--- scripts/generate.py | 15 ++++++++------- 2 files changed, 49 insertions(+), 10 deletions(-) diff --git a/scripts/blog.py b/scripts/blog.py index d177201..d2d6d83 100644 --- a/scripts/blog.py +++ b/scripts/blog.py @@ -22,6 +22,7 @@ import shutil import traceback import time import re +import sqlite3 from typing import List from bs4 import BeautifulSoup as bs4 @@ -63,6 +64,7 @@ JINJA_ENV = jinja2.Environment( autoescape=jinja2.select_autoescape() ) +PARSER_NAMESPACE = 'codigoparallevar.com/blog' WATCH = True if os.getenv('WATCH_AND_REBUILD', '1') == '0': WATCH = False @@ -176,6 +178,12 @@ def get_out_path(front_matter): return out_path +def create_db(path): + db = sqlite3.connect(path) + db.execute('CREATE VIRTUAL TABLE IF NOT EXISTS note_search USING fts5(note_id, title, body, top_level_title, is_done, is_todo, parser_namespace, url, tokenize="trigram");') + db.execute('DELETE FROM note_search WHERE parser_namespace = ?;', (PARSER_NAMESPACE,)) + return db + def load_all(top_dir_relative): top = os.path.abspath(top_dir_relative) @@ -456,10 +464,39 @@ def render_rss(docs, dest_top): f.write(result) -def regen_all(source_top, dest_top, docs=None): +def regen_all(source_top, dest_top, docs=None, db=None): if docs is None: docs = load_all(source_top) + cur = db.cursor() + cleaned_db = False + + try: + cur.execute('DELETE FROM note_search WHERE parser_namespace = ?;', (PARSER_NAMESPACE,)) + cleaned_db = True + except sqlite3.OperationalError as err: + if WATCH: + logging.warning("Error pre-cleaning DB, search won't be updated") + else: + raise + + # Save posts to DB + for (doc, front_matter, out_path) in docs.values(): + cur.execute('''INSERT INTO note_search(note_id, title, body, top_level_title, is_done, is_todo, parser_namespace, url) VALUES (?, ?, ?, ?, ?, ?, ?, ?);''', + ( + out_path, + front_matter['title'], + doc, + front_matter['title'], + False, + False, + PARSER_NAMESPACE, + out_path + '/index.html', + )) + + cur.close() + db.commit() + # Render posts for (doc, front_matter, out_path) in docs.values(): doc_full_path = os.path.join(dest_top, out_path) @@ -513,7 +550,8 @@ def main(source_top, dest_top): ## Initial load t0 = time.time() logging.info("Initial load...") - docs = regen_all(source_top, dest_top) + db = create_db(os.path.join(dest_top, '..', 'db.sqlite3')) + docs = regen_all(source_top, dest_top, db=db) logging.info("Initial load completed in {:.2f}s".format(time.time() - t0)) if not WATCH: @@ -557,7 +595,7 @@ def main(source_top, dest_top): if is_static_resource: logging.info("Updated static resources in {:.2f}s".format(time.time() - t0)) else: - docs = regen_all(source_top, dest_top, docs) + docs = regen_all(source_top, dest_top, docs, db=db) logging.info("Updated all in {:.2f}s".format(time.time() - t0)) else: diff --git a/scripts/generate.py b/scripts/generate.py index cc18e2a..4ceaf8c 100644 --- a/scripts/generate.py +++ b/scripts/generate.py @@ -46,6 +46,7 @@ IMG_EXTENSIONS = set([ "gif", ]) SKIPPED_TAGS = set(['attach']) +PARSER_NAMESPACE = 'codigoparallevar.com/notes' WATCH = True if os.getenv('WATCH_AND_REBUILD', '1') == '0': @@ -88,11 +89,9 @@ def is_git_path(path): return any([chunk == ".git" for chunk in path.split(os.sep)]) def create_db(path): - if os.path.exists(path): - os.unlink(path) - db = sqlite3.connect(path) - db.execute('CREATE VIRTUAL TABLE note_search USING fts5(note_id, title, body, top_level_title, is_done, is_todo, tokenize="trigram");') + db.execute('CREATE VIRTUAL TABLE IF NOT EXISTS note_search USING fts5(note_id, title, body, top_level_title, is_done, is_todo, parser_namespace, url tokenize="trigram");') + db.execute('DELETE FROM note_search WHERE parser_namespace = ?;', (PARSER_NAMESPACE,)) return db def load_all(top_dir_relative): @@ -126,7 +125,7 @@ def regen_all(src_top, dest_top, *, docs=None, db=None): cleaned_db = False try: - cur.execute('DELETE FROM note_search;') + cur.execute('DELETE FROM note_search WHERE parser_namespace = ?;', (PARSER_NAMESPACE,)) cleaned_db = True except sqlite3.OperationalError as err: if WATCH: @@ -262,7 +261,7 @@ def regen_all(src_top, dest_top, *, docs=None, db=None): topLevelHeadline = topLevelHeadline.parent # Save for full-text-search - cur.execute('''INSERT INTO note_search(note_id, title, body, top_level_title, is_done, is_todo) VALUES (?, ?, ?, ?, ?, ?);''', + cur.execute('''INSERT INTO note_search(note_id, title, body, top_level_title, is_done, is_todo, parser_namespace, url) VALUES (?, ?, ?, ?, ?, ?, ?, ?);''', ( headline.id, headline.title.get_text(), @@ -270,6 +269,8 @@ def regen_all(src_top, dest_top, *, docs=None, db=None): topLevelHeadline.title.get_text(), headline.is_done, headline.is_todo, + PARSER_NAMESPACE, + headline.id + '.node.html', )) # Update graph, replace document ids with headline ids @@ -356,7 +357,7 @@ def main(src_top, dest_top): t0 = time.time() os.makedirs(dest_top, exist_ok=True) - db = create_db(os.path.join(dest_top, 'db.sqlite3')) + db = create_db(os.path.join(dest_top, '..', 'db.sqlite3')) docs = regen_all(src_top, dest_top, db=db) if not WATCH: From 28122c3c31684ae0d2c8a9f93c90aa9fb62aa976 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Mart=C3=ADnez=20Portela?= Date: Thu, 5 Oct 2023 08:53:50 +0200 Subject: [PATCH 02/20] Use dark-syntax.css only when dark mode is selected. --- static/dark-syntax.css | 167 +++++++++++++++++++++-------------------- 1 file changed, 85 insertions(+), 82 deletions(-) diff --git a/static/dark-syntax.css b/static/dark-syntax.css index 2938240..afdbd46 100644 --- a/static/dark-syntax.css +++ b/static/dark-syntax.css @@ -1,82 +1,85 @@ -pre { line-height: 125%; } -td.linenos .normal { color: inherit; background-color: transparent; padding-left: 5px; padding-right: 5px; } -span.linenos { color: inherit; background-color: transparent; padding-left: 5px; padding-right: 5px; } -td.linenos .special { color: #000000; background-color: #ffffc0; padding-left: 5px; padding-right: 5px; } -span.linenos.special { color: #000000; background-color: #ffffc0; padding-left: 5px; padding-right: 5px; } -.hll { background-color: #49483e } -.c { color: #75715e } /* Comment */ -.err { color: #960050; background-color: #1e0010 } /* Error */ -.esc { color: #f8f8f2 } /* Escape */ -.g { color: #f8f8f2 } /* Generic */ -.k { color: #66d9ef } /* Keyword */ -.l { color: #ae81ff } /* Literal */ -.n { color: #f8f8f2 } /* Name */ -.o { color: #f92672 } /* Operator */ -.x { color: #f8f8f2 } /* Other */ -.p { color: #f8f8f2 } /* Punctuation */ -.ch { color: #75715e } /* Comment.Hashbang */ -.cm { color: #75715e } /* Comment.Multiline */ -.cp { color: #75715e } /* Comment.Preproc */ -.cpf { color: #75715e } /* Comment.PreprocFile */ -.c1 { color: #75715e } /* Comment.Single */ -.cs { color: #75715e } /* Comment.Special */ -.gd { color: #f92672 } /* Generic.Deleted */ -.ge { color: #f8f8f2; font-style: italic } /* Generic.Emph */ -.gr { color: #f8f8f2 } /* Generic.Error */ -.gh { color: #f8f8f2 } /* Generic.Heading */ -.gi { color: #a6e22e } /* Generic.Inserted */ -.go { color: #66d9ef } /* Generic.Output */ -.gp { color: #f92672; font-weight: bold } /* Generic.Prompt */ -.gs { color: #f8f8f2; font-weight: bold } /* Generic.Strong */ -.gu { color: #75715e } /* Generic.Subheading */ -.gt { color: #f8f8f2 } /* Generic.Traceback */ -.kc { color: #66d9ef } /* Keyword.Constant */ -.kd { color: #66d9ef } /* Keyword.Declaration */ -.kn { color: #f92672 } /* Keyword.Namespace */ -.kp { color: #66d9ef } /* Keyword.Pseudo */ -.kr { color: #66d9ef } /* Keyword.Reserved */ -.kt { color: #66d9ef } /* Keyword.Type */ -.ld { color: #e6db74 } /* Literal.Date */ -.m { color: #ae81ff } /* Literal.Number */ -.s { color: #e6db74 } /* Literal.String */ -.na { color: #a6e22e } /* Name.Attribute */ -.nb { color: #f8f8f2 } /* Name.Builtin */ -.nc { color: #a6e22e } /* Name.Class */ -.no { color: #66d9ef } /* Name.Constant */ -.nd { color: #a6e22e } /* Name.Decorator */ -.ni { color: #f8f8f2 } /* Name.Entity */ -.ne { color: #a6e22e } /* Name.Exception */ -.nf { color: #a6e22e } /* Name.Function */ -.nl { color: #f8f8f2 } /* Name.Label */ -.nn { color: #f8f8f2 } /* Name.Namespace */ -.nx { color: #a6e22e } /* Name.Other */ -.py { color: #f8f8f2 } /* Name.Property */ -.nt { color: #f92672 } /* Name.Tag */ -.nv { color: #f8f8f2 } /* Name.Variable */ -.ow { color: #f92672 } /* Operator.Word */ -.w { color: #f8f8f2 } /* Text.Whitespace */ -.mb { color: #ae81ff } /* Literal.Number.Bin */ -.mf { color: #ae81ff } /* Literal.Number.Float */ -.mh { color: #ae81ff } /* Literal.Number.Hex */ -.mi { color: #ae81ff } /* Literal.Number.Integer */ -.mo { color: #ae81ff } /* Literal.Number.Oct */ -.sa { color: #e6db74 } /* Literal.String.Affix */ -.sb { color: #e6db74 } /* Literal.String.Backtick */ -.sc { color: #e6db74 } /* Literal.String.Char */ -.dl { color: #e6db74 } /* Literal.String.Delimiter */ -.sd { color: #e6db74 } /* Literal.String.Doc */ -.s2 { color: #e6db74 } /* Literal.String.Double */ -.se { color: #ae81ff } /* Literal.String.Escape */ -.sh { color: #e6db74 } /* Literal.String.Heredoc */ -.si { color: #e6db74 } /* Literal.String.Interpol */ -.sx { color: #e6db74 } /* Literal.String.Other */ -.sr { color: #e6db74 } /* Literal.String.Regex */ -.s1 { color: #e6db74 } /* Literal.String.Single */ -.ss { color: #e6db74 } /* Literal.String.Symbol */ -.bp { color: #f8f8f2 } /* Name.Builtin.Pseudo */ -.fm { color: #a6e22e } /* Name.Function.Magic */ -.vc { color: #f8f8f2 } /* Name.Variable.Class */ -.vg { color: #f8f8f2 } /* Name.Variable.Global */ -.vi { color: #f8f8f2 } /* Name.Variable.Instance */ -.vm { color: #f8f8f2 } /* Name.Variable.Magic */ -.il { color: #ae81ff } /* Literal.Number.Integer.Long */ +/* Dark mode. */ +@media (prefers-color-scheme: dark) { + pre { line-height: 125%; } + td.linenos .normal { color: inherit; background-color: transparent; padding-left: 5px; padding-right: 5px; } + span.linenos { color: inherit; background-color: transparent; padding-left: 5px; padding-right: 5px; } + td.linenos .special { color: #000000; background-color: #ffffc0; padding-left: 5px; padding-right: 5px; } + span.linenos.special { color: #000000; background-color: #ffffc0; padding-left: 5px; padding-right: 5px; } + .hll { background-color: #49483e } + .c { color: #75715e } /* Comment */ + .err { color: #960050; background-color: #1e0010 } /* Error */ + .esc { color: #f8f8f2 } /* Escape */ + .g { color: #f8f8f2 } /* Generic */ + .k { color: #66d9ef } /* Keyword */ + .l { color: #ae81ff } /* Literal */ + .n { color: #f8f8f2 } /* Name */ + .o { color: #f92672 } /* Operator */ + .x { color: #f8f8f2 } /* Other */ + .p { color: #f8f8f2 } /* Punctuation */ + .ch { color: #75715e } /* Comment.Hashbang */ + .cm { color: #75715e } /* Comment.Multiline */ + .cp { color: #75715e } /* Comment.Preproc */ + .cpf { color: #75715e } /* Comment.PreprocFile */ + .c1 { color: #75715e } /* Comment.Single */ + .cs { color: #75715e } /* Comment.Special */ + .gd { color: #f92672 } /* Generic.Deleted */ + .ge { color: #f8f8f2; font-style: italic } /* Generic.Emph */ + .gr { color: #f8f8f2 } /* Generic.Error */ + .gh { color: #f8f8f2 } /* Generic.Heading */ + .gi { color: #a6e22e } /* Generic.Inserted */ + .go { color: #66d9ef } /* Generic.Output */ + .gp { color: #f92672; font-weight: bold } /* Generic.Prompt */ + .gs { color: #f8f8f2; font-weight: bold } /* Generic.Strong */ + .gu { color: #75715e } /* Generic.Subheading */ + .gt { color: #f8f8f2 } /* Generic.Traceback */ + .kc { color: #66d9ef } /* Keyword.Constant */ + .kd { color: #66d9ef } /* Keyword.Declaration */ + .kn { color: #f92672 } /* Keyword.Namespace */ + .kp { color: #66d9ef } /* Keyword.Pseudo */ + .kr { color: #66d9ef } /* Keyword.Reserved */ + .kt { color: #66d9ef } /* Keyword.Type */ + .ld { color: #e6db74 } /* Literal.Date */ + .m { color: #ae81ff } /* Literal.Number */ + .s { color: #e6db74 } /* Literal.String */ + .na { color: #a6e22e } /* Name.Attribute */ + .nb { color: #f8f8f2 } /* Name.Builtin */ + .nc { color: #a6e22e } /* Name.Class */ + .no { color: #66d9ef } /* Name.Constant */ + .nd { color: #a6e22e } /* Name.Decorator */ + .ni { color: #f8f8f2 } /* Name.Entity */ + .ne { color: #a6e22e } /* Name.Exception */ + .nf { color: #a6e22e } /* Name.Function */ + .nl { color: #f8f8f2 } /* Name.Label */ + .nn { color: #f8f8f2 } /* Name.Namespace */ + .nx { color: #a6e22e } /* Name.Other */ + .py { color: #f8f8f2 } /* Name.Property */ + .nt { color: #f92672 } /* Name.Tag */ + .nv { color: #f8f8f2 } /* Name.Variable */ + .ow { color: #f92672 } /* Operator.Word */ + .w { color: #f8f8f2 } /* Text.Whitespace */ + .mb { color: #ae81ff } /* Literal.Number.Bin */ + .mf { color: #ae81ff } /* Literal.Number.Float */ + .mh { color: #ae81ff } /* Literal.Number.Hex */ + .mi { color: #ae81ff } /* Literal.Number.Integer */ + .mo { color: #ae81ff } /* Literal.Number.Oct */ + .sa { color: #e6db74 } /* Literal.String.Affix */ + .sb { color: #e6db74 } /* Literal.String.Backtick */ + .sc { color: #e6db74 } /* Literal.String.Char */ + .dl { color: #e6db74 } /* Literal.String.Delimiter */ + .sd { color: #e6db74 } /* Literal.String.Doc */ + .s2 { color: #e6db74 } /* Literal.String.Double */ + .se { color: #ae81ff } /* Literal.String.Escape */ + .sh { color: #e6db74 } /* Literal.String.Heredoc */ + .si { color: #e6db74 } /* Literal.String.Interpol */ + .sx { color: #e6db74 } /* Literal.String.Other */ + .sr { color: #e6db74 } /* Literal.String.Regex */ + .s1 { color: #e6db74 } /* Literal.String.Single */ + .ss { color: #e6db74 } /* Literal.String.Symbol */ + .bp { color: #f8f8f2 } /* Name.Builtin.Pseudo */ + .fm { color: #a6e22e } /* Name.Function.Magic */ + .vc { color: #f8f8f2 } /* Name.Variable.Class */ + .vg { color: #f8f8f2 } /* Name.Variable.Global */ + .vi { color: #f8f8f2 } /* Name.Variable.Instance */ + .vm { color: #f8f8f2 } /* Name.Variable.Magic */ + .il { color: #ae81ff } /* Literal.Number.Integer.Long */ +} From 89e50a631000bcb8922578580471feed302e8b4e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Mart=C3=ADnez=20Portela?= Date: Wed, 1 Nov 2023 23:37:57 +0100 Subject: [PATCH 03/20] Fix color scheme on dark mode. --- static/homepage.html | 4 +++- static/style.css | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/static/homepage.html b/static/homepage.html index a0ac06a..d7437ba 100644 --- a/static/homepage.html +++ b/static/homepage.html @@ -6,11 +6,13 @@