WIP: Save blog & notes data on FTS search DB.

README.md

@@ -1,29 +0,0 @@
-# Codigo para llevar's generator
-
-This is the static site generator used to build [Codigo Para Llevar](https://codigoparallevar.com/) (my personal site). It contains:
-
-- A markdown blog (with content ported from acrylamid). Saved as `/blog/`.
-- A set of org-mode based notes. Saved as `/notes/`.
-
-It also copies over some static assets (css, js, fonts).
-
-The scripts are hardcoded with the hostnames and paths for my own site, so you might want to update them.
-
-General documentation is in progress and might be replaced little by little by the more interactive [org-web-editor](https://code.codigoparallevar.com/kenkeiras/org-web-editor) once that one (1) supports all the features here and (2) has support for building static sites.
-
-## Instructions
-
-Generally, what you want to do is to run `make` once to prepare the static files, then run this to generate the notes.
-
-```bash
-mkdir -p _gen
-WATCH_AND_REBUILD=0 python3 scripts/generate.py <path to your notes> _gen/notes [<DEFAULT SUBPATH (usually 'public', '.' to ignore)>]
-```
-
-Use `WATCH_AND_REBUILD=1` (or empty) for automatic rebuilds.
-
-## Filtering
-
-This won't render **all** notes, but try to select the PUBLIC ones and skip the PRIVATE ones.
-
-PUBLIC files are contained on the DEFAULT_SUBPATH, PRIVATE headlines have the `:private:` tag.

scripts/blog.py

@@ -22,6 +22,7 @@ import shutil
 import traceback
 import time
 import re
+import sqlite3
 from typing import List
 
 from bs4 import BeautifulSoup as bs4
@@ -63,6 +64,7 @@ JINJA_ENV = jinja2.Environment(
     autoescape=jinja2.select_autoescape()
 )
 
+PARSER_NAMESPACE = 'codigoparallevar.com/blog'
 WATCH = True
 if os.getenv('WATCH_AND_REBUILD', '1') == '0':
     WATCH = False
@@ -176,6 +178,12 @@ def get_out_path(front_matter):
     return out_path
 
 
+def create_db(path):
+    db = sqlite3.connect(path)
+    db.execute('CREATE VIRTUAL TABLE IF NOT EXISTS note_search USING fts5(note_id, title, body, top_level_title, is_done, is_todo, parser_namespace, url,  tokenize="trigram");')
+    db.execute('DELETE FROM note_search WHERE parser_namespace = ?;', (PARSER_NAMESPACE,))
+    return db
+
 def load_all(top_dir_relative):
     top = os.path.abspath(top_dir_relative)
 
@@ -456,10 +464,39 @@ def render_rss(docs, dest_top):
         f.write(result)
 
 
-def regen_all(source_top, dest_top, docs=None):
+def regen_all(source_top, dest_top, docs=None, db=None):
     if docs is None:
         docs = load_all(source_top)
 
+    cur = db.cursor()
+    cleaned_db = False
+
+    try:
+        cur.execute('DELETE FROM note_search WHERE parser_namespace = ?;', (PARSER_NAMESPACE,))
+        cleaned_db = True
+    except sqlite3.OperationalError as err:
+        if WATCH:
+            logging.warning("Error pre-cleaning DB, search won't be updated")
+        else:
+            raise
+
+    # Save posts to DB
+    for (doc, front_matter, out_path) in docs.values():
+        cur.execute('''INSERT INTO note_search(note_id, title, body, top_level_title, is_done, is_todo, parser_namespace, url) VALUES (?, ?, ?, ?, ?, ?, ?, ?);''',
+                    (
+                        out_path,
+                        front_matter['title'],
+                        doc,
+                        front_matter['title'],
+                        False,
+                        False,
+                        PARSER_NAMESPACE,
+                        out_path + '/index.html',
+                    ))
+
+    cur.close()
+    db.commit()
+
     # Render posts
     for (doc, front_matter, out_path) in docs.values():
         doc_full_path = os.path.join(dest_top, out_path)
@@ -513,7 +550,8 @@ def main(source_top, dest_top):
     ## Initial load
     t0 = time.time()
     logging.info("Initial load...")
-    docs = regen_all(source_top, dest_top)
+    db = create_db(os.path.join(dest_top, '..', 'db.sqlite3'))
+    docs = regen_all(source_top, dest_top, db=db)
     logging.info("Initial load completed in {:.2f}s".format(time.time() - t0))
 
     if not WATCH:
@@ -557,7 +595,7 @@ def main(source_top, dest_top):
             if is_static_resource:
                 logging.info("Updated static resources in {:.2f}s".format(time.time() - t0))
             else:
-                docs = regen_all(source_top, dest_top, docs)
+                docs = regen_all(source_top, dest_top, docs, db=db)
                 logging.info("Updated all in {:.2f}s".format(time.time() - t0))
 
         else:

scripts/generate.py

@@ -46,14 +46,14 @@ IMG_EXTENSIONS = set([
     "gif",
 ])
 SKIPPED_TAGS = set(['attach'])
-DEFAULT_SUBPATH = "public"
+PARSER_NAMESPACE = 'codigoparallevar.com/notes'
 
 WATCH = True
 if os.getenv('WATCH_AND_REBUILD', '1') == '0':
     WATCH = False
 
 MIN_HIDDEN_HEADLINE_LEVEL = 2
-INDEX_ID = os.getenv("INDEX_ID", "ea48ec1d-f9d4-4fb7-b39a-faa7b6e2ba95")
+INDEX_ID = "ea48ec1d-f9d4-4fb7-b39a-faa7b6e2ba95"
 SITE_NAME = "Código para llevar"
 
 MONITORED_EVENT_TYPES = (
@@ -89,11 +89,9 @@ def is_git_path(path):
     return any([chunk == ".git" for chunk in path.split(os.sep)])
 
 def create_db(path):
-    if os.path.exists(path):
-        os.unlink(path)
-
     db = sqlite3.connect(path)
-    db.execute('CREATE VIRTUAL TABLE note_search USING fts5(note_id, title, body, top_level_title, is_done, is_todo, tokenize="trigram");')
+    db.execute('CREATE VIRTUAL TABLE IF NOT EXISTS note_search USING fts5(note_id, title, body, top_level_title, is_done, is_todo, parser_namespace, url tokenize="trigram");')
+    db.execute('DELETE FROM note_search WHERE parser_namespace = ?;', (PARSER_NAMESPACE,))
     return db
 
 def load_all(top_dir_relative):
@@ -109,9 +107,7 @@ def load_all(top_dir_relative):
             path = os.path.join(root, name)
 
             try:
-                doc = load_org(open(path),
-                               environment={"org-todo-keywords": "TODO(t) NEXT(n) MEETING(m/!) Q(q) PAUSED(p!/!) EVENT(e/!) SOMETIME(s) WAITING(w@/!) TRACK(r/!) | DISCARDED(x@/!) VALIDATING(v!/!) DONE(d!/!)"},
-                               extra_cautious=True)
+                doc = load_org(open(path), extra_cautious=True)
                 docs.append(doc)
             except Exception as err:
                 import traceback
@@ -123,28 +119,13 @@ def load_all(top_dir_relative):
     logging.info("Collected {} files".format(len(docs)))
     return docs
 
-def remove_non_public_headlines(doc: org_rw.OrgDoc | org_rw.Headline):
-    if isinstance(doc, org_rw.OrgDoc):
-        doc.headlines = list(filter_private_headlines(doc.headlines))
-        for hl in doc.headlines:
-            remove_non_public_headlines(hl)
-    else:
-        doc.children = list(filter_private_headlines(doc.children))
-        for hl in doc.children:
-            remove_non_public_headlines(hl)
-
-def filter_private_headlines(headlines):
-    for hl in headlines:
-        if 'private' not in hl.tags:
-            yield hl
-
-def regen_all(src_top, dest_top, subpath, *, docs=None, db=None):
+def regen_all(src_top, dest_top, *, docs=None, db=None):
     files_generated = 0
     cur = db.cursor()
     cleaned_db = False
 
     try:
-        cur.execute('DELETE FROM note_search;')
+        cur.execute('DELETE FROM note_search WHERE parser_namespace = ?;', (PARSER_NAMESPACE,))
         cleaned_db = True
     except sqlite3.OperationalError as err:
         if WATCH:
@@ -165,12 +146,10 @@ def regen_all(src_top, dest_top, subpath, *, docs=None, db=None):
     main_headline_to_docid = {}
     for doc in docs:
         relpath = os.path.relpath(doc.path, src_top)
-
-        remove_non_public_headlines(doc)
         changed = False
         headlines = list(doc.getAllHeadlines())
         related = None
-        if not relpath.startswith(subpath + "/"):
+        if not relpath.startswith("public/"):
             # print("Skip:", relpath)
             continue
 
@@ -282,7 +261,7 @@ def regen_all(src_top, dest_top, subpath, *, docs=None, db=None):
             topLevelHeadline = topLevelHeadline.parent
 
         # Save for full-text-search
-        cur.execute('''INSERT INTO note_search(note_id, title, body, top_level_title, is_done, is_todo) VALUES (?, ?, ?, ?, ?, ?);''',
+        cur.execute('''INSERT INTO note_search(note_id, title, body, top_level_title, is_done, is_todo, parser_namespace, url) VALUES (?, ?, ?, ?, ?, ?, ?, ?);''',
                     (
                         headline.id,
                         headline.title.get_text(),
@@ -290,6 +269,8 @@ def regen_all(src_top, dest_top, subpath, *, docs=None, db=None):
                         topLevelHeadline.title.get_text(),
                         headline.is_done,
                         headline.is_todo,
+                        PARSER_NAMESPACE,
+                        headline.id + '.node.html',
                     ))
 
     # Update graph, replace document ids with headline ids
@@ -369,15 +350,15 @@ def regen_all(src_top, dest_top, subpath, *, docs=None, db=None):
                             dirs_exist_ok=True)
 
 
-def main(src_top, dest_top, subpath):
+def main(src_top, dest_top):
     notifier = inotify.adapters.InotifyTrees([src_top, STATIC_PATH])
 
     ## Initial load
     t0 = time.time()
 
     os.makedirs(dest_top, exist_ok=True)
-    db = create_db(os.path.join(dest_top, 'db.sqlite3'))
-    docs = regen_all(src_top, dest_top, subpath=subpath, db=db)
+    db = create_db(os.path.join(dest_top, '..', 'db.sqlite3'))
+    docs = regen_all(src_top, dest_top, db=db)
 
     if not WATCH:
         logging.info("Build completed in {:.2f}s".format(time.time() - t0))
@@ -395,7 +376,7 @@ def main(src_top, dest_top, subpath):
         print("CHANGED: {}".format(filepath))
         t0 = time.time()
         try:
-            docs = regen_all(src_top, dest_top, subpath=subpath, docs=docs, db=db)
+            docs = regen_all(src_top, dest_top, docs=docs, db=db)
         except:
             logging.error(traceback.format_exc())
             logging.error("Loading new templates failed 😿")
@@ -493,7 +474,7 @@ def render_block(content, acc, _class, is_code):
     acc.append('</pre>')
 
 def unindent(content):
-    base_indentation = min([0] + [
+    base_indentation = min([
         len(l) - len(l.lstrip(' '))
         for l in content.split('\n')
         if len(l.strip()) > 0
@@ -734,11 +715,8 @@ def render_connections(headline_id, content, graph, doc_to_headline_remapping):
     #     return
 
     logging.info("Generating centered graph for {}".format(headline_id))
-    try:
     svg = gen_centered_graph.gen(headline_id, graph['nodes'], doc_to_headline_remapping)
     content.append("<div class='connections'>{}</div>".format(svg))
-    except:
-        logging.warning("Broken reference on headline ID={}".format(headline_id))
 
 def render(headline, doc, graph, headlineLevel, doc_to_headline_remapping):
     try:
@@ -758,10 +736,10 @@ def render(headline, doc, graph, headlineLevel, doc_to_headline_remapping):
         content.append(render(child, doc, headlineLevel=headlineLevel+1, graph=graph,
                               doc_to_headline_remapping=doc_to_headline_remapping))
 
-    if headline.state is None or headline.state.get('name') is None:
+    if headline.state is None:
         state = ""
     else:
-        state = f'<span class="state todo-{headline.is_todo} state-{headline.state["name"]}">{headline.state["name"]}</span>'
+        state = f'<span class="state todo-{headline.is_todo} state-{headline.state}">{headline.state}</span>'
 
     if headline.is_todo:
         todo_state = "todo"
@@ -848,13 +826,9 @@ def save_changes(doc):
 
 
 if __name__ == "__main__":
-    if len(sys.argv) not in (3, 4):
-        print("Usage: {} SOURCE_TOP DEST_TOP <SUBPATH>".format(sys.argv[0]))
+    if len(sys.argv) != 3:
+        print("Usage: {} SOURCE_TOP DEST_TOP".format(sys.argv[0]))
         exit(0)
 
     logging.basicConfig(level=logging.INFO, format="%(levelname)-8s %(message)s")
-    subpath = DEFAULT_SUBPATH
-
-    if len(sys.argv) == 4:
-        subpath = sys.argv[3]
-    exit(main(sys.argv[1], sys.argv[2], subpath=subpath))
+    exit(main(sys.argv[1], sys.argv[2]))

static/dark-syntax.css

@@ -1,5 +1,3 @@
-/* Dark mode. */
-@media (prefers-color-scheme: dark) {
 pre { line-height: 125%; }
 td.linenos .normal { color: inherit; background-color: transparent; padding-left: 5px; padding-right: 5px; }
 span.linenos { color: inherit; background-color: transparent; padding-left: 5px; padding-right: 5px; }
@@ -82,4 +80,3 @@
 .vi { color: #f8f8f2 } /* Name.Variable.Instance */
 .vm { color: #f8f8f2 } /* Name.Variable.Magic */
 .il { color: #ae81ff } /* Literal.Number.Integer.Long */
-}

static/homepage.html

@@ -6,13 +6,11 @@
         <meta name="viewport" content="width=device-width, initial-scale=1">
         <style>
           body {
-             background-color: white;
              font-family: sans-serif;
              margin: 0 auto;
              width: fit-content;
              max-width: 100ex;
              padding: 0 1ex;
-             color: black;
           }
           .header h1 {
              text-align: center;
@@ -47,7 +45,7 @@
              border-right: 1px solid #000;
          }
          @media (prefers-color-scheme: dark) {
-             body {
+             html {
                  background-color: #1d1f21;
                  color: #fafafe;
              }
@@ -74,12 +72,6 @@
 
         </div>
         <div class="links">
-            <a href="/notes">
-              <section>
-                <h2>Notes</h2>
-                <p>Some publicly-visible notes from a sort of knowledge graph that I use as information dump.</p>
-              </section>
-            </a>
             <section>
                 <h2><a href="/blog">Blog</a></h2>
                 <p>
@@ -108,24 +100,12 @@
                   </ul>
                 </p>
             </section>
+            <a href="/notes">
               <section>
-                <h2>Talks / Slides</h2>
-                <p>
-                  <ul>
-                    <li>
-                      Malleable Software
-                      (<a href="/slides/hackliza2024/software-maleable/software-maleable.odp">galician, </a>
-                      for <a href="https://hackliza.gal">Hackliza</a>
-                      <a href="/slides/hackliza2024/software-maleable/software-maleable.pdf">[PDF]</a>
-                      <a href="/slides/hackliza2024/software-maleable/software-maleable.odp">[ODP]</a>)
-                      (<a href="/slides/eslibre2024/software-maleable.odp">spanish,</a>
-                      for <a href="https://eslib.re/2024/">esLibre 2024</a>
-                      <a href="/slides/eslibre2024/software-maleable.pdf">[PDF]</a>
-                      <a href="/slides/eslibre2024/software-maleable.odp">[ODP]</a>).
-                    </li>
-                  </ul>
-                </p>
+                <h2>Notes</h2>
+                <p>Some publicly-visible notes from a sort of knowledge graph that I use as information dump.</p>
               </section>
+            </a>
             <!-- section>
                 <h2>Projects</h2>
                 <p>
@@ -136,7 +116,7 @@
             <section id="social">
                 <h2>Find me</h2>
                 <p>
-                    <a href="https://social.codigoparallevar.com/@kenkeiras">ActivityPub</a>
+                    <a href="https://social.codigoparallevar.com/@kenkeiras">Mastodon</a>
                     <a href="https://github.com/kenkeiras">GitHub</a>
                     <a href="https://gitlab.com/kenkeiras">GitLab</a>
                     <a href="https://programaker.com/users/kenkeiras">PrograMaker</a>

static/style.css

@@ -10,8 +10,6 @@ body {
     max-width: 80ex;
     margin: 0 auto;
     padding: 0.5ex 1ex;
-    background-color: white;
-    color: black;
 }
 
 body.blog {
@@ -334,11 +332,6 @@ h1.title .state.todo-True {
 h1.title .state.todo-False {
     background-color: rgba(0,255,0,0.25);
 }
-h1.title .state.todo-True.state-SOMETIME {
-    background-color: #ddd;
-    color: black;
-}
-
 
 h1.title .tags {
     float: right;
@@ -375,7 +368,6 @@ a.internal::after {
 }
 a.external::after {
     content: ' ↗';
-    vertical-align: top;
 }
 
 /* Markup */
@@ -588,7 +580,7 @@ tr.__table-separator {
 
 /* Dark mode. */
 @media (prefers-color-scheme: dark) {
-    html, body {
+    html {
         background-color: #1d1f21;
         color: #fafafe;
     }