diff --git a/scripts/generate.py b/scripts/generate.py index 70d2de3..3dcaee6 100644 --- a/scripts/generate.py +++ b/scripts/generate.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 +import sqlite3 import time import json import html @@ -45,6 +46,14 @@ STATIC_PATH = os.path.join(ROOT_DIR, 'static') def is_git_path(path): return any([chunk == ".git" for chunk in path.split(os.sep)]) +def create_db(path): + if os.path.exists(path): + os.unlink(path) + + db = sqlite3.connect(path) + db.execute('CREATE VIRTUAL TABLE note_search USING fts5(note_id, title, body);') + return db + def load_all(top_dir_relative): top = os.path.abspath(top_dir_relative) @@ -70,8 +79,10 @@ def load_all(top_dir_relative): logging.info("Collected {} files".format(len(docs))) return docs -def regen_all(src_top, dest_top, docs=None): +def regen_all(src_top, dest_top, *, docs=None, db=None): files_generated = 0 + cur = db.cursor() + cur.execute('DELETE FROM note_search;') docs = load_all(src_top) doc_to_headline_remapping = {} @@ -176,6 +187,15 @@ def regen_all(src_top, dest_top, docs=None): "depth": headline.depth, } + # Save for full-text-search + cur.execute('''INSERT INTO note_search(note_id, title, body) VALUES (?, ?, ?);''', + ( + headline.id, + headline.title.get_text(), + ''.join(headline.get_contents('raw')), + )) + + # Render HTML with open(endpath, "wt") as f: f.write(as_document(render(headline, doc, headlineLevel=0), org_rw.token_list_to_plaintext(headline.title.contents))) @@ -206,7 +226,8 @@ def regen_all(src_top, dest_top, docs=None): f.write(source.replace('', json.dumps(graph))) logging.info("Generated {} files".format(files_generated)) - + cur.close() + db.commit() def main(src_top, dest_top): notifier = inotify.adapters.InotifyTrees([src_top, STATIC_PATH]) @@ -214,7 +235,8 @@ def main(src_top, dest_top): ## Initial load t0 = time.time() - docs = regen_all(src_top, dest_top) + db = create_db(os.path.join(dest_top, 'db.sqlite3')) + docs = regen_all(src_top, dest_top, db=db) logging.info("Initial load completed in {:.2f}s".format(time.time() - t0)) ## Updating @@ -228,7 +250,7 @@ def main(src_top, dest_top): print("CHANGED: {}".format(filepath)) t0 = time.time() try: - docs = regen_all(src_top, dest_top, docs) + docs = regen_all(src_top, dest_top, docs=docs, db=db) except: logging.error(traceback.format_exc()) logging.error("Loading new templates failed 😿")