From abfd4b16c5aac3838a30206b3d3c34d1858a6eb4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Mart=C3=ADnez=20Portela?= Date: Tue, 3 Oct 2023 00:07:45 +0200 Subject: [PATCH 1/6] Fix blog linking. --- scripts/blog.py | 5 +++-- static/article.tmpl.html | 2 +- static/article_list.tmpl.html | 2 +- static/blog_index.tmpl.html | 2 +- static/category_list.tmpl.html | 2 +- 5 files changed, 7 insertions(+), 6 deletions(-) diff --git a/scripts/blog.py b/scripts/blog.py index 36e4a36..eb1b440 100644 --- a/scripts/blog.py +++ b/scripts/blog.py @@ -122,6 +122,7 @@ def slugify(title): slug = unidecode(title).lower() slug = SLUG_REMOVE_RE.sub('', slug) slug = SLUG_HYPHENATE_RE.sub('-', slug) + slug = slug.strip('-') return slug.strip() @@ -167,7 +168,7 @@ def get_out_path(front_matter): out_path = os.path.join(str(front_matter['date'].year), front_matter['slug']) if front_matter.get('lang', LANG_PRIORITY[0]) != LANG_PRIORITY[0]: - out_path = os.path.join(str(front_matter['date'].year), front_matter['lang'], front_matter['slug']) + out_path = os.path.join(front_matter['lang'], str(front_matter['date'].year), front_matter['slug']) return out_path @@ -388,7 +389,7 @@ def render_categories(docs, dest_top): result = CATEGORY_LIST_TEMPLATE.render( posts=posts, ) - path = os.path.join(dest_top, "tags", tag, "index.html") + path = os.path.join(dest_top, "tags", tag.replace('/', '_'), "index.html") os.makedirs(os.path.dirname(path), exist_ok=True) with open(path, 'wt') as f: f.write(result) diff --git a/static/article.tmpl.html b/static/article.tmpl.html index 6383710..d4a02c1 100644 --- a/static/article.tmpl.html +++ b/static/article.tmpl.html @@ -38,7 +38,7 @@ diff --git a/static/article_list.tmpl.html b/static/article_list.tmpl.html index 0e52252..0618924 100644 --- a/static/article_list.tmpl.html +++ b/static/article_list.tmpl.html @@ -42,7 +42,7 @@
diff --git a/static/blog_index.tmpl.html b/static/blog_index.tmpl.html index b1939c7..3bc6d8f 100644 --- a/static/blog_index.tmpl.html +++ b/static/blog_index.tmpl.html @@ -40,7 +40,7 @@ diff --git a/static/category_list.tmpl.html b/static/category_list.tmpl.html index 893b5a3..1b19771 100644 --- a/static/category_list.tmpl.html +++ b/static/category_list.tmpl.html @@ -42,7 +42,7 @@
From 650b16df32337ecb28fed320c593edd28a75e266 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Mart=C3=ADnez=20Portela?= Date: Tue, 3 Oct 2023 00:07:57 +0200 Subject: [PATCH 2/6] Add no-watch build on blog. --- scripts/blog.py | 8 ++++++++ scripts/upload.sh | 4 ++++ 2 files changed, 12 insertions(+) diff --git a/scripts/blog.py b/scripts/blog.py index eb1b440..1e48008 100644 --- a/scripts/blog.py +++ b/scripts/blog.py @@ -63,6 +63,10 @@ JINJA_ENV = jinja2.Environment( autoescape=jinja2.select_autoescape() ) +WATCH = True +if os.getenv('WATCH_AND_REBUILD', '1') == '0': + WATCH = False + def update_statics(): global ARTICLE_TEMPLATE ARTICLE_TEMPLATE = JINJA_ENV.get_template(ARTICLE_TEMPLATE_NAME) @@ -521,6 +525,10 @@ def main(source_top, dest_top): docs = regen_all(source_top, dest_top) logging.info("Initial load completed in {:.2f}s".format(time.time() - t0)) + if not WATCH: + logging.info("Build completed in {:.2f}s".format(time.time() - t0)) + return 0 + ## Updating for event in notifier.event_gen(yield_nones=False): (ev, types, directory, file) = event diff --git a/scripts/upload.sh b/scripts/upload.sh index 482106c..1cc17cf 100644 --- a/scripts/upload.sh +++ b/scripts/upload.sh @@ -13,11 +13,15 @@ cd ../scripts rm -Rf ../_gen/notes WATCH_AND_REBUILD=0 python3 generate.py ~/.logs/brain ../_gen/notes +rm -Rf ../_gen/blog +WATCH_AND_REBUILD=0 python3 blog.py ~/cloud/nextcloud/blog/posts/ ../_gen/blog + # Upload notes cd ../_gen rsync -HPaz static/ --delete-after --exclude='*.html' root@codigoparallevar.com:/mnt/vols/misc/codigoparallevar/static/ rsync -HPaz notes/ --delete-after --exclude='xapian' --exclude='*.sqlite3' root@codigoparallevar.com:/mnt/vols/misc/codigoparallevar/notes/ rsync -HPaz notes/db.sqlite3 root@codigoparallevar.com:/mnt/vols/misc/codigoparallevar-api/ +rsync -HPaz blog/ --delete-after --exclude='xapian' --exclude='*.sqlite3' root@codigoparallevar.com:/mnt/vols/misc/codigoparallevar/blog/ # Restart API server ssh root@codigoparallevar.com docker restart notes-api-server From bd644e378814b735c49492c35fc85bd732b8d5a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Mart=C3=ADnez=20Portela?= Date: Tue, 3 Oct 2023 00:08:07 +0200 Subject: [PATCH 3/6] No need to re-link videos or hrefs. --- scripts/blog.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/scripts/blog.py b/scripts/blog.py index 1e48008..9ba6cd0 100644 --- a/scripts/blog.py +++ b/scripts/blog.py @@ -296,15 +296,6 @@ def summarize(doc): for child in summary.children: result.append(child) - # Update summary links and hrefs - for v in result.find_all('video') + result.find_all('image'): - if 'src' in v.attrs and ':' not in v['src']: - v['src'] = '/blog/' + v['src'].lstrip('/') - - for v in result.find_all('a'): - if 'href' in v.attrs and ':' not in v['href']: - v['href'] = '/blog/' + v['href'].lstrip('/') - return result def render_index(docs, dest_top): From c588187ae35cb3c414668e9f96f44c27455d5960 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Mart=C3=ADnez=20Portela?= Date: Tue, 3 Oct 2023 00:08:26 +0200 Subject: [PATCH 4/6] Fix directory creation on re-render. --- scripts/blog.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/blog.py b/scripts/blog.py index 9ba6cd0..f80dd02 100644 --- a/scripts/blog.py +++ b/scripts/blog.py @@ -573,7 +573,7 @@ def main(source_top, dest_top): docs[filepath] = (doc, front_matter, out_path) doc_full_path = os.path.join(dest_top, out_path) print("Updated: {}.html".format(doc_full_path)) - os.makedirs(os.path.dirname(doc_full_path), exist_ok=True) + os.makedirs(os.path.dirname(doc_full_path + '/index.html'), exist_ok=True) # print("==", doc_full_path) with open(doc_full_path + '/index.html', 'wt') as f: try: From 600e737767466dcc80127a25ce7f8602ed60ea47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Mart=C3=ADnez=20Portela?= Date: Tue, 3 Oct 2023 00:08:39 +0200 Subject: [PATCH 5/6] Re-render archives on file changes. --- scripts/blog.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/blog.py b/scripts/blog.py index f80dd02..d177201 100644 --- a/scripts/blog.py +++ b/scripts/blog.py @@ -578,6 +578,7 @@ def main(source_top, dest_top): with open(doc_full_path + '/index.html', 'wt') as f: try: render_article(doc, front_matter, f, out_path) + render_archive(docs, dest_top) except: logging.error(traceback.format_exc()) logging.error("Rendering failed 😿") From 23f8fcefe56523a3b25c1d68c4f96f385d6340ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Mart=C3=ADnez=20Portela?= Date: Tue, 3 Oct 2023 00:09:08 +0200 Subject: [PATCH 6/6] Improve link checking. --- scripts/test-links.py | 36 ++++++++++++++++++++++-------------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/scripts/test-links.py b/scripts/test-links.py index 5f142e2..1fbd25e 100644 --- a/scripts/test-links.py +++ b/scripts/test-links.py @@ -3,6 +3,7 @@ import logging import os import sys +import urllib.parse from bs4 import BeautifulSoup as bs4 @@ -19,25 +20,32 @@ def main(files_top): print("\r{} files".format(len(found_files)), end='', flush=True) print() - found_broken = False + found_broken = 0 for fpath in tqdm(found_files): with open(fpath) as f: tree = bs4(f.read(), features='lxml', parser='html5') - for link in tree.find_all('a'): - if 'href' not in link.attrs: - continue - if ':' in link['href']: - continue - if link['href'].startswith('/'): - target = link['href'] # TODO: Find a better way to model the root - else: - target = os.path.join(os.path.dirname(fpath), link['href']) - if os.path.isdir(target): - pass - elif not os.path.exists(target): - print("[{}] -[ error ]-> {} | {}".format(fpath, target, link['href'])) + + for tag, attr in [('a', 'href'), ('img', 'src'), ('audio', 'src'), ('video', 'src')]: + for link in tree.find_all(tag): + if attr not in link.attrs: + continue + link.attrs[attr] = link.attrs[attr].split('#')[0] + if not link.attrs[attr]: + continue + if ':' in link[attr]: + continue + if link[attr].startswith('/'): + target = os.path.join(os.path.abspath(files_top), urllib.parse.unquote(link[attr].lstrip('/'))) + else: + target = os.path.join(os.path.dirname(fpath), urllib.parse.unquote(link[attr])) + if os.path.isdir(target): + pass + elif not os.path.exists(target): + print("[{}] -[ error ]-> {} | {}".format(fpath, target, link[attr])) + found_broken += 1 if found_broken: + print(f"Found {found_broken} broken links") exit(1) else: exit(0)