From 49a5ec3df2f3b3f8ae301af64cb9d9b25b7e4285 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Mart=C3=ADnez=20Portela?= Date: Fri, 9 Jun 2023 20:54:01 +0200 Subject: [PATCH 01/11] Experiment with centered graph drawing. --- scripts/gen-centered-graph.py | 125 ++++++++++++++++++++++++++++++++++ scripts/generate.py | 19 ++++-- static/style.css | 22 ++++++ 3 files changed, 160 insertions(+), 6 deletions(-) create mode 100644 scripts/gen-centered-graph.py diff --git a/scripts/gen-centered-graph.py b/scripts/gen-centered-graph.py new file mode 100644 index 0000000..ef4ff18 --- /dev/null +++ b/scripts/gen-centered-graph.py @@ -0,0 +1,125 @@ +import requests +import sys + +url = 'http://localhost:8000/notes/graph.json' +reference_node = sys.argv[1] +out = sys.argv[2] + +g = requests.get(url).json() +centered_graph = { reference_node: g[reference_node] } +del g[reference_node] +new_nodes = True + +in_emacs_tree = { + reference_node: set(), +} + +while new_nodes: + new_nodes = False + removed = set() + for k, v in g.items(): + for link in v["links"]: + if link["target"].startswith("id:"): + link["target"] = link["target"][3:] + if link['target'] in centered_graph and link.get('relation') == 'in': + centered_graph[k] = v + + for l in v["links"]: + if l.get('relation') == 'in': + t = l['target'] + if t.startswith("id:"): + t = t[3:] + + if '[' in t: + # Special case, to be handled on org_rw + continue + + if t not in in_emacs_tree: + in_emacs_tree[t] = set() + in_emacs_tree[t].add(k) + + v['links'] = [ + l for l in v["links"] + if l.get('relation') != 'in' + ] + + + removed.add(k) + new_nodes = True + break + for k in removed: + del g[k] + +in_emacs = set(centered_graph.keys()) + + +# One more round for the rest, not requiring "in" +for k, v in g.items(): + for link in v["links"]: + if link["target"].startswith("id:"): + link["target"] = link["target"][3:] + if link['target'] in in_emacs: + centered_graph[k] = v + removed.add(k) + +g = centered_graph + +f = open('graph.dot', 'wt') +f.write('digraph {\n') +# f.write('bgcolor="#222222"\n') +# f.write('fontcolor="#ffffff"\n') +f.write('maxiter=1000\n') +f.write('splines=curved\n') +# f.write('splines=spline\n') # Not supported with edges to cluster +f.write('node[shape=rect]\n') +# f.write('edge[color="#ffffff"]\n') + +def draw_subgraph(node_id): + f.write("subgraph cluster_{} {{\n".format(node_id.replace("-", "_"))) + f.write('URL="./{}.node.html"\n'.format(node_id)) + # f.write('color="#ffffff"\n') + + f.write("label=\"{}\"\n".format(g[node_id]['title'].replace("\"", "'"))) + f.write("\n") + + # print("T: {}".format(in_emacs_tree), file=sys.stderr) + for k in in_emacs_tree[node_id]: + v = g[k] + print("_" + k.replace("-", "_") + "[label=\"" + v["title"].replace("\"", "'") + "\", URL=\"" + k + ".node.html\"];", file=f) + + if k in in_emacs_tree: + draw_subgraph(k) + + f.write("\n}") + +draw_subgraph(reference_node) + +for k, v in g.items(): + if k not in in_emacs: + print("_" + k.replace("-", "_") + "[label=\"" + v["title"].replace("\"", "'") + "\", URL=\"" + k + ".node.html\"];", file=f) + +for k, v in g.items(): + for link in v["links"]: + if link["target"].startswith("id:"): + link["target"] = link["target"][3:] + + if '[' in link['target']: + # Special case, to be handled on org_rw + continue + if link['target'] not in g: + # Irrelevant + continue + if link['target'] in in_emacs_tree: + t = 'cluster_{}'.format(link['target'].replace("-", "_")) + else: + t = "_" + link["target"].replace("-", "_") + print("_" + k.replace("-", "_") + "->" + t, file=f) + +f.write('}\n') +# dot graph.dot -Tsvg graph.svg + +f.close() + +import subprocess +subprocess.call("fdp graph.dot -Tsvg -o '{}'".format(out), shell=True) +# return "graph.svg" diff --git a/scripts/generate.py b/scripts/generate.py index 7c28db6..b818a9a 100644 --- a/scripts/generate.py +++ b/scripts/generate.py @@ -336,6 +336,7 @@ def regen_all(src_top, dest_top, *, docs=None, db=None): os.makedirs(attachments_dir, exist_ok=True) for base in base_dirs: data_dir = os.path.join(src_top, base, 'data') + logging.info("Copying attachments from: {}".format(data_dir)) if not os.path.exists(data_dir): continue for subdir in os.listdir(data_dir): @@ -706,12 +707,18 @@ def render_connections(headline_id, content, graph): if headline_id not in graph['backlinks']: return - content.append("") + # if headline_id != 'aa29be89-70e7-4465-91ed-361cf0ce62f2': + # return + + # TODO: Cache results + # TODO: Avoid querying graph API on script + logging.info("Generating centered graph for {}".format(headline_id)) + import subprocess + this_dir = os.path.dirname(os.path.abspath(__file__)) + os.makedirs('cache', exist_ok=True) + subprocess.check_call(['python3', os.path.join(this_dir, 'gen-centered-graph.py'), headline_id, 'cache/' + headline_id + '.svg']) + with open('cache/' + headline_id + '.svg') as f: + content.append("
{}
".format(f.read())) def render(headline, doc, graph, headlineLevel): try: diff --git a/static/style.css b/static/style.css index 1fe59e4..413279d 100644 --- a/static/style.css +++ b/static/style.css @@ -480,6 +480,17 @@ tr.__table-separator { border-bottom: 0.5ex solid black; } +.connections svg { + max-width: 100%; + height: auto; +} + +.connections svg #graph0 > polygon { + /* Main box */ + fill: transparent; + stroke: none; +} + /* Side-to-side */ @media (min-width: 120ex) { body:not(.no-toc) { @@ -617,4 +628,15 @@ tr.__table-separator { tr.__table-separator { border-bottom: 0.5ex solid #eee; } + + .connections svg polygon { + stroke: white; + fill: #222; + } + .connections svg text { + fill: white; + } + .connections svg path { + stroke: white; + } } From 3f5ec66c3da4404bd0ecc5064eebe8b72d3d6381 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Mart=C3=ADnez=20Portela?= Date: Fri, 9 Jun 2023 21:04:04 +0200 Subject: [PATCH 02/11] Fix: render graph.json, a dependency, before node-centered-graph. --- scripts/generate.py | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/scripts/generate.py b/scripts/generate.py index b818a9a..026a960 100644 --- a/scripts/generate.py +++ b/scripts/generate.py @@ -288,6 +288,19 @@ def regen_all(src_top, dest_top, *, docs=None, db=None): backlink_graph[main_headline_id] = set() backlink_graph[main_headline_id].add(backlink) + # Output graph files + graphpath = os.path.join(dest_top, "graph.json") + graph_explorer_path = os.path.join(dest_top, "graph.html") + with open(graphpath, "wt") as f: + json.dump(obj=graph, fp=f, indent=2) + graph_explorer_path = os.path.join(dest_top, "graph.html") + with open(graph_explorer_path, 'wt') as f: + with open(os.path.join(os.path.dirname(os.path.abspath(dest_top)), '..', 'static', 'graph_explorer.html'), 'rt') as template: + source = template.read() + f.write(source.replace('', + json.dumps(graph))) + logging.info("Generated {} files".format(files_generated)) + # Render docs after we've built the graph # Render main headlines full_graph_info = { "nodes": graph, "backlinks": backlink_graph, "main_headlines": main_headlines_by_path } @@ -298,7 +311,6 @@ def regen_all(src_top, dest_top, *, docs=None, db=None): f.write(render_as_document(main_headline, main_headline.doc, headlineLevel=0, graph=full_graph_info, title=org_rw.token_list_to_plaintext(main_headline.title.contents))) - # Render all headlines for headline in all_headlines: endpath = os.path.join(dest_top, headline.id + ".node.html") @@ -316,18 +328,6 @@ def regen_all(src_top, dest_top, *, docs=None, db=None): title=org_rw.token_list_to_plaintext(headline.title.contents))) files_generated += 1 - # Output graph files - graphpath = os.path.join(dest_top, "graph.json") - graph_explorer_path = os.path.join(dest_top, "graph.html") - with open(graphpath, "wt") as f: - json.dump(obj=graph, fp=f, indent=2) - graph_explorer_path = os.path.join(dest_top, "graph.html") - with open(graph_explorer_path, 'wt') as f: - with open(os.path.join(os.path.dirname(os.path.abspath(dest_top)), '..', 'static', 'graph_explorer.html'), 'rt') as template: - source = template.read() - f.write(source.replace('', - json.dumps(graph))) - logging.info("Generated {} files".format(files_generated)) cur.close() db.commit() @@ -712,13 +712,17 @@ def render_connections(headline_id, content, graph): # TODO: Cache results # TODO: Avoid querying graph API on script + # TODO: Properly render outgouing links logging.info("Generating centered graph for {}".format(headline_id)) import subprocess this_dir = os.path.dirname(os.path.abspath(__file__)) os.makedirs('cache', exist_ok=True) subprocess.check_call(['python3', os.path.join(this_dir, 'gen-centered-graph.py'), headline_id, 'cache/' + headline_id + '.svg']) - with open('cache/' + headline_id + '.svg') as f: - content.append("
{}
".format(f.read())) + try: + with open('cache/' + headline_id + '.svg') as f: + content.append("
{}
".format(f.read())) + except FileNotFoundError: + logging.exception('Graph file not produced on headline: "{}"'.format(headline_id)) def render(headline, doc, graph, headlineLevel): try: From 135423b8e5b35c6991fabe7a1315c398be1c84c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Mart=C3=ADnez=20Portela?= Date: Sat, 10 Jun 2023 15:34:44 +0200 Subject: [PATCH 03/11] Wrap graph generation in python code, remove API dependency. --- scripts/gen-centered-graph.py | 125 --------------------------------- scripts/gen_centered_graph.py | 126 ++++++++++++++++++++++++++++++++++ scripts/generate.py | 15 +--- scripts/ops_cache.py | 0 4 files changed, 129 insertions(+), 137 deletions(-) delete mode 100644 scripts/gen-centered-graph.py create mode 100644 scripts/gen_centered_graph.py create mode 100644 scripts/ops_cache.py diff --git a/scripts/gen-centered-graph.py b/scripts/gen-centered-graph.py deleted file mode 100644 index ef4ff18..0000000 --- a/scripts/gen-centered-graph.py +++ /dev/null @@ -1,125 +0,0 @@ -import requests -import sys - -url = 'http://localhost:8000/notes/graph.json' -reference_node = sys.argv[1] -out = sys.argv[2] - -g = requests.get(url).json() -centered_graph = { reference_node: g[reference_node] } -del g[reference_node] -new_nodes = True - -in_emacs_tree = { - reference_node: set(), -} - -while new_nodes: - new_nodes = False - removed = set() - for k, v in g.items(): - for link in v["links"]: - if link["target"].startswith("id:"): - link["target"] = link["target"][3:] - if link['target'] in centered_graph and link.get('relation') == 'in': - centered_graph[k] = v - - for l in v["links"]: - if l.get('relation') == 'in': - t = l['target'] - if t.startswith("id:"): - t = t[3:] - - if '[' in t: - # Special case, to be handled on org_rw - continue - - if t not in in_emacs_tree: - in_emacs_tree[t] = set() - in_emacs_tree[t].add(k) - - v['links'] = [ - l for l in v["links"] - if l.get('relation') != 'in' - ] - - - removed.add(k) - new_nodes = True - break - for k in removed: - del g[k] - -in_emacs = set(centered_graph.keys()) - - -# One more round for the rest, not requiring "in" -for k, v in g.items(): - for link in v["links"]: - if link["target"].startswith("id:"): - link["target"] = link["target"][3:] - if link['target'] in in_emacs: - centered_graph[k] = v - removed.add(k) - -g = centered_graph - -f = open('graph.dot', 'wt') -f.write('digraph {\n') -# f.write('bgcolor="#222222"\n') -# f.write('fontcolor="#ffffff"\n') -f.write('maxiter=1000\n') -f.write('splines=curved\n') -# f.write('splines=spline\n') # Not supported with edges to cluster -f.write('node[shape=rect]\n') -# f.write('edge[color="#ffffff"]\n') - -def draw_subgraph(node_id): - f.write("subgraph cluster_{} {{\n".format(node_id.replace("-", "_"))) - f.write('URL="./{}.node.html"\n'.format(node_id)) - # f.write('color="#ffffff"\n') - - f.write("label=\"{}\"\n".format(g[node_id]['title'].replace("\"", "'"))) - f.write("\n") - - # print("T: {}".format(in_emacs_tree), file=sys.stderr) - for k in in_emacs_tree[node_id]: - v = g[k] - print("_" + k.replace("-", "_") + "[label=\"" + v["title"].replace("\"", "'") + "\", URL=\"" + k + ".node.html\"];", file=f) - - if k in in_emacs_tree: - draw_subgraph(k) - - f.write("\n}") - -draw_subgraph(reference_node) - -for k, v in g.items(): - if k not in in_emacs: - print("_" + k.replace("-", "_") + "[label=\"" + v["title"].replace("\"", "'") + "\", URL=\"" + k + ".node.html\"];", file=f) - -for k, v in g.items(): - for link in v["links"]: - if link["target"].startswith("id:"): - link["target"] = link["target"][3:] - - if '[' in link['target']: - # Special case, to be handled on org_rw - continue - if link['target'] not in g: - # Irrelevant - continue - if link['target'] in in_emacs_tree: - t = 'cluster_{}'.format(link['target'].replace("-", "_")) - else: - t = "_" + link["target"].replace("-", "_") - print("_" + k.replace("-", "_") + "->" + t, file=f) - -f.write('}\n') -# dot graph.dot -Tsvg graph.svg - -f.close() - -import subprocess -subprocess.call("fdp graph.dot -Tsvg -o '{}'".format(out), shell=True) -# return "graph.svg" diff --git a/scripts/gen_centered_graph.py b/scripts/gen_centered_graph.py new file mode 100644 index 0000000..8176e16 --- /dev/null +++ b/scripts/gen_centered_graph.py @@ -0,0 +1,126 @@ +import requests +import sys +import subprocess +import ops_cache +import copy +import tempfile + +# TODO: Cache results +# TODO: Properly render outgouing links + +# @ops_cache() +def gen(headline_id, graph): + reference_node = headline_id + + g = copy.deepcopy(graph) + centered_graph = { reference_node: g[reference_node] } + del g[reference_node] + new_nodes = True + + in_emacs_tree = { + reference_node: set(), + } + + while new_nodes: + new_nodes = False + removed = set() + for k, v in g.items(): + for link in v["links"]: + if link["target"].startswith("id:"): + link["target"] = link["target"][3:] + if link['target'] in centered_graph and link.get('relation') == 'in': + centered_graph[k] = v + + for l in v["links"]: + if l.get('relation') == 'in': + t = l['target'] + if t.startswith("id:"): + t = t[3:] + + if '[' in t: + # Special case, to be handled on org_rw + continue + + if t not in in_emacs_tree: + in_emacs_tree[t] = set() + in_emacs_tree[t].add(k) + + v['links'] = [ + l for l in v["links"] + if l.get('relation') != 'in' + ] + + + removed.add(k) + new_nodes = True + break + for k in removed: + del g[k] + + in_emacs = set(centered_graph.keys()) + + # One more round for the rest, not requiring "in" + for k, v in g.items(): + for link in v["links"]: + if link["target"].startswith("id:"): + link["target"] = link["target"][3:] + if link['target'] in in_emacs: + centered_graph[k] = v + removed.add(k) + + g = centered_graph + + with tempfile.NamedTemporaryFile(suffix='.dot', mode='wt') as f: + f.write('digraph {\n') + f.write('maxiter=1000\n') + f.write('splines=curved\n') + # f.write('splines=spline\n') # Not supported with edges to cluster + f.write('node[shape=rect]\n') + + def draw_subgraph(node_id): + f.write("subgraph cluster_{} {{\n".format(node_id.replace("-", "_"))) + f.write('URL="./{}.node.html"\n'.format(node_id)) + + f.write("label=\"{}\"\n".format(g[node_id]['title'].replace("\"", "'"))) + f.write("\n") + + # print("T: {}".format(in_emacs_tree), file=sys.stderr) + for k in in_emacs_tree[node_id]: + v = g[k] + print("_" + k.replace("-", "_") + "[label=\"" + v["title"].replace("\"", "'") + "\", URL=\"" + k + ".node.html\"];", file=f) + + if k in in_emacs_tree: + draw_subgraph(k) + + f.write("\n}") + + draw_subgraph(reference_node) + + for k, v in g.items(): + if k not in in_emacs: + print("_" + k.replace("-", "_") + "[label=\"" + v["title"].replace("\"", "'") + "\", URL=\"" + k + ".node.html\"];", file=f) + + for k, v in g.items(): + for link in v["links"]: + if link["target"].startswith("id:"): + link["target"] = link["target"][3:] + + if '[' in link['target']: + # Special case, to be handled on org_rw + continue + if link['target'] not in g: + # Irrelevant + continue + if link['target'] in in_emacs_tree: + t = 'cluster_{}'.format(link['target'].replace("-", "_")) + else: + t = "_" + link["target"].replace("-", "_") + print("_" + k.replace("-", "_") + "->" + t, file=f) + + f.write('}\n') + f.close() + + with tempfile.NamedTemporaryFile(suffix='.svg') as fsvg: + subprocess.call("fdp graph.dot -Tsvg -o '{}'".format(fsvg.name), shell=True) + fsvg.seek(0) + return fsvg.read().decode() diff --git a/scripts/generate.py b/scripts/generate.py index 026a960..c6ecc3d 100644 --- a/scripts/generate.py +++ b/scripts/generate.py @@ -25,6 +25,7 @@ from org_rw import token_list_to_raw import pygments import pygments.lexers import pygments.formatters +import gen_centered_graph # Set custom states for state in ("NEXT", "MEETING", "Q", "PAUSED", "SOMETIME", "TRACK", "WAITING"): @@ -710,19 +711,9 @@ def render_connections(headline_id, content, graph): # if headline_id != 'aa29be89-70e7-4465-91ed-361cf0ce62f2': # return - # TODO: Cache results - # TODO: Avoid querying graph API on script - # TODO: Properly render outgouing links logging.info("Generating centered graph for {}".format(headline_id)) - import subprocess - this_dir = os.path.dirname(os.path.abspath(__file__)) - os.makedirs('cache', exist_ok=True) - subprocess.check_call(['python3', os.path.join(this_dir, 'gen-centered-graph.py'), headline_id, 'cache/' + headline_id + '.svg']) - try: - with open('cache/' + headline_id + '.svg') as f: - content.append("
{}
".format(f.read())) - except FileNotFoundError: - logging.exception('Graph file not produced on headline: "{}"'.format(headline_id)) + svg = gen_centered_graph.gen(headline_id, graph['nodes']) + content.append("
{}
".format(svg)) def render(headline, doc, graph, headlineLevel): try: diff --git a/scripts/ops_cache.py b/scripts/ops_cache.py new file mode 100644 index 0000000..e69de29 From fa789984f4c66b6e0c27a46550c4f6b40a492357 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Mart=C3=ADnez=20Portela?= Date: Sat, 10 Jun 2023 15:55:43 +0200 Subject: [PATCH 04/11] Implement long-operation cache. --- scripts/gen_centered_graph.py | 3 +- scripts/ops_cache.py | 75 +++++++++++++++++++++++++++++++++++ 2 files changed, 76 insertions(+), 2 deletions(-) diff --git a/scripts/gen_centered_graph.py b/scripts/gen_centered_graph.py index 8176e16..fce246e 100644 --- a/scripts/gen_centered_graph.py +++ b/scripts/gen_centered_graph.py @@ -5,10 +5,9 @@ import ops_cache import copy import tempfile -# TODO: Cache results # TODO: Properly render outgouing links -# @ops_cache() +@ops_cache.cache def gen(headline_id, graph): reference_node = headline_id diff --git a/scripts/ops_cache.py b/scripts/ops_cache.py index e69de29..45c1431 100644 --- a/scripts/ops_cache.py +++ b/scripts/ops_cache.py @@ -0,0 +1,75 @@ +import sqlite3 +import json +import logging +from typing import Optional +import xdg +import os +import datetime + +CACHE_DB: Optional[sqlite3.Connection] = None +CACHE_PATH = os.path.join(xdg.xdg_cache_home(), 'codigoparallevar', 'ops.sqlite3') + +def init_db(): + global CACHE_DB + + os.makedirs(os.path.dirname(CACHE_PATH), exist_ok=True) + CACHE_DB = sqlite3.connect(CACHE_PATH) + + cur = CACHE_DB.cursor() + cur.execute('''CREATE TABLE IF NOT EXISTS ops( + in_val TEXT PRIMARY KEY, + code TEXT, + out_val TEXT, + added_at DateTime + ); + ''') + CACHE_DB.commit() + cur.close() + +def query_cache(in_val, code): + if CACHE_DB is None: + init_db() + assert CACHE_DB is not None + cur = CACHE_DB.cursor() + cur.execute('''SELECT out_val FROM ops WHERE in_val = ? AND code = ?''', (in_val, code)) + + # Should return only one result, right? 🤷 + results = cur.fetchall() + assert len(results) < 2 + if len(results) == 0: + return None + else: + return results[0][0] + +def save_cache(in_val, code, out_val): + if CACHE_DB is None: + init_db() + assert CACHE_DB is not None + cur = CACHE_DB.cursor() + cur.execute(''' + INSERT INTO ops(in_val, code, out_val, added_at) + VALUES (?, ?, ?, ?);''', + (in_val, code, out_val, datetime.datetime.now())) + CACHE_DB.commit() + cur.close() + +def cache(fun): + fun_code = fun.__code__.co_code.decode('latin-1') + def wrapped(*kargs, **kwargs): + in_val = json.dumps({ + 'kargs': kargs, + 'kwargs': kwargs, + 'fun_code': fun_code, + }) + + cache_result = query_cache(in_val, fun_code) + found_in_cache = cache_result is not None + if not found_in_cache: + out_val = fun(*kargs, **kwargs) + save_cache(in_val, fun_code, out_val) + else: + out_val = cache_result + + logging.info("{} bytes in, {} bytes out (in_cache: {})".format(len(in_val), len(out_val), found_in_cache)) + return out_val + return wrapped From 960a3693d35c477e53e42ceb9a0768653c7e5270 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Mart=C3=ADnez=20Portela?= Date: Tue, 13 Jun 2023 00:00:35 +0200 Subject: [PATCH 05/11] Fix typo on intermediate dot file name. --- scripts/gen_centered_graph.py | 8 +++----- scripts/generate.py | 2 +- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/scripts/gen_centered_graph.py b/scripts/gen_centered_graph.py index fce246e..f39b755 100644 --- a/scripts/gen_centered_graph.py +++ b/scripts/gen_centered_graph.py @@ -1,12 +1,10 @@ -import requests -import sys import subprocess import ops_cache import copy import tempfile +import os # TODO: Properly render outgouing links - @ops_cache.cache def gen(headline_id, graph): reference_node = headline_id @@ -117,9 +115,9 @@ def gen(headline_id, graph): print("_" + k.replace("-", "_") + "->" + t, file=f) f.write('}\n') - f.close() + f.flush() with tempfile.NamedTemporaryFile(suffix='.svg') as fsvg: - subprocess.call("fdp graph.dot -Tsvg -o '{}'".format(fsvg.name), shell=True) + subprocess.call(['fdp', f.name, '-Tsvg', '-o', fsvg.name]) fsvg.seek(0) return fsvg.read().decode() diff --git a/scripts/generate.py b/scripts/generate.py index c6ecc3d..cff7b37 100644 --- a/scripts/generate.py +++ b/scripts/generate.py @@ -725,7 +725,7 @@ def render(headline, doc, graph, headlineLevel): content = [] render_tree(dom, content, headline, graph) - if headline.id: + if headline.id and headlineLevel == 0: render_connections(headline.id, content, graph) for child in headline.children: From da20c14ae774c4d3a34c89b663fd75a36b2ad16d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Mart=C3=ADnez=20Portela?= Date: Sun, 18 Jun 2023 23:24:33 +0200 Subject: [PATCH 06/11] Fix generation of graphs where top-level headline has no backlines. --- scripts/generate.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/scripts/generate.py b/scripts/generate.py index cff7b37..c9d7b15 100644 --- a/scripts/generate.py +++ b/scripts/generate.py @@ -705,9 +705,6 @@ def render_toc_headline(headline, acc): def render_connections(headline_id, content, graph): - if headline_id not in graph['backlinks']: - return - # if headline_id != 'aa29be89-70e7-4465-91ed-361cf0ce62f2': # return From 539240079fa0c914a27a024ff8f189a3273f8612 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Mart=C3=ADnez=20Portela?= Date: Sun, 18 Jun 2023 23:56:30 +0200 Subject: [PATCH 07/11] Fix rendering of outgoing links on graph. See: http://127.0.0.1:8000/notes/343fe43b-f687-4f83-8171-c966a6887898.node.html#343fe43b-f687-4f83-8171-c966a6887898 --- scripts/gen_centered_graph.py | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/scripts/gen_centered_graph.py b/scripts/gen_centered_graph.py index f39b755..40840b9 100644 --- a/scripts/gen_centered_graph.py +++ b/scripts/gen_centered_graph.py @@ -4,7 +4,7 @@ import copy import tempfile import os -# TODO: Properly render outgouing links + @ops_cache.cache def gen(headline_id, graph): reference_node = headline_id @@ -17,6 +17,7 @@ def gen(headline_id, graph): in_emacs_tree = { reference_node: set(), } + linked_from_internal = set() while new_nodes: new_nodes = False @@ -46,7 +47,11 @@ def gen(headline_id, graph): l for l in v["links"] if l.get('relation') != 'in' ] - + for l in v['links']: + lt = l['target'] + if lt.startswith("id:"): + lt = lt[3:] + linked_from_internal.add(lt) removed.add(k) new_nodes = True @@ -58,12 +63,17 @@ def gen(headline_id, graph): # One more round for the rest, not requiring "in" for k, v in g.items(): + backlinked = False for link in v["links"]: if link["target"].startswith("id:"): link["target"] = link["target"][3:] if link['target'] in in_emacs: centered_graph[k] = v + backlinked = True removed.add(k) + if not backlinked and (k in linked_from_internal): + centered_graph[k] = v + removed.add(k) g = centered_graph @@ -76,20 +86,22 @@ def gen(headline_id, graph): def draw_subgraph(node_id): f.write("subgraph cluster_{} {{\n".format(node_id.replace("-", "_"))) - f.write('URL="./{}.node.html"\n'.format(node_id)) + f.write(' URL="./{}.node.html"\n'.format(node_id)) - f.write("label=\"{}\"\n".format(g[node_id]['title'].replace("\"", "'"))) + f.write(" label=\"{}\"\n".format(g[node_id]['title'].replace("\"", "'"))) f.write("\n") # print("T: {}".format(in_emacs_tree), file=sys.stderr) for k in in_emacs_tree[node_id]: v = g[k] - print("_" + k.replace("-", "_") + "[label=\"" + v["title"].replace("\"", "'") + "\", URL=\"" + k + ".node.html\"];", file=f) if k in in_emacs_tree: draw_subgraph(k) + else: + print(" _" + k.replace("-", "_") + "[label=\"" + v["title"].replace("\"", "'") + "\", URL=\"" + k + ".node.html\"];", file=f) - f.write("\n}") + + f.write("\n}\n") draw_subgraph(reference_node) @@ -98,6 +110,10 @@ def gen(headline_id, graph): print("_" + k.replace("-", "_") + "[label=\"" + v["title"].replace("\"", "'") + "\", URL=\"" + k + ".node.html\"];", file=f) for k, v in g.items(): + link_src = '_' + k.replace("-", "_") + if k in in_emacs_tree: + link_src = 'cluster_{}'.format(k.replace("-", "_")) + for link in v["links"]: if link["target"].startswith("id:"): link["target"] = link["target"][3:] @@ -112,7 +128,7 @@ def gen(headline_id, graph): t = 'cluster_{}'.format(link['target'].replace("-", "_")) else: t = "_" + link["target"].replace("-", "_") - print("_" + k.replace("-", "_") + "->" + t, file=f) + print(link_src + "->" + t, file=f) f.write('}\n') f.flush() From d6c8b9f3db3852c4609667c45eabf4bc1c7ef21a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Mart=C3=ADnez=20Portela?= Date: Mon, 19 Jun 2023 00:07:40 +0200 Subject: [PATCH 08/11] Fix links originating from top-level of graph center. --- scripts/gen_centered_graph.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/scripts/gen_centered_graph.py b/scripts/gen_centered_graph.py index 40840b9..6806e1a 100644 --- a/scripts/gen_centered_graph.py +++ b/scripts/gen_centered_graph.py @@ -9,15 +9,20 @@ import os def gen(headline_id, graph): reference_node = headline_id + linked_from_internal = set() g = copy.deepcopy(graph) centered_graph = { reference_node: g[reference_node] } + for l in g[reference_node]['links']: + lt = l['target'] + if lt.startswith("id:"): + lt = lt[3:] + linked_from_internal.add(lt) del g[reference_node] new_nodes = True in_emacs_tree = { reference_node: set(), } - linked_from_internal = set() while new_nodes: new_nodes = False From dee465f6af8250caa1d5754c06c9c6a5da5e47a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Mart=C3=ADnez=20Portela?= Date: Tue, 20 Jun 2023 00:09:08 +0200 Subject: [PATCH 09/11] Remap document IDs. --- scripts/gen_centered_graph.py | 12 +++++++++++- scripts/generate.py | 20 +++++++++++++------- 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/scripts/gen_centered_graph.py b/scripts/gen_centered_graph.py index 6806e1a..fb825b7 100644 --- a/scripts/gen_centered_graph.py +++ b/scripts/gen_centered_graph.py @@ -6,11 +6,15 @@ import os @ops_cache.cache -def gen(headline_id, graph): +def gen(headline_id, graph, doc_to_headline_remapping): reference_node = headline_id linked_from_internal = set() g = copy.deepcopy(graph) + + if 'id:' + reference_node in doc_to_headline_remapping: + reference_node = doc_to_headline_remapping['id:' + reference_node].split(':', 1)[1] + centered_graph = { reference_node: g[reference_node] } for l in g[reference_node]['links']: lt = l['target'] @@ -28,6 +32,9 @@ def gen(headline_id, graph): new_nodes = False removed = set() for k, v in g.items(): + if 'id:' + k in doc_to_headline_remapping: + k = doc_to_headline_remapping['id:' + k].split(':', 1)[1] + for link in v["links"]: if link["target"].startswith("id:"): link["target"] = link["target"][3:] @@ -68,6 +75,9 @@ def gen(headline_id, graph): # One more round for the rest, not requiring "in" for k, v in g.items(): + if 'id:' + k in doc_to_headline_remapping: + k = doc_to_headline_remapping['id:' + k].split(':', 1)[1] + backlinked = False for link in v["links"]: if link["target"].startswith("id:"): diff --git a/scripts/generate.py b/scripts/generate.py index c9d7b15..252efc8 100644 --- a/scripts/generate.py +++ b/scripts/generate.py @@ -310,6 +310,7 @@ def regen_all(src_top, dest_top, *, docs=None, db=None): endpath = os.path.join(dest_top, main_headline.doc.id + ".node.html") with open(endpath, "wt") as f: f.write(render_as_document(main_headline, main_headline.doc, headlineLevel=0, graph=full_graph_info, + doc_to_headline_remapping=doc_to_headline_remapping, title=org_rw.token_list_to_plaintext(main_headline.title.contents))) # Render all headlines @@ -319,6 +320,7 @@ def regen_all(src_top, dest_top, *, docs=None, db=None): # Render HTML with open(endpath, "wt") as f: f.write(render_as_document(headline, headline.doc, headlineLevel=0, graph=full_graph_info, + doc_to_headline_remapping=doc_to_headline_remapping, title=org_rw.token_list_to_plaintext(headline.title.contents))) files_generated += 1 @@ -326,6 +328,7 @@ def regen_all(src_top, dest_top, *, docs=None, db=None): index_endpath = os.path.join(dest_top, "index.html") with open(index_endpath, "wt") as f: f.write(render_as_document(headline, headline.doc, headlineLevel=0, graph=full_graph_info, + doc_to_headline_remapping=doc_to_headline_remapping, title=org_rw.token_list_to_plaintext(headline.title.contents))) files_generated += 1 @@ -653,7 +656,7 @@ def render_inline(tree, f, headline, graph): return ''.join(acc) -def render_as_document(headline, doc, headlineLevel, graph, title): +def render_as_document(headline, doc, headlineLevel, graph, title, doc_to_headline_remapping): if isinstance(headline.parent, org_rw.Headline): topLevelHeadline = headline.parent while isinstance(topLevelHeadline.parent, org_rw.Headline): @@ -676,7 +679,9 @@ def render_as_document(headline, doc, headlineLevel, graph, title): """ else: - return as_document(render(headline, doc, graph=graph, headlineLevel=headlineLevel), title, render_toc(doc)) + return as_document(render(headline, doc, graph=graph, headlineLevel=headlineLevel, + doc_to_headline_remapping=doc_to_headline_remapping), + title, render_toc(doc)) def render_toc(doc): acc = ['
    '] @@ -704,15 +709,15 @@ def render_toc_headline(headline, acc): -def render_connections(headline_id, content, graph): +def render_connections(headline_id, content, graph, doc_to_headline_remapping): # if headline_id != 'aa29be89-70e7-4465-91ed-361cf0ce62f2': # return logging.info("Generating centered graph for {}".format(headline_id)) - svg = gen_centered_graph.gen(headline_id, graph['nodes']) + svg = gen_centered_graph.gen(headline_id, graph['nodes'], doc_to_headline_remapping) content.append("
    {}
    ".format(svg)) -def render(headline, doc, graph, headlineLevel): +def render(headline, doc, graph, headlineLevel, doc_to_headline_remapping): try: dom = headline.as_dom() except: @@ -723,10 +728,11 @@ def render(headline, doc, graph, headlineLevel): content = [] render_tree(dom, content, headline, graph) if headline.id and headlineLevel == 0: - render_connections(headline.id, content, graph) + render_connections(headline.id, content, graph, doc_to_headline_remapping=doc_to_headline_remapping) for child in headline.children: - content.append(render(child, doc, headlineLevel=headlineLevel+1, graph=graph)) + content.append(render(child, doc, headlineLevel=headlineLevel+1, graph=graph, + doc_to_headline_remapping=doc_to_headline_remapping)) if headline.state is None: state = "" From 9784f78f1c61ff1194ee5e68c4d923ded18e59d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Mart=C3=ADnez=20Portela?= Date: Tue, 20 Jun 2023 00:15:41 +0200 Subject: [PATCH 10/11] Deduplicate graph edges. --- scripts/gen_centered_graph.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/gen_centered_graph.py b/scripts/gen_centered_graph.py index fb825b7..ab997e8 100644 --- a/scripts/gen_centered_graph.py +++ b/scripts/gen_centered_graph.py @@ -93,7 +93,7 @@ def gen(headline_id, graph, doc_to_headline_remapping): g = centered_graph with tempfile.NamedTemporaryFile(suffix='.dot', mode='wt') as f: - f.write('digraph {\n') + f.write('strict digraph {\n') f.write('maxiter=1000\n') f.write('splines=curved\n') # f.write('splines=spline\n') # Not supported with edges to cluster From 661a5e0cf85b09d240d49c9e4a1833ad16097337 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Mart=C3=ADnez=20Portela?= Date: Mon, 26 Jun 2023 23:56:32 +0200 Subject: [PATCH 11/11] Add centered graph as first element. --- scripts/generate.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/generate.py b/scripts/generate.py index 252efc8..cc18e2a 100644 --- a/scripts/generate.py +++ b/scripts/generate.py @@ -726,10 +726,11 @@ def render(headline, doc, graph, headlineLevel, doc_to_headline_remapping): print_tree(dom, indentation=2, headline=headline) content = [] - render_tree(dom, content, headline, graph) if headline.id and headlineLevel == 0: render_connections(headline.id, content, graph, doc_to_headline_remapping=doc_to_headline_remapping) + render_tree(dom, content, headline, graph) + for child in headline.children: content.append(render(child, doc, headlineLevel=headlineLevel+1, graph=graph, doc_to_headline_remapping=doc_to_headline_remapping))