Allow rendering of links that require graph knowledge.

- Fix rendering of `./filename.org` links.

scripts/generate.py

@@ -92,12 +92,19 @@ def regen_all(src_top, dest_top, *, docs=None, db=None):
     doc_to_headline_remapping = {}
 
     os.makedirs(dest_top, exist_ok=True)
-    graph = {}
+
+    ## Build headline list
+    # This includes a virtual headline for ID-referenced documents.
+    all_headlines = []
+    main_headlines_by_path = {}
     for doc in docs:
         relpath = os.path.relpath(doc.path, src_top)
         changed = False
         headlines = list(doc.getAllHeadlines())
         related = None
+        if not relpath.startswith("public/"):
+            # print("Skip:", relpath)
+            continue
 
         i = len(headlines)
         while i > 0:
@@ -123,10 +130,7 @@ def regen_all(src_top, dest_top, *, docs=None, db=None):
             print("Updated", relpath)
             save_changes(doc)
 
-        if not relpath.startswith("public/"):
+        all_headlines.extend(headlines)
-            # print("Skip:", relpath)
-            continue
-
         main_headline = None
         topHeadlines = doc.getTopHeadlines()
 
@@ -134,14 +138,10 @@ def regen_all(src_top, dest_top, *, docs=None, db=None):
             or (len(topHeadlines) == 2 and related is not None)):
 
             main_headline = [h for h in topHeadlines if h != related][0]
+            main_headlines_by_path[doc.path] = main_headline
             if doc.id is not None:
-                endpath = os.path.join(dest_top, doc.id + ".node.html")
+                doc_to_headline_remapping['id:' + doc.id] = 'id:' + main_headline.id
-                with open(endpath, "wt") as f:
+                files_generated += 1
-                    doc_to_headline_remapping['id:' + doc.id] = 'id:' + main_headline.id
-
-                    f.write(render_as_document(main_headline, doc, headlineLevel=0,
-                                               title=org_rw.token_list_to_plaintext(main_headline.title.contents)))
-                    files_generated += 1
         elif doc.id is not None:
             logging.error("Cannot render document from id: {}. {} headlines {} related".format(
                 relpath,
@@ -149,75 +149,62 @@ def regen_all(src_top, dest_top, *, docs=None, db=None):
                 'with' if related is not None else 'without'
             ))
 
-        for headline in headlines:
+    graph = {}
-            endpath = os.path.join(dest_top, headline.id + ".node.html")
+    # Build graph
+    for headline in all_headlines:
+        links = []
+        headline_links = list(headline.get_links())
+        if headline == main_headline and related is not None:
+            headline_links.extend(list(related.get_links()))
 
-            links = []
+        for l in headline_links:
-            headline_links = list(headline.get_links())
+            if l.value.startswith('http://') or l.value.startswith('https://'):
-            if headline == main_headline and related is not None:
+                pass # Ignore for now, external URL
-                headline_links.extend(list(related.get_links()))
+            elif l.value.startswith('id:'):
+                links.append({'target': l.value})
+            elif l.value.startswith('attachment:'):
+                pass # Ignore, attachment
+            elif l.value.startswith('file:'):
+                pass # Ignore, attachment
+            elif l.value.startswith('notmuch:'):
+                pass # Ignore, mail
+            elif l.value.startswith('orgit-rev:'):
+                pass # Ignore, mail
+            elif l.value.startswith('*'):
+                pass # Ignore, internal
+            elif not ':' in l.value.split()[0]:
+                pass # Ignore, internal
+            elif l.value.startswith('./'):
+                pass # TODO: Properly handle
+            else:
+                logging.warning('On document {}, unknown link to {}'.format(doc.path, l.value))
 
-            for l in headline_links:
+        if headline.parent:
-                if l.value.startswith('http://') or l.value.startswith('https://'):
+            if isinstance(headline.parent, org_rw.Headline):
-                    pass # Ignore for now, external URL
+                links.append({
-                elif l.value.startswith('id:'):
+                    "target": headline.parent.id,
-                    links.append({'target': l.value})
+                    "relation": "in"
-                elif l.value.startswith('attachment:'):
+                })
-                    pass # Ignore, attachment
+        graph[headline.id] = {
-                elif l.value.startswith('file:'):
+            "title": org_rw.org_rw.token_list_to_plaintext(headline.title.contents).strip(),
-                    pass # Ignore, attachment
+            "links": links,
-                elif l.value.startswith('notmuch:'):
+            "depth": headline.depth,
-                    pass # Ignore, mail
+        }
-                elif l.value.startswith('orgit-rev:'):
-                    pass # Ignore, mail
-                elif l.value.startswith('*'):
-                    pass # Ignore, internal
-                elif not ':' in l.value.split()[0]:
-                    pass # Ignore, internal
-                elif l.value.startswith('./'):
-                    pass # TODO: Properly handle
-                else:
-                    logging.warning('On document {}, unknown link to {}'.format(doc.path, l.value))
 
-            if headline.parent:
+        topLevelHeadline = headline
-                if isinstance(headline.parent, org_rw.Headline):
+        while isinstance(topLevelHeadline.parent, org_rw.Headline):
-                    links.append({
+            topLevelHeadline = topLevelHeadline.parent
-                        "target": headline.parent.id,
-                        "relation": "in"
-                    })
-            graph[headline.id] = {
-                "title": org_rw.org_rw.token_list_to_plaintext(headline.title.contents).strip(),
-                "links": links,
-                "depth": headline.depth,
-            }
 
-            topLevelHeadline = headline
+        # Save for full-text-search
-            while isinstance(topLevelHeadline.parent, org_rw.Headline):
+        cur.execute('''INSERT INTO note_search(note_id, title, body, top_level_title, is_done, is_todo) VALUES (?, ?, ?, ?, ?, ?);''',
-                topLevelHeadline = topLevelHeadline.parent
+                    (
-
+                        headline.id,
-            # Save for full-text-search
+                        headline.title.get_text(),
-            cur.execute('''INSERT INTO note_search(note_id, title, body, top_level_title, is_done, is_todo) VALUES (?, ?, ?, ?, ?, ?);''',
+                        ''.join(headline.get_contents('raw')),
-                        (
+                        topLevelHeadline.title.get_text(),
-                            headline.id,
+                        headline.is_done,
-                            headline.title.get_text(),
+                        headline.is_todo,
-                            ''.join(headline.get_contents('raw')),
+                    ))
-                            topLevelHeadline.title.get_text(),
-                            headline.is_done,
-                            headline.is_todo,
-                        ))
-
-            # Render HTML
-            with open(endpath, "wt") as f:
-                f.write(render_as_document(headline, doc, headlineLevel=0,
-                                           title=org_rw.token_list_to_plaintext(headline.title.contents)))
-                files_generated += 1
-
-            if headline.id == INDEX_ID:
-                index_endpath = os.path.join(dest_top, "index.html")
-                with open(index_endpath, "wt") as f:
-                    f.write(render_as_document(headline, doc, headlineLevel=0,
-                                               title=org_rw.token_list_to_plaintext(headline.title.contents)))
-                    files_generated += 1
 
     # Update graph, replace document ids with headline ids
     for headline_data in graph.values():
@@ -225,6 +212,34 @@ def regen_all(src_top, dest_top, *, docs=None, db=None):
             if link['target'] in doc_to_headline_remapping:
                 link['target'] = doc_to_headline_remapping[link['target']]
 
+    # Render docs after we've built the graph
+    # Render main headlines
+    full_graph_info = { "nodes": graph, "main_headlines": main_headlines_by_path }
+    for _docpath, main_headline in main_headlines_by_path.items():
+        if main_headline.doc.id:
+            endpath = os.path.join(dest_top, main_headline.doc.id + ".node.html")
+            with open(endpath, "wt") as f:
+                f.write(render_as_document(main_headline, main_headline.doc, headlineLevel=0, graph=full_graph_info,
+                                           title=org_rw.token_list_to_plaintext(main_headline.title.contents)))
+
+
+    # Render all headlines
+    for headline in all_headlines:
+        endpath = os.path.join(dest_top, headline.id + ".node.html")
+
+        # Render HTML
+        with open(endpath, "wt") as f:
+            f.write(render_as_document(headline, headline.doc, headlineLevel=0, graph=full_graph_info,
+                                       title=org_rw.token_list_to_plaintext(headline.title.contents)))
+            files_generated += 1
+
+        if headline.id == INDEX_ID:
+            index_endpath = os.path.join(dest_top, "index.html")
+            with open(index_endpath, "wt") as f:
+                f.write(render_as_document(headline, headline.doc, headlineLevel=0, graph=full_graph_info,
+                                           title=org_rw.token_list_to_plaintext(headline.title.contents)))
+                files_generated += 1
+
     # Output graph files
     graphpath = os.path.join(dest_top, "graph.json")
     graph_explorer_path = os.path.join(dest_top, "graph.html")
@@ -297,29 +312,29 @@ def print_element(element, indentation, headline):
         print_tree(element, indentation, headline)
 
 
-def render_property_drawer(element, acc):
+def render_property_drawer(element, acc, headline, graph):
     pass
 
 
-def render_logbook_drawer(element, acc):
+def render_logbook_drawer(element, acc, headline, graph):
     pass
 
 
-def render_property_node(element, acc):
+def render_property_node(element, acc, headline, graph):
     pass
 
 
-def render_list_group(element, acc):
+def render_list_group(element, acc, headline, graph):
     acc.append("<ul>")
-    render_tree(element.children, acc)
+    render_tree(element.children, acc, headline, graph)
     acc.append("</ul>")
 
-def render_table(element, acc):
+def render_table(element, acc, graph, headline):
     acc.append("<table>")
-    render_tree(element.children, acc)
+    render_tree(element.children, acc, headline, graph)
     acc.append("</table>")
 
-def render_table_row(element, acc):
+def render_table_row(element, acc, headline, graph):
     acc.append("<tr>")
     for cell in element.cells:
         acc.append("<td>")
@@ -327,22 +342,22 @@ def render_table_row(element, acc):
         acc.append("</td>")
     acc.append("</tr>")
 
-def render_table_separator_row(element, acc):
+def render_table_separator_row(element, acc, headline, graph):
     acc.append("<tr class='__table-separator'></tr>")
 
-def render_list_item(element, acc):
+def render_list_item(element, acc, headline, graph):
     acc.append("<li>")
     if element.tag is not None:
         acc.append("<span class='tag'>")
-        render_text_tokens(element.tag, acc)
+        render_text_tokens(element.tag, acc, headline, graph)
         acc.append("</span>")
 
     acc.append("<span class='item'>")
-    render_text_tokens(element.content, acc)
+    render_text_tokens(element.content, acc, headline, graph)
     acc.append("</span></li>")
 
 
-def render_code_block(element, acc):
+def render_code_block(element, acc, headline, graph):
     acc.append('<pre class="{}"><code>'.format(element.subtype.lower()))
     content = html.escape(element.lines)
 
@@ -360,23 +375,23 @@ def render_code_block(element, acc):
     acc.append('\n'.join(content_lines))
     acc.append('</code></pre>')
 
-def render_results_block(element, acc):
+def render_results_block(element, acc, headline, graph):
     # TODO:
     # acc.append('<pre class="results"><code>')
     # render_tree(element.children, acc)
     # acc.append('</code></pre>')
     pass
 
-def render_org_text(element, acc):
+def render_org_text(element, acc, headline, graph):
     as_dom = org_rw.text_to_dom(element.contents, element)
-    render_text_tokens(as_dom, acc)
+    render_text_tokens(as_dom, acc, headline, graph)
 
-def render_text(element, acc):
+def render_text(element, acc, headline, graph):
     acc.append('<div class="text">')
-    render_text_tokens(element.content, acc)
+    render_text_tokens(element.content, acc, headline, graph)
     acc.append('</div>')
 
-def render_text_tokens(tokens, acc):
+def render_text_tokens(tokens, acc, headline, graph):
     acc.append('<p>')
     for chunk in tokens:
         if isinstance(chunk, str):
@@ -386,6 +401,30 @@ def render_text_tokens(tokens, acc):
             link_target = chunk.value
             if link_target.startswith('id:'):
                 link_target = './' + link_target[3:] + '.node.html'
+            elif link_target.startswith('./') or link_target.startswith('../'):
+                if '::' in link_target:
+                    logging.warn('Not implemented headline links to other files. Used on {}'.format(link_target))
+
+                else:
+                    target_path = os.path.abspath(os.path.join(os.path.dirname(headline.doc.path), link_target))
+                    if target_path not in graph['main_headlines']:
+                        logging.warn('Link to doc not in graph: {}'.format(target_path))
+                    else:
+                        link_target = './' + graph['main_headlines'][target_path].id + '.node.html'
+            elif link_target.startswith('attachment:'):
+                logging.warn('Not implemented `attachment:` links. Used on {}'.format(link_target))
+            elif link_target.startswith('git://'):
+                logging.warn('Not implemented `git://`. Used on {}'.format(link_target))
+            elif link_target.startswith('* '):
+                logging.warn('Not implemented `* Headline` links. Used on {}'.format(link_target))
+            else:
+                if not (
+                    link_target.startswith('https://')
+                    or link_target.startswith('http://')
+                    or link_target.startswith('/')
+                ):
+                    raise NotImplementedError('Unknown link type: {}'
+                                              .format(link_target))
             description = chunk.description
             if description is None:
                 description = chunk.value
@@ -399,7 +438,7 @@ def render_text_tokens(tokens, acc):
     acc.append('</p>')
 
 
-def render_tag(element, acc):
+def render_tag(element, acc, headline, graph):
     return {
         dom.PropertyDrawerNode: render_property_drawer,
         dom.LogbookDrawerNode: render_logbook_drawer,
@@ -413,20 +452,20 @@ def render_tag(element, acc):
         dom.Text: render_text,
         dom.ResultsDrawerNode: render_results_block,
         org_rw.Text: render_org_text,
-    }[type(element)](element, acc)
+    }[type(element)](element, acc, headline, graph)
 
 
-def render_tree(tree, acc):
+def render_tree(tree, acc, headline, graph):
     for element in tree:
-        render_tag(element, acc)
+        render_tag(element, acc, headline, graph)
 
-def render_inline(tree, f):
+def render_inline(tree, f, headline, graph):
     acc = []
-    f(tree, acc)
+    f(tree, acc, headline, graph)
     return ''.join(acc)
 
 
-def render_as_document(headline, doc, headlineLevel, title):
+def render_as_document(headline, doc, headlineLevel, graph, title):
     if isinstance(headline.parent, org_rw.Headline):
         topLevelHeadline = headline.parent
         while isinstance(topLevelHeadline.parent, org_rw.Headline):
@@ -448,9 +487,9 @@ def render_as_document(headline, doc, headlineLevel, title):
 </html>
         """
     else:
-        return as_document(render(headline, doc, headlineLevel), title)
+        return as_document(render(headline, doc, graph=graph, headlineLevel=headlineLevel), title)
 
-def render(headline, doc, headlineLevel):
+def render(headline, doc, graph, headlineLevel):
     try:
         dom = headline.as_dom()
     except:
@@ -459,9 +498,9 @@ def render(headline, doc, headlineLevel):
     print_tree(dom, indentation=2, headline=headline)
 
     content = []
-    render_tree(dom, content)
+    render_tree(dom, content, headline, graph)
     for child in headline.children:
-        content.append(render(child, doc, headlineLevel=headlineLevel+1))
+        content.append(render(child, doc, headlineLevel=headlineLevel+1, graph=graph))
 
     if headline.state is None:
         state = ""
@@ -483,7 +522,7 @@ def render(headline, doc, headlineLevel):
     #     display_state = 'expanded'
     display_state = 'expanded'
 
-    title = render_inline(headline.title, render_tag)
+    title = render_inline(headline.title, render_tag, headline, graph)
 
     if headlineLevel > 0:
         title = f"<a href=\"javascript:toggle_expand('{html.escape(headline.id)}')\">{title}</a>"