diff --git a/scripts/generate.py b/scripts/generate.py index b6e0bc4..48d803f 100644 --- a/scripts/generate.py +++ b/scripts/generate.py @@ -92,12 +92,19 @@ def regen_all(src_top, dest_top, *, docs=None, db=None): doc_to_headline_remapping = {} os.makedirs(dest_top, exist_ok=True) - graph = {} + + ## Build headline list + # This includes a virtual headline for ID-referenced documents. + all_headlines = [] + main_headlines_by_path = {} for doc in docs: relpath = os.path.relpath(doc.path, src_top) changed = False headlines = list(doc.getAllHeadlines()) related = None + if not relpath.startswith("public/"): + # print("Skip:", relpath) + continue i = len(headlines) while i > 0: @@ -123,10 +130,7 @@ def regen_all(src_top, dest_top, *, docs=None, db=None): print("Updated", relpath) save_changes(doc) - if not relpath.startswith("public/"): - # print("Skip:", relpath) - continue - + all_headlines.extend(headlines) main_headline = None topHeadlines = doc.getTopHeadlines() @@ -134,14 +138,10 @@ def regen_all(src_top, dest_top, *, docs=None, db=None): or (len(topHeadlines) == 2 and related is not None)): main_headline = [h for h in topHeadlines if h != related][0] + main_headlines_by_path[doc.path] = main_headline if doc.id is not None: - endpath = os.path.join(dest_top, doc.id + ".node.html") - with open(endpath, "wt") as f: - doc_to_headline_remapping['id:' + doc.id] = 'id:' + main_headline.id - - f.write(render_as_document(main_headline, doc, headlineLevel=0, - title=org_rw.token_list_to_plaintext(main_headline.title.contents))) - files_generated += 1 + doc_to_headline_remapping['id:' + doc.id] = 'id:' + main_headline.id + files_generated += 1 elif doc.id is not None: logging.error("Cannot render document from id: {}. {} headlines {} related".format( relpath, @@ -149,75 +149,62 @@ def regen_all(src_top, dest_top, *, docs=None, db=None): 'with' if related is not None else 'without' )) - for headline in headlines: - endpath = os.path.join(dest_top, headline.id + ".node.html") + graph = {} + # Build graph + for headline in all_headlines: + links = [] + headline_links = list(headline.get_links()) + if headline == main_headline and related is not None: + headline_links.extend(list(related.get_links())) - links = [] - headline_links = list(headline.get_links()) - if headline == main_headline and related is not None: - headline_links.extend(list(related.get_links())) + for l in headline_links: + if l.value.startswith('http://') or l.value.startswith('https://'): + pass # Ignore for now, external URL + elif l.value.startswith('id:'): + links.append({'target': l.value}) + elif l.value.startswith('attachment:'): + pass # Ignore, attachment + elif l.value.startswith('file:'): + pass # Ignore, attachment + elif l.value.startswith('notmuch:'): + pass # Ignore, mail + elif l.value.startswith('orgit-rev:'): + pass # Ignore, mail + elif l.value.startswith('*'): + pass # Ignore, internal + elif not ':' in l.value.split()[0]: + pass # Ignore, internal + elif l.value.startswith('./'): + pass # TODO: Properly handle + else: + logging.warning('On document {}, unknown link to {}'.format(doc.path, l.value)) - for l in headline_links: - if l.value.startswith('http://') or l.value.startswith('https://'): - pass # Ignore for now, external URL - elif l.value.startswith('id:'): - links.append({'target': l.value}) - elif l.value.startswith('attachment:'): - pass # Ignore, attachment - elif l.value.startswith('file:'): - pass # Ignore, attachment - elif l.value.startswith('notmuch:'): - pass # Ignore, mail - elif l.value.startswith('orgit-rev:'): - pass # Ignore, mail - elif l.value.startswith('*'): - pass # Ignore, internal - elif not ':' in l.value.split()[0]: - pass # Ignore, internal - elif l.value.startswith('./'): - pass # TODO: Properly handle - else: - logging.warning('On document {}, unknown link to {}'.format(doc.path, l.value)) + if headline.parent: + if isinstance(headline.parent, org_rw.Headline): + links.append({ + "target": headline.parent.id, + "relation": "in" + }) + graph[headline.id] = { + "title": org_rw.org_rw.token_list_to_plaintext(headline.title.contents).strip(), + "links": links, + "depth": headline.depth, + } - if headline.parent: - if isinstance(headline.parent, org_rw.Headline): - links.append({ - "target": headline.parent.id, - "relation": "in" - }) - graph[headline.id] = { - "title": org_rw.org_rw.token_list_to_plaintext(headline.title.contents).strip(), - "links": links, - "depth": headline.depth, - } + topLevelHeadline = headline + while isinstance(topLevelHeadline.parent, org_rw.Headline): + topLevelHeadline = topLevelHeadline.parent - topLevelHeadline = headline - while isinstance(topLevelHeadline.parent, org_rw.Headline): - topLevelHeadline = topLevelHeadline.parent - - # Save for full-text-search - cur.execute('''INSERT INTO note_search(note_id, title, body, top_level_title, is_done, is_todo) VALUES (?, ?, ?, ?, ?, ?);''', - ( - headline.id, - headline.title.get_text(), - ''.join(headline.get_contents('raw')), - topLevelHeadline.title.get_text(), - headline.is_done, - headline.is_todo, - )) - - # Render HTML - with open(endpath, "wt") as f: - f.write(render_as_document(headline, doc, headlineLevel=0, - title=org_rw.token_list_to_plaintext(headline.title.contents))) - files_generated += 1 - - if headline.id == INDEX_ID: - index_endpath = os.path.join(dest_top, "index.html") - with open(index_endpath, "wt") as f: - f.write(render_as_document(headline, doc, headlineLevel=0, - title=org_rw.token_list_to_plaintext(headline.title.contents))) - files_generated += 1 + # Save for full-text-search + cur.execute('''INSERT INTO note_search(note_id, title, body, top_level_title, is_done, is_todo) VALUES (?, ?, ?, ?, ?, ?);''', + ( + headline.id, + headline.title.get_text(), + ''.join(headline.get_contents('raw')), + topLevelHeadline.title.get_text(), + headline.is_done, + headline.is_todo, + )) # Update graph, replace document ids with headline ids for headline_data in graph.values(): @@ -225,6 +212,34 @@ def regen_all(src_top, dest_top, *, docs=None, db=None): if link['target'] in doc_to_headline_remapping: link['target'] = doc_to_headline_remapping[link['target']] + # Render docs after we've built the graph + # Render main headlines + full_graph_info = { "nodes": graph, "main_headlines": main_headlines_by_path } + for _docpath, main_headline in main_headlines_by_path.items(): + if main_headline.doc.id: + endpath = os.path.join(dest_top, main_headline.doc.id + ".node.html") + with open(endpath, "wt") as f: + f.write(render_as_document(main_headline, main_headline.doc, headlineLevel=0, graph=full_graph_info, + title=org_rw.token_list_to_plaintext(main_headline.title.contents))) + + + # Render all headlines + for headline in all_headlines: + endpath = os.path.join(dest_top, headline.id + ".node.html") + + # Render HTML + with open(endpath, "wt") as f: + f.write(render_as_document(headline, headline.doc, headlineLevel=0, graph=full_graph_info, + title=org_rw.token_list_to_plaintext(headline.title.contents))) + files_generated += 1 + + if headline.id == INDEX_ID: + index_endpath = os.path.join(dest_top, "index.html") + with open(index_endpath, "wt") as f: + f.write(render_as_document(headline, headline.doc, headlineLevel=0, graph=full_graph_info, + title=org_rw.token_list_to_plaintext(headline.title.contents))) + files_generated += 1 + # Output graph files graphpath = os.path.join(dest_top, "graph.json") graph_explorer_path = os.path.join(dest_top, "graph.html") @@ -297,29 +312,29 @@ def print_element(element, indentation, headline): print_tree(element, indentation, headline) -def render_property_drawer(element, acc): +def render_property_drawer(element, acc, headline, graph): pass -def render_logbook_drawer(element, acc): +def render_logbook_drawer(element, acc, headline, graph): pass -def render_property_node(element, acc): +def render_property_node(element, acc, headline, graph): pass -def render_list_group(element, acc): +def render_list_group(element, acc, headline, graph): acc.append("") -def render_table(element, acc): +def render_table(element, acc, graph, headline): acc.append("") - render_tree(element.children, acc) + render_tree(element.children, acc, headline, graph) acc.append("
") -def render_table_row(element, acc): +def render_table_row(element, acc, headline, graph): acc.append("") for cell in element.cells: acc.append("") @@ -327,22 +342,22 @@ def render_table_row(element, acc): acc.append("") acc.append("") -def render_table_separator_row(element, acc): +def render_table_separator_row(element, acc, headline, graph): acc.append("") -def render_list_item(element, acc): +def render_list_item(element, acc, headline, graph): acc.append("
  • ") if element.tag is not None: acc.append("") - render_text_tokens(element.tag, acc) + render_text_tokens(element.tag, acc, headline, graph) acc.append("") acc.append("") - render_text_tokens(element.content, acc) + render_text_tokens(element.content, acc, headline, graph) acc.append("
    • ") -def render_code_block(element, acc): +def render_code_block(element, acc, headline, graph): acc.append('
    '.format(element.subtype.lower()))
     content = html.escape(element.lines)
 
@@ -360,23 +375,23 @@ def render_code_block(element, acc):
     acc.append('\n'.join(content_lines))
     acc.append('
    ') -def render_results_block(element, acc): +def render_results_block(element, acc, headline, graph): # TODO: # acc.append('
    ')
     # render_tree(element.children, acc)
     # acc.append('
    ') pass -def render_org_text(element, acc): +def render_org_text(element, acc, headline, graph): as_dom = org_rw.text_to_dom(element.contents, element) - render_text_tokens(as_dom, acc) + render_text_tokens(as_dom, acc, headline, graph) -def render_text(element, acc): +def render_text(element, acc, headline, graph): acc.append('
    ') - render_text_tokens(element.content, acc) + render_text_tokens(element.content, acc, headline, graph) acc.append('
    ') -def render_text_tokens(tokens, acc): +def render_text_tokens(tokens, acc, headline, graph): acc.append('

    ') for chunk in tokens: if isinstance(chunk, str): @@ -386,6 +401,30 @@ def render_text_tokens(tokens, acc): link_target = chunk.value if link_target.startswith('id:'): link_target = './' + link_target[3:] + '.node.html' + elif link_target.startswith('./') or link_target.startswith('../'): + if '::' in link_target: + logging.warn('Not implemented headline links to other files. Used on {}'.format(link_target)) + + else: + target_path = os.path.abspath(os.path.join(os.path.dirname(headline.doc.path), link_target)) + if target_path not in graph['main_headlines']: + logging.warn('Link to doc not in graph: {}'.format(target_path)) + else: + link_target = './' + graph['main_headlines'][target_path].id + '.node.html' + elif link_target.startswith('attachment:'): + logging.warn('Not implemented `attachment:` links. Used on {}'.format(link_target)) + elif link_target.startswith('git://'): + logging.warn('Not implemented `git://`. Used on {}'.format(link_target)) + elif link_target.startswith('* '): + logging.warn('Not implemented `* Headline` links. Used on {}'.format(link_target)) + else: + if not ( + link_target.startswith('https://') + or link_target.startswith('http://') + or link_target.startswith('/') + ): + raise NotImplementedError('Unknown link type: {}' + .format(link_target)) description = chunk.description if description is None: description = chunk.value @@ -399,7 +438,7 @@ def render_text_tokens(tokens, acc): acc.append('

    ') -def render_tag(element, acc): +def render_tag(element, acc, headline, graph): return { dom.PropertyDrawerNode: render_property_drawer, dom.LogbookDrawerNode: render_logbook_drawer, @@ -413,20 +452,20 @@ def render_tag(element, acc): dom.Text: render_text, dom.ResultsDrawerNode: render_results_block, org_rw.Text: render_org_text, - }[type(element)](element, acc) + }[type(element)](element, acc, headline, graph) -def render_tree(tree, acc): +def render_tree(tree, acc, headline, graph): for element in tree: - render_tag(element, acc) + render_tag(element, acc, headline, graph) -def render_inline(tree, f): +def render_inline(tree, f, headline, graph): acc = [] - f(tree, acc) + f(tree, acc, headline, graph) return ''.join(acc) -def render_as_document(headline, doc, headlineLevel, title): +def render_as_document(headline, doc, headlineLevel, graph, title): if isinstance(headline.parent, org_rw.Headline): topLevelHeadline = headline.parent while isinstance(topLevelHeadline.parent, org_rw.Headline): @@ -448,9 +487,9 @@ def render_as_document(headline, doc, headlineLevel, title): """ else: - return as_document(render(headline, doc, headlineLevel), title) + return as_document(render(headline, doc, graph=graph, headlineLevel=headlineLevel), title) -def render(headline, doc, headlineLevel): +def render(headline, doc, graph, headlineLevel): try: dom = headline.as_dom() except: @@ -459,9 +498,9 @@ def render(headline, doc, headlineLevel): print_tree(dom, indentation=2, headline=headline) content = [] - render_tree(dom, content) + render_tree(dom, content, headline, graph) for child in headline.children: - content.append(render(child, doc, headlineLevel=headlineLevel+1)) + content.append(render(child, doc, headlineLevel=headlineLevel+1, graph=graph)) if headline.state is None: state = "" @@ -483,7 +522,7 @@ def render(headline, doc, headlineLevel): # display_state = 'expanded' display_state = 'expanded' - title = render_inline(headline.title, render_tag) + title = render_inline(headline.title, render_tag, headline, graph) if headlineLevel > 0: title = f"{title}"