From bb7d438e0d5515205d474fcc8104faf3ded6cf1f Mon Sep 17 00:00:00 2001 From: kenkeiras Date: Sun, 1 Oct 2017 17:10:50 +0200 Subject: [PATCH] Add session context, increase logging. --- .gitignore | 2 +- naive-nlu/tree_nlu/cli.py | 24 ++++----- naive-nlu/tree_nlu/parsing.py | 75 +++++++++++++------------- naive-nlu/tree_nlu/session/org_mode.py | 29 ++++++++-- naive-nlu/tree_nlu/test.py | 9 ++-- naive-nlu/tree_nlu/tests/basic.py | 4 +- naive-nlu/tree_nlu/tests/gac_100.py | 37 ++++++------- 7 files changed, 103 insertions(+), 77 deletions(-) diff --git a/.gitignore b/.gitignore index 474c6f3..961205f 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,4 @@ *.ba?k *.pyc __pycache__ -treeNLU-*session-*.org +treeNLU-*session*.org diff --git a/naive-nlu/tree_nlu/cli.py b/naive-nlu/tree_nlu/cli.py index 82e55ea..7434d12 100644 --- a/naive-nlu/tree_nlu/cli.py +++ b/naive-nlu/tree_nlu/cli.py @@ -34,7 +34,6 @@ def main(): while True: try: data = input("> ").strip() - session().log_step(data, 0) except EOFError: print("bye") break @@ -50,16 +49,17 @@ def main(): show_samples(knowledge) continue - ret = knowledge.process(data) - if ret: - result, _, _ = ret - if not is_modifiable_property(result): - print("<", result) + with session().log(data): + ret = knowledge.process(data) + if ret: + result, _, _ = ret + if not is_modifiable_property(result): + print("<", result) + else: + result.setter() + print("OK") + elif ret is None: + print("- Couldn't understand that, oops... -") else: - result.setter() - print("OK") - elif ret is None: - print("- Couldn't understand that, oops... -") - else: - print("Unhandled response:", ret) + print("Unhandled response:", ret) print("< Bye!") diff --git a/naive-nlu/tree_nlu/parsing.py b/naive-nlu/tree_nlu/parsing.py index f8369a0..18cfdb5 100644 --- a/naive-nlu/tree_nlu/parsing.py +++ b/naive-nlu/tree_nlu/parsing.py @@ -97,45 +97,46 @@ def integrate_language(knowledge_base, example): break for position, atom in lower_levels: - session().annotate("\x1b[1mSelecting\x1b[0m: {}".format(atom)) - similar = get_similar_tree(knowledge_base, atom, tokens) - remix, (start_bounds, end_bounds) = build_remix_matrix(knowledge_base, tokens, atom, similar) + with session().log(atom): + session().annotate("\x1b[1mSelecting\x1b[0m: {}".format(atom)) + similar = get_similar_tree(knowledge_base, atom, tokens) + remix, (start_bounds, end_bounds) = build_remix_matrix(knowledge_base, tokens, atom, similar) - after_remix = apply_remix(tokens[len(start_bounds):-len(end_bounds)], remix) - session().annotate("--FIND MIX--") - session().annotate("-MIX- | {}".format(remix)) - session().annotate("-FRM- | {}".format(tokens)) - session().annotate("-AFT- | {}".format(after_remix)) + after_remix = apply_remix(tokens[len(start_bounds):-len(end_bounds)], remix) + session().annotate("--FIND MIX--") + session().annotate("-MIX- | {}".format(remix)) + session().annotate("-FRM- | {}".format(tokens)) + session().annotate("-AFT- | {}".format(after_remix)) - session().annotate("--- TEMPLATE ---") + session().annotate("--- TEMPLATE ---") - _, matcher, result = make_template(knowledge_base, after_remix, atom) - session().annotate("Tx: {}".format(after_remix)) - session().annotate("Mx: {}".format(matcher)) - session().annotate("Rx: {}".format(result)) - session().annotate("Sx: {}".format(start_bounds)) - session().annotate("Ex: {}".format(end_bounds)) + _, matcher, result = make_template(knowledge_base, after_remix, atom) + session().annotate("Tx: {}".format(after_remix)) + session().annotate("Mx: {}".format(matcher)) + session().annotate("Rx: {}".format(result)) + session().annotate("Sx: {}".format(start_bounds)) + session().annotate("Ex: {}".format(end_bounds)) - assert(len(after_remix) + len(start_bounds) + len(end_bounds) == len(tokens)) - session().annotate( " +-> {}".format(after_remix)) - subquery_type = knowledge_evaluation.get_subquery_type(knowledge_base.knowledge, atom) - session().annotate(r" \-> <{}>".format(subquery_type)) + assert(len(after_remix) + len(start_bounds) + len(end_bounds) == len(tokens)) + session().annotate( " +-> {}".format(after_remix)) + subquery_type = knowledge_evaluation.get_subquery_type(knowledge_base.knowledge, atom) + session().annotate(r" \-> <{}>".format(subquery_type)) - # Clean remaining tokens - new_tokens = list(tokens) - offset = len(start_bounds) - for _ in range(len(remix)): - new_tokens.pop(offset) + # Clean remaining tokens + new_tokens = list(tokens) + offset = len(start_bounds) + for _ in range(len(remix)): + new_tokens.pop(offset) - # TODO: Get a specific types for... types - new_tokens.insert(offset, (subquery_type, remix)) - tokens = new_tokens + # TODO: Get a specific types for... types + new_tokens.insert(offset, (subquery_type, remix)) + tokens = new_tokens - resolved_parsed = replace_position(resolved_parsed, position, offset) - session().annotate("RP: {}".format(resolved_parsed)) - session().annotate("AT: {}".format(atom)) - session().annotate("#########") + resolved_parsed = replace_position(resolved_parsed, position, offset) + session().annotate("RP: {}".format(resolved_parsed)) + session().annotate("AT: {}".format(atom)) + session().annotate("#########") tokens, matcher, result = make_template(knowledge_base, tokens, resolved_parsed) @@ -298,12 +299,12 @@ def get_similar_tree(knowledge_base, atom, tokens): return None for i, possibility in enumerate(sorted_possibilities): - session().annotate('---- POSSIBILITY #{} ----'.format(i)) - similar_matcher, similar_result, similar_result_resolved, _, _ = possibility - session().annotate('AST: {}'.format(similar_result)) - session().annotate('Based on: {}'.format(similar_matcher)) - session().annotate('Results on: {}'.format(similar_result_resolved)) - session().annotate('---------------------') + with session().log(possibility): + similar_matcher, similar_result, similar_result_resolved, _, _ = possibility + session().annotate('AST: {}'.format(similar_result)) + session().annotate('Based on: {}'.format(similar_matcher)) + session().annotate('Results on: {}'.format(similar_result_resolved)) + session().annotate('---------------------') return sorted_possibilities[0] diff --git a/naive-nlu/tree_nlu/session/org_mode.py b/naive-nlu/tree_nlu/session/org_mode.py index 13ee3ed..e25600c 100644 --- a/naive-nlu/tree_nlu/session/org_mode.py +++ b/naive-nlu/tree_nlu/session/org_mode.py @@ -25,21 +25,40 @@ def global_session(): return SESSION +class LevelContext: + def __init__(self, increaser, decreaser): + self.increaser = increaser + self.decreaser = decreaser + + def __enter__(self): + self.increaser() + + def __exit__(self, _type, _value, _traceback): + self.decreaser() + + class OrgModeSession: def __init__(self, fname): self.f = open(fname, 'wt') - self.last_level = 0 + self.level = 0 def annotate(self, annotation): self.f.write("{indentation} {data}\n".format( - indentation=' ' * (self.last_level + 2 + 1), + indentation=' ' * (self.level + 2 + 1), data=annotation)) - def log_step(self, string, level): + def log(self, string): self.f.write("{indentation} {data}\n".format( - indentation='*' * (level + 1), + indentation='*' * (self.level + 1), data=string)) - self.last_level = level + + return LevelContext(self.inc_level, self.dec_level) + + def inc_level(self): + self.level += 1 + + def dec_level(self): + self.level -= 1 def close(self): self.f.close() diff --git a/naive-nlu/tree_nlu/test.py b/naive-nlu/tree_nlu/test.py index ee048e4..f4313f9 100644 --- a/naive-nlu/tree_nlu/test.py +++ b/naive-nlu/tree_nlu/test.py @@ -16,13 +16,15 @@ tests = ( def gen_session_name(): - now = datetime.datetime.utcnow() - return "treeNLU-test-session-{}.org".format( - now.strftime("%y_%m_%d %H:%M:%S_%f")) + return "treeNLU-test-session.org" def main(): org_mode.create_global_session(gen_session_name()) + + now = datetime.datetime.utcnow() + org_mode.global_session().annotate("Ran on {}".format( + now.strftime("%y_%m_%d %H:%M:%S_%f"))) failed = False for test_name, test_module in tests: try: @@ -39,6 +41,7 @@ def main(): failed = True traceback.print_exc() raise + org_mode.global_session().close() if failed: exit(1) diff --git a/naive-nlu/tree_nlu/tests/basic.py b/naive-nlu/tree_nlu/tests/basic.py index 4fc7e48..4c8184e 100644 --- a/naive-nlu/tree_nlu/tests/basic.py +++ b/naive-nlu/tree_nlu/tests/basic.py @@ -125,7 +125,9 @@ def main(): knowledge=base_knowledge, ) - differences = knowledge.train(examples) + for example in examples: + with session().log(example['text']): + differences = knowledge.train([example]) session().annotate("----") session().annotate(differences()) diff --git a/naive-nlu/tree_nlu/tests/gac_100.py b/naive-nlu/tree_nlu/tests/gac_100.py index 5e09abc..2a7b259 100644 --- a/naive-nlu/tree_nlu/tests/gac_100.py +++ b/naive-nlu/tree_nlu/tests/gac_100.py @@ -696,19 +696,20 @@ def main(): for i, (example_type, data) in enumerate(examples): if example_type == 'full_example': - session().log_step(data['affirmation'], 0) affirmation = { 'text': data['affirmation'], 'parsed': data['parsed'][1], } question = data - show_progbar(i, total, data['affirmation']) - differences = knowledge.train([affirmation]) + with session().log(data['affirmation']): + show_progbar(i, total, data['affirmation']) + differences = knowledge.train([affirmation]) - show_progbar(i, total, data['text']) - differences = knowledge.train([question]) - session().annotate(differences()) + with session().log(data['text']): + show_progbar(i, total, data['text']) + differences = knowledge.train([question]) + session().annotate(differences()) result, _, _ = knowledge.process(data['text']) @@ -720,20 +721,20 @@ def main(): raise AssertionError('{} is not {}'.format(result, data['answer'])) elif example_type == 'text_example': - session().log_step(data['affirmation'], 0) + with session().log(data['affirmation']): + show_progbar(i, total, data['affirmation']) + affirmation = data['affirmation'] + session().annotate("Processing affirmation: {}".format(affirmation)) + _, _, _ = knowledge.process(affirmation) - show_progbar(i, total, data['affirmation']) - affirmation = data['affirmation'] - session().annotate("Processing affirmation: {}".format(affirmation)) - _, _, _ = knowledge.process(affirmation) + with session().log(data['question']): + show_progbar(i, total, data['question']) + question = data['question'] + session().annotate("Processing question : {}".format(question)) + result, _, _ = knowledge.process(question) - show_progbar(i, total, data['question']) - question = data['question'] - session().annotate("Processing question : {}".format(question)) - result, _, _ = knowledge.process(question) - - if result != data['answer']: - raise AssertionError('{} is not {}'.format(result, data['answer'])) + if result != data['answer']: + raise AssertionError('{} is not {}'.format(result, data['answer'])) else: raise NotImplementedError('Example type: {}'.format(example_type))