Dig deeper in cut-by-token approach.

2018-04-15 17:47:04 +02:00 · 2018-04-15 17:47:04 +02:00 · 998a183fd2
commit 998a183fd2
parent d601ae3f83
4 changed files with 86 additions and 16 deletions
--- a/naive-nlu/tree_nlu/tests/tokenization.py
+++ b/naive-nlu/tree_nlu/tests/tokenization.py
@ -65,6 +65,8 @@ def main():
            with session().log(example['text']):
                tokens = list(knowledge.tokenize(example['text']))

+                print(tokens)
+                print(example['tokens'])
                assert example['tokens'] == tokens

        else: