Add (non-passing) tokenization.

naive-nlu/tree_nlu/tests/basic.py

@@ -3,6 +3,7 @@ import json
 
 from ..knowledge_base import KnowledgeBase
 from ..modifiable_property import is_modifiable_property
+from ..utils.tokenization import train_basic_tokenization
 
 examples = [
     {
@@ -107,6 +108,9 @@ base_knowledge = {
     'swim': {
         "groups": {'verb'},
     },
+    'planet': {
+        'groups': {'noun'}
+    }
 }
 
 def test_assumption(expectedResponse, knowledge, query):
@@ -125,6 +129,8 @@ def main():
         knowledge=base_knowledge,
     )
 
+    train_basic_tokenization(knowledge)
+
     for example in examples:
         with session().log(example['text']):
             differences = knowledge.train([example])