Add (non-passing) tokenization.

naive-nlu/tree_nlu/utils/tokenization.py

@@ -0,0 +1,19 @@
+BASIC_TOKENIZATION_EXAMPLES = (
+    ({
+        "text": 'cat',
+        "tokens": ['cat'],
+    }),
+    ({
+        "text": 'text separated by spaces',
+        "tokens": ['text', 'separated', 'by', 'spaces'],
+    }),
+    ({
+        "text": 'is earth a planet?',
+        "tokens": ['is', 'earth', 'a', 'planet', '?'],
+    }),
+)
+
+
+def train_basic_tokenization(knowledge_base):
+    for example in BASIC_TOKENIZATION_EXAMPLES:
+        knowledge_base.train_tokenizer(example)