BASIC_TOKENIZATION_EXAMPLES = (
    ({
        "text": 'cat',
        "tokens": ['cat'],
    }),
    ({
        "text": 'text separated by spaces',
        "tokens": ['text', 'separated', 'by', 'spaces'],
    }),
    ({
        "text": 'is earth a planet?',
        "tokens": ['is', 'earth', 'a', 'planet', '?'],
    }),
)


def train_basic_tokenization(knowledge_base):
    for example in BASIC_TOKENIZATION_EXAMPLES:
        knowledge_base.train_tokenizer(example)