From 5297158110c1c6594c89c101b636bf37b96d84a2 Mon Sep 17 00:00:00 2001 From: kenkeiras Date: Tue, 23 May 2017 19:04:10 +0200 Subject: [PATCH] Package as `tree_nlu`. --- naive-nlu/setup.py | 15 ++ naive-nlu/test.py | 157 +----------------- naive-nlu/tree_nlu/__init__.py | 0 naive-nlu/{ => tree_nlu}/depth_meter.py | 2 +- naive-nlu/{ => tree_nlu}/knowledge_base.py | 7 +- .../{ => tree_nlu}/knowledge_evaluation.py | 2 +- .../{ => tree_nlu}/modifiable_property.py | 0 naive-nlu/{ => tree_nlu}/parameters.py | 0 naive-nlu/{ => tree_nlu}/parsing.py | 8 +- naive-nlu/tree_nlu/test.py | 157 ++++++++++++++++++ 10 files changed, 184 insertions(+), 164 deletions(-) create mode 100644 naive-nlu/setup.py create mode 100644 naive-nlu/tree_nlu/__init__.py rename naive-nlu/{ => tree_nlu}/depth_meter.py (92%) rename naive-nlu/{ => tree_nlu}/knowledge_base.py (96%) rename naive-nlu/{ => tree_nlu}/knowledge_evaluation.py (99%) rename naive-nlu/{ => tree_nlu}/modifiable_property.py (100%) rename naive-nlu/{ => tree_nlu}/parameters.py (100%) rename naive-nlu/{ => tree_nlu}/parsing.py (98%) create mode 100644 naive-nlu/tree_nlu/test.py diff --git a/naive-nlu/setup.py b/naive-nlu/setup.py new file mode 100644 index 0000000..8fdc33b --- /dev/null +++ b/naive-nlu/setup.py @@ -0,0 +1,15 @@ +from setuptools import setup + +setup(name='tree_nlu', + version='0.1', + description='Naïve AST based NLU.', + author='kenkeiras', + author_email='kenkeiras@codigoparallevar.com', + packages=['tree_nlu'], + scripts=[ + ], + include_package_data=True, + install_requires = [ + 'jsondiff', + ], + zip_safe=False) diff --git a/naive-nlu/test.py b/naive-nlu/test.py index 45fd229..740652e 100644 --- a/naive-nlu/test.py +++ b/naive-nlu/test.py @@ -1,157 +1,4 @@ -import json -import logging - -logging.getLogger().setLevel(logging.INFO) - -from knowledge_base import KnowledgeBase -from modifiable_property import is_modifiable_property - -examples = [ - { - "text": "icecream is cold", - "parsed": ("exists-property-with-value", 'icecream', 'cold'), - }, - { - "text": "is icecream cold?", - "parsed": ("question", ("exists-property-with-value", 'icecream', 'cold')) - }, - { - "text": "lava is dangerous", - "parsed": ("exists-property-with-value", 'lava', 'dangerous') - }, - { - "text": "is lava dangerous?", - "parsed": ("question", ("exists-property-with-value", 'lava', 'dangerous')), - }, - { - "text": "earth is a planet", - "parsed": ("pertenence-to-group", 'earth', 'planet'), - }, - { - "text": "io is a moon", - "parsed": ("pertenence-to-group", 'io', 'moon'), - }, - { - "text": "is earth a moon?", - "parsed": ("question", ("pertenence-to-group", 'earth', 'moon')), - }, - { - "text": "Green is a color", - "parsed": ("pertenence-to-group", 'green', 'color'), - }, - { - "text": "a plane can fly", - "parsed": ("has-capacity", 'plane', 'fly') - }, - { - "text": "a wale can swim", - "parsed": ("has-capacity", 'wale', 'swim') - }, - { - "text": "if earth is a planet, it is big", - "parsed": ("implies", - ("pertenence-to-group", 'earth', 'planet'), - ("exists-property-with-value", 'earth', 'big')), - }, -] - -base_knowledge = { - 'icecream': { - "groups": set(['noun', 'object', 'comestible', 'sweet']), - }, - 'lava': { - "groups": set(['noun', 'object']), - }, - 'earth': { - "groups": set(['noun', 'object', 'planet']), - }, - 'io': { - "groups": set(['noun', 'object']), - }, - 'green': { - "groups": set(['noun', 'color', 'concept']), - }, - 'plane': { - "groups": set(['noun', 'object', 'vehicle', 'fast']), - }, - 'car': { - "groups": set(['noun', 'object', 'vehicle', 'slow-ish']), - }, - 'wale': { - "groups": set(['noun', 'object', 'living-being']), - }, - 'cold': { - "groups": set(['property', 'temperature']), - "as_property": "temperature", - }, - 'dangerous': { - "groups": set(['property']), - "as_property": "safety", - }, - 'planet': { - "groups": set(['noun', 'group']), - }, - 'moon': { - "groups": set(['noun', 'group']), - }, - 'color': { - "groups": set(['property', 'group']), - }, - 'fly': { - "groups": set(['verb']), - }, - 'swim': { - "groups": set(['verb']), - }, -} - - -def test_assumption(expectedResponse, knowledge, query): - logging.info("Query: {}".format(query['text'])) - logging.info("Expected: {}".format(expectedResponse)) - - result, abstract_tree, diff = knowledge.process(query['text']) - end_result = result.getter() if is_modifiable_property(result) else result - - logging.info("\x1b[0;3{}mResult: {}\x1b[0m".format("1" if end_result != expectedResponse else "2", end_result)) - assert(end_result == expectedResponse) - - -def main(): - knowledge = KnowledgeBase( - knowledge=base_knowledge, - ) - - differences = knowledge.train(examples) - - logging.info("----") - logging.info(differences()) - logging.info("----") - - test_assumption(True, knowledge, {'text': 'earth is a planet'}) - test_assumption(True, knowledge, {'text': 'is lava dangerous?'}) - for test in [{'text': 'a bus can run'}, {'text': 'io is a moon'}]: - row = test['text'] - result, inferred_tree, differences = knowledge.process(row) - - logging.info("result:", result) - logging.info(differences()) - logging.info("---") - logging.info('-----') - logging.info(json.dumps(sorted(knowledge.knowledge.keys()), indent=4)) - logging.info('-----') - - queryTrue = { - "text": "is io a moon?", - "parsed": ("question", ("pertenence-to-group", "io", "moon")) - } - queryFalse = { - "text": "is io a planet?", - "parsed": ("question", ("pertenence-to-group", "io", "planet")) - } - - test_assumption(False, knowledge, queryFalse) - test_assumption(True, knowledge, queryTrue) +from tree_nlu import test if __name__ == '__main__': - main() + test.main() diff --git a/naive-nlu/tree_nlu/__init__.py b/naive-nlu/tree_nlu/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/naive-nlu/depth_meter.py b/naive-nlu/tree_nlu/depth_meter.py similarity index 92% rename from naive-nlu/depth_meter.py rename to naive-nlu/tree_nlu/depth_meter.py index db1dab9..3ee624f 100644 --- a/naive-nlu/depth_meter.py +++ b/naive-nlu/tree_nlu/depth_meter.py @@ -1,5 +1,5 @@ import sys -import parameters +from . import parameters def show_depth(depth: int, zoom: int=2): offset = int((parameters.MAX_RECURSIONS - depth) / (2 / zoom)) diff --git a/naive-nlu/knowledge_base.py b/naive-nlu/tree_nlu/knowledge_base.py similarity index 96% rename from naive-nlu/knowledge_base.py rename to naive-nlu/tree_nlu/knowledge_base.py index 31fe4d2..4c27700 100644 --- a/naive-nlu/knowledge_base.py +++ b/naive-nlu/tree_nlu/knowledge_base.py @@ -1,9 +1,10 @@ import copy import logging -import parsing -import knowledge_evaluation -from modifiable_property import is_modifiable_property + +from . import parsing +from . import knowledge_evaluation +from .modifiable_property import is_modifiable_property def diff_knowledge(before, after): diff --git a/naive-nlu/knowledge_evaluation.py b/naive-nlu/tree_nlu/knowledge_evaluation.py similarity index 99% rename from naive-nlu/knowledge_evaluation.py rename to naive-nlu/tree_nlu/knowledge_evaluation.py index 4485adb..a24c07d 100644 --- a/naive-nlu/knowledge_evaluation.py +++ b/naive-nlu/tree_nlu/knowledge_evaluation.py @@ -1,4 +1,4 @@ -from modifiable_property import ( +from .modifiable_property import ( ModifiableProperty, ModifiablePropertyWithAst, is_modifiable_property, diff --git a/naive-nlu/modifiable_property.py b/naive-nlu/tree_nlu/modifiable_property.py similarity index 100% rename from naive-nlu/modifiable_property.py rename to naive-nlu/tree_nlu/modifiable_property.py diff --git a/naive-nlu/parameters.py b/naive-nlu/tree_nlu/parameters.py similarity index 100% rename from naive-nlu/parameters.py rename to naive-nlu/tree_nlu/parameters.py diff --git a/naive-nlu/parsing.py b/naive-nlu/tree_nlu/parsing.py similarity index 98% rename from naive-nlu/parsing.py rename to naive-nlu/tree_nlu/parsing.py index 3b6bf09..fa16a33 100644 --- a/naive-nlu/parsing.py +++ b/naive-nlu/tree_nlu/parsing.py @@ -1,16 +1,16 @@ #!/usr/bin/env python -import knowledge_evaluation +from . import knowledge_evaluation -import depth_meter +from . import depth_meter import logging import re import copy from functools import reduce from typing import List -from modifiable_property import ModifiableProperty -import parameters +from .modifiable_property import ModifiableProperty +from . import parameters # TODO: more flexible tokenization def to_tokens(text): diff --git a/naive-nlu/tree_nlu/test.py b/naive-nlu/tree_nlu/test.py new file mode 100644 index 0000000..fbd24d8 --- /dev/null +++ b/naive-nlu/tree_nlu/test.py @@ -0,0 +1,157 @@ +import json +import logging + +logging.getLogger().setLevel(logging.INFO) + +from .knowledge_base import KnowledgeBase +from .modifiable_property import is_modifiable_property + +examples = [ + { + "text": "icecream is cold", + "parsed": ("exists-property-with-value", 'icecream', 'cold'), + }, + { + "text": "is icecream cold?", + "parsed": ("question", ("exists-property-with-value", 'icecream', 'cold')) + }, + { + "text": "lava is dangerous", + "parsed": ("exists-property-with-value", 'lava', 'dangerous') + }, + { + "text": "is lava dangerous?", + "parsed": ("question", ("exists-property-with-value", 'lava', 'dangerous')), + }, + { + "text": "earth is a planet", + "parsed": ("pertenence-to-group", 'earth', 'planet'), + }, + { + "text": "io is a moon", + "parsed": ("pertenence-to-group", 'io', 'moon'), + }, + { + "text": "is earth a moon?", + "parsed": ("question", ("pertenence-to-group", 'earth', 'moon')), + }, + { + "text": "Green is a color", + "parsed": ("pertenence-to-group", 'green', 'color'), + }, + { + "text": "a plane can fly", + "parsed": ("has-capacity", 'plane', 'fly') + }, + { + "text": "a wale can swim", + "parsed": ("has-capacity", 'wale', 'swim') + }, + { + "text": "if earth is a planet, it is big", + "parsed": ("implies", + ("pertenence-to-group", 'earth', 'planet'), + ("exists-property-with-value", 'earth', 'big')), + }, +] + +base_knowledge = { + 'icecream': { + "groups": set(['noun', 'object', 'comestible', 'sweet']), + }, + 'lava': { + "groups": set(['noun', 'object']), + }, + 'earth': { + "groups": set(['noun', 'object', 'planet']), + }, + 'io': { + "groups": set(['noun', 'object']), + }, + 'green': { + "groups": set(['noun', 'color', 'concept']), + }, + 'plane': { + "groups": set(['noun', 'object', 'vehicle', 'fast']), + }, + 'car': { + "groups": set(['noun', 'object', 'vehicle', 'slow-ish']), + }, + 'wale': { + "groups": set(['noun', 'object', 'living-being']), + }, + 'cold': { + "groups": set(['property', 'temperature']), + "as_property": "temperature", + }, + 'dangerous': { + "groups": set(['property']), + "as_property": "safety", + }, + 'planet': { + "groups": set(['noun', 'group']), + }, + 'moon': { + "groups": set(['noun', 'group']), + }, + 'color': { + "groups": set(['property', 'group']), + }, + 'fly': { + "groups": set(['verb']), + }, + 'swim': { + "groups": set(['verb']), + }, +} + + +def test_assumption(expectedResponse, knowledge, query): + logging.info("Query: {}".format(query['text'])) + logging.info("Expected: {}".format(expectedResponse)) + + result, abstract_tree, diff = knowledge.process(query['text']) + end_result = result.getter() if is_modifiable_property(result) else result + + logging.info("\x1b[0;3{}mResult: {}\x1b[0m".format("1" if end_result != expectedResponse else "2", end_result)) + assert(end_result == expectedResponse) + + +def main(): + knowledge = KnowledgeBase( + knowledge=base_knowledge, + ) + + differences = knowledge.train(examples) + + logging.info("----") + logging.info(differences()) + logging.info("----") + + test_assumption(True, knowledge, {'text': 'earth is a planet'}) + test_assumption(True, knowledge, {'text': 'is lava dangerous?'}) + for test in [{'text': 'a bus can run'}, {'text': 'io is a moon'}]: + row = test['text'] + result, inferred_tree, differences = knowledge.process(row) + + logging.info("result:", result) + logging.info(differences()) + logging.info("---") + logging.info('-----') + logging.info(json.dumps(sorted(knowledge.knowledge.keys()), indent=4)) + logging.info('-----') + + queryTrue = { + "text": "is io a moon?", + "parsed": ("question", ("pertenence-to-group", "io", "moon")) + } + queryFalse = { + "text": "is io a planet?", + "parsed": ("question", ("pertenence-to-group", "io", "planet")) + } + + test_assumption(False, knowledge, queryFalse) + test_assumption(True, knowledge, queryTrue) + +if __name__ == '__main__': + main()