diff --git a/naive-nlu/setup.py b/naive-nlu/setup.py new file mode 100644 index 0000000..8fdc33b --- /dev/null +++ b/naive-nlu/setup.py @@ -0,0 +1,15 @@ +from setuptools import setup + +setup(name='tree_nlu', + version='0.1', + description='Naïve AST based NLU.', + author='kenkeiras', + author_email='kenkeiras@codigoparallevar.com', + packages=['tree_nlu'], + scripts=[ + ], + include_package_data=True, + install_requires = [ + 'jsondiff', + ], + zip_safe=False) diff --git a/naive-nlu/test.py b/naive-nlu/test.py index 347f6e7..740652e 100644 --- a/naive-nlu/test.py +++ b/naive-nlu/test.py @@ -1,151 +1,4 @@ -import json -import logging - -logging.getLogger().setLevel(logging.INFO) - -from knowledge_base import KnowledgeBase -from modifiable_property import is_modifiable_property - -examples = [ - { - "text": "icecream is cold", - "parsed": ("exists-property-with-value", 'icecream', 'cold'), - }, - { - "text": "is icecream cold?", - "parsed": ("question", ("exists-property-with-value", 'icecream', 'cold')) - }, - { - "text": "lava is dangerous", - "parsed": ("exists-property-with-value", 'lava', 'dangerous') - }, - { - "text": "is lava dangerous?", - "parsed": ("question", ("exists-property-with-value", 'lava', 'dangerous')), - }, - { - "text": "earth is a planet", - "parsed": ("pertenence-to-group", 'earth', 'planet'), - }, - { - "text": "io is a moon", - "parsed": ("pertenence-to-group", 'io', 'moon'), - }, - { - "text": "is earth a moon?", - "parsed": ("question", ("pertenence-to-group", 'earth', 'moon')), - }, - { - "text": "Green is a color", - "parsed": ("pertenence-to-group", 'green', 'color'), - }, - { - "text": "a plane can fly", - "parsed": ("has-capacity", 'plane', 'fly') - }, - { - "text": "a wale can swim", - "parsed": ("has-capacity", 'wale', 'swim') - }, -] - -base_knowledge = { - 'icecream': { - "groups": set(['noun', 'object', 'comestible', 'sweet']), - }, - 'lava': { - "groups": set(['noun', 'object']), - }, - 'earth': { - "groups": set(['noun', 'object', 'planet']), - }, - 'io': { - "groups": set(['noun', 'object']), - }, - 'green': { - "groups": set(['noun', 'color', 'concept']), - }, - 'plane': { - "groups": set(['noun', 'object', 'vehicle', 'fast']), - }, - 'car': { - "groups": set(['noun', 'object', 'vehicle', 'slow-ish']), - }, - 'wale': { - "groups": set(['noun', 'object', 'living-being']), - }, - 'cold': { - "groups": set(['property', 'temperature']), - "as_property": "temperature", - }, - 'dangerous': { - "groups": set(['property']), - "as_property": "safety", - }, - 'planet': { - "groups": set(['noun', 'group']), - }, - 'moon': { - "groups": set(['noun', 'group']), - }, - 'color': { - "groups": set(['property', 'group']), - }, - 'fly': { - "groups": set(['verb']), - }, - 'swim': { - "groups": set(['verb']), - }, -} - - -def test_assumption(expectedResponse, knowledge, query): - logging.info("Query: {}".format(query['text'])) - logging.info("Expected: {}".format(expectedResponse)) - - result, abstract_tree, diff = knowledge.process(query['text']) - end_result = result.getter() if is_modifiable_property(result) else result - - logging.info("\x1b[0;3{}mResult: {}\x1b[0m".format("1" if end_result != expectedResponse else "2", end_result)) - assert(end_result == expectedResponse) - - -def main(): - knowledge = KnowledgeBase( - knowledge=base_knowledge, - ) - - differences = knowledge.train(examples) - - logging.info("----") - logging.info(differences()) - logging.info("----") - - test_assumption(True, knowledge, {'text': 'earth is a planet'}) - test_assumption(True, knowledge, {'text': 'is lava dangerous?'}) - for test in [{'text': 'a bus can run'}, {'text': 'io is a moon'}]: - row = test['text'] - result, inferred_tree, differences = knowledge.process(row) - - logging.info("result:", result) - logging.info(differences()) - logging.info("---") - logging.info('-----') - logging.info(json.dumps(sorted(knowledge.knowledge.keys()), indent=4)) - logging.info('-----') - - queryTrue = { - "text": "is io a moon?", - "parsed": ("question", ("pertenence-to-group", "io", "moon")) - } - queryFalse = { - "text": "is io a planet?", - "parsed": ("question", ("pertenence-to-group", "io", "planet")) - } - - test_assumption(False, knowledge, queryFalse) - test_assumption(True, knowledge, queryTrue) +from tree_nlu import test if __name__ == '__main__': - main() + test.main() diff --git a/naive-nlu/tree_nlu/__init__.py b/naive-nlu/tree_nlu/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/naive-nlu/depth_meter.py b/naive-nlu/tree_nlu/depth_meter.py similarity index 92% rename from naive-nlu/depth_meter.py rename to naive-nlu/tree_nlu/depth_meter.py index db1dab9..3ee624f 100644 --- a/naive-nlu/depth_meter.py +++ b/naive-nlu/tree_nlu/depth_meter.py @@ -1,5 +1,5 @@ import sys -import parameters +from . import parameters def show_depth(depth: int, zoom: int=2): offset = int((parameters.MAX_RECURSIONS - depth) / (2 / zoom)) diff --git a/naive-nlu/knowledge_base.py b/naive-nlu/tree_nlu/knowledge_base.py similarity index 86% rename from naive-nlu/knowledge_base.py rename to naive-nlu/tree_nlu/knowledge_base.py index 31fe4d2..31c84a1 100644 --- a/naive-nlu/knowledge_base.py +++ b/naive-nlu/tree_nlu/knowledge_base.py @@ -1,9 +1,10 @@ import copy import logging -import parsing -import knowledge_evaluation -from modifiable_property import is_modifiable_property + +from . import parsing +from . import knowledge_evaluation +from .modifiable_property import is_modifiable_property def diff_knowledge(before, after): @@ -23,6 +24,13 @@ class KnowledgeBase(object): # Parse everything parsed_examples = [] for example in examples: + # If there's parsed data, leverage it ASAP + if 'parsed' in example: + result = knowledge_evaluation.integrate_information(self.knowledge, { + "parsed": example['parsed'], + }) + self.act_upon(result) + logging.info("\x1b[7;32m> {} \x1b[0m".format(example)) tokens, decomposition, inferred_tree = parsing.integrate_language(self, example) logging.info(tokens) @@ -48,6 +56,7 @@ class KnowledgeBase(object): def process(self, row): + row = row.lower() knowledge_before = copy.deepcopy(self.knowledge) logging.info("\x1b[7;32m> {} \x1b[0m".format(row)) tokens = parsing.to_tokens(row) diff --git a/naive-nlu/knowledge_evaluation.py b/naive-nlu/tree_nlu/knowledge_evaluation.py similarity index 56% rename from naive-nlu/knowledge_evaluation.py rename to naive-nlu/tree_nlu/knowledge_evaluation.py index 4485adb..524bd56 100644 --- a/naive-nlu/knowledge_evaluation.py +++ b/naive-nlu/tree_nlu/knowledge_evaluation.py @@ -1,4 +1,4 @@ -from modifiable_property import ( +from .modifiable_property import ( ModifiableProperty, ModifiablePropertyWithAst, is_modifiable_property, @@ -41,17 +41,43 @@ def get_subquery_type(knowledge_base, atom): def property_for_value(knowledge_base, value): - return knowledge_base[value]['as_property'] + if value in knowledge_base: + # Annotate the property as property + groups = knowledge_base[value].get('groups', {'property'}) + groups.add('property') + knowledge_base[value]['groups'] = groups + + # And find the property "name" + if 'as_property' in knowledge_base[value]: + return knowledge_base[value]['as_property'] + + return knowledge_base[value].get('groups', {'property'}) + else: + # Consider that any property is... a property + knowledge_base[value] = {'groups': {'property'}} + return {'property'} def modifiable_property_from_property(prop, path, value): def getter(): nonlocal prop, path, value - return (path in prop) and prop[path] == value + if isinstance(path, set): + # If the property is from a set, it's true if any possible + # path has a element as true + return any(map(lambda possible_path: ((possible_path in prop) + and + (prop[possible_path] == value)), + path)) + else: + return (path in prop) and prop[path] == value def setter(): nonlocal prop, path, value - prop[path] = value + if isinstance(path, set): + for possible_path in path: + prop[possible_path] = value + else: + prop[path] = value return ModifiableProperty( getter=getter, @@ -87,22 +113,42 @@ def modifiable_element_for_existance_in_set(container, set_name, element): setter=setter, ) +def modifiable_element_for_existance_in_group(container, element, backlink, set_name='groups'): + def getter(): + nonlocal container, element, backlink, set_name + return (set_name in container) and (element in container[set_name]) + + def setter(): + nonlocal container, set_name, element + backlink['groups'].add(set_name) + return container[set_name].add(element) + + return ModifiableProperty( + getter=getter, + setter=setter, + ) + def pertenence_to_group(knowledge_base, elements, subj, group): subj = resolve(knowledge_base, elements, subj) group = resolve(knowledge_base, elements, group) if subj not in knowledge_base: - knowledge_base[subj] = {} + knowledge_base[subj] = {'groups': set()} if "groups" not in knowledge_base[subj]: knowledge_base[subj]["groups"] = set() - return modifiable_element_for_existance_in_set( - container=knowledge_base[subj], - set_name="groups", - element=group - ) + if group not in knowledge_base: + knowledge_base[group] = {'groups': set()} + if "groups" not in knowledge_base[group]: + knowledge_base[group]["groups"] = set() + + return modifiable_element_for_existance_in_group( + container=knowledge_base[subj], + element=group, + backlink=knowledge_base[group], + ) def has_capacity(knowledge_base, elements, subj, capacity): subj = resolve(knowledge_base, elements, subj) @@ -128,12 +174,48 @@ def question(knowledge_base, elements, subj): return subj.getter() return subj +def implies(knowledge_base, elements, precedent, consequent): + precedent = resolve(knowledge_base, elements, precedent) + consequent = resolve(knowledge_base, elements, consequent) + + if precedent not in knowledge_base: + knowledge_base[precedent] = {'groups': set()} + + if "implications" not in knowledge_base[precedent]: + knowledge_base[precedent]["implications"] = set() + + return modifiable_element_for_existance_in_set( + container=knowledge_base[precedent], + set_name="implications", + element=consequent + ) + + +def property_has_value(knowledge_base, elements, subj, prop, value): + subj = resolve(knowledge_base, elements, subj) + prop = resolve(knowledge_base, elements, prop) + value = resolve(knowledge_base, elements, value) + + if subj not in knowledge_base: + knowledge_base[subj] = {'groups': set()} + + if prop not in knowledge_base[subj]: + knowledge_base[subj][prop] = set() + + return modifiable_element_for_existance_in_set( + container=knowledge_base[subj], + set_name=prop, + element=value + ) + knowledge_ingestion = { "exists-property-with-value": exists_property_with_value, "pertenence-to-group": pertenence_to_group, "has-capacity": has_capacity, "question": question, + "implies": implies, + "property-has-value": property_has_value, } diff --git a/naive-nlu/modifiable_property.py b/naive-nlu/tree_nlu/modifiable_property.py similarity index 100% rename from naive-nlu/modifiable_property.py rename to naive-nlu/tree_nlu/modifiable_property.py diff --git a/naive-nlu/parameters.py b/naive-nlu/tree_nlu/parameters.py similarity index 100% rename from naive-nlu/parameters.py rename to naive-nlu/tree_nlu/parameters.py diff --git a/naive-nlu/parsing.py b/naive-nlu/tree_nlu/parsing.py similarity index 77% rename from naive-nlu/parsing.py rename to naive-nlu/tree_nlu/parsing.py index 3b6bf09..265bd59 100644 --- a/naive-nlu/parsing.py +++ b/naive-nlu/tree_nlu/parsing.py @@ -1,16 +1,16 @@ #!/usr/bin/env python -import knowledge_evaluation +from . import knowledge_evaluation -import depth_meter +from . import depth_meter import logging import re import copy from functools import reduce -from typing import List -from modifiable_property import ModifiableProperty -import parameters +from typing import List, Dict +from .modifiable_property import ModifiableProperty +from . import parameters # TODO: more flexible tokenization def to_tokens(text): @@ -88,13 +88,15 @@ def integrate_language(knowledge_base, example): for position, atom in lower_levels: logging.debug("\x1b[1mSelecting\x1b[0m: {}".format(atom)) - similar = get_similar_tree(knowledge_base, atom) + similar = get_similar_tree(knowledge_base, atom, tokens) remix, (start_bounds, end_bounds) = build_remix_matrix(knowledge_base, tokens, atom, similar) _, matcher, result = make_template(knowledge_base, tokens, atom) logging.debug("Tx: {}".format(tokens)) logging.debug("Mx: {}".format(matcher)) logging.debug("Rx: {}".format(result)) logging.debug("Remix: {}".format(remix)) + logging.debug("Sx: {}".format(start_bounds)) + logging.debug("Ex: {}".format(end_bounds)) after_remix = apply_remix(tokens[len(start_bounds):-len(end_bounds)], remix) assert(len(after_remix) + len(start_bounds) + len(end_bounds) == len(tokens)) @@ -127,16 +129,22 @@ def integrate_language(knowledge_base, example): def apply_remix(tokens, remix): rebuilt = [] for i in remix: - rebuilt.append(tokens[i]) + if isinstance(i, int): + if i >= len(tokens): + return None + rebuilt.append(tokens[i]) + else: + assert(isinstance(i, str)) + rebuilt.append(i) return rebuilt def build_remix_matrix(knowledge_base, tokens, atom, similar): tokens = list(tokens) tokens, matcher, result = make_template(knowledge_base, tokens, atom) - similar_matcher, similar_result, similar_result_resolved, _ = similar + similar_matcher, similar_result, similar_result_resolved, _, _ = similar - start_bounds, end_bounds = find_bounds(matcher, similar_matcher) + start_bounds, end_bounds = find_bounds(knowledge_base, matcher, similar_matcher) for i, element in (end_bounds + start_bounds[::-1]): matcher.pop(i) @@ -154,13 +162,14 @@ def get_possible_remixes(knowledge_base, matcher, similar_matcher): for element in matcher: logging.debug("- {}".format(element)) logging.debug("+ {}".format(similar_matcher)) - assert(element in similar_matcher or isinstance(element, dict)) - - if isinstance(element, dict): - indexes = all_matching_indexes(knowledge_base, similar_matcher, element) + if element in similar_matcher or isinstance(element, dict): + if isinstance(element, dict): + indexes = all_matching_indexes(knowledge_base, similar_matcher, element) + else: + indexes = all_indexes(similar_matcher, element) + matrix.append(indexes) else: - indexes = all_indexes(similar_matcher, element) - matrix.append(indexes) + matrix.append([element]) # TODO: do some scoring to find the most "interesting combination" return [list(x) for x in list(zip(*matrix))] @@ -190,13 +199,21 @@ def all_matching_indexes(knowledge_base, collection, element): instance = knowledge_base.knowledge[instance]["groups"] intersection = set(instance) & set(element) - if len(intersection) > 0: + if (len(intersection) > 0 or (0 == len(instance) == len(element))): indexes.append((i, intersection)) return [x[0] for x in sorted(indexes, key=lambda x: len(x[1]), reverse=True)] -def find_bounds(matcher, similar_matcher): +def element_matches_groups(knowledge, element: Dict, groups): + if isinstance(groups, str) and groups in knowledge: + return len(knowledge[element].get("groups", set()) & element['groups']) > 0 + elif isinstance(groups, dict): + return len(element.get("groups", set()) & element['groups']) > 0 + return False + + +def find_bounds(knowledge, matcher, similar_matcher): start_bounds = [] for i, element in enumerate(matcher): if element in similar_matcher: @@ -206,7 +223,15 @@ def find_bounds(matcher, similar_matcher): end_bounds = [] for i, element in enumerate(matcher[::-1]): - if element in similar_matcher: + in_similar = False + if isinstance(element, str): + in_similar = element in similar_matcher + elif isinstance(element, dict): + in_similar = any(map(lambda groups: element_matches_groups(knowledge.knowledge, + element, groups), + similar_matcher)) + + if in_similar: break else: end_bounds.append((len(matcher) - (i + 1), element)) @@ -214,7 +239,7 @@ def find_bounds(matcher, similar_matcher): return start_bounds, end_bounds -def get_similar_tree(knowledge_base, atom): +def get_similar_tree(knowledge_base, atom, tokens): possibilities = [] # Find matching possibilities @@ -238,12 +263,17 @@ def get_similar_tree(knowledge_base, atom): raw)) # TODO: Probably should take into account the categories of the elements in the "intake" ([0]) element - score = sum([resolved[i] == atom[i] + atom_score = sum([resolved[i] == atom[i] for i in range(min(len(resolved), len(atom)))]) - sorted_possibilities.append((raw, possibility, resolved, score)) - sorted_possibilities = sorted(sorted_possibilities, key=lambda p: p[3], reverse=True) + token_score = sum([similar_token in tokens + for similar_token + in raw]) + + sorted_possibilities.append((raw, possibility, resolved, atom_score, token_score)) + + sorted_possibilities = sorted(sorted_possibilities, key=lambda p: p[3] * 100 + p[4], reverse=True) if len(sorted_possibilities) < 1: return None @@ -268,14 +298,23 @@ def get_matching(sample, other): x[0][i][0] == sample[0][i][0], other)) - return [sample[0][x] if isinstance(sample[0][x], str) - else - sample[0][x] if isinstance(sample[0][x], tuple) - else {'groups': sample[0][x]['groups'] & reduce(lambda a, b: a & b, - map(lambda y: y[0][x]['groups'], - other))} - for x - in range(l)] + matching = [] + for x in range(l): # Generate the combination of this and other(s) matcher + first_sample_data = sample[0][x] + if isinstance(first_sample_data, str): + matching.append(first_sample_data) + elif isinstance(first_sample_data, tuple): + matching.append(first_sample_data) + else: + this_groups = sample[0][x]['groups'] + if len(other) > 0: + other_groups = reduce(lambda a, b: a & b, + map(lambda y: y[0][x]['groups'], + other)) + this_groups = this_groups & other_groups + + matching.append({'groups': this_groups}) + return matching def reprocess_language_knowledge(knowledge_base, examples): @@ -294,8 +333,16 @@ def reprocess_language_knowledge(knowledge_base, examples): def reverse_remix(tree_section, remix): result_section = [] + offset = 0 for origin in remix: - result_section.append(copy.deepcopy(tree_section[origin])) + if isinstance(origin, int): + if origin >= len(tree_section): + return None + + result_section.append(copy.deepcopy(tree_section[origin + offset])) + else: + assert(isinstance(origin, str)) + offset += 1 return result_section + tree_section[len(remix):] @@ -332,6 +379,9 @@ def resolve_fit(knowledge, fit, remaining_recursions): else: ((result_type, remixer), tokens) = element remixed_tokens = reverse_remix(tokens, remixer) + if remixed_tokens is None: + return None + minifit = get_fit(knowledge, remixed_tokens, remaining_recursions - 1) if minifit is None: return None diff --git a/naive-nlu/tree_nlu/test.py b/naive-nlu/tree_nlu/test.py new file mode 100644 index 0000000..810e3c8 --- /dev/null +++ b/naive-nlu/tree_nlu/test.py @@ -0,0 +1,34 @@ +import traceback +import logging +from .tests import basic +from .tests import gac_100 + +logging.getLogger().setLevel(logging.ERROR) + +tests = ( + ("basic", basic), + ("gac 100", gac_100), +) + +def main(): + failed = False + for test_name, test_module in tests: + try: + test_module.main() + print(" \x1b[1;32m✓\x1b[0m {}".format(test_name)) + except AssertionError as ae: + print(" \x1b[1;31m✗\x1b[0m {}{}".format(test_name, + ('\n [Assertion] {}'.format(ae.args[0])) if len(ae.args) > 0 + else '')) + failed = True + + except Exception as e: + print(" \x1b[1;7;31m!\x1b[0m {}\n [Exception] {}".format(test_name, e)) + failed = True + traceback.print_exc() + + if failed: + exit(1) + +if __name__ == '__main__': + main() diff --git a/naive-nlu/tree_nlu/tests/basic.py b/naive-nlu/tree_nlu/tests/basic.py new file mode 100644 index 0000000..450e7e0 --- /dev/null +++ b/naive-nlu/tree_nlu/tests/basic.py @@ -0,0 +1,151 @@ +import logging +import json + +from ..knowledge_base import KnowledgeBase +from ..modifiable_property import is_modifiable_property + +examples = [ + { + "text": "icecream is cold", + "parsed": ("exists-property-with-value", 'icecream', 'cold'), + }, + { + "text": "is icecream cold?", + "parsed": ("question", ("exists-property-with-value", 'icecream', 'cold')) + }, + { + "text": "lava is dangerous", + "parsed": ("exists-property-with-value", 'lava', 'dangerous') + }, + { + "text": "is lava dangerous?", + "parsed": ("question", ("exists-property-with-value", 'lava', 'dangerous')), + }, + { + "text": "earth is a planet", + "parsed": ("pertenence-to-group", 'earth', 'planet'), + }, + { + "text": "io is a moon", + "parsed": ("pertenence-to-group", 'io', 'moon'), + }, + { + "text": "is earth a moon?", + "parsed": ("question", ("pertenence-to-group", 'earth', 'moon')), + }, + { + "text": "Green is a color", + "parsed": ("pertenence-to-group", 'green', 'color'), + }, + { + "text": "a plane can fly", + "parsed": ("has-capacity", 'plane', 'fly') + }, + { + "text": "a wale can swim", + "parsed": ("has-capacity", 'wale', 'swim') + }, + # { + # "text": "if earth is a planet, it is big", + # "parsed": ("implies", + # ("pertenence-to-group", 'earth', 'planet'), + # ("exists-property-with-value", 'earth', 'big')), + # }, +] + +base_knowledge = { + 'icecream': { + "groups": {'noun', 'object', 'comestible', 'sweet'}, + }, + 'lava': { + "groups": {'noun', 'object'}, + }, + 'earth': { + "groups": {'noun', 'object', 'planet'}, + }, + 'io': { + "groups": {'noun', 'object'}, + }, + 'green': { + "groups": {'noun', 'color', 'concept'}, + }, + 'plane': { + "groups": {'noun', 'object', 'vehicle', 'fast'}, + }, + 'car': { + "groups": {'noun', 'object', 'vehicle', 'slow-ish'}, + }, + 'wale': { + "groups": {'noun', 'object', 'living-being'}, + }, + 'cold': { + "groups": {'property', 'temperature'}, + "as_property": "temperature", + }, + 'dangerous': { + "groups": {'property'}, + "as_property": "safety", + }, + 'planet': { + "groups": {'noun', 'group'}, + }, + 'moon': { + "groups": {'noun', 'group'}, + }, + 'color': { + "groups": {'property', 'group'}, + }, + 'fly': { + "groups": {'verb'}, + }, + 'swim': { + "groups": {'verb'}, + }, +} + +def test_assumption(expectedResponse, knowledge, query): + logging.info("Query: {}".format(query['text'])) + logging.info("Expected: {}".format(expectedResponse)) + + result, abstract_tree, diff = knowledge.process(query['text']) + end_result = result.getter() if is_modifiable_property(result) else result + + logging.info("\x1b[0;3{}mResult: {}\x1b[0m".format("1" if end_result != expectedResponse else "2", end_result)) + if end_result != expectedResponse: + raise AssertionError('{} is not {}'.format(end_result, expectedResponse)) + +def main(): + knowledge = KnowledgeBase( + knowledge=base_knowledge, + ) + + differences = knowledge.train(examples) + + logging.info("----") + logging.info(differences()) + logging.info("----") + + test_assumption(True, knowledge, {'text': 'earth is a planet'}) + test_assumption(True, knowledge, {'text': 'is lava dangerous?'}) + for test in [{'text': 'a bus can run'}, {'text': 'io is a moon'}]: + row = test['text'] + result, inferred_tree, differences = knowledge.process(row) + + logging.info("result:", result) + logging.info(differences()) + logging.info("---") + logging.info('-----') + logging.info(json.dumps(sorted(knowledge.knowledge.keys()), indent=4)) + logging.info('-----') + + queryTrue = { + "text": "is io a moon?", + "parsed": ("question", ("pertenence-to-group", "io", "moon")) + } + queryFalse = { + "text": "is io a planet?", + "parsed": ("question", ("pertenence-to-group", "io", "planet")) + } + + test_assumption(False, knowledge, queryFalse) + test_assumption(True, knowledge, queryTrue) diff --git a/naive-nlu/tree_nlu/tests/gac_100.py b/naive-nlu/tree_nlu/tests/gac_100.py new file mode 100644 index 0000000..b2c31e0 --- /dev/null +++ b/naive-nlu/tree_nlu/tests/gac_100.py @@ -0,0 +1,715 @@ +from ..knowledge_base import KnowledgeBase +from ..utils.visuals import show_progbar + +def _assert(args): + assert(args) + +def _assert_msg(args, msg): + assert(args, msg) + +examples = [ + ('full_example', + { + "text": "is icecream cold?", + "affirmation": "icecream is cold", + "parsed": ("question", + ("exists-property-with-value", 'icecream', 'cold')), + "answer": True, + "after_execution": [( + lambda knowledge: _assert('cold' in knowledge.knowledge['icecream']['property']) + ),], + }), + ('full_example', + { + "text": "is earth a planet?", + "affirmation": "earth is a planet", + "parsed": ("question", + ("pertenence-to-group", 'earth', 'planet')), + "answer": True, + "after_execution": [( + lambda knowledge: _assert('planet' in knowledge.knowledge['earth']['groups']) + ),], + }), + ('full_example', + { + "text": "Is green a color?", + "affirmation": "green is a color", + "parsed": ("question", + ("pertenence-to-group", 'green', 'color')), + "answer": True, + "after_execution": [( + lambda knowledge: _assert('color' in knowledge.knowledge['green']['groups']) + ),], + }), + ('full_example', + { + "text": "do airplanes fly?", + "affirmation": "airplanes fly", + "parsed": ("question", + ("has-capacity", 'plane', 'fly')), + "answer": True, + "after_execution": [( + lambda knowledge: _assert('fly' in knowledge.knowledge['plane']['capacities']) + ),], + }), + ('full_example', + { + "text": "Is it hot during the summer?", + "affirmation": "it is hot during summer", + "parsed": ("question", + ("implies", 'summer', 'hot')), + "answer": True, + "after_execution": [( + lambda knowledge: _assert('hot' in knowledge.knowledge['summer']['implications']) + ),], + }), + ('full_example', + { + "text": "is chile in south america ?", + "affirmation": "chile is in south america", + "parsed": ("question", + ("property-has-value", 'chile', 'location', 'south america')), + "answer": True, + "after_execution": [( + lambda knowledge: _assert('south america' in knowledge.knowledge['chile']['location']) + ),], + }), + ('full_example', + { + "text": "Was Socrates a man?", + "affirmation": "Socrates was a man", + "parsed": ("question", + ("pertenence-to-group", 'socrates', 'man')), + "answer": True, + "after_execution": [( + lambda knowledge: _assert('man' in knowledge.knowledge['socrates']['groups']) + ),], + }), + # { + # "text": "Computers use electricity?", + # "affirmation": "Computers use electricity?", + # "parsed": (), + # "answer": None, + # }, + # { + # "text": "The dominant language in france is french?", + # "affirmation": "The dominant language in france is french?", + # "parsed": (), + # "answer": None, + # }, + # { + # "text": "was abraham lincoln once president of the united states?", + # "affirmation": "was abraham lincoln once president of the united states?", + # "parsed": (), + # "answer": None, + # }, + ('text_example', + { + "question": "Is milk white?", + "affirmation": "milk is white", + "answer": True, + }), + # { + # "text": "do people have emotions?", + # "affirmation": "do people have emotions?", + # "parsed": (), + # "answer": None, + # }, + # { + # "text": "do objects appear smaller as they move away from you?", + # "affirmation": "do objects appear smaller as they move away from you?", + # "parsed": (), + # "answer": None, + # }, + # { + # "text": "Does the human species have a male and female gender?", + # "affirmation": "Does the human species have a male and female gender?", + # "parsed": (), + # "answer": None, + # }, + # { + # "text": "Is a mountain mostly made of rock?", + # "affirmation": "Is a mountain mostly made of rock?", + # "parsed": (), + # "answer": None, + # }, + # { + # "text": "is sun microsystems a computer company?", + # "affirmation": "is sun microsystems a computer company?", + # "parsed": (), + # "answer": None, + # }, + # { + # "text": "Do you see with your eyes and smell with your nose?", + # "affirmation": "Do you see with your eyes and smell with your nose?", + # "parsed": (), + # "answer": None, + # }, + # { + # "text": "Is smoking bad for your health?", + # "affirmation": "Is smoking bad for your health?", + # "parsed": (), + # "answer": None, + # }, + # { + # "text": "Does a dog have four legs?", + # "affirmation": "Does a dog have four legs?", + # "parsed": (), + # "answer": None, + # }, + # { + # "text": "Do mammals have hearts?", + # "affirmation": "Do mammals have hearts?", + # "parsed": (), + # "answer": None, + # }, + # { + # "text": "is the Earth a planet?", + # "affirmation": "is the Earth a planet?", + # "parsed": (), + # "answer": None, + # }, + # { + # "text": "Is water a liquid?", + # "affirmation": "Is water a liquid?", + # "parsed": (), + # "answer": None, + # }, + # { + # "text": "Is Bugs Bunny a cartoon character?", + # "affirmation": "Is Bugs Bunny a cartoon character?", + # "parsed": (), + # "answer": None, + # }, + # { + # "text": "Do Humans communicate by Telephone?", + # "affirmation": "Do Humans communicate by Telephone?", + # "parsed": (), + # "answer": None, + # }, + # { + # "text": "is beer a drink ?", + # "affirmation": "is beer a drink ?", + # "parsed": (), + # "answer": None, + # }, + # { + # "text": "are there 12 months in a year?", + # "affirmation": "are there 12 months in a year?", + # "parsed": (), + # "answer": None, + # }, + # { + # "text": "does the sun hurt your eyes when you look at it?", + # "affirmation": "does the sun hurt your eyes when you look at it?", + # "parsed": (), + # "answer": None, + # }, + # { + # "text": "Do most cars have doors?", + # "affirmation": "Do most cars have doors?", + # "parsed": (), + # "answer": None, + # }, + # { + # "text": "is orange both a fruit and a colour?", + # "affirmation": "is orange both a fruit and a colour?", + # "parsed": (), + # "answer": None, + # }, + # { + # "text": "Is water a necessity?", + # "affirmation": "Is water a necessity?", + # "parsed": (), + # "answer": None, + # }, + # { + # "text": "Do CDs have better quality sound than Cassettes?", + # "affirmation": "Do CDs have better quality sound than Cassettes?", + # "parsed": (), + # "answer": None, + # }, + # { + # "text": "do animals die?", + # "affirmation": "do animals die?", + # "parsed": (), + # "answer": None, + # }, + # { + # "text": "Is the arctic cold?", + # "affirmation": "Is the arctic cold?", + # "parsed": (), + # "answer": None, + # }, + # { + # "text": "Do people have 2 eyes?", + # "affirmation": "Do people have 2 eyes?", + # "parsed": (), + # "answer": None, + # }, + # { + # "text": "does a person have a brain?", + # "affirmation": "does a person have a brain?", + # "parsed": (), + # "answer": None, + # }, + # { + # "text": "Is the rain wet?", + # "affirmation": "Is the rain wet?", + # "parsed": (), + # "answer": None, + # }, + # { + # "text": "Is division a mathematical operation?", + # "affirmation": "Is division a mathematical operation?", + # "parsed": (), + # "answer": None, + # }, + # { + # "text": "is 400 greater than 399?", + # "affirmation": "is 400 greater than 399?", + # "parsed": (), + # "answer": None, + # }, + # { + # "text": "is magenta a color?", + # "affirmation": "is magenta a color?", + # "parsed": (), + # "answer": None, + # }, + # { + # "text": "Are books educational?", + # "affirmation": "Are books educational?", + # "parsed": (), + # "answer": None, + # }, + # { + # "text": "Was the Great Wall of China built by humans?", + # "affirmation": "Was the Great Wall of China built by humans?", + # "parsed": (), + # "answer": None, + # }, + # { + # "text": "Are pianos musical instruments?", + # "affirmation": "Are pianos musical instruments?", + # "parsed": (), + # "answer": None, + # }, + # { + # "text": "Has Bill Clinton been President of the United States?", + # "affirmation": "Has Bill Clinton been President of the United States?", + # "parsed": (), + # "answer": None, + # }, + # { + # "text": "Is a whale a mammal?", + # "affirmation": "Is a whale a mammal?", + # "parsed": (), + # "answer": None, + # }, + # { + # "text": "Are lemons yellow?", + # "affirmation": "Are lemons yellow?", + # "parsed": (), + # "answer": None, + # }, + # { + # "text": "Is the South Pole cold?", + # "affirmation": "Is the South Pole cold?", + # "parsed": (), + # "answer": None, + # }, + # { + # "text": "Is Africa warm?", + # "affirmation": "Is Africa warm?", + # "parsed": (), + # "answer": None, + # }, + # { + # "text": "Is Antarctica cold?", + # "affirmation": "Is Antarctica cold?", + # "parsed": (), + # "answer": None, + # }, + # { + # "text": "Is rock is generally harder than wood?", + # "affirmation": "Is rock is generally harder than wood?", + # "parsed": (), + # "answer": None, + # }, + # { + # "text": "Do dogs chase cats?", + # "affirmation": "Do dogs chase cats?", + # "parsed": (), + # "answer": None, + # }, + # { + # "text": "can humans die from cold temperatures?", + # "affirmation": "can humans die from cold temperatures?", + # "parsed": (), + # "answer": None, + # }, + # { + # "text": "do people enjoy conversation?", + # "affirmation": "do people enjoy conversation?", + # "parsed": (), + # "answer": None, + # }, + # { + # "text": "Is Bill Clinton the President of the United States?", + # "affirmation": "Is Bill Clinton the President of the United States?", + # "parsed": (), + # "answer": None, + # }, + # { + # "text": "Are books a good source of information?", + # "affirmation": "Are books a good source of information?", + # "parsed": (), + # "answer": None, + # }, + # { + # "text": "are friends different than enemies?", + # "affirmation": "are friends different than enemies?", + # "parsed": (), + # "answer": None, + # }, + # { + # "text": "are people alive?", + # "affirmation": "are people alive?", + # "parsed": (), + # "answer": None, + # }, + # { + # "text": "Do triangles have 3 sides?", + # "affirmation": "Do triangles have 3 sides?", + # "parsed": (), + # "answer": None, + # }, + # { + # "text": "Is Ice cream cold?", + # "affirmation": "Is Ice cream cold?", + # "parsed": (), + # "answer": None, + # }, + # { + # "text": "Are all sides of a square the same length?", + # "affirmation": "Are all sides of a square the same length?", + # "parsed": (), + # "answer": None, + # }, + # { + # "text": "Do all people eat food?", + # "affirmation": "Do all people eat food?", + # "parsed": (), + # "answer": None, + # }, + # { + # "text": "do dentists repair teeth?", + # "affirmation": "do dentists repair teeth?", + # "parsed": (), + # "answer": None, + # }, + # { + # "text": "Is America bigger than Japan?", + # "affirmation": "Is America bigger than Japan?", + # "parsed": (), + # "answer": None, + # }, + # { + # "text": "Do all triangles have three sides?", + # "affirmation": "Do all triangles have three sides?", + # "parsed": (), + # "answer": None, + # }, + # { + # "text": "A grocery store sales food?", + # "affirmation": "A grocery store sales food?", + # "parsed": (), + # "answer": None, + # }, + # { + # "text": "Does a sunburn cause pain?", + # "affirmation": "Does a sunburn cause pain?", + # "parsed": (), + # "answer": None, + # }, + # { + # "text": "Is a computer an invention?", + # "affirmation": "Is a computer an invention?", + # "parsed": (), + # "answer": None, + # }, + # { + # "text": "have humans visited the moon?", + # "affirmation": "have humans visited the moon?", + # "parsed": (), + # "answer": None, + # }, + # { + # "text": "Are there people in India?", + # "affirmation": "Are there people in India?", + # "parsed": (), + # "answer": None, + # }, + # { + # "text": "Was Einstein a genius?", + # "affirmation": "Was Einstein a genius?", + # "parsed": (), + # "answer": None, + # }, + # { + # "text": "Are we on the planet earth?", + # "affirmation": "Are we on the planet earth?", + # "parsed": (), + # "answer": None, + # }, + # { + # "text": "do people comb their hair in the morning?", + # "affirmation": "do people comb their hair in the morning?", + # "parsed": (), + # "answer": None, + # }, + # { + # "text": "Does it hurt to lose a friend?", + # "affirmation": "Does it hurt to lose a friend?", + # "parsed": (), + # "answer": None, + # }, + # { + # "text": "Are there people on the earth?", + # "affirmation": "Are there people on the earth?", + # "parsed": (), + # "answer": None, + # }, + # { + # "text": "Was George Washington a president of the United States of America?", + # "affirmation": "Was George Washington a president of the United States of America?", + # "parsed": (), + # "answer": None, + # }, + # { + # "text": "Does an ocean have salt water in it?", + # "affirmation": "Does an ocean have salt water in it?", + # "parsed": (), + # "answer": None, + # }, + # { + # "text": "Is night darker than day?", + # "affirmation": "Is night darker than day?", + # "parsed": (), + # "answer": None, + # }, + # { + # "text": "Does a triangle have three sides?", + # "affirmation": "Does a triangle have three sides?", + # "parsed": (), + # "answer": None, + # }, + # { + # "text": "Are peaches fruit?", + # "affirmation": "Are peaches fruit?", + # "parsed": (), + # "answer": None, + # }, + # { + # "text": "Do people urinate?", + # "affirmation": "Do people urinate?", + # "parsed": (), + # "answer": None, + # }, + # { + # "text": "Is Germany located in Europe?", + # "affirmation": "Is Germany located in Europe?", + # "parsed": (), + # "answer": None, + # }, + # { + # "text": "Do mirrors reflect light?", + # "affirmation": "Do mirrors reflect light?", + # "parsed": (), + # "answer": None, + # }, + # { + # "text": "Are people born naked?", + # "affirmation": "Are people born naked?", + # "parsed": (), + # "answer": None, + # }, + # { + # "text": "Is it hot near the equator?", + # "affirmation": "Is it hot near the equator?", + # "parsed": (), + # "answer": None, + # }, + # { + # "text": "is paper made from trees?", + # "affirmation": "is paper made from trees?", + # "parsed": (), + # "answer": None, + # }, + # { + # "text": "Can a female have children?", + # "affirmation": "Can a female have children?", + # "parsed": (), + # "answer": None, + # }, + # { + # "text": "Are people born every day?", + # "affirmation": "Are people born every day?", + # "parsed": (), + # "answer": None, + # }, + # { + # "text": "Are shoes worn on the feet?", + # "affirmation": "Are shoes worn on the feet?", + # "parsed": (), + # "answer": None, + # }, + # { + # "text": "does it get wet when it rains?", + # "affirmation": "does it get wet when it rains?", + # "parsed": (), + # "answer": None, + # }, + # { + # "text": "Are there plants and insects in the rainforest which have no names?", + # "affirmation": "Are there plants and insects in the rainforest which have no names?", + # "parsed": (), + # "answer": None, + # }, + # { + # "text": "Do people eat pigs?", + # "affirmation": "Do people eat pigs?", + # "parsed": (), + # "answer": None, + # }, + # { + # "text": "Do businessmen wear ties?", + # "affirmation": "Do businessmen wear ties?", + # "parsed": (), + # "answer": None, + # }, + # { + # "text": "Is New York in the United States?", + # "affirmation": "Is New York in the United States?", + # "parsed": (), + # "answer": None, + # }, + # { + # "text": "Are humans more intelligent than ants?", + # "affirmation": "Are humans more intelligent than ants?", + # "parsed": (), + # "answer": None, + # }, + # { + # "text": "Are ravens black?", + # "affirmation": "Are ravens black?", + # "parsed": (), + # "answer": None, + # }, + # { + # "text": "Are there rats on ships?", + # "affirmation": "Are there rats on ships?", + # "parsed": (), + # "answer": None, + # }, + # { + # "text": "are lions animals?", + # "affirmation": "are lions animals?", + # "parsed": (), + # "answer": None, + # }, + # { + # "text": "6 is greater than 5?", + # "affirmation": "6 is greater than 5?", + # "parsed": (), + # "answer": None, + # }, + # { + # "text": "Is water made of hydrogen and oxygen?", + # "affirmation": "Is water made of hydrogen and oxygen?", + # "parsed": (), + # "answer": None, + # }, + # { + # "text": "is the sky blue on a clear day?", + # "affirmation": "is the sky blue on a clear day?", + # "parsed": (), + # "answer": None, + # }, + # { + # "text": "Do most people work during the day?", + # "affirmation": "Do most people work during the day?", + # "parsed": (), + # "answer": None, + # }, +] + +base_knowledge = { + 'icecream': { + "groups": {'noun', 'object', 'comestible', 'sweet'}, + }, + 'hot': { + "groups": {'property', 'temperature'}, + }, + 'summer': { + "groups": {'epoch'}, + }, + 'planet': { + "groups": {'noun', 'group'}, + }, + 'green': { + "groups": {'noun', 'color', 'concept'}, + }, + 'fly': { + "groups": {'verb'}, + }, +} + +def main(): + knowledge = KnowledgeBase( + knowledge=base_knowledge, + ) + + total = len(examples) + + for i, (example_type, data) in enumerate(examples): + if example_type == 'full_example': + affirmation = { + 'text': data['affirmation'], + 'parsed': data['parsed'][1], + } + question = data + + show_progbar(i, total, data['affirmation']) + differences = knowledge.train([affirmation]) + + show_progbar(i, total, data['text']) + differences = knowledge.train([question]) + + result, _, _ = knowledge.process(data['text']) + + if result != data['answer']: + raise AssertionError('{} is not {}'.format(result, data['answer'])) + + if "after_execution" in data: + for f in data["after_execution"]: + f(knowledge) + + elif example_type == 'text_example': + show_progbar(i, total, data['affirmation']) + affirmation = data['affirmation'] + + show_progbar(i, total, data['question']) + question = data['question'] + + _, _, _ = knowledge.process(affirmation) + result, _, _ = knowledge.process(question) + + if result != data['answer']: + raise AssertionError('{} is not {}'.format(result, data['answer'])) + + else: + raise NotImplementedError('Example type: {}'.format(example_type)) + + print("\r\x1b[K", end='') diff --git a/naive-nlu/tree_nlu/utils/json_dumper.py b/naive-nlu/tree_nlu/utils/json_dumper.py new file mode 100644 index 0000000..061dd68 --- /dev/null +++ b/naive-nlu/tree_nlu/utils/json_dumper.py @@ -0,0 +1,4 @@ +def dumper(obj): + if isinstance(obj, set): + return list(obj) + return obj diff --git a/naive-nlu/tree_nlu/utils/visuals.py b/naive-nlu/tree_nlu/utils/visuals.py new file mode 100644 index 0000000..a6dd611 --- /dev/null +++ b/naive-nlu/tree_nlu/utils/visuals.py @@ -0,0 +1,15 @@ +def show_progbar(done, total, msg=''): + total_blocks = 10 + blocks_done = (done * total_blocks) // total + blocks_to_go = total_blocks - blocks_done + + print('\r\x1b[K' # Go to the start of the line + '\x1b[0m' # Restart the "style" + '|' # Put the first "|" + + blocks_done * '█' # Completed blocks + + blocks_to_go * ' ' # Uncompleted blocks + + '\x1b[7m|\x1b[0m' # End the bar + + ' ' + + msg # Add message + + '\r' # Go back to the start + , end='')