Package as tree_nlu.
This commit is contained in:
parent
ec17fca6cf
commit
5297158110
15
naive-nlu/setup.py
Normal file
15
naive-nlu/setup.py
Normal file
@ -0,0 +1,15 @@
|
||||
from setuptools import setup
|
||||
|
||||
setup(name='tree_nlu',
|
||||
version='0.1',
|
||||
description='Naïve AST based NLU.',
|
||||
author='kenkeiras',
|
||||
author_email='kenkeiras@codigoparallevar.com',
|
||||
packages=['tree_nlu'],
|
||||
scripts=[
|
||||
],
|
||||
include_package_data=True,
|
||||
install_requires = [
|
||||
'jsondiff',
|
||||
],
|
||||
zip_safe=False)
|
||||
@ -1,157 +1,4 @@
|
||||
import json
|
||||
import logging
|
||||
|
||||
logging.getLogger().setLevel(logging.INFO)
|
||||
|
||||
from knowledge_base import KnowledgeBase
|
||||
from modifiable_property import is_modifiable_property
|
||||
|
||||
examples = [
|
||||
{
|
||||
"text": "icecream is cold",
|
||||
"parsed": ("exists-property-with-value", 'icecream', 'cold'),
|
||||
},
|
||||
{
|
||||
"text": "is icecream cold?",
|
||||
"parsed": ("question", ("exists-property-with-value", 'icecream', 'cold'))
|
||||
},
|
||||
{
|
||||
"text": "lava is dangerous",
|
||||
"parsed": ("exists-property-with-value", 'lava', 'dangerous')
|
||||
},
|
||||
{
|
||||
"text": "is lava dangerous?",
|
||||
"parsed": ("question", ("exists-property-with-value", 'lava', 'dangerous')),
|
||||
},
|
||||
{
|
||||
"text": "earth is a planet",
|
||||
"parsed": ("pertenence-to-group", 'earth', 'planet'),
|
||||
},
|
||||
{
|
||||
"text": "io is a moon",
|
||||
"parsed": ("pertenence-to-group", 'io', 'moon'),
|
||||
},
|
||||
{
|
||||
"text": "is earth a moon?",
|
||||
"parsed": ("question", ("pertenence-to-group", 'earth', 'moon')),
|
||||
},
|
||||
{
|
||||
"text": "Green is a color",
|
||||
"parsed": ("pertenence-to-group", 'green', 'color'),
|
||||
},
|
||||
{
|
||||
"text": "a plane can fly",
|
||||
"parsed": ("has-capacity", 'plane', 'fly')
|
||||
},
|
||||
{
|
||||
"text": "a wale can swim",
|
||||
"parsed": ("has-capacity", 'wale', 'swim')
|
||||
},
|
||||
{
|
||||
"text": "if earth is a planet, it is big",
|
||||
"parsed": ("implies",
|
||||
("pertenence-to-group", 'earth', 'planet'),
|
||||
("exists-property-with-value", 'earth', 'big')),
|
||||
},
|
||||
]
|
||||
|
||||
base_knowledge = {
|
||||
'icecream': {
|
||||
"groups": set(['noun', 'object', 'comestible', 'sweet']),
|
||||
},
|
||||
'lava': {
|
||||
"groups": set(['noun', 'object']),
|
||||
},
|
||||
'earth': {
|
||||
"groups": set(['noun', 'object', 'planet']),
|
||||
},
|
||||
'io': {
|
||||
"groups": set(['noun', 'object']),
|
||||
},
|
||||
'green': {
|
||||
"groups": set(['noun', 'color', 'concept']),
|
||||
},
|
||||
'plane': {
|
||||
"groups": set(['noun', 'object', 'vehicle', 'fast']),
|
||||
},
|
||||
'car': {
|
||||
"groups": set(['noun', 'object', 'vehicle', 'slow-ish']),
|
||||
},
|
||||
'wale': {
|
||||
"groups": set(['noun', 'object', 'living-being']),
|
||||
},
|
||||
'cold': {
|
||||
"groups": set(['property', 'temperature']),
|
||||
"as_property": "temperature",
|
||||
},
|
||||
'dangerous': {
|
||||
"groups": set(['property']),
|
||||
"as_property": "safety",
|
||||
},
|
||||
'planet': {
|
||||
"groups": set(['noun', 'group']),
|
||||
},
|
||||
'moon': {
|
||||
"groups": set(['noun', 'group']),
|
||||
},
|
||||
'color': {
|
||||
"groups": set(['property', 'group']),
|
||||
},
|
||||
'fly': {
|
||||
"groups": set(['verb']),
|
||||
},
|
||||
'swim': {
|
||||
"groups": set(['verb']),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def test_assumption(expectedResponse, knowledge, query):
|
||||
logging.info("Query: {}".format(query['text']))
|
||||
logging.info("Expected: {}".format(expectedResponse))
|
||||
|
||||
result, abstract_tree, diff = knowledge.process(query['text'])
|
||||
end_result = result.getter() if is_modifiable_property(result) else result
|
||||
|
||||
logging.info("\x1b[0;3{}mResult: {}\x1b[0m".format("1" if end_result != expectedResponse else "2", end_result))
|
||||
assert(end_result == expectedResponse)
|
||||
|
||||
|
||||
def main():
|
||||
knowledge = KnowledgeBase(
|
||||
knowledge=base_knowledge,
|
||||
)
|
||||
|
||||
differences = knowledge.train(examples)
|
||||
|
||||
logging.info("----")
|
||||
logging.info(differences())
|
||||
logging.info("----")
|
||||
|
||||
test_assumption(True, knowledge, {'text': 'earth is a planet'})
|
||||
test_assumption(True, knowledge, {'text': 'is lava dangerous?'})
|
||||
for test in [{'text': 'a bus can run'}, {'text': 'io is a moon'}]:
|
||||
row = test['text']
|
||||
result, inferred_tree, differences = knowledge.process(row)
|
||||
|
||||
logging.info("result:", result)
|
||||
logging.info(differences())
|
||||
logging.info("---")
|
||||
logging.info('-----')
|
||||
logging.info(json.dumps(sorted(knowledge.knowledge.keys()), indent=4))
|
||||
logging.info('-----')
|
||||
|
||||
queryTrue = {
|
||||
"text": "is io a moon?",
|
||||
"parsed": ("question", ("pertenence-to-group", "io", "moon"))
|
||||
}
|
||||
queryFalse = {
|
||||
"text": "is io a planet?",
|
||||
"parsed": ("question", ("pertenence-to-group", "io", "planet"))
|
||||
}
|
||||
|
||||
test_assumption(False, knowledge, queryFalse)
|
||||
test_assumption(True, knowledge, queryTrue)
|
||||
from tree_nlu import test
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
test.main()
|
||||
|
||||
0
naive-nlu/tree_nlu/__init__.py
Normal file
0
naive-nlu/tree_nlu/__init__.py
Normal file
@ -1,5 +1,5 @@
|
||||
import sys
|
||||
import parameters
|
||||
from . import parameters
|
||||
|
||||
def show_depth(depth: int, zoom: int=2):
|
||||
offset = int((parameters.MAX_RECURSIONS - depth) / (2 / zoom))
|
||||
@ -1,9 +1,10 @@
|
||||
import copy
|
||||
|
||||
import logging
|
||||
import parsing
|
||||
import knowledge_evaluation
|
||||
from modifiable_property import is_modifiable_property
|
||||
|
||||
from . import parsing
|
||||
from . import knowledge_evaluation
|
||||
from .modifiable_property import is_modifiable_property
|
||||
|
||||
|
||||
def diff_knowledge(before, after):
|
||||
@ -1,4 +1,4 @@
|
||||
from modifiable_property import (
|
||||
from .modifiable_property import (
|
||||
ModifiableProperty,
|
||||
ModifiablePropertyWithAst,
|
||||
is_modifiable_property,
|
||||
@ -1,16 +1,16 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
import knowledge_evaluation
|
||||
from . import knowledge_evaluation
|
||||
|
||||
import depth_meter
|
||||
from . import depth_meter
|
||||
import logging
|
||||
import re
|
||||
import copy
|
||||
|
||||
from functools import reduce
|
||||
from typing import List
|
||||
from modifiable_property import ModifiableProperty
|
||||
import parameters
|
||||
from .modifiable_property import ModifiableProperty
|
||||
from . import parameters
|
||||
|
||||
# TODO: more flexible tokenization
|
||||
def to_tokens(text):
|
||||
157
naive-nlu/tree_nlu/test.py
Normal file
157
naive-nlu/tree_nlu/test.py
Normal file
@ -0,0 +1,157 @@
|
||||
import json
|
||||
import logging
|
||||
|
||||
logging.getLogger().setLevel(logging.INFO)
|
||||
|
||||
from .knowledge_base import KnowledgeBase
|
||||
from .modifiable_property import is_modifiable_property
|
||||
|
||||
examples = [
|
||||
{
|
||||
"text": "icecream is cold",
|
||||
"parsed": ("exists-property-with-value", 'icecream', 'cold'),
|
||||
},
|
||||
{
|
||||
"text": "is icecream cold?",
|
||||
"parsed": ("question", ("exists-property-with-value", 'icecream', 'cold'))
|
||||
},
|
||||
{
|
||||
"text": "lava is dangerous",
|
||||
"parsed": ("exists-property-with-value", 'lava', 'dangerous')
|
||||
},
|
||||
{
|
||||
"text": "is lava dangerous?",
|
||||
"parsed": ("question", ("exists-property-with-value", 'lava', 'dangerous')),
|
||||
},
|
||||
{
|
||||
"text": "earth is a planet",
|
||||
"parsed": ("pertenence-to-group", 'earth', 'planet'),
|
||||
},
|
||||
{
|
||||
"text": "io is a moon",
|
||||
"parsed": ("pertenence-to-group", 'io', 'moon'),
|
||||
},
|
||||
{
|
||||
"text": "is earth a moon?",
|
||||
"parsed": ("question", ("pertenence-to-group", 'earth', 'moon')),
|
||||
},
|
||||
{
|
||||
"text": "Green is a color",
|
||||
"parsed": ("pertenence-to-group", 'green', 'color'),
|
||||
},
|
||||
{
|
||||
"text": "a plane can fly",
|
||||
"parsed": ("has-capacity", 'plane', 'fly')
|
||||
},
|
||||
{
|
||||
"text": "a wale can swim",
|
||||
"parsed": ("has-capacity", 'wale', 'swim')
|
||||
},
|
||||
{
|
||||
"text": "if earth is a planet, it is big",
|
||||
"parsed": ("implies",
|
||||
("pertenence-to-group", 'earth', 'planet'),
|
||||
("exists-property-with-value", 'earth', 'big')),
|
||||
},
|
||||
]
|
||||
|
||||
base_knowledge = {
|
||||
'icecream': {
|
||||
"groups": set(['noun', 'object', 'comestible', 'sweet']),
|
||||
},
|
||||
'lava': {
|
||||
"groups": set(['noun', 'object']),
|
||||
},
|
||||
'earth': {
|
||||
"groups": set(['noun', 'object', 'planet']),
|
||||
},
|
||||
'io': {
|
||||
"groups": set(['noun', 'object']),
|
||||
},
|
||||
'green': {
|
||||
"groups": set(['noun', 'color', 'concept']),
|
||||
},
|
||||
'plane': {
|
||||
"groups": set(['noun', 'object', 'vehicle', 'fast']),
|
||||
},
|
||||
'car': {
|
||||
"groups": set(['noun', 'object', 'vehicle', 'slow-ish']),
|
||||
},
|
||||
'wale': {
|
||||
"groups": set(['noun', 'object', 'living-being']),
|
||||
},
|
||||
'cold': {
|
||||
"groups": set(['property', 'temperature']),
|
||||
"as_property": "temperature",
|
||||
},
|
||||
'dangerous': {
|
||||
"groups": set(['property']),
|
||||
"as_property": "safety",
|
||||
},
|
||||
'planet': {
|
||||
"groups": set(['noun', 'group']),
|
||||
},
|
||||
'moon': {
|
||||
"groups": set(['noun', 'group']),
|
||||
},
|
||||
'color': {
|
||||
"groups": set(['property', 'group']),
|
||||
},
|
||||
'fly': {
|
||||
"groups": set(['verb']),
|
||||
},
|
||||
'swim': {
|
||||
"groups": set(['verb']),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def test_assumption(expectedResponse, knowledge, query):
|
||||
logging.info("Query: {}".format(query['text']))
|
||||
logging.info("Expected: {}".format(expectedResponse))
|
||||
|
||||
result, abstract_tree, diff = knowledge.process(query['text'])
|
||||
end_result = result.getter() if is_modifiable_property(result) else result
|
||||
|
||||
logging.info("\x1b[0;3{}mResult: {}\x1b[0m".format("1" if end_result != expectedResponse else "2", end_result))
|
||||
assert(end_result == expectedResponse)
|
||||
|
||||
|
||||
def main():
|
||||
knowledge = KnowledgeBase(
|
||||
knowledge=base_knowledge,
|
||||
)
|
||||
|
||||
differences = knowledge.train(examples)
|
||||
|
||||
logging.info("----")
|
||||
logging.info(differences())
|
||||
logging.info("----")
|
||||
|
||||
test_assumption(True, knowledge, {'text': 'earth is a planet'})
|
||||
test_assumption(True, knowledge, {'text': 'is lava dangerous?'})
|
||||
for test in [{'text': 'a bus can run'}, {'text': 'io is a moon'}]:
|
||||
row = test['text']
|
||||
result, inferred_tree, differences = knowledge.process(row)
|
||||
|
||||
logging.info("result:", result)
|
||||
logging.info(differences())
|
||||
logging.info("---")
|
||||
logging.info('-----')
|
||||
logging.info(json.dumps(sorted(knowledge.knowledge.keys()), indent=4))
|
||||
logging.info('-----')
|
||||
|
||||
queryTrue = {
|
||||
"text": "is io a moon?",
|
||||
"parsed": ("question", ("pertenence-to-group", "io", "moon"))
|
||||
}
|
||||
queryFalse = {
|
||||
"text": "is io a planet?",
|
||||
"parsed": ("question", ("pertenence-to-group", "io", "planet"))
|
||||
}
|
||||
|
||||
test_assumption(False, knowledge, queryFalse)
|
||||
test_assumption(True, knowledge, queryTrue)
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Loading…
Reference in New Issue
Block a user