Pass test using layer structure.

This commit is contained in:
kenkeiras 2018-04-24 23:01:36 +02:00
parent a444766c7c
commit 1ded981099
5 changed files with 36 additions and 24 deletions

View file

@ -9,6 +9,7 @@ from typing import List, Dict
from ..modifiable_property import ModifiableProperty
from .. import parameters
from ..atoms import Atom, a, is_atom
from .. import knowledge_evaluation
def make_template(knowledge_base, tokens, parsed):
matcher = list(tokens)
@ -97,8 +98,9 @@ def integrate_language(knowledge_base, example):
result = build_remix_matrix(knowledge_base, tokens, atom, similar)
if result is not None:
break
else:
raise Exception('Similar not found')
return
remix, (start_bounds, end_bounds) = result
after_remix = apply_remix(tokens[len(start_bounds):-len(end_bounds)], remix)

View file

@ -8,4 +8,7 @@ class ParsingLayer:
yield from parsing.integrate_language(knowledge_base, example)
def train(self, knowledge_base, example):
assert False
assert False
def process(self, knowledge_base, input):
yield from parsing.get_fit(knowledge_base, input)

View file

@ -29,14 +29,20 @@ class TokenizationLayer:
def integrate(self, knowledge_base, data):
assert knowledge_base is self.knowledge_base
print(data)
assert 'text' in data
with session().log("Tokenize: {}".format(data['text'])):
for tokens in tokenization.to_tokens(self, data['text']):
data_with_row = copy.copy(data)
data_with_row['tokens'] = tokens
print(data_with_row)
yield data_with_row
tokens = self.tokenize(data['text'])
data_with_row = copy.copy(data)
data_with_row['tokens'] = tokens
yield data_with_row
# with session().log("Tokenize: {}".format(data['text'])):
# for tokens in tokenization.to_tokens(self, data['text']):
# data_with_row = copy.copy(data)
# data_with_row['tokens'] = tokens
# yield data_with_row
def process(self, knowledge_base, row):
yield self.tokenize(row)
def tokenize(self, row, return_one=True):