Pass test using layer structure.
This commit is contained in:
parent
a444766c7c
commit
1ded981099
5 changed files with 36 additions and 24 deletions
|
@ -9,6 +9,7 @@ from typing import List, Dict
|
|||
from ..modifiable_property import ModifiableProperty
|
||||
from .. import parameters
|
||||
from ..atoms import Atom, a, is_atom
|
||||
from .. import knowledge_evaluation
|
||||
|
||||
def make_template(knowledge_base, tokens, parsed):
|
||||
matcher = list(tokens)
|
||||
|
@ -97,8 +98,9 @@ def integrate_language(knowledge_base, example):
|
|||
result = build_remix_matrix(knowledge_base, tokens, atom, similar)
|
||||
if result is not None:
|
||||
break
|
||||
else:
|
||||
raise Exception('Similar not found')
|
||||
|
||||
return
|
||||
remix, (start_bounds, end_bounds) = result
|
||||
|
||||
after_remix = apply_remix(tokens[len(start_bounds):-len(end_bounds)], remix)
|
||||
|
|
|
@ -8,4 +8,7 @@ class ParsingLayer:
|
|||
yield from parsing.integrate_language(knowledge_base, example)
|
||||
|
||||
def train(self, knowledge_base, example):
|
||||
assert False
|
||||
assert False
|
||||
|
||||
def process(self, knowledge_base, input):
|
||||
yield from parsing.get_fit(knowledge_base, input)
|
|
@ -29,14 +29,20 @@ class TokenizationLayer:
|
|||
def integrate(self, knowledge_base, data):
|
||||
assert knowledge_base is self.knowledge_base
|
||||
|
||||
print(data)
|
||||
assert 'text' in data
|
||||
with session().log("Tokenize: {}".format(data['text'])):
|
||||
for tokens in tokenization.to_tokens(self, data['text']):
|
||||
data_with_row = copy.copy(data)
|
||||
data_with_row['tokens'] = tokens
|
||||
print(data_with_row)
|
||||
yield data_with_row
|
||||
tokens = self.tokenize(data['text'])
|
||||
data_with_row = copy.copy(data)
|
||||
data_with_row['tokens'] = tokens
|
||||
yield data_with_row
|
||||
|
||||
# with session().log("Tokenize: {}".format(data['text'])):
|
||||
# for tokens in tokenization.to_tokens(self, data['text']):
|
||||
# data_with_row = copy.copy(data)
|
||||
# data_with_row['tokens'] = tokens
|
||||
# yield data_with_row
|
||||
|
||||
def process(self, knowledge_base, row):
|
||||
yield self.tokenize(row)
|
||||
|
||||
|
||||
def tokenize(self, row, return_one=True):
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue