kenkeiras/lang-model
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Increase exploration, remove unnecessary initial knowledge.

Browse Source
This commit is contained in:
kenkeiras 2017-10-02 23:37:20 +02:00
parent a0810dd0e8
commit 75174e1736
2 changed files with 106 additions and 96 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

184
naive-nlu/tree_nlu/parsing.py
View File

@ -28,7 +28,7 @@ def make_template(knowledge_base, tokens, parsed):
if word in template: if word in template:
template[template.index(word)] = i template[template.index(word)] = i
matcher[i] = { matcher[i] = {
'groups': set(knowledge_base.knowledge[word]['groups']) 'groups': set(knowledge_base.knowledge.get(word, {}).get('groups', set())),
} }
return tokens, matcher, template return tokens, matcher, template
@ -98,8 +98,15 @@ def integrate_language(knowledge_base, example):
for position, atom in lower_levels: for position, atom in lower_levels:
with session().log("Atom {}".format(atom)): with session().log("Atom {}".format(atom)):
similar = get_similar_tree(knowledge_base, atom, tokens) similars = get_similar_tree(knowledge_base, atom, tokens)
remix, (start_bounds, end_bounds) = build_remix_matrix(knowledge_base, tokens, atom, similar) for similar in similars:
result = build_remix_matrix(knowledge_base, tokens, atom, similar)
if result is not None:
break
if result is None:
raise Exception("No match found")
remix, (start_bounds, end_bounds) = result
after_remix = apply_remix(tokens[len(start_bounds):-len(end_bounds)], remix) after_remix = apply_remix(tokens[len(start_bounds):-len(end_bounds)], remix)
session().annotate("--FIND MIX--") session().annotate("--FIND MIX--")
@ -161,38 +168,47 @@ def apply_remix(tokens, remix):
def build_remix_matrix(knowledge_base, tokens, atom, similar): def build_remix_matrix(knowledge_base, tokens, atom, similar):
tokens = list(tokens) tokens = list(tokens)
tokens, matcher, result = make_template(knowledge_base, tokens, atom) with session().log("Remix matrix for {} - {}".format(tokens, atom)):
similar_matcher, similar_result, similar_result_resolved, _, _ = similar tokens, matcher, result = make_template(knowledge_base, tokens, atom)
similar_matcher, similar_result, similar_result_resolved, _, _ = similar
start_bounds, end_bounds = find_bounds(knowledge_base, matcher, similar_matcher) start_bounds, end_bounds = find_bounds(knowledge_base, matcher, similar_matcher)
for i, element in (end_bounds + start_bounds[::-1]): for i, element in (end_bounds + start_bounds[::-1]):
matcher.pop(i) matcher.pop(i)
tokens.pop(i) tokens.pop(i)
possible_remixes = get_possible_remixes(knowledge_base, matcher, similar_matcher) possible_remixes = get_possible_remixes(knowledge_base, matcher, similar_matcher)
chosen_remix = possible_remixes[0] session().annotate("Possible remixes: {}".format(possible_remixes))
if len(possible_remixes) < 1:
return None
return chosen_remix, (start_bounds, end_bounds) chosen_remix = possible_remixes[0]
return chosen_remix, (start_bounds, end_bounds)
def get_possible_remixes(knowledge_base, matcher, similar_matcher): def get_possible_remixes(knowledge_base, matcher, similar_matcher):
matrix = [] matrix = []
for element in matcher: with session().log("Possible remixes from matcher: {}".format(matcher)):
session().annotate("- {}".format(element)) for element in matcher:
session().annotate("+ {}".format(similar_matcher)) with session().log("Element `{}`".format(element)):
if element in similar_matcher or isinstance(element, dict): session().annotate("Similar `{}`".format(similar_matcher))
if isinstance(element, dict): if element in similar_matcher or isinstance(element, dict):
indexes = all_matching_indexes(knowledge_base, similar_matcher, element) if isinstance(element, dict):
else: indexes = all_matching_indexes(knowledge_base, similar_matcher, element)
indexes = all_indexes(similar_matcher, element) session().annotate("Dict element matching: {}".format(indexes))
matrix.append(indexes) else:
else: indexes = all_indexes(similar_matcher, element)
matrix.append([element]) session().annotate("* element matching: {}".format(indexes))
matrix.append(indexes)
else:
session().annotate("`else` element matching: [element]")
matrix.append([element])
# TODO: do some scoring to find the most "interesting combination" # TODO: do some scoring to find the most "interesting combination"
return [list(x) for x in list(zip(*matrix))] return [list(x) for x in list(zip(*matrix))]
def all_indexes(collection, element): def all_indexes(collection, element):
@ -298,12 +314,14 @@ def get_similar_tree(knowledge_base, atom, tokens):
return None return None
for i, possibility in enumerate(sorted_possibilities): for i, possibility in enumerate(sorted_possibilities):
similar_matcher, similar_result, similar_result_resolved, _, _ = possibility similar_matcher, similar_result, similar_result_resolved, _atom_score, _token_score = possibility
with session().log("Like {}".format(similar_matcher)): with session().log("Like {}".format(similar_matcher)):
session().annotate('Results on: {}'.format(similar_result_resolved))
session().annotate('AST: {}'.format(similar_result)) session().annotate('AST: {}'.format(similar_result))
session().annotate('Results on: {}'.format(similar_result_resolved))
session().annotate('Atom score: {}'.format(_atom_score))
session().annotate('Token score: {}'.format(_token_score))
return sorted_possibilities[0] return sorted_possibilities
# TODO: unroll this mess # TODO: unroll this mess
@ -375,14 +393,14 @@ def reverse_remix(tree_section, remix):
def get_fit(knowledge, tokens, remaining_recursions=parameters.MAX_RECURSIONS): def get_fit(knowledge, tokens, remaining_recursions=parameters.MAX_RECURSIONS):
results = [] results = []
for matcher, ast in knowledge.trained: for matcher, ast in knowledge.trained:
result = match_fit(knowledge, tokens, matcher, ast, with session().log("{} <- {}".format(matcher, tokens)):
remaining_recursions) result = match_fit(knowledge, tokens, matcher, ast,
remaining_recursions)
if result is not None: if result is not None:
results.append(result) with session().log("Result: {}".format(result)):
session().annotate("XXX {}".format(result)) results.append(result)
session().annotate(' - ' + '\n - '.join(map(str, results)))
if len(results) > 0: if len(results) > 0:
return results[0] return results[0]
@ -407,19 +425,20 @@ def resolve_fit(knowledge, fit, remaining_recursions):
if is_definite_minisegment(element): if is_definite_minisegment(element):
fitted.append(element) fitted.append(element)
else: else:
((result_type, remixer), tokens) = element with session().log("Resolving fit of `{}`".format(element)):
remixed_tokens = reverse_remix(tokens, remixer) ((result_type, remixer), tokens) = element
if remixed_tokens is None: remixed_tokens = reverse_remix(tokens, remixer)
return None if remixed_tokens is None:
return None
minifit = get_fit(knowledge, remixed_tokens, remaining_recursions - 1) minifit = get_fit(knowledge, remixed_tokens, remaining_recursions - 1)
if minifit is None: if minifit is None:
return None return None
minitokens, miniast = minifit minitokens, miniast = minifit
session().annotate(" AST | {}".format(miniast)) session().annotate(" AST | {}".format(miniast))
subproperty = knowledge_evaluation.resolve(knowledge.knowledge, minitokens, miniast) subproperty = knowledge_evaluation.resolve(knowledge.knowledge, minitokens, miniast)
fitted.append(subproperty) fitted.append(subproperty)
return fitted return fitted
@ -430,33 +449,38 @@ def match_fit(knowledge, tokens, matcher, ast, remaining_recursions):
session().annotate(indent + 'T> {}'.format(tokens)) session().annotate(indent + 'T> {}'.format(tokens))
session().annotate(indent + 'M> {}'.format(matcher)) session().annotate(indent + 'M> {}'.format(matcher))
for minisegment in matcher: for minisegment in matcher:
possibilities_after_round = [] with session().log("Minisegment `{}`".format(minisegment)):
session().annotate(indent + "MS {}".format(minisegment)) possibilities_after_round = []
for matched_tokens, remaining_tokens in segment_possibilities: for matched_tokens, remaining_tokens in segment_possibilities:
if len(remaining_tokens) < 1: if len(remaining_tokens) < 1:
continue continue
session().annotate(indent + "RT {}".format(remaining_tokens[0])) session().annotate(indent + "RT {}".format(remaining_tokens[0]))
session().annotate(indent + "DEF {}".format(is_definite_minisegment(minisegment))) session().annotate(indent + "DEF {}".format(is_definite_minisegment(minisegment)))
if is_definite_minisegment(minisegment): if is_definite_minisegment(minisegment):
# What if not match -----< # What if not match -----<
if match_token(knowledge, remaining_tokens[0], minisegment): if match_token(knowledge, remaining_tokens[0], minisegment):
possibilities_after_round.append(( possibilities_after_round.append((
matched_tokens + [remaining_tokens[0]], matched_tokens + [remaining_tokens[0]],
remaining_tokens[1:] remaining_tokens[1:]
)) ))
else:
# What if not match!!!!!!-----<
# TODO: optimize this with a look ahead
for i in range(1, len(tokens)):
possibilities_after_round.append((
matched_tokens + [(minisegment, remaining_tokens[:i])],
remaining_tokens[i:]
))
session().annotate(indent + "## PA {}".format(possibilities_after_round))
else: else:
# What if not match!!!!!!-----< segment_possibilities = possibilities_after_round
# TODO: optimize this with a look ahead for possibility in segment_possibilities:
for i in range(1, len(tokens)): with session().log("Possibility: `{}`".format(possibility)):
possibilities_after_round.append(( pass
matched_tokens + [(minisegment, remaining_tokens[:i])], if len(segment_possibilities) < 1:
remaining_tokens[i:] with session().log("NO POSSIBLE"):
)) pass
session().annotate(indent + "## PA {}".format(possibilities_after_round))
else:
segment_possibilities = possibilities_after_round
session().annotate(">>>> {}".format(len(segment_possibilities)))
fully_matched_segments = [(matched, remaining) fully_matched_segments = [(matched, remaining)
for (matched, remaining) for (matched, remaining)
@ -464,15 +488,19 @@ def match_fit(knowledge, tokens, matcher, ast, remaining_recursions):
if len(remaining) == 0] if len(remaining) == 0]
resolved_fits = [] resolved_fits = []
for fit, _ in fully_matched_segments: with session().log("Full matches"):
session().annotate(indent + "::: {}".format(fit)) # REMIXES HAVE TO BE APPLIED BEFORE!!! for fit, _ in fully_matched_segments:
with session().log(fit): # REMIXES HAVE TO BE APPLIED BEFORE!!!
pass
session().annotate(indent + '*' * 20) with session().log("Resolutions"):
for fit, _ in fully_matched_segments: for fit, _ in fully_matched_segments:
session().annotate(indent + "::: {}".format(fit)) # REMIXES HAVE TO BE APPLIED BEFORE!!! with session().log("Resolving {}".format(fit)): # REMIXES HAVE TO BE APPLIED BEFORE!!!
resolved_fit = resolve_fit(knowledge, fit, remaining_recursions) resolved_fit = resolve_fit(knowledge, fit, remaining_recursions)
if resolved_fit is not None: if resolved_fit is not None:
resolved_fits.append(resolved_fit) resolved_fits.append(resolved_fit)
else:
session().annotate("Not resolved")
if len(resolved_fits) == 0: if len(resolved_fits) == 0:
return None return None

18
naive-nlu/tree_nlu/tests/gac_100.py
View File

@ -655,24 +655,9 @@ examples = [
] ]
base_knowledge = { base_knowledge = {
'icecream': {
"groups": {'noun', 'object', 'comestible', 'sweet'},
},
'hot': {
"groups": {'property', 'temperature'},
},
'summer': { 'summer': {
"groups": {'epoch'}, "groups": {'epoch'},
}, },
'planet': {
"groups": {'noun', 'group'},
},
'green': {
"groups": {'noun', 'color', 'concept'},
},
'milk': {
"groups": {'noun'},
},
'fly': { 'fly': {
"groups": {'verb'}, "groups": {'verb'},
}, },
@ -682,9 +667,6 @@ base_knowledge = {
'electricity': { 'electricity': {
"groups": {'power'}, "groups": {'power'},
}, },
'french': {
"groups": {'language'},
}
} }
def main(): def main():