From b0eb864a259149cca1667dc6ddc0c136062d251f Mon Sep 17 00:00:00 2001 From: Damian at mba Date: Sun, 23 Oct 2022 23:01:53 +0200 Subject: [PATCH] move attention weighting operations to postfix --- ldm/invoke/prompt_parser.py | 112 +++++++++++++++----------- tests/test_prompt_parser.py | 152 +++++++++++++++++++----------------- 2 files changed, 147 insertions(+), 117 deletions(-) diff --git a/ldm/invoke/prompt_parser.py b/ldm/invoke/prompt_parser.py index d5ebd18dfc..48d08a7908 100644 --- a/ldm/invoke/prompt_parser.py +++ b/ldm/invoke/prompt_parser.py @@ -353,9 +353,34 @@ def build_parser_syntax(attention_plus_base: float, attention_minus_base: float) else: raise PromptParser.ParsingException("Cannot make fragment from " + str(x)) + def build_escaped_word_parser(escaped_chars_to_ignore: str): + terms = [] + for c in escaped_chars_to_ignore: + terms.append(pp.Literal('\\'+c)) + terms.append( + #pp.CharsNotIn(string.whitespace + escaped_chars_to_ignore, exact=1) + pp.Word(pp.printables, exclude_chars=string.whitespace + escaped_chars_to_ignore) + ) + return pp.Combine(pp.OneOrMore( + pp.MatchFirst(terms) + )) + + def build_escaped_word_parser_charbychar(escaped_chars_to_ignore: str): + escapes = [] + for c in escaped_chars_to_ignore: + escapes.append(pp.Literal('\\'+c)) + return pp.Combine(pp.OneOrMore( + pp.MatchFirst(escapes + [pp.CharsNotIn( + string.whitespace + escaped_chars_to_ignore, + exact=1 + )]) + )) + + + def parse_fragment_str(x, in_quotes: bool=False, in_parens: bool=False): + #print(f"parsing fragment string \"{x}\"") fragment_string = x[0] - #print(f"parsing fragment string \"{fragment_string}\"") if len(fragment_string.strip()) == 0: return Fragment('') @@ -401,59 +426,55 @@ def build_parser_syntax(attention_plus_base: float, attention_minus_base: float) parenthesized_fragment.set_name('parenthesized_fragment').set_debug(False) debug_attention = False - # attention control of the form +(phrase) / -(phrase) / (phrase) + # attention control of the form (phrase)+ / (phrase)+ / (phrase) # phrase can be multiple words, can have multiple +/- signs to increase the effect or type a floating point or integer weight - attention_head = (number | pp.Word('+') | pp.Word('-'))\ - .set_name("attention_head")\ - .set_debug(False) - word_inside_attention = pp.Combine(pp.OneOrMore( - pp.Literal('\\)') | pp.Literal('\\(') | pp.Literal('\\"') | - pp.Word(pp.printables, exclude_chars=string.whitespace + '\\()"') - )).set_name('word_inside_attention') attention_with_parens = pp.Forward() + attention_without_parens = pp.Forward() - attention_with_parens_delimited_list = pp.OneOrMore(pp.Or([ - quoted_fragment.copy().set_debug(debug_attention), - attention.copy().set_debug(debug_attention), - cross_attention_substitute, - word_inside_attention.set_debug(debug_attention) - #pp.White() - ]).set_name('delim_inner').set_debug(debug_attention)) - # have to disable ignore_expr here to prevent pyparsing from stripping off quote marks - attention_with_parens_body = pp.nested_expr(content=attention_with_parens_delimited_list, - ignore_expr=None#((pp.Literal("\\(") | pp.Literal('\\)'))) - ) - attention_with_parens_body.set_debug(debug_attention) - attention_with_parens << (attention_head + attention_with_parens_body) + attention_with_parens_foot = (number | pp.Word('+') | pp.Word('-'))\ + .set_name("attention_foot")\ + .set_debug(False) + attention_with_parens <<= pp.Group( + lparen + + pp.ZeroOrMore(quoted_fragment | attention_with_parens | parenthesized_fragment | cross_attention_substitute | attention_without_parens | + (pp.Empty() + build_escaped_word_parser_charbychar('()')).set_name('undecorated_word').set_debug(debug_attention)#.set_parse_action(lambda t: t[0]) + ) + + rparen + attention_with_parens_foot) attention_with_parens.set_name('attention_with_parens').set_debug(debug_attention) - attention_without_parens = (pp.Word('+') | pp.Word('-')) + (quoted_fragment | word_inside_attention) + attention_without_parens_foot = pp.Or(pp.Word('+') | pp.Word('-')).set_name('attention_without_parens_foots') + attention_without_parens <<= pp.Group( + (quoted_fragment.copy().set_name('attention_quoted_fragment_without_parens').set_debug(debug_attention) + attention_without_parens_foot) | + pp.Combine(build_escaped_word_parser_charbychar('()+-')).set_name('attention_word_without_parens').set_debug(debug_attention)#.set_parse_action(lambda x: print('escapéd', x)) + + attention_without_parens_foot)#.leave_whitespace() attention_without_parens.set_name('attention_without_parens').set_debug(debug_attention) - attention << (attention_with_parens | attention_without_parens) + + attention << pp.MatchFirst([attention_with_parens, + attention_without_parens + ]) attention.set_name('attention') def make_attention(x): - #print("making Attention from", x) - weight = 1 - # number(str) - if type(x[0]) is float or type(x[0]) is int: - weight = float(x[0]) - # +(str) or -(str) or +str or -str - elif type(x[0]) is str: - base = attention_plus_base if x[0][0] == '+' else attention_minus_base - weight = pow(base, len(x[0])) - if type(x[1]) is list or type(x[1]) is pp.ParseResults: - return Attention(weight=weight, children=[(Fragment(x) if type(x) is str else x) for x in x[1]]) - elif type(x[1]) is str: - return Attention(weight=weight, children=[Fragment(x[1])]) - elif type(x[1]) is Fragment: - return Attention(weight=weight, children=[x[1]]) - raise PromptParser.ParsingException(f"Don't know how to make attention with children {x[1]}") + #print("entered make_attention with", x) + children = x[0][:-1] + weight_raw = x[0][-1] + weight = 1.0 + if type(weight_raw) is float or type(weight_raw) is int: + weight = weight_raw + elif type(weight_raw) is str: + base = attention_plus_base if weight_raw[0] == '+' else attention_minus_base + weight = pow(base, len(weight_raw)) + + #print("making Attention from", children, "with weight", weight) + + return Attention(weight=weight, children=[(Fragment(x) if type(x) is str else x) for x in children]) attention_with_parens.set_parse_action(make_attention) attention_without_parens.set_parse_action(make_attention) + #print("parsing test:", attention_with_parens.parse_string("mountain (man)1.1")) + # cross-attention control empty_string = ((lparen + rparen) | pp.Literal('""').suppress() | @@ -487,10 +508,10 @@ def build_parser_syntax(attention_plus_base: float, attention_minus_base: float) cross_attention_substitute.set_name('cross_attention_substitute').set_debug(debug_cross_attention_control) def make_cross_attention_substitute(x): - print("making cacs for", x[0], "->", x[1], "with options", x.as_dict()) + #print("making cacs for", x[0], "->", x[1], "with options", x.as_dict()) #if len(x>2): cacs = CrossAttentionControlSubstitute(x[0], x[1], options=x.as_dict()) - print("made", cacs) + #print("made", cacs) return cacs cross_attention_substitute.set_parse_action(make_cross_attention_substitute) @@ -511,10 +532,11 @@ def build_parser_syntax(attention_plus_base: float, attention_minus_base: float) (quotes + pp.ZeroOrMore(pp.Word(string.whitespace)) + quotes)).set_debug(False).set_name('empty') # root prompt definition - prompt = ((pp.OneOrMore(prompt_part | quoted_fragment) | empty) + pp.StringEnd()) \ + prompt = (pp.OneOrMore(pp.Or([prompt_part, quoted_fragment, empty])) + pp.StringEnd()) \ .set_parse_action(lambda x: Prompt(x)) - + #print("parsing test:", prompt.parse_string("spaced eyes--")) + #print("parsing test:", prompt.parse_string("eyes--")) # weighted blend of prompts # ("promptA", "promptB").blend(a, b) where "promptA" and "promptB" are valid prompts and a and b are float or @@ -536,7 +558,7 @@ def build_parser_syntax(attention_plus_base: float, attention_minus_base: float) quoted_prompt = pp.dbl_quoted_string.set_parse_action(make_prompt_from_quoted_string) quoted_prompt.set_name('quoted_prompt') - debug_blend=True + debug_blend=False blend_terms = pp.delimited_list(quoted_prompt).set_name('blend_terms').set_debug(debug_blend) blend_weights = (pp.delimited_list(number) + pp.Optional(pp.Char(",").suppress() + "no_normalize")).set_name('blend_weights').set_debug(debug_blend) blend = pp.Group(lparen + pp.Group(blend_terms) + rparen diff --git a/tests/test_prompt_parser.py b/tests/test_prompt_parser.py index 02644012d8..203b95ddf0 100644 --- a/tests/test_prompt_parser.py +++ b/tests/test_prompt_parser.py @@ -34,27 +34,28 @@ class PromptParserTestCase(unittest.TestCase): self.assertEqual(make_weighted_conjunction([("fire, flames , fire", 1)]), parse_prompt("fire, flames , fire")) def test_attention(self): - self.assertEqual(make_weighted_conjunction([('flames', 0.5)]), parse_prompt("0.5(flames)")) - self.assertEqual(make_weighted_conjunction([('fire flames', 0.5)]), parse_prompt("0.5(fire flames)")) - self.assertEqual(make_weighted_conjunction([('flames', 1.1)]), parse_prompt("+(flames)")) - self.assertEqual(make_weighted_conjunction([('flames', 0.9)]), parse_prompt("-(flames)")) - self.assertEqual(make_weighted_conjunction([('fire', 1), ('flames', 0.5)]), parse_prompt("fire 0.5(flames)")) - self.assertEqual(make_weighted_conjunction([('flames', pow(1.1, 2))]), parse_prompt("++(flames)")) - self.assertEqual(make_weighted_conjunction([('flames', pow(0.9, 2))]), parse_prompt("--(flames)")) - self.assertEqual(make_weighted_conjunction([('flowers', pow(0.9, 3)), ('flames', pow(1.1, 3))]), parse_prompt("---(flowers) +++flames")) - self.assertEqual(make_weighted_conjunction([('flowers', pow(0.9, 3)), ('flames', pow(1.1, 3))]), parse_prompt("---(flowers) +++flames")) - self.assertEqual(make_weighted_conjunction([('flowers', pow(0.9, 3)), ('flames+', pow(1.1, 3))]), - parse_prompt("---(flowers) +++flames+")) + self.assertEqual(make_weighted_conjunction([('flames', 0.5)]), parse_prompt("(flames)0.5")) + self.assertEqual(make_weighted_conjunction([('fire flames', 0.5)]), parse_prompt("(fire flames)0.5")) + self.assertEqual(make_weighted_conjunction([('flames', 1.1)]), parse_prompt("(flames)+")) + self.assertEqual(make_weighted_conjunction([('flames', 1.1)]), parse_prompt("flames+")) + self.assertEqual(make_weighted_conjunction([('flames', 1.1)]), parse_prompt("\"flames\"+")) + self.assertEqual(make_weighted_conjunction([('flames', 0.9)]), parse_prompt("(flames)-")) + self.assertEqual(make_weighted_conjunction([('flames', 0.9)]), parse_prompt("flames-")) + self.assertEqual(make_weighted_conjunction([('flames', 0.9)]), parse_prompt("\"flames\"-")) + self.assertEqual(make_weighted_conjunction([('fire', 1), ('flames', 0.5)]), parse_prompt("fire (flames)0.5")) + self.assertEqual(make_weighted_conjunction([('flames', pow(1.1, 2))]), parse_prompt("(flames)++")) + self.assertEqual(make_weighted_conjunction([('flames', pow(0.9, 2))]), parse_prompt("(flames)--")) + self.assertEqual(make_weighted_conjunction([('flowers', pow(0.9, 3)), ('flames', pow(1.1, 3))]), parse_prompt("(flowers)--- flames+++")) self.assertEqual(make_weighted_conjunction([('pretty flowers', 1.1)]), - parse_prompt("+(pretty flowers)")) + parse_prompt("(pretty flowers)+")) self.assertEqual(make_weighted_conjunction([('pretty flowers', 1.1), (', the flames are too hot', 1)]), - parse_prompt("+(pretty flowers), the flames are too hot")) + parse_prompt("(pretty flowers)+, the flames are too hot")) def test_no_parens_attention_runon(self): - self.assertEqual(make_weighted_conjunction([('fire', pow(1.1, 2)), ('flames', 1.0)]), parse_prompt("++fire flames")) - self.assertEqual(make_weighted_conjunction([('fire', pow(0.9, 2)), ('flames', 1.0)]), parse_prompt("--fire flames")) - self.assertEqual(make_weighted_conjunction([('flowers', 1.0), ('fire', pow(1.1, 2)), ('flames', 1.0)]), parse_prompt("flowers ++fire flames")) - self.assertEqual(make_weighted_conjunction([('flowers', 1.0), ('fire', pow(0.9, 2)), ('flames', 1.0)]), parse_prompt("flowers --fire flames")) + self.assertEqual(make_weighted_conjunction([('fire', 1.0), ('flames', pow(1.1, 2))]), parse_prompt("fire flames++")) + self.assertEqual(make_weighted_conjunction([('fire', 1.0), ('flames', pow(0.9, 2))]), parse_prompt("fire flames--")) + self.assertEqual(make_weighted_conjunction([('flowers', 1.0), ('fire', pow(1.1, 2)), ('flames', 1.0)]), parse_prompt("flowers fire++ flames")) + self.assertEqual(make_weighted_conjunction([('flowers', 1.0), ('fire', pow(0.9, 2)), ('flames', 1.0)]), parse_prompt("flowers fire-- flames")) def test_explicit_conjunction(self): @@ -62,7 +63,7 @@ class PromptParserTestCase(unittest.TestCase): self.assertEqual(Conjunction([FlattenedPrompt([('fire', 1.0)]), FlattenedPrompt([('flames', 1.0)])]), parse_prompt('("fire", "flames").and()')) self.assertEqual( Conjunction([FlattenedPrompt([('fire flames', 1.0)]), FlattenedPrompt([('mountain man', 1.0)])]), parse_prompt('("fire flames", "mountain man").and()')) - self.assertEqual(Conjunction([FlattenedPrompt([('fire', 2.0)]), FlattenedPrompt([('flames', 0.9)])]), parse_prompt('("2.0(fire)", "-flames").and()')) + self.assertEqual(Conjunction([FlattenedPrompt([('fire', 2.0)]), FlattenedPrompt([('flames', 0.9)])]), parse_prompt('("(fire)2.0", "flames-").and()')) self.assertEqual(Conjunction([FlattenedPrompt([('fire', 1.0)]), FlattenedPrompt([('flames', 1.0)]), FlattenedPrompt([('mountain man', 1.0)])]), parse_prompt('("fire", "flames", "mountain man").and()')) @@ -75,8 +76,11 @@ class PromptParserTestCase(unittest.TestCase): parse_prompt('("fire", "flames").and(2,1,2)') def test_complex_conjunction(self): + + #print(parse_prompt("a person with a hat (riding a bicycle.swap(skateboard))++")) + self.assertEqual(Conjunction([FlattenedPrompt([("mountain man", 1.0)]), FlattenedPrompt([("a person with a hat", 1.0), ("riding a bicycle", pow(1.1,2))])], weights=[0.5, 0.5]), - parse_prompt("(\"mountain man\", \"a person with a hat ++(riding a bicycle)\").and(0.5, 0.5)")) + parse_prompt("(\"mountain man\", \"a person with a hat (riding a bicycle)++\").and(0.5, 0.5)")) self.assertEqual(Conjunction([FlattenedPrompt([("mountain man", 1.0)]), FlattenedPrompt([("a person with a hat", 1.0), ("riding a", 1.1*1.1), @@ -85,7 +89,7 @@ class PromptParserTestCase(unittest.TestCase): [Fragment("skateboard", pow(1.1,2))]) ]) ], weights=[0.5, 0.5]), - parse_prompt("(\"mountain man\", \"a person with a hat ++(riding a bicycle.swap(skateboard))\").and(0.5, 0.5)")) + parse_prompt("(\"mountain man\", \"a person with a hat (riding a bicycle.swap(skateboard))++\").and(0.5, 0.5)")) def test_badly_formed(self): def make_untouched_prompt(prompt): @@ -95,24 +99,25 @@ class PromptParserTestCase(unittest.TestCase): self.assertEqual(make_untouched_prompt(prompt), parse_prompt(prompt)) assert_if_prompt_string_not_untouched('a test prompt') - assert_if_prompt_string_not_untouched('a badly formed test+ prompt') + # todo handle this + #assert_if_prompt_string_not_untouched('a badly formed +test prompt') with self.assertRaises(pyparsing.ParseException): parse_prompt('a badly (formed test prompt') #with self.assertRaises(pyparsing.ParseException): with self.assertRaises(pyparsing.ParseException): - parse_prompt('a badly (formed test+ prompt') + parse_prompt('a badly (formed +test prompt') with self.assertRaises(pyparsing.ParseException): - parse_prompt('a badly (formed test+ )prompt') + parse_prompt('a badly (formed +test )prompt') with self.assertRaises(pyparsing.ParseException): - parse_prompt('a badly (formed test+ )prompt') + parse_prompt('a badly (formed +test )prompt') with self.assertRaises(pyparsing.ParseException): - parse_prompt('(((a badly (formed test+ )prompt') + parse_prompt('(((a badly (formed +test )prompt') with self.assertRaises(pyparsing.ParseException): - parse_prompt('(a (ba)dly (f)ormed test+ prompt') + parse_prompt('(a (ba)dly (f)ormed +test prompt') with self.assertRaises(pyparsing.ParseException): - parse_prompt('(a (ba)dly (f)ormed test+ +prompt') + parse_prompt('(a (ba)dly (f)ormed +test +prompt') with self.assertRaises(pyparsing.ParseException): - parse_prompt('("((a badly (formed test+ ").blend(1.0)') + parse_prompt('("((a badly (formed +test ").blend(1.0)') def test_blend(self): @@ -129,7 +134,7 @@ class PromptParserTestCase(unittest.TestCase): FlattenedPrompt([('fire flames', 1.0), ('hot', pow(1.1, 2))]), FlattenedPrompt([('hi', 1.0)])], weights=[0.7, 0.3, 1.0])]), - parse_prompt("(\"fire\", \"fire flames ++(hot)\", \"hi\").blend(0.7, 0.3, 1.0)") + parse_prompt("(\"fire\", \"fire flames (hot)++\", \"hi\").blend(0.7, 0.3, 1.0)") ) # blend a single entry is not a failure self.assertEqual(Conjunction([Blend([FlattenedPrompt([('fire', 1.0)])], [0.7])]), @@ -156,17 +161,17 @@ class PromptParserTestCase(unittest.TestCase): self.assertEqual( Conjunction([Blend([FlattenedPrompt([('mountain, man, hairy', 1)]), FlattenedPrompt([('face, teeth,', 1), ('eyes', 0.9*0.9)])], weights=[1.0,-1.0])]), - parse_prompt('("mountain, man, hairy", "face, teeth, --eyes").blend(1,-1)') + parse_prompt('("mountain, man, hairy", "face, teeth, eyes--").blend(1,-1)') ) def test_nested(self): self.assertEqual(make_weighted_conjunction([('fire', 1.0), ('flames', 2.0), ('trees', 3.0)]), - parse_prompt('fire 2.0(flames 1.5(trees))')) + parse_prompt('fire (flames (trees)1.5)2.0')) self.assertEqual(Conjunction([Blend(prompts=[FlattenedPrompt([('fire', 1.0), ('flames', 1.2100000000000002)]), FlattenedPrompt([('mountain', 1.0), ('man', 2.0)])], weights=[1.0, 1.0])]), - parse_prompt('("fire ++(flames)", "mountain 2(man)").blend(1,1)')) + parse_prompt('("fire (flames)++", "mountain (man)2").blend(1,1)')) def test_cross_attention_control(self): @@ -237,15 +242,15 @@ class PromptParserTestCase(unittest.TestCase): flames_to_trees_fire = Conjunction([FlattenedPrompt([ CrossAttentionControlSubstitute([Fragment('flames',0.5)], [Fragment('trees',0.7)]), Fragment(',', 1), Fragment('fire', 2.0)])]) - self.assertEqual(flames_to_trees_fire, parse_prompt('"0.5(flames)".swap("0.7(trees)"), 2.0(fire)')) + self.assertEqual(flames_to_trees_fire, parse_prompt('"(flames)0.5".swap("(trees)0.7"), (fire)2.0')) flames_to_trees_fire = Conjunction([FlattenedPrompt([ CrossAttentionControlSubstitute([Fragment('fire',0.5), Fragment('flames',0.25)], [Fragment('trees',0.7)]), Fragment(',', 1), Fragment('fire', 2.0)])]) - self.assertEqual(flames_to_trees_fire, parse_prompt('"0.5(fire 0.5(flames))".swap("0.7(trees)"), 2.0(fire)')) + self.assertEqual(flames_to_trees_fire, parse_prompt('"(fire (flames)0.5)0.5".swap("(trees)0.7"), (fire)2.0')) flames_to_trees_fire = Conjunction([FlattenedPrompt([ CrossAttentionControlSubstitute([Fragment('fire',0.5), Fragment('flames',0.25)], [Fragment('trees',0.7), Fragment('houses', 1)]), Fragment(',', 1), Fragment('fire', 2.0)])]) - self.assertEqual(flames_to_trees_fire, parse_prompt('"0.5(fire 0.5(flames))".swap("0.7(trees) houses"), 2.0(fire)')) + self.assertEqual(flames_to_trees_fire, parse_prompt('"(fire (flames)0.5)0.5".swap("(trees)0.7 houses"), (fire)2.0')) def test_cross_attention_control_options(self): self.assertEqual(Conjunction([ @@ -271,48 +276,48 @@ class PromptParserTestCase(unittest.TestCase): self.assertEqual(make_basic_conjunction(['mountain (man)']),parse_prompt('mountain \(man\)')) self.assertEqual(make_basic_conjunction(['mountain (man )']),parse_prompt('mountain (\(man)\)')) self.assertEqual(make_basic_conjunction(['mountain (man)']),parse_prompt('mountain (\(man\))')) - self.assertEqual(make_weighted_conjunction([('mountain', 1), ('(man)', 1.1)]), parse_prompt('mountain +(\(man\))')) - self.assertEqual(make_weighted_conjunction([('mountain', 1), ('(man)', 1.1)]), parse_prompt('"mountain" +(\(man\))')) - self.assertEqual(make_weighted_conjunction([('"mountain"', 1), ('(man)', 1.1)]), parse_prompt('\\"mountain\\" +(\(man\))')) + self.assertEqual(make_weighted_conjunction([('mountain', 1), ('(man)', 1.1)]), parse_prompt('mountain (\(man\))+')) + self.assertEqual(make_weighted_conjunction([('mountain', 1), ('(man)', 1.1)]), parse_prompt('"mountain" (\(man\))+')) + self.assertEqual(make_weighted_conjunction([('"mountain"', 1), ('(man)', 1.1)]), parse_prompt('\\"mountain\\" (\(man\))+')) # same weights for each are combined into one - self.assertEqual(make_weighted_conjunction([('"mountain" (man)', 1.1)]), parse_prompt('+(\\"mountain\\") +(\(man\))')) - self.assertEqual(make_weighted_conjunction([('"mountain"', 1.1), ('(man)', 0.9)]), parse_prompt('+(\\"mountain\\") -(\(man\))')) + self.assertEqual(make_weighted_conjunction([('"mountain" (man)', 1.1)]), parse_prompt('(\\"mountain\\")+ (\(man\))+')) + self.assertEqual(make_weighted_conjunction([('"mountain"', 1.1), ('(man)', 0.9)]), parse_prompt('(\\"mountain\\")+ (\(man\))-')) - self.assertEqual(make_weighted_conjunction([('mountain', 1), ('\(man\)', 1.1)]),parse_prompt('mountain 1.1(\(man\))')) - self.assertEqual(make_weighted_conjunction([('mountain', 1), ('\(man\)', 1.1)]),parse_prompt('"mountain" 1.1(\(man\))')) - self.assertEqual(make_weighted_conjunction([('"mountain"', 1), ('\(man\)', 1.1)]),parse_prompt('\\"mountain\\" 1.1(\(man\))')) + self.assertEqual(make_weighted_conjunction([('mountain', 1), ('\(man\)', 1.1)]),parse_prompt('mountain (\(man\))1.1')) + self.assertEqual(make_weighted_conjunction([('mountain', 1), ('\(man\)', 1.1)]),parse_prompt('"mountain" (\(man\))1.1')) + self.assertEqual(make_weighted_conjunction([('"mountain"', 1), ('\(man\)', 1.1)]),parse_prompt('\\"mountain\\" (\(man\))1.1')) # same weights for each are combined into one - self.assertEqual(make_weighted_conjunction([('\\"mountain\\" \(man\)', 1.1)]),parse_prompt('+(\\"mountain\\") 1.1(\(man\))')) - self.assertEqual(make_weighted_conjunction([('\\"mountain\\"', 1.1), ('\(man\)', 0.9)]),parse_prompt('1.1(\\"mountain\\") 0.9(\(man\))')) + self.assertEqual(make_weighted_conjunction([('\\"mountain\\" \(man\)', 1.1)]),parse_prompt('(\\"mountain\\")+ (\(man\))1.1')) + self.assertEqual(make_weighted_conjunction([('\\"mountain\\"', 1.1), ('\(man\)', 0.9)]),parse_prompt('(\\"mountain\\")1.1 (\(man\))0.9')) - self.assertEqual(make_weighted_conjunction([('hairy', 1), ('mountain', 1.1), ('\(man\)', 1.1*1.1)]),parse_prompt('hairy +(mountain +(\(man\)))')) - self.assertEqual(make_weighted_conjunction([('hairy', 1), ('\(man\)', 1.1*1.1), ('mountain', 1.1)]),parse_prompt('hairy +(1.1(\(man\)) "mountain")')) - self.assertEqual(make_weighted_conjunction([('hairy', 1), ('mountain', 1.1), ('\(man\)', 1.1*1.1)]),parse_prompt('hairy +("mountain" 1.1(\(man\)) )')) - self.assertEqual(make_weighted_conjunction([('hairy', 1), ('mountain, man', 1.1)]),parse_prompt('hairy +("mountain, man")')) - self.assertEqual(make_weighted_conjunction([('hairy', 1), ('mountain, man with a', 1.1), ('beard', 1.1*1.1)]), parse_prompt('hairy +("mountain, man" with a +beard)')) - self.assertEqual(make_weighted_conjunction([('hairy', 1), ('mountain, man with a', 1.1), ('beard', 1.1*2.0)]), parse_prompt('hairy +("mountain, man" with a 2.0(beard))')) - self.assertEqual(make_weighted_conjunction([('hairy', 1), ('mountain, \"man\" with a', 1.1), ('beard', 1.1*2.0)]), parse_prompt('hairy +("mountain, \\"man\\"" with a 2.0(beard))')) - self.assertEqual(make_weighted_conjunction([('hairy', 1), ('mountain, m\"an\" with a', 1.1), ('beard', 1.1*2.0)]), parse_prompt('hairy +("mountain, m\\"an\\"" with a 2.0(beard))')) + self.assertEqual(make_weighted_conjunction([('hairy', 1), ('mountain', 1.1), ('\(man\)', 1.1*1.1)]),parse_prompt('hairy (mountain (\(man\))+)+')) + self.assertEqual(make_weighted_conjunction([('hairy', 1), ('\(man\)', 1.1*1.1), ('mountain', 1.1)]),parse_prompt('hairy ((\(man\))1.1 "mountain")+')) + self.assertEqual(make_weighted_conjunction([('hairy', 1), ('mountain', 1.1), ('\(man\)', 1.1*1.1)]),parse_prompt('hairy ("mountain" (\(man\))1.1 )+')) + self.assertEqual(make_weighted_conjunction([('hairy', 1), ('mountain, man', 1.1)]),parse_prompt('hairy ("mountain, man")+')) + self.assertEqual(make_weighted_conjunction([('hairy', 1), ('mountain, man with a', 1.1), ('beard', 1.1*1.1)]), parse_prompt('hairy ("mountain, man" with a beard+)+')) + self.assertEqual(make_weighted_conjunction([('hairy', 1), ('mountain, man with a', 1.1), ('beard', 1.1*2.0)]), parse_prompt('hairy ("mountain, man" with a (beard)2.0)+')) + self.assertEqual(make_weighted_conjunction([('hairy', 1), ('mountain, \"man\" with a', 1.1), ('beard', 1.1*2.0)]), parse_prompt('hairy ("mountain, \\"man\\"" with a (beard)2.0)+')) + self.assertEqual(make_weighted_conjunction([('hairy', 1), ('mountain, m\"an\" with a', 1.1), ('beard', 1.1*2.0)]), parse_prompt('hairy ("mountain, m\\"an\\"" with a (beard)2.0)+')) - self.assertEqual(make_weighted_conjunction([('hairy', 1), ('mountain, \"man (with a', 1.1), ('beard', 1.1*2.0)]), parse_prompt('hairy +("mountain, \\\"man\" \(with a 2.0(beard))')) - self.assertEqual(make_weighted_conjunction([('hairy', 1), ('mountain, \"man w(ith a', 1.1), ('beard', 1.1*2.0)]), parse_prompt('hairy +("mountain, \\\"man\" w\(ith a 2.0(beard))')) - self.assertEqual(make_weighted_conjunction([('hairy', 1), ('mountain, \"man with( a', 1.1), ('beard', 1.1*2.0)]), parse_prompt('hairy +("mountain, \\\"man\" with\( a 2.0(beard))')) - self.assertEqual(make_weighted_conjunction([('hairy', 1), ('mountain, \"man )with a', 1.1), ('beard', 1.1*2.0)]), parse_prompt('hairy +("mountain, \\\"man\" \)with a 2.0(beard))')) - self.assertEqual(make_weighted_conjunction([('hairy', 1), ('mountain, \"man w)ith a', 1.1), ('beard', 1.1*2.0)]), parse_prompt('hairy +("mountain, \\\"man\" w\)ith a 2.0(beard))')) - self.assertEqual(make_weighted_conjunction([('hairy', 1), ('mountain, \"man with) a', 1.1), ('beard', 1.1*2.0)]), parse_prompt('hairy +("mountain, \\\"man\" with\) a 2.0(beard))')) - self.assertEqual(make_weighted_conjunction([('hairy', 1), ('mou)ntain, \"man (wit(h a', 1.1), ('beard', 1.1*2.0)]), parse_prompt('hairy +("mou\)ntain, \\\"man\" \(wit\(h a 2.0(beard))')) - self.assertEqual(make_weighted_conjunction([('hai(ry', 1), ('mountain, \"man w)ith a', 1.1), ('beard', 1.1*2.0)]), parse_prompt('hai\(ry +("mountain, \\\"man\" w\)ith a 2.0(beard))')) - self.assertEqual(make_weighted_conjunction([('hairy((', 1), ('mountain, \"man with a', 1.1), ('beard', 1.1*2.0)]), parse_prompt('hairy\(\( +("mountain, \\\"man\" with a 2.0(beard))')) + self.assertEqual(make_weighted_conjunction([('hairy', 1), ('mountain, \"man (with a', 1.1), ('beard', 1.1*2.0)]), parse_prompt('hairy ("mountain, \\\"man\" \(with a (beard)2.0)+')) + self.assertEqual(make_weighted_conjunction([('hairy', 1), ('mountain, \"man w(ith a', 1.1), ('beard', 1.1*2.0)]), parse_prompt('hairy ("mountain, \\\"man\" w\(ith a (beard)2.0)+')) + self.assertEqual(make_weighted_conjunction([('hairy', 1), ('mountain, \"man with( a', 1.1), ('beard', 1.1*2.0)]), parse_prompt('hairy ("mountain, \\\"man\" with\( a (beard)2.0)+')) + self.assertEqual(make_weighted_conjunction([('hairy', 1), ('mountain, \"man )with a', 1.1), ('beard', 1.1*2.0)]), parse_prompt('hairy ("mountain, \\\"man\" \)with a (beard)2.0)+')) + self.assertEqual(make_weighted_conjunction([('hairy', 1), ('mountain, \"man w)ith a', 1.1), ('beard', 1.1*2.0)]), parse_prompt('hairy ("mountain, \\\"man\" w\)ith a (beard)2.0)+')) + self.assertEqual(make_weighted_conjunction([('hairy', 1), ('mountain, \"man with) a', 1.1), ('beard', 1.1*2.0)]), parse_prompt('hairy ("mountain, \\\"man\" with\) a (beard)2.0)+')) + self.assertEqual(make_weighted_conjunction([('hairy', 1), ('mou)ntain, \"man (wit(h a', 1.1), ('beard', 1.1*2.0)]), parse_prompt('hairy ("mou\)ntain, \\\"man\" \(wit\(h a (beard)2.0)+')) + self.assertEqual(make_weighted_conjunction([('hai(ry', 1), ('mountain, \"man w)ith a', 1.1), ('beard', 1.1*2.0)]), parse_prompt('hai\(ry ("mountain, \\\"man\" w\)ith a (beard)2.0)+')) + self.assertEqual(make_weighted_conjunction([('hairy((', 1), ('mountain, \"man with a', 1.1), ('beard', 1.1*2.0)]), parse_prompt('hairy\(\( ("mountain, \\\"man\" with a (beard)2.0)+')) - self.assertEqual(make_weighted_conjunction([('mountain, \"man (with a', 1.1), ('beard', 1.1*2.0), ('hairy', 1)]), parse_prompt('+("mountain, \\\"man\" \(with a 2.0(beard)) hairy')) - self.assertEqual(make_weighted_conjunction([('mountain, \"man w(ith a', 1.1), ('beard', 1.1*2.0), ('hairy', 1)]), parse_prompt('+("mountain, \\\"man\" w\(ith a 2.0(beard))hairy')) - self.assertEqual(make_weighted_conjunction([('mountain, \"man with( a', 1.1), ('beard', 1.1*2.0), ('hairy', 1)]), parse_prompt('+("mountain, \\\"man\" with\( a 2.0(beard)) hairy')) - self.assertEqual(make_weighted_conjunction([('mountain, \"man )with a', 1.1), ('beard', 1.1*2.0), ('hairy', 1)]), parse_prompt('+("mountain, \\\"man\" \)with a 2.0(beard)) hairy')) - self.assertEqual(make_weighted_conjunction([('mountain, \"man w)ith a', 1.1), ('beard', 1.1*2.0), ('hairy', 1)]), parse_prompt('+("mountain, \\\"man\" w\)ith a 2.0(beard)) hairy')) - self.assertEqual(make_weighted_conjunction([('mountain, \"man with) a', 1.1), ('beard', 1.1*2.0), ('hairy', 1)]), parse_prompt(' +("mountain, \\\"man\" with\) a 2.0(beard)) hairy')) - self.assertEqual(make_weighted_conjunction([('mou)ntain, \"man (wit(h a', 1.1), ('beard', 1.1*2.0), ('hairy', 1)]), parse_prompt('+("mou\)ntain, \\\"man\" \(wit\(h a 2.0(beard)) hairy')) - self.assertEqual(make_weighted_conjunction([('mountain, \"man w)ith a', 1.1), ('beard', 1.1*2.0), ('hai(ry', 1)]), parse_prompt('+("mountain, \\\"man\" w\)ith a 2.0(beard)) hai\(ry ')) - self.assertEqual(make_weighted_conjunction([('mountain, \"man with a', 1.1), ('beard', 1.1*2.0), ('hairy((', 1)]), parse_prompt('+("mountain, \\\"man\" with a 2.0(beard)) hairy\(\( ')) + self.assertEqual(make_weighted_conjunction([('mountain, \"man (with a', 1.1), ('beard', 1.1*2.0), ('hairy', 1)]), parse_prompt('("mountain, \\\"man\" \(with a (beard)2.0)+ hairy')) + self.assertEqual(make_weighted_conjunction([('mountain, \"man w(ith a', 1.1), ('beard', 1.1*2.0), ('hairy', 1)]), parse_prompt('("mountain, \\\"man\" w\(ith a (beard)2.0)+hairy')) + self.assertEqual(make_weighted_conjunction([('mountain, \"man with( a', 1.1), ('beard', 1.1*2.0), ('hairy', 1)]), parse_prompt('("mountain, \\\"man\" with\( a (beard)2.0)+ hairy')) + self.assertEqual(make_weighted_conjunction([('mountain, \"man )with a', 1.1), ('beard', 1.1*2.0), ('hairy', 1)]), parse_prompt('("mountain, \\\"man\" \)with a (beard)2.0)+ hairy')) + self.assertEqual(make_weighted_conjunction([('mountain, \"man w)ith a', 1.1), ('beard', 1.1*2.0), ('hairy', 1)]), parse_prompt('("mountain, \\\"man\" w\)ith a (beard)2.0)+ hairy')) + self.assertEqual(make_weighted_conjunction([('mountain, \"man with) a', 1.1), ('beard', 1.1*2.0), ('hairy', 1)]), parse_prompt(' ("mountain, \\\"man\" with\) a (beard)2.0)+ hairy')) + self.assertEqual(make_weighted_conjunction([('mou)ntain, \"man (wit(h a', 1.1), ('beard', 1.1*2.0), ('hairy', 1)]), parse_prompt('("mou\)ntain, \\\"man\" \(wit\(h a (beard)2.0)+ hairy')) + self.assertEqual(make_weighted_conjunction([('mountain, \"man w)ith a', 1.1), ('beard', 1.1*2.0), ('hai(ry', 1)]), parse_prompt('("mountain, \\\"man\" w\)ith a (beard)2.0)+ hai\(ry ')) + self.assertEqual(make_weighted_conjunction([('mountain, \"man with a', 1.1), ('beard', 1.1*2.0), ('hairy((', 1)]), parse_prompt('("mountain, \\\"man\" with a (beard)2.0)+ hairy\(\( ')) def test_cross_attention_escaping(self): @@ -339,7 +344,10 @@ class PromptParserTestCase(unittest.TestCase): parse_prompt('mountain (\(\(\().swap(m\(on\)\)key)')) def test_single(self): - self.assertEqual(make_weighted_conjunction([('hairy', 1), ('mou)ntain, \"man (wit(h a', 1.1), ('beard', 1.1*2.0)]), parse_prompt('hairy +("mou\)ntain, \\\"man\" \(wit\(h a 2.0(beard))')) + # todo handle this + #self.assertEqual(make_basic_conjunction(['a badly formed +test prompt']), + # parse_prompt('a badly formed +test prompt')) + pass if __name__ == '__main__':