From 2db8216e68bb71a05c358c9268775f1e4589447b Mon Sep 17 00:00:00 2001 From: msg-bq Date: Sat, 24 May 2025 11:37:06 +0800 Subject: [PATCH 1/6] add depth in cfg derivation --- .../build_labels/random_cfg_deriver.py | 169 ++++++++++++++---- generate_dataset/gen_utils/access_llm.py | 2 + 2 files changed, 137 insertions(+), 34 deletions(-) diff --git a/generate_dataset/build_labels/random_cfg_deriver.py b/generate_dataset/build_labels/random_cfg_deriver.py index 9a7e9f1..6f29cff 100644 --- a/generate_dataset/build_labels/random_cfg_deriver.py +++ b/generate_dataset/build_labels/random_cfg_deriver.py @@ -2,9 +2,10 @@ #### Grace Hadiyanto #### CS439 FA14 import asyncio +import re import sys import random - +from typing import Literal from .instance_funcs import get_concept_instance from .derivation_funcs import * # 因为是别人的包，我就按原始的方式直接import *了 @@ -26,45 +27,143 @@ def _parse_alt_rule(line): return rule[1:] -def _find_references(current_string): - variable_references = [] - begin_index = 0 - end_index = 0 - is_variable = False - for i, c in enumerate(current_string): +SymbolType = Literal['variable', 'terminal'] + + +class Symbol: + def __init__(self, content: str, depth: int = 0): + self.name = content + self.type = self._get_type() + self.depth = depth + + def _get_type(self) -> SymbolType: + end_index = 0 + is_variable = False + + for i, c in enumerate(self.name): + if c == _NON_TERMINAL_SYMBOL and not is_variable: + is_variable = True + begin_index = i + end_index = i + elif (c.isupper() or c in extra_nonterminal_chars) and is_variable: + end_index += 1 + elif is_variable: + raise SyntaxError( + 'non-terminal symbol should be only consist of upper characters or some other' + f' characters in extra_nonterminal_chars variables. But given {self.name} ' + f'at position {i} with context ' + f'{self.name[max(i - 5, 0): min(i + 5, len(self.name))].replace("*space", " ")}' + ) + + if is_variable: + return 'variable' + else: + return 'terminal' + + def __eq__(self, other): + if isinstance(other, Symbol): + return self.name == other.name and self.type == other.type + elif isinstance(other, str): + return self.name == other + else: + raise TypeError("Invalid type for comparison") + + def __str__(self): + return str((self.name, self.type, self.depth)) + + +class ExpandStr: + _delimiter: list[str] = [' '] + + def __init__(self, string: str = None, symbols: list[Symbol] = None, init_depth: int = 0): + if string is None and symbols is None: + raise ValueError("Either string or symbols must be provided") + + self.symbols = self._convert(string) if string else symbols + + for s in self.symbols: + s.depth += init_depth + + def _split(self, string: str) -> list[str]: + return re.split('|'.join(self._delimiter), string) + + def _convert(self, string: str) -> list[Symbol]: + texts = self._split(string) + return [Symbol(t) for t in texts] + + def __add__(self, other): + if isinstance(other, str): + self.symbols.extend(self._convert(other)) + elif isinstance(other, Symbol): + self.symbols.append(other) + elif isinstance(other, ExpandStr): + self.symbols.extend(other.symbols) + else: + raise TypeError("Invalid type for addition") + + def __getitem__(self, item): + if isinstance(item, int): + return self.symbols[item] + elif isinstance(item, slice): + return ExpandStr(symbols=self.symbols[item], init_depth=0) + + def __iter__(self): + return iter(self.symbols) + + def _trans_str(self): + return ' '.join(symbol.name for symbol in self.symbols) + + def __eq__(self, other): + if isinstance(other, ExpandStr): + return self._trans_str() == other._trans_str() + elif isinstance(other, str): + return self._trans_str() == other + else: + raise TypeError("Invalid type for comparison") + + def __str__(self): + return self._trans_str() + + +def _find_references(current_string: ExpandStr) -> list[Symbol]: + variable_references = [s for s in current_string if s.type == 'variable'] + # begin_index = 0 + # end_index = 0 + # is_variable = False + # for i, symbol in enumerate(current_string): # If the character is an upper case and the variable flag is off, we've # hit the beginning of a new variable, so turn the variable flag on, and # reset the begin and end index. - if c == _NON_TERMINAL_SYMBOL and not is_variable: - is_variable = True - begin_index = i - end_index = i + # if symbol.type == 'variable': + # is_variable = True + # begin_index = i + # end_index = i # Here we increment the end index of the current variable reference # we are counting. - elif (c.isupper() or c in extra_nonterminal_chars) and is_variable: - end_index += 1 + # elif (symbol.isupper() or symbol in extra_nonterminal_chars) and is_variable: + # end_index += 1 # If the character is not an upper case alphabet character and the # variable flag is on, it's the end of our variable reference substring. # Save the reference to the list and reset the variable flag, begin, and # end index. - elif is_variable: - if c == ' ': - variable_references.append(VariableReference(current_string, begin_index, end_index)) - is_variable = False - begin_index = i - end_index = i - else: # 不允许非终止符直接续终止符相关的形式，所以直接抛异常 - raise SyntaxError( - 'non-terminal symbol should be only consist of upper characters or some other' - f' characters in extra_nonterminal_chars variables. But given {current_string} ' - f'at position {i} with context ' - f'{current_string[max(i - 5, 0): min(i + 5, len(current_string))].replace("*space", " ")}' - ) + # elif is_variable: + # if symbol == ' ': + # variable_references.append(symbol) + # is_variable = False + # begin_index = i + # end_index = i + # else: # 不允许非终止符直接续终止符相关的形式，所以直接抛异常 + # raise SyntaxError( + # 'non-terminal symbol should be only consist of upper characters or some other' + # f' characters in extra_nonterminal_chars variables. But given {current_string} ' + # f'at position {i} with context ' + # f'{current_string[max(i - 5, 0): min(i + 5, len(current_string))].replace("*space", " ")}' + # ) # If we've enumerated through the whole string and the variable flag is on, # there's a variable that hasn't been added to the list, so add it. - if is_variable: - variable_references.append(VariableReference(current_string, begin_index, end_index)) + # if is_variable: + # variable_references.append(VariableReference(current_string, begin_index, end_index)) return variable_references @@ -81,7 +180,7 @@ def _roulette_choice_rule(rules: list[str]) -> str: return random.choices(rules, weights=probability, k=1)[0] -async def _derive_string(current_string, grammar): +async def _derive_string(current_string: ExpandStr, grammar) -> ExpandStr: # While the string is not fully lower case(i.e. contains rules to be replaced # with productions): # 1. find variable references in the current string @@ -97,11 +196,12 @@ def _avoid_spurious_non_terminal_symbol(terminal_symbol): # variable_references = [] updated_string = None while True: - variable_references = _find_references(current_string) + variable_references: list[Symbol] = _find_references(current_string) if not variable_references: break - random_variable = random.choice(variable_references) # hack: 这个地方酌情优化，未经优化的random会导致大量重复 + variable_index = random.randint(0, len(variable_references)-1) + random_variable: Symbol = variable_references[variable_index] # random_production = random.choice(grammar.variable_dict[random_variable.name].rules) random_production = _roulette_choice_rule(grammar.variable_dict[random_variable.name].rules) @@ -115,8 +215,9 @@ def _avoid_spurious_non_terminal_symbol(terminal_symbol): terminal_symbols[i] = concept_instance random_production = ' '.join(terminal_symbols) - updated_string = (current_string[:random_variable.start_index] + random_production + - current_string[random_variable.end_index + 1:]) + production_expand = ExpandStr(random_production, init_depth=random_variable.depth+1) + updated_string = (current_string[:variable_index] + production_expand + + current_string[variable_index + 1:]) print('In "{}" replacing "{}" with "{}" to obtain "{}"'.format(current_string, random_variable.name, random_production, @@ -189,4 +290,4 @@ async def generate_expressions(n: int) -> list: if __name__ == '__main__': exps = asyncio.run(generate_expressions(n=2)) print("============") - print(exps) + print(exps) \ No newline at end of file diff --git a/generate_dataset/gen_utils/access_llm.py b/generate_dataset/gen_utils/access_llm.py index 0b9746b..7819717 100644 --- a/generate_dataset/gen_utils/access_llm.py +++ b/generate_dataset/gen_utils/access_llm.py @@ -36,6 +36,7 @@ # ) async def async_query_gpt(user_prompt: str, model=model, temperature=NOT_GIVEN) -> str: + return "sds" response = await aclient_gpt.chat.completions.create( model=model, messages=[ @@ -49,6 +50,7 @@ async def async_query_gpt(user_prompt: str, model=model, temperature=NOT_GIVEN) def query_gpt(user_prompt: str, model=model, temperature=NOT_GIVEN) -> str: + return '' if isinstance(user_prompt, str): prompt = [{"role": "user", "content": user_prompt}] else: From 8cb9fd8dc051f35865ae969bebba952d482c5108 Mon Sep 17 00:00:00 2001 From: msg-bq Date: Sat, 24 May 2025 11:40:00 +0800 Subject: [PATCH 2/6] Update random_cfg_deriver.py --- generate_dataset/build_labels/random_cfg_deriver.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/generate_dataset/build_labels/random_cfg_deriver.py b/generate_dataset/build_labels/random_cfg_deriver.py index 6f29cff..40a8496 100644 --- a/generate_dataset/build_labels/random_cfg_deriver.py +++ b/generate_dataset/build_labels/random_cfg_deriver.py @@ -215,7 +215,7 @@ def _avoid_spurious_non_terminal_symbol(terminal_symbol): terminal_symbols[i] = concept_instance random_production = ' '.join(terminal_symbols) - production_expand = ExpandStr(random_production, init_depth=random_variable.depth+1) + production_expand = ExpandbStr(random_production, init_depth=random_variable.depth+1) updated_string = (current_string[:variable_index] + production_expand + current_string[variable_index + 1:]) print('In "{}" replacing "{}" with "{}" to obtain "{}"'.format(current_string, @@ -277,7 +277,8 @@ async def generate_expressions(n: int) -> list: # Derive a random string from the grammar until all the variables are used final_exps = set() while True: - final_string = await _derive_string(start_string, the_grammar) # todo: 最好这里过滤下全null + start_expand_str = ExpandStr(string=start_string) + final_string = await _derive_string(start_expand_str, the_grammar) # todo: 最好这里过滤下全null final_exps.add(final_string) print('FINAL STRING:\n{}'.format(final_string)) From f4ed8e70384fffc6e67070876195a9cba43c6910 Mon Sep 17 00:00:00 2001 From: msg-bq Date: Sat, 24 May 2025 11:44:40 +0800 Subject: [PATCH 3/6] fix index error --- generate_dataset/build_labels/random_cfg_deriver.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/generate_dataset/build_labels/random_cfg_deriver.py b/generate_dataset/build_labels/random_cfg_deriver.py index 40a8496..06ac311 100644 --- a/generate_dataset/build_labels/random_cfg_deriver.py +++ b/generate_dataset/build_labels/random_cfg_deriver.py @@ -125,8 +125,8 @@ def __str__(self): return self._trans_str() -def _find_references(current_string: ExpandStr) -> list[Symbol]: - variable_references = [s for s in current_string if s.type == 'variable'] +def _find_references(current_string: ExpandStr) -> list[tuple[int, Symbol]]: + variable_references = [(i, s) for i, s in enumerate(current_string) if s.type == 'variable'] # begin_index = 0 # end_index = 0 # is_variable = False @@ -196,12 +196,11 @@ def _avoid_spurious_non_terminal_symbol(terminal_symbol): # variable_references = [] updated_string = None while True: - variable_references: list[Symbol] = _find_references(current_string) + variable_references: list[tuple[int, Symbol]] = _find_references(current_string) if not variable_references: break - variable_index = random.randint(0, len(variable_references)-1) - random_variable: Symbol = variable_references[variable_index] + variable_index, random_variable = random.choice(variable_references) # random_production = random.choice(grammar.variable_dict[random_variable.name].rules) random_production = _roulette_choice_rule(grammar.variable_dict[random_variable.name].rules) @@ -215,7 +214,7 @@ def _avoid_spurious_non_terminal_symbol(terminal_symbol): terminal_symbols[i] = concept_instance random_production = ' '.join(terminal_symbols) - production_expand = ExpandbStr(random_production, init_depth=random_variable.depth+1) + production_expand = ExpandStr(random_production, init_depth=random_variable.depth+1) updated_string = (current_string[:variable_index] + production_expand + current_string[variable_index + 1:]) print('In "{}" replacing "{}" with "{}" to obtain "{}"'.format(current_string, From 497e23db58b74e2c1445a3d640e1d0559b1f037e Mon Sep 17 00:00:00 2001 From: msg-bq Date: Sat, 24 May 2025 11:53:05 +0800 Subject: [PATCH 4/6] add return for __add__ --- generate_dataset/build_labels/random_cfg_deriver.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/generate_dataset/build_labels/random_cfg_deriver.py b/generate_dataset/build_labels/random_cfg_deriver.py index 06ac311..c7901ae 100644 --- a/generate_dataset/build_labels/random_cfg_deriver.py +++ b/generate_dataset/build_labels/random_cfg_deriver.py @@ -101,6 +101,8 @@ def __add__(self, other): else: raise TypeError("Invalid type for addition") + return self + def __getitem__(self, item): if isinstance(item, int): return self.symbols[item] From 7c7b2fc5a036c9ff59d955c5d1d053259f0ecca1 Mon Sep 17 00:00:00 2001 From: msg-bq Date: Sat, 24 May 2025 11:57:47 +0800 Subject: [PATCH 5/6] add extra info for _roulette_choice_rule --- generate_dataset/build_labels/random_cfg_deriver.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/generate_dataset/build_labels/random_cfg_deriver.py b/generate_dataset/build_labels/random_cfg_deriver.py index c7901ae..9091d85 100644 --- a/generate_dataset/build_labels/random_cfg_deriver.py +++ b/generate_dataset/build_labels/random_cfg_deriver.py @@ -169,7 +169,7 @@ def _find_references(current_string: ExpandStr) -> list[tuple[int, Symbol]]: return variable_references -def _roulette_choice_rule(rules: list[str]) -> str: +def _roulette_choice_rule(current_variable: Symbol, rules: list[str]) -> str: """ 给定一些可能的derivation的rules，以更高的概率选择非嵌套规则，避免嵌套层级过深 todo: 怎么才能让使用者快速注意到这个函数，然后决定自己想不想用或者更改为random.choice @@ -204,7 +204,7 @@ def _avoid_spurious_non_terminal_symbol(terminal_symbol): variable_index, random_variable = random.choice(variable_references) # random_production = random.choice(grammar.variable_dict[random_variable.name].rules) - random_production = _roulette_choice_rule(grammar.variable_dict[random_variable.name].rules) + random_production = _roulette_choice_rule(random_variable, grammar.variable_dict[random_variable.name].rules) terminal_symbols = random_production.split() for i, terminal_symbol in enumerate(terminal_symbols): From a6ec622194fbe4f78871b16e39f09dc735f7a649 Mon Sep 17 00:00:00 2001 From: msg-bq Date: Sat, 24 May 2025 12:38:13 +0800 Subject: [PATCH 6/6] remove redundant code and fix type errors --- .../build_labels/random_cfg_deriver.py | 45 +++---------------- generate_dataset/gen_utils/access_llm.py | 2 +- 2 files changed, 6 insertions(+), 41 deletions(-) diff --git a/generate_dataset/build_labels/random_cfg_deriver.py b/generate_dataset/build_labels/random_cfg_deriver.py index 9091d85..a0a8a97 100644 --- a/generate_dataset/build_labels/random_cfg_deriver.py +++ b/generate_dataset/build_labels/random_cfg_deriver.py @@ -115,6 +115,10 @@ def __iter__(self): def _trans_str(self): return ' '.join(symbol.name for symbol in self.symbols) + @property + def text(self): + return self._trans_str() + def __eq__(self, other): if isinstance(other, ExpandStr): return self._trans_str() == other._trans_str() @@ -129,43 +133,6 @@ def __str__(self): def _find_references(current_string: ExpandStr) -> list[tuple[int, Symbol]]: variable_references = [(i, s) for i, s in enumerate(current_string) if s.type == 'variable'] - # begin_index = 0 - # end_index = 0 - # is_variable = False - # for i, symbol in enumerate(current_string): - # If the character is an upper case and the variable flag is off, we've - # hit the beginning of a new variable, so turn the variable flag on, and - # reset the begin and end index. - # if symbol.type == 'variable': - # is_variable = True - # begin_index = i - # end_index = i - # Here we increment the end index of the current variable reference - # we are counting. - # elif (symbol.isupper() or symbol in extra_nonterminal_chars) and is_variable: - # end_index += 1 - # If the character is not an upper case alphabet character and the - # variable flag is on, it's the end of our variable reference substring. - # Save the reference to the list and reset the variable flag, begin, and - # end index. - # elif is_variable: - # if symbol == ' ': - # variable_references.append(symbol) - # is_variable = False - # begin_index = i - # end_index = i - # else: # 不允许非终止符直接续终止符相关的形式，所以直接抛异常 - # raise SyntaxError( - # 'non-terminal symbol should be only consist of upper characters or some other' - # f' characters in extra_nonterminal_chars variables. But given {current_string} ' - # f'at position {i} with context ' - # f'{current_string[max(i - 5, 0): min(i + 5, len(current_string))].replace("*space", " ")}' - # ) - - # If we've enumerated through the whole string and the variable flag is on, - # there's a variable that hasn't been added to the list, so add it. - # if is_variable: - # variable_references.append(VariableReference(current_string, begin_index, end_index)) return variable_references @@ -203,7 +170,6 @@ def _avoid_spurious_non_terminal_symbol(terminal_symbol): break variable_index, random_variable = random.choice(variable_references) - # random_production = random.choice(grammar.variable_dict[random_variable.name].rules) random_production = _roulette_choice_rule(random_variable, grammar.variable_dict[random_variable.name].rules) terminal_symbols = random_production.split() @@ -211,7 +177,6 @@ def _avoid_spurious_non_terminal_symbol(terminal_symbol): if terminal_symbol.endswith('_generate'): concept_name = terminal_symbol[:len(terminal_symbol) - len('_generate')] concept_instance: str = await get_concept_instance(concept_name.lower().replace('_', ' ')) - # todo: 每次都调取一遍llm太慢了，也容易重复。其实一次性就能返回10个，然后囤起来就是了。 concept_instance = concept_instance.replace(' ', '*space') terminal_symbols[i] = concept_instance random_production = ' '.join(terminal_symbols) @@ -280,7 +245,7 @@ async def generate_expressions(n: int) -> list: while True: start_expand_str = ExpandStr(string=start_string) final_string = await _derive_string(start_expand_str, the_grammar) # todo: 最好这里过滤下全null - final_exps.add(final_string) + final_exps.add(final_string.text) print('FINAL STRING:\n{}'.format(final_string)) if len(final_exps) > n: diff --git a/generate_dataset/gen_utils/access_llm.py b/generate_dataset/gen_utils/access_llm.py index 7819717..9f1744d 100644 --- a/generate_dataset/gen_utils/access_llm.py +++ b/generate_dataset/gen_utils/access_llm.py @@ -36,7 +36,7 @@ # ) async def async_query_gpt(user_prompt: str, model=model, temperature=NOT_GIVEN) -> str: - return "sds" + return "['example']" response = await aclient_gpt.chat.completions.create( model=model, messages=[