add Conic10k data generating part

.gitignore

@@ -12,3 +12,4 @@
 /.idea
 *.xml
 /generate_dataset/build_labels/instance_funcs/llm_instance_datadir
+.idea/workspace.xml

generate_dataset/ad_hoc_(can_ignore)/concepts.json

@@ -0,0 +1,25 @@
+{
+    "Curve": "曲线",
+    "Conicsection": "\\in Curve 圆锥曲线",
+    "Line": "\\in Curve 直线",
+    "Linesegment": "\\in Curve 线段",
+    "Ray": "\\in Curve 射线",
+    "Hyperbola": "\\in ConicSection 双曲线",
+    "Circle": "\\in ConicSection 圆",
+    "Parabola": "\\in ConicSection 抛物线",
+    "Ellipse": "\\in ConicSection 椭圆",
+    "Point": "二维坐标系中的点",
+    "Origin": "\\in Point 二维坐标系的原点",
+    "Axis": "二维坐标系中的坐标轴",
+    "Xaxis": "\\in axis 二维坐标系中的横坐标",
+    "Yaxis": "\\in axis 二维坐标系中的纵坐标",
+    "Vector": "二维向量",
+    "Angle": "角",
+    "Number": "数",
+    "Real": "\\in Number 实数",
+    "Rad": "弧度",
+    "Degree": "度",
+    "Pi": "圆周率",
+    "Pm": "±",
+    "Triangle": "三角形"
+}

generate_dataset/ad_hoc_(can_ignore)/operators.json

@@ -0,0 +1,92 @@
+{
+    "Coordinate": "Coordinate(p: Point) -> (Number, Number)：二维坐标系中点p的坐标。",
+    "Xcoordinate": "XCoordinate(p: Point) -> Number：二维坐标系中点p的横坐标。",
+    "Ycoordinate": "YCoordinate(p: Point) -> Number：二维坐标系中点p的纵坐标。",
+    "Locusequation": "LocusEquation(p: Point) -> Expression：点p的轨迹方程。",
+    "Locus": "Locus(p: Point) -> Curve：点p的轨迹。",
+    "Quadrant": "Quadrant(p: Point) -> Number：点p所在的象限。",
+    "Focus": "Focus(c: ConicSection) -> Union[Set[point], Point]：圆锥曲线c的焦点，如果有多个焦点则返回集合，否则返回该点。",
+    "Rightfocus": "RightFocus(c: ConicSection) -> Point：圆锥曲线c的右焦点。",
+    "Leftfocus": "LeftFocus(c: ConicSection) -> Point：圆锥曲线c的左焦点。",
+    "Lowerfocus": "LowerFocus(c: ConicSection) -> Point：圆锥曲线c的下焦点。",
+    "Upperfocus": "UpperFocus(c: ConicSection) -> Point：圆锥曲线c的上焦点。",
+    "Focallength": "FocalLength(c: ConicSection) -> Number：圆锥曲线c的焦距。",
+    "Halffocallength": "HalfFocalLength(c: ConicSection) -> Number：圆锥曲线c的半焦距。",
+    "Symmetryaxis": "SymmetryAxis(c: ConicSection) -> Axis：圆锥曲线c的对称轴。",
+    "Eccentricity": "Eccentricity(c: ConicSection) -> Number：圆锥曲线c的离心率。",
+    "Vertex": "Vertex(c: ConicSection) -> Union[Set[Point], Point]：圆锥曲线c的顶点，如果有多个顶点则返回集合，否则返回该点。",
+    "Uppervertex": "UpperVertex(c: Union[Ellipse,Hyperbola]) -> Point：椭圆或双曲线c的上顶点。",
+    "Lowervertex": "LowerVertex(c: Union[Ellipse,Hyperbola]) -> Point：椭圆或双曲线c的下顶点。",
+    "Leftvertex": "LeftVertex(c: Union[Ellipse, Hyperbola]) -> Point：椭圆或双曲线c的左顶点。",
+    "Rightvertex": "RightVertex(c: Union[Ellipse, Hyperbola]) -> Point：椭圆或双曲线c的右顶点。",
+    "Directrix": "Directrix(c: ConicSection) -> Union[Set[Line], Line]：圆锥曲线c的准线，如果有多条准线则返回集合，否则返回该条准线。",
+    "Leftdirectrix": "LeftDirectrix(c: ConicSection) -> Line：圆锥曲线c的左准线。",
+    "Rightdirectrix": "RightDirectrix(c: ConicSection) -> Line：圆锥曲线c的右准线。",
+    "Majoraxis": "MajorAxis(g: Ellipse) -> LineSegment：椭圆g的长轴。",
+    "Minoraxis": "MinorAxis(g: Ellipse) -> LineSegment：椭圆g的短轴。",
+    "Realaxis": "RealAxis(c: Hyperbola) -> LineSegment：双曲线c的实轴。",
+    "Imageinaryaxis": "ImageinaryAxis(c: Hyperbola) -> LineSegment：双曲线c的虚轴。",
+    "Leftpart": "LeftPart(c: Hyperbola) -> Curve：双曲线c的左支部分。",
+    "Rightpart": "RightPart(c: Hyperbola) -> Curve：双曲线c的右支部分。",
+    "Asymptote": "Asymptote(c: Hyperbola) -> Set[Line]：双曲线c的两条渐近线。",
+    "Diameter": "Diameter(c: Circle) -> Number：圆c的直径。",
+    "Radius": "Radius(c: Circle) -> Number：圆c的半径。",
+    "Perimeter": "Perimeter(c: Union[Circle, Ellipse]) -> Number：圆c的周长。",
+    "Center": "Center(c: Union[Circle, Ellipse]) -> Point：圆或椭圆c的中心点。",
+    "Expression": "Expression(c: Curve)：曲线c的方程。",
+    "Lineof": "LineOf(p1: Point, p2: Point) -> Line：穿过p1、p2两点的一条直线。",
+    "Slope": "Slope(l: Line) -> Number：直线l的斜率。",
+    "Inclination": "Inclination(l: Line) -> Number：直线l的倾斜角。",
+    "Intercept": "Intercept(l: Line, a: Union[xAxis, yAxis]) -> Number：直线l在坐标轴上的截距。",
+    "Linesegmentof": "LineSegmentOf(p1: Point, p2: Point) -> LineSegment：连接p1、p2两点的线段。",
+    "Length": "Length(l: LineSegment) -> Number：线段l的长度。",
+    "Midpoint": "MidPoint(l: LineSegment) -> Point：线段l的中点。",
+    "Overlappingline": "OverlappingLine(l: LineSegment) -> Line：线段l所在的直线。",
+    "Perpendicularbisector": "PerpendicularBisector(l: LineSegment) -> Line：线段l的垂直平分线。",
+    "Endpoint": "Endpoint(l: Union[LineSegment, Ray]) -> Union[Set[Point], Point]：线段或射线l的端点，如果有多个端点则返回集合，否则返回该点。",
+    "Projection": "Projection(p: Point, l: Union[Line, Axis]) -> Point：点p在l上的投影。",
+    "Interceptchord": "InterceptChord(l: Line, c: ConicSection) -> LineSegment：直线l被圆锥曲线c截取的线段。",
+    "Triangleof": "TriangleOf(p1: Point, p2: Point, p3: Point) -> Triangle：以p1、p2、p3为顶点的三角形。",
+    "Inscribedcircle": "InscribedCircle(t: Triangle) -> Circle：三角形t的内切圆。",
+    "Circumcircle": "CircumCircle(t: Triangle) -> Circle：三角形t的外接圆。",
+    "Incenter": "Incenter(t: Triangle) -> Point：三角形t的内心。",
+    "Orthocenter": "Orthocenter(t: Triangle) -> Point：三角形t的垂心。",
+    "Circumcenter": "Circumcenter(t: Triangle) -> Point：三角形t的外心。",
+    "Angleof": "AngleOf(p1: Point, p2: Point, p3: Point) -> Number：以p2为顶点的∠p1p2p3的角度值。",
+    "Vectorof": "VectorOf(p1: Point, p2: Point) -> Vector：由点p1指向点p2的向量。",
+    "Dotproduct": "DotProduct(v1: Vector, v2: Vector) -> Number：向量v1与向量v2的点积。",
+    "Numintersection": "NumIntersection(a: Union[Axis, Curve], b: Union[Axis, Curve]) -> Number：a与b的交点的数量。",
+    "Intersection": "Intersection(a: Union[Axis, Curve], b: Union[Axis, Curve]) -> Union[Set[point], Point]：a与b的交点，如果有多个交点则返回集合，否则返回该点。",
+    "Footpoint": "FootPoint(l1: Union[Line, LineSegment], l2: Union[Line, LineSegment]) -> Point：线l1与线l2互相垂直，获取其垂足。",
+    "Distance": "Distance(a: Union[Point, Line], b: Union[Point, Line]) -> Number：a与b之间的距离。",
+    "Area": "Area(c: Union[Circle, Ellipse, Triangle]) -> Number：c的面积。",
+    "Tangentpoint": "TangentPoint(l: Line, c: Circle) -> Point：直线l与曲线c相切于c上的一点。",
+    "Tangentofpoint": "TangentOfPoint(p: Point, c: Curve) -> Line：过点p作出的曲线c的切线。",
+    "Tangentonpoint": "TangentOnPoint(p: Point, c: Curve) -> Line：过曲线c上的一点p作曲线c的切线。",
+    "Oneof": "OneOf(s: Set[T]) -> T：获取集合s中的一个元素。",
+    "Abs": "Abs(n: Union[Number, LineSegment]) -> Number：n的绝对值。",
+    "Lg": "lg(n: Number) -> Number：数n的log值。",
+    "Interreciprocal": "InterReciprocal(n: Number) -> Number：数n的倒数。",
+    "Sqrt": "sqrt(n: Number) -> Number：数n的平方根。",
+    "Max": "Max(n: Number) -> Number：变量n的最大值。",
+    "Min": "Min(n: Number) -> Number：变量n的最小值。",
+    "Range": "Range(n: Number) -> (Number, Number)：变量n的取值范围。",
+    "Whenmin": "WhenMin(n: Number)：当变量n取最小值时。",
+    "Whenmax": "WhenMax(n: Number)：当变量n取最大值时。",
+    "Applyunit": "ApplyUnit(n: Number, u: Union[degree, rad])：以n数值，u为单位的值。",
+    "In": "In(n: Number, (r1: Number, r2: Number))：变量n的值位于区间(r1,r2)。",
+    "Between": "Between(p1: Point, p2: Point, p3: Point)：点p1位于点p2和点p3之间。",
+    "Isintersect": "IsIntersect(a: Union[Axis, Curve], b: Union[axis, Curve])：a与b相交。",
+    "Isperpendicular": "IsPerpendicular(l1: Union[Line, LineSegment, xAxis, yAxis], l2: Union[Line, LineSegment, xAxis, yAxis])：l1与l2互相垂直。",
+    "Isparallel": "IsParallel(l1: Union[Line, LineSegment, xAxis, yAxis], l2: Union[Line, LineSegment, xAxis, yAxis])：l1与l2互相平行。",
+    "Isdiameter": "IsDiameter(l: LineSegment, c: Circle)：线段l是圆c的直径。",
+    "IschordOf": "IsChordOf(l: LineSegment, c: ConicSection)：线段l是圆锥曲线c的弦。",
+    "Istangent": "IsTangent(l: Line, c: ConicSection)：直线l是圆锥曲线c的切线。",
+    "Isinscribedcircle": "IsInscribedCircle(c: Circle, t: Triangle)：圆c是三角形t的内切圆。",
+    "Iscircumcircle": "IsCircumCircle(c: Circle, t: Triangle)：圆c是三角形t的外接圆。",
+    "Isintangent": "IsInTangent(c1: Circle, c2: Circle)：圆c1与圆c2内切。",
+    "Isouttangent": "IsOutTangent(c1: Circle, c2: Circle)：圆c1与圆c2外切。",
+    "Pointoncurve": "PointOnCurve(p: Point, c: Union[Axis, Curve])：点p在c上。",
+    "And": "And(a: Assersion, b: Assertion)：断言逻辑表达式a与b同时成立。",
+    "Negation": "Negation(a: Assertion)：断言逻辑表达式a不成立。"
+}

generate_dataset/ad_hoc_(can_ignore)/problems/_filter.py

@@ -0,0 +1,23 @@
+# 其中一个小实验进行的数据过滤，但是合正常流程和最终的实验无关
+
+import json
+
+# 读取原始数据
+with open(r"D:\桌面\6023\generate_dataset\semantic_parsing-q_upload_change\generate_dataset\assertions_output"
+          r"\merge_tuples\conic10k_output_1_problem_new.json", "r", encoding="utf-8") as f:
+    datas = json.load(f)
+
+res = []
+for data in datas:
+    declaration = data["Declarations"]
+    if "Triangle" in declaration:  # hack(zcl): 这个是因为cfg里不小心新增了Triangle这个concept，conic10k里没有，
+        # 但因为已经生成自然语言问题了，就先这样特殊处理了
+        continue
+    else:
+        res.append(data)
+
+with open(
+        r"D:\桌面\6023\generate_dataset\semantic_parsing-q_upload_change\generate_dataset\assertions_output"
+        r"\synthetic_datas.json",
+        "w", encoding="utf-8") as f:
+    json.dump(res, f, indent=4, ensure_ascii=False)

generate_dataset/ad_hoc_(can_ignore)/problems/filter.py

@@ -0,0 +1,17 @@
+import json
+import random
+
+# 读取原始数据
+with open(r"D:\桌面\6023\generate_dataset\semantic_parsing-q_upload_change\generate_dataset\assertions_output\merge_tuples\conic10k_output_1_problem_new.json", "r", encoding="utf-8") as f:
+    datas = json.load(f)
+
+res = []
+for data in datas:
+    declaratiosn = data["Declarations"]
+    if "Triangle" in declaratiosn:
+        continue
+    else:
+        res.append(data)
+
+with open(r"D:\桌面\6023\generate_dataset\semantic_parsing-q_upload_change\generate_dataset\assertions_output\synthetic_datas.json", "w", encoding="utf-8") as f:
+    json.dump(res, f, indent=4, ensure_ascii=False)

generate_dataset/ad_hoc_(can_ignore)/problems/sample_datas.py

@@ -0,0 +1,31 @@
+import json
+import random
+
+# 读取原始数据
+with open(r"D:\桌面\6023\generate_dataset\semantic_parsing-q_upload_change\generate_dataset\assertions_output\synthetic_datas.json", "r", encoding="utf-8") as f:
+    data = json.load(f)
+
+assert len(data) == 10456, f"数据数量为 {len(data)}，不是 16500"
+
+# 分层参数设置
+num_layers = 8              # 每层 500 条
+layer_size = 1307
+layers = [data[i * layer_size:(i + 1) * layer_size] for i in range(num_layers)]
+
+# 需要的总样本数
+sample_sizes = [2614, 5228, 7842, 10456]
+
+# 每层采样比例 = 样本数 / 总数据数
+def stratified_sampling(layers, total_samples, seed=42):
+    random.seed(seed)
+    samples = []
+    per_layer = total_samples // len(layers)  # 均匀抽样数量
+    for layer in layers:
+        samples.extend(random.sample(layer, per_layer))
+    return samples
+
+# 按不同目标数量生成样本
+for size in sample_sizes:
+    sampled = stratified_sampling(layers, size)
+    with open(f"D:\\桌面\\6023\\generate_dataset\\semantic_parsing-q_upload_change\\generate_dataset\\assertions_output\\sample_{size}.json", "w", encoding="utf-8") as f:
+        json.dump(sampled, f, ensure_ascii=False, indent=2)

generate_dataset/ad_hoc_(can_ignore)/random_merge.py

@@ -0,0 +1,162 @@
+# hack:  # fixme: 这个文件本身应当通过START ← Assertions + Assertions ← Assertion Assertions来处理
+# 这个文件只能算是特殊处理的方案。但既然已经写了就放在这里
+
+# 随机组合assertions，得到一个问题
+import random
+import json
+
+
+def preprocess_data(data: list[dict]) -> list[dict]:
+    """预处理数据，过滤掉declarations或facts为None的元素"""
+    return [elem for elem in data if
+            elem.get('declarations') is not None and elem.get('facts') is not None and elem.get('facts').count('(') < 3]
+
+
+def parse_declarations(decl_str: str) -> dict[str, str]:
+    """解析declarations字符串为字典，过滤数字声明并标准化类型名称"""
+    if decl_str is None:
+        return {}
+    decl_str = decl_str.strip()
+    if not decl_str:
+        return {}
+    decls = {}
+    parts = [p.strip() for p in decl_str.split(';') if p.strip()]
+    for part in parts:
+        try:
+            var, typ = part.split(':')
+            var = var.strip()
+            typ = typ.strip()
+            # 过滤掉纯数字的变量名
+            if var.isdigit():
+                continue
+            elif var.startswith('-') and var.lstrip('-').isdigit():
+                continue
+            elif var.lower() == "true" or var.lower() == "false":
+                continue
+            # 标准化类型名称：首字母大写，其余小写
+            typ = typ.capitalize()
+            decls[var] = typ
+        except ValueError:
+            continue  # 忽略格式错误的声明
+    return decls
+
+
+def check_conflict(decls1: dict[str, str], decls2: dict[str, str]) -> bool:
+    """检查两个declarations字典是否有冲突"""
+    common_vars = set(decls1.keys()) & set(decls2.keys())
+    for var in common_vars:
+        if decls1[var] != decls2[var]:
+            return True
+    return False
+
+
+def combine_elements(elements: list[dict]) -> tuple[dict[str, str] | None, list[str] | None]:
+    """合并多个元素的declarations和facts，返回合并后的declarations和facts列表"""
+    combined_decls = {}
+    facts = []
+    for elem in elements:
+        decls = parse_declarations(elem.get('declarations'))
+        # 检查冲突
+        if check_conflict(combined_decls, decls):
+            return None, None
+        combined_decls.update(decls)
+        fact = elem.get('facts')
+        if fact is not None:
+            facts.append(fact)
+    return combined_decls, facts
+
+
+def transform_fact_to_question(fact: str) -> tuple[str, str | None] | None:
+    """
+    将 fact 的右式替换为 '?'，并返回替换后的 fact 和被替换的右式变量（如果有）。
+    例如：
+    - 输入 "DotProduct(q, e) = -8" → 返回 ("DotProduct(q, e) = ?", None)
+    - 输入 "MajorAxis(k) = o" → 返回 ("MajorAxis(k) = ?", "o")
+    """
+    if '=' not in fact:
+        return None  # 忽略格式错误的 fact
+    left, right = fact.split('=', 1)
+    right = right.strip()
+
+    # 检查右式是否是变量（由字母组成，不含运算符或括号）
+    right_var = right if right.isalpha() else None
+    transformed_fact = f"{left.strip()} = ?"
+    return transformed_fact, right_var
+
+
+def transform_combination(combination: dict) -> dict:
+    """
+    对单个组合进行转换：
+    1. 随机选择一个 fact，将其右式替换为 '?'
+    2. 如果被替换的右式是一个变量，则从 declarations 中移除它
+    """
+    facts = combination["facts"]
+    if not facts:
+        return combination  # 无 facts 可转换
+
+    # 随机选择一个 fact 进行转换
+    selected_index = random.randint(0, len(facts) - 1)
+    selected_fact = facts[selected_index]
+
+    # 替换右式为 '?'
+    transformed_fact, right_var = transform_fact_to_question(selected_fact)
+    if transformed_fact is None:
+        return combination  # 转换失败
+
+    # 更新 facts
+    new_facts = facts.copy()
+    new_facts[selected_index] = transformed_fact
+
+    # 更新 declarations（如果右式是变量）
+    new_declarations = combination["declarations"].copy()
+    if right_var is not None and right_var in new_declarations:
+        del new_declarations[right_var]
+
+    return {
+        "id": combination["id"],
+        "declarations": new_declarations,
+        "facts": new_facts
+    }
+
+
+def find_valid_combinations(data: list[dict], min_size=2, max_size=5, max_attempts=5000, max_tuples=10) -> list[dict]:
+    """随机寻找有效的元素组合，确保declarations不超过8个"""
+    valid_combinations = []
+    attempts = 0
+    data = preprocess_data(data)  # 预处理数据
+    while attempts < max_attempts and len(valid_combinations) < max_tuples:
+        attempts += 1
+        k = random.randint(min_size, max_size)
+        if k > len(data):
+            continue
+        selected = random.sample(data, k)
+        combined_decls, combined_facts = combine_elements(selected)
+        if combined_decls is not None and combined_facts and len(combined_decls) <= 8:
+            valid_combinations.append({
+                "id": len(valid_combinations) + 1,
+                "declarations": combined_decls,
+                "facts": combined_facts
+            })
+    return valid_combinations
+
+
+def main():
+    file = open(
+        r'D:\桌面\6023\generate_dataset\semantic_parsing-q_upload_change\generate_dataset\assertions_output\assertions'
+        r'\conic10k_output_1.jsonl',
+        'r', encoding='utf-8')
+    data = json.load(file)
+    valid_combinations = find_valid_combinations(data, max_tuples=100000)
+
+    # 对每个组合进行转换（随机选择一个 fact 替换为 '?'）
+    transformed_combinations = [transform_combination(comb) for comb in valid_combinations]
+
+    new_file = open(
+        r'D:\桌面\6023\generate_dataset\semantic_parsing-q_upload_change\generate_dataset\assertions_output'
+        r'\merge_assertions\conic10k_output_1_new.json',
+        'w', encoding='utf-8')
+    json.dump(transformed_combinations, new_file, indent=4, ensure_ascii=False)
+
+
+if __name__ == "__main__":
+    main()