lm-eval-harness Task 配置与注册 源码精读

一句话总结

# Task  = 一个 YAML 文件 = 一个评测任务
# doc   = 数据集中的一行 = 一道题
# Instance = 一次模型调用请求 = 一道题拆出来的一个或多个请求

# 关系：
# 1 个 Task 包含 N 条 doc（来自 test.jsonl）
# 1 条 doc 产生 1 个或多个 Instance（取决于 output_type）

用具体例子理解

假设你运行 --tasks gsm8k_5shot_generation,arc_challenge_local，数据集各有 500 / 1000 条：

# ═══════════════════════════════════════════════════
# Task 层：每个 YAML 对应一个 Task 对象
# ═══════════════════════════════════════════════════
task_dict = {
    "gsm8k_5shot_generation": Task对象A,   # ← gsm8k_5shot_generation.yaml
    "arc_challenge_local":    Task对象B,   # ← arc_challenge.yaml
}

# Task 对象里有什么？
Task对象A.config          # TaskConfig（YAML 解析后的所有配置）
Task对象A.dataset         # HF Dataset（test.jsonl 加载后的数据集）
Task对象A.instances       # List[Instance]（所有请求，后面详解）
Task对象A.OUTPUT_TYPE     # "generate_until"

# ═══════════════════════════════════════════════════
# doc 层：数据集中的每一行就是一个 doc
# ═══════════════════════════════════════════════════

# GSM8K 有 500 条 doc：
doc_0 = {"prompt": "Janet's ducks...", "label": "18"}       # doc_id=0
doc_1 = {"prompt": "A store sells...",  "label": "42"}      # doc_id=1
...
doc_499 = {"prompt": "If x+y=10...",    "label": "7"}       # doc_id=499

# ARC 有 1000 条 doc：
doc_0 = {"question": "Which factor...", "choices": {...}, "answerKey": "A"}
...

# doc 就是一个普通 Python 字典，字段名取决于你的 test.jsonl

# ═══════════════════════════════════════════════════
# Instance 层：doc → Instance 的映射关系
# ═══════════════════════════════════════════════════

# ── generate_until：1 doc → 1 Instance ──
# GSM8K 500 条 doc → 500 个 Instance
Task对象A.instances = [
    Instance(request_type="generate_until", doc=doc_0, idx=0, doc_id=0, ...),
    Instance(request_type="generate_until", doc=doc_1, idx=0, doc_id=1, ...),
    ...  # 共 500 个
]

# ── multiple_choice：1 doc → N 个 Instance（N=选项数）──
# ARC 1000 条 doc，每条有 4 个选项 → 4000 个 Instance
Task对象B.instances = [
    # doc_0 的 4 个 Instance（每个选项一个 loglikelihood 请求）
    Instance(request_type="loglikelihood", doc=doc_0, idx=0, doc_id=0, ...),  # 选项 A
    Instance(request_type="loglikelihood", doc=doc_0, idx=1, doc_id=0, ...),  # 选项 B
    Instance(request_type="loglikelihood", doc=doc_0, idx=2, doc_id=0, ...),  # 选项 C
    Instance(request_type="loglikelihood", doc=doc_0, idx=3, doc_id=0, ...),  # 选项 D
    # doc_1 的 4 个 Instance
    Instance(request_type="loglikelihood", doc=doc_1, idx=0, doc_id=1, ...),  # 选项 A
    Instance(request_type="loglikelihood", doc=doc_1, idx=1, doc_id=1, ...),  # 选项 B
    ...  # 共 4000 个
]
# idx 是选项索引（0~3），doc_id 是样本编号（0~999）
# 评分时用 doc_id 把同一道题的 4 个 Instance 找回来

evaluate() 循环中如何用 doc_id 关联

# evaluator.py 的评分循环（伪代码简化版）：
for doc_id, doc in enumerate(task.test_docs()):   # 遍历每条 doc

    # 用 doc_id 从 4000 个 Instance 中筛出属于这道题的
    requests = [x for x in task.instances if x.doc_id == doc_id]
    requests.sort(key=lambda x: x.idx)  # 按选项索引排序

    # generate_until: requests = [Instance_0]         → 1 个
    # multiple_choice: requests = [Inst_0, Inst_1, Inst_2, Inst_3] → 4 个

    # 提取模型输出
    results = [req.filtered_resps[key] for req in requests]

    # generate_until: results = ["18"]
    # multiple_choice: results = [(-1.23,T,".."), (-4.56,F,".."), (-5.78,F,".."), (-6.12,F,"..")]
    #                             选项A的ll     选项B的ll       选项C的ll       选项D的ll

    # 传给 process_results 计算这一道题的分数
    metrics = task.process_results(doc, results)   # → {"acc": 1.0, "acc_norm": 1.0}

三层关系总结图

# ┌─────────────────────────────────────────────────┐
# │                   Task（评测任务）                │
# │  gsm8k_5shot_generation.yaml                    │
# │  output_type: generate_until                    │
# │                                                 │
# │  ┌─ doc_0 ─────────────────────────────────┐    │
# │  │ {"prompt":"Janet's ducks...","label":"18"} │  │
# │  │                                         │    │
# │  │  └→ Instance(doc_id=0, idx=0)           │    │
# │  │     resps=["The answer is: 18."]        │    │
# │  │     filtered_resps={"get-answer":"18"}   │    │
# │  └─────────────────────────────────────────┘    │
# │                                                 │
# │  ┌─ doc_1 ─────────────────────────────────┐    │
# │  │ {"prompt":"A store sells...","label":"42"} │  │
# │  │                                         │    │
# │  │  └→ Instance(doc_id=1, idx=0)           │    │
# │  └─────────────────────────────────────────┘    │
# │  ...（共 500 条 doc → 500 个 Instance）         │
# └─────────────────────────────────────────────────┘
#
# ┌─────────────────────────────────────────────────┐
# │                   Task（评测任务）                │
# │  arc_challenge.yaml                             │
# │  output_type: multiple_choice                   │
# │                                                 │
# │  ┌─ doc_0 ─────────────────────────────────┐    │
# │  │ {"question":"Which factor...","answerKey":"A"}│
# │  │                                         │    │
# │  │  ├→ Instance(doc_id=0, idx=0) 选项A     │    │
# │  │  ├→ Instance(doc_id=0, idx=1) 选项B     │    │
# │  │  ├→ Instance(doc_id=0, idx=2) 选项C     │    │
# │  │  └→ Instance(doc_id=0, idx=3) 选项D     │    │
# │  └─────────────────────────────────────────┘    │
# │  ...（共 1000 条 doc → 4000 个 Instance）       │
# └─────────────────────────────────────────────────┘

# ═══ Step 1: apply_filters ═══
# 500 个 Instance，每个的 resps 是模型原始生成文本
# filter pipeline "get-answer": regex → regex → take_first
instance[0].filtered_resps = {"get-answer": "18"}
instance[1].filtered_resps = {"get-answer": "42"}
instance[2].filtered_resps = {"get-answer": "7"}
...
instance[499].filtered_resps = {"get-answer": "156"}

# ═══ Step 2: process_results 逐条计算 ═══
# GSM8K 没有自定义 process_results，框架用 metric_list 中的 exact_match
# 对每条：比较 filtered_resps 与 doc_to_target(doc)

# doc 0: filtered="18", target="18" → exact_match=1.0 ✓
# doc 1: filtered="42", target="35" → exact_match=0.0 ✗
# doc 2: filtered="7",  target="7"  → exact_match=1.0 ✓
# ...
# vals[("gsm8k","get-answer","exact_match")] = [1.0, 0.0, 1.0, ..., 0.0]

# ═══ Step 3: aggregation ═══
# metric_list 指定 aggregation: mean
# mean([1.0, 0.0, 1.0, ..., 0.0]) = 0.7234
# stderr = bootstrap([1.0, 0.0, ...], iters=1000) = 0.0128

# ═══ 最终输出 ═══
{
    "gsm8k_5shot_generation": {
        "exact_match,get-answer": 0.7234,
        "exact_match_stderr,get-answer": 0.0128
    }
}
# 注意 key 格式："指标名,filter名"
# 如果有多个 filter pipeline，每个都会独立产生一组指标

RULER（长文本评测）的 YAML 同时使用了三个 !function 引用，是理解"哪个字段在哪一步被调用"的最佳示例：

# ruler_32k_cwe.yaml
task: ruler_32k_cwe
dataset_path: json
dataset_kwargs:
  data_dir: /data/.../ruler/ruler_32k_cwe
output_type: generate_until
doc_to_text: input                            # ← 纯字段名
process_docs: !function utils.process_docs    # ← !function ①
doc_to_target: !function utils.doc_to_target  # ← !function ②
process_results: !function utils.process_results  # ← !function ③
metric_list:
  - metric: em
    aggregation: mean
    higher_is_better: true
generation_kwargs:
  until: ["\n\n"]
  max_gen_toks: 64

!function ① process_docs — 阶段二 · 数据加载后立即调用

# utils.py
def process_docs(dataset):
    def _process(doc):
        return {
            "input": doc["input"],       # 保留长文本 prompt
            "outputs": doc["outputs"],   # 保留答案列表（可能多个正确答案）
        }
    return dataset.map(_process)

# 调用时机：load_dataset() 之后，遍历 doc 之前
# 输入：整个 HF Dataset（所有样本）
# 输出：只保留 input 和 outputs 两个字段的 Dataset
# 效果：丢弃原始数据中不需要的字段，标准化字段名

!function ② doc_to_target — 阶段三 · Prompt 构建 + 阶段五 · 评分

# utils.py
def doc_to_target(doc):
    return ' '.join(doc['outputs'])

# 调用时机 A（Prompt 构建）：fewshot_context 中拼 few-shot 示例时
#   输入：doc = {"input": "...长文...", "outputs": ["Paris", "France"]}
#   输出："Paris France"（作为 few-shot 示例的 target 文本）

# 调用时机 B（评分）：process_results 内部对比时
#   RULER 用自定义 process_results，所以此时不直接调用 doc_to_target
#   而是直接读 doc["outputs"]

# 注意：doc_to_target 在管线中可能被调用多次！
#   - 每条 fewshot 示例调用一次（拼答案）
#   - loglikelihood/mc 的 construct_requests 调用一次（构建 continuation）
#   - generate_until 不在 construct_requests 中调用，但评分时可能用到

!function ③ process_results — 阶段五 · 模型推理结束后

# utils.py
def process_results(doc, results):
    preds, refs = results, doc["outputs"]
    preds = preds[0]    # results = ["模型生成的文本"]，取第一个
    em = sum([1.0 if r.lower() in preds.lower() else 0.0 for r in refs]) / len(refs)
    return {"em": em}

# 调用时机：apply_filters() 之后，evaluate() 循环中逐条调用
# 输入：
#   doc = {"input": "...长文...", "outputs": ["Paris", "France"]}
#   results = ["Paris is the capital"]    ← filtered_resps 的值
# 执行：
#   refs = ["Paris", "France"]
#   "paris" in "paris is the capital" → 1.0  ✓
#   "france" in "paris is the capital" → 0.0  ✗
#   em = (1.0 + 0.0) / 2 = 0.5
# 输出：{"em": 0.5}
#
# 然后 metric_list 中 aggregation: mean 对所有样本的 em 取平均 → 最终分数

三个 !function 的调用顺序

# 时间线：
# ① process_docs(dataset)     ← 数据加载后，最先执行，且只执行一次
#       ↓
# （以下对每条 doc 循环）
# ② doc_to_target(doc)        ← fewshot_context 中拼 few-shot 示例时调用
#       ↓
#    doc_to_text(doc)          ← fewshot_context 中获取当前 doc 的 prompt
#       ↓
#    construct_requests()      ← 创建 Instance
#       ↓
#    LM 推理                   ← 模型生成文本
#       ↓
#    apply_filters()           ← 无 filter_list 则跳过
#       ↓
# ③ process_results(doc, results)  ← 对模型输出评分，最后执行

Stage 0 — 原始数据（test.jsonl 中的一行）

{"prompt": "Janet's ducks lay 16 eggs per day. She eats three for breakfast every morning and bakes muffins for her friends every day with four. She sells every duck egg at the farmers' market daily for $2. How much in dollars does she make every day at the farmers' market?\nThe answer is:", "label": "18"}

这就是数据集中的一条原始文档 doc，是一个 Python 字典。

Stage 1 — doc_to_text(doc) → prompt 字符串

YAML 配置 doc_to_text: prompt（纯字段名模式），框架执行：

# 框架内部等价于：
text = doc["prompt"]
# 结果：
"Janet's ducks lay 16 eggs per day. She eats three for breakfast every morning and bakes muffins for her friends every day with four. She sells every duck egg at the farmers' market daily for $2. How much in dollars does she make every day at the farmers' market?\nThe answer is:"

Stage 2 — fewshot_context(doc, num_fewshot=0) → ctx 字典

因为 num_fewshot: 0（zero-shot），fewshot_context 直接把 description（空字符串）和 doc_to_text 的结果拼接：

# fewshot_context 返回值（ctx）—— 注意是字典，不是纯字符串：
{
    "text": "Janet's ducks lay 16 eggs per day...The answer is:"
}
# 如果 num_fewshot > 0，text 会在前面拼上 few-shot 示例：
# {"text": "Q: What is 2+2?\nThe answer is: 4\n\nQ: ...\n\n实际问题的 prompt"}

Stage 3 — construct_requests(doc, ctx) → Instance 对象

output_type: generate_until，框架创建一个 Instance：

Instance(
    request_type = "generate_until",
    doc = {"prompt": "Janet's ducks...", "label": "18"},   # 原始文档保留
    arguments = (
        {"text": "Janet's ducks lay 16 eggs...The answer is:"},  # ctx
        {"until": ["\n\n"], "do_sample": false, "temperature": 0.00001}  # generation_kwargs
    ),
    idx = 0,
    metadata = ("gsm8k_5shot_generation", 0, 1),  # (task_name, doc_id, repeats)
    resps = [],             # 待填充
    filtered_resps = {},    # 待填充
)

Stage 4 — 模型推理 LM.generate_until() → resps

vllm_norm 将 ctx 转为 HTTP payload 发送给 vLLM 服务，返回的生成文本存入 resps：

# 模型生成文本经 thinking token 处理后写入 instance.resps：
instance.resps = [
    "She sells 16 - 3 - 4 = 9 duck eggs a day. 9 * $2 = $18. The answer is: 18."
]
# resps 是列表，长度 = repeats（默认 1）

Stage 5 — apply_filters() → filtered_resps

按 filter_list 中配置的 pipeline 顺序执行过滤器：

# 输入（instance.resps 列表）：
["She sells 16 - 3 - 4 = 9 duck eggs a day. 9 * $2 = $18. The answer is: 18."]

# Step 1  regex "The answer is: (.*)."
#   捕获组 → ["18"]

# Step 2  regex "(\\-?[0-9\\.\\,]+)"
#   提取纯数字 → ["18"]

# Step 3  take_first
#   列表 → 单值 → "18"

# 最终写入：
instance.filtered_resps["get-answer"] = "18"

Stage 6 — process_results(doc, results) → 指标字典

GSM8K 使用默认的 exact_match，未配置自定义 process_results：

# results = ["18"]（filtered_resps 中的值）
# doc_to_target(doc) = doc["label"] = "18"

# exact_match 计算（考虑 ignore_case、regexes_to_ignore）：
#   prediction = "18"  vs  reference = "18"  → 完全匹配
# 返回：
{"exact_match": 1.0}

Stage 7 — aggregation → 最终分数

# 所有样本的 exact_match 值：[1.0, 0.0, 1.0, 1.0, 0.0, ...]
# aggregation: mean → 取平均
# 最终结果：
{
    "gsm8k_5shot_generation": {
        "exact_match": 0.7234,        # 准确率
        "exact_match_stderr": 0.0128  # 标准误差
    }
}

Stage 0 — 原始数据（test.jsonl 中的一行）

{
    "question": "Which factor will most likely cause a person to develop a fever?",
    "choices": {
        "text": ["a]viral infection", "an its injury", "a bee sting", "a broken bone"],
        "label": ["A", "B", "C", "D"]
    },
    "answerKey": "A"
}

Stage 1 — doc_to_text / doc_to_target / doc_to_choice 渲染

三个 Jinja2 模板分别渲染：

# doc_to_text: "Question: {{question}}\nAnswer:"
text = "Question: Which factor will most likely cause a person to develop a fever?\nAnswer:"

# doc_to_target: "{{choices.label.index(answerKey)}}"
target = 0    # choices.label = ["A","B","C","D"], answerKey = "A", index = 0

# doc_to_choice: "{{choices.text}}"
choices = ["a viral infection", "an insect injury", "a bee sting", "a broken bone"]

Stage 2 — fewshot_context(doc, num_fewshot=25) → ctx 字典

从 train split 中随机采样 25 条示例，拼接成 few-shot prompt：

# ContextSampler 对每条 few-shot 示例的处理：
#   doc_to_target(fewshot_doc) 返回 int（如 0）
#   但 sampler 会自动转为选项文本：
#     doc_to_choice(fewshot_doc)[0] → "a viral infection"
# 见 samplers.py:76: str(self.doc_to_choice(doc)[self.doc_to_target(doc)])

# fewshot_context 返回值（ctx）：
{
    "text": "Question: What is the most common cause of earthquakes?\nAnswer: tectonic plate movement\n\nQuestion: Which best describes a mineral?\nAnswer: a naturally occurring inorganic solid\n\n... (共 25 个示例，target 都是选项文字而非索引数字) ...\n\nQuestion: Which factor will most likely cause a person to develop a fever?\nAnswer:"
}
# 每个 few-shot 示例 = doc_to_text(doc) + target_delimiter + choices[target_index]
# 示例间用 fewshot_delimiter（默认 "\n\n"）分隔

Stage 3 — construct_requests(doc, ctx) → Instance 列表

output_type: multiple_choice，框架为每个选项创建一个 Instance（共 4 个）：

# 将 ctx 与每个 choice 拼接，创建 4 个 loglikelihood 请求：
[
    Instance(
        request_type = "loglikelihood",
        doc = {原始 doc},
        arguments = (
            {"text": "...25 fewshot examples...\nQuestion: Which factor...?\nAnswer:"},  # ctx
            " a viral infection",     # target_delimiter + choice[0]
            {...generation_kwargs}
        ),
        idx = 0,   # 第 0 个选项
    ),
    Instance(
        request_type = "loglikelihood",
        arguments = (ctx, " an insect injury", ...),
        idx = 1,   # 第 1 个选项
    ),
    Instance(
        request_type = "loglikelihood",
        arguments = (ctx, " a bee sting", ...),
        idx = 2,   # 第 2 个选项
    ),
    Instance(
        request_type = "loglikelihood",
        arguments = (ctx, " a broken bone", ...),
        idx = 3,   # 第 3 个选项
    ),
]
# 注意：arguments 的第二个元素前有 target_delimiter（默认 " "）

Stage 4 — 模型推理 LM.loglikelihood() → resps

对每个 Instance，模型计算 continuation（选项文本）在给定 context 条件下的 log probability：

# ═══════════════════════════════════════════════════
# loglikelihood 计算原理（以选项 "a viral infection" 为例）
# ═══════════════════════════════════════════════════

# 1. 拼接完整文本：context + continuation
full_text = "Question: ...?\nAnswer:" + " a viral infection"

# 2. 分别 tokenize context 和 continuation
context_tokens    = [tok1, tok2, ..., tok_N]     # N 个 token
continuation_tokens = [tok_A, tok_B, tok_C]       # 3 个 token（" a", " viral", " infection"）
all_tokens = context_tokens + continuation_tokens  # 拼在一起

# 3. 构建 mask：context 位置=0，continuation 位置=1
mask = [0]*N + [1, 1, 1]

# 4. 发送给 vLLM，设置 echo=True + logprobs=1 + max_tokens=1
#    → 模型返回每个 token 位置的 log probability
#    不生成新 token，只是计算已有 token 的概率

# 5. 只累加 continuation 部分的 log prob（mask=1 的位置）
continuation_logprobs = log_p(" a") + log_p(" viral") + log_p(" infection")
#                     = (-0.31)     + (-0.52)         + (-0.40)
#                     = -1.23

# ═══════════════════════════════════════════════════
# is_greedy 判断原理
# ═══════════════════════════════════════════════════

# 对 continuation 的每个 token 位置，检查：
#   该位置概率最高的 token（top_token） == 实际的 continuation token？
# 如果每个位置都匹配 → is_greedy = True

# 位置 N+0: top_token=" a"         == 实际 token " a"         ✓
# 位置 N+1: top_token=" viral"     == 实际 token " viral"     ✓
# 位置 N+2: top_token=" infection" == 实际 token " infection" ✓
# → is_greedy = True （模型贪心解码恰好会生成这个选项）

# 如果某个位置 top_token ≠ 实际 token → is_greedy = False
# 例如：位置 N+1 的 top_token 是 " bacterial" 而非 " viral" → False

4 个选项的计算结果：

# 每个 Instance 的 resps = [(continuation_logprobs, is_greedy, info_dict)]
instances[0].resps = [(-1.23, True,  "[{...}]")]  # " a viral infection"   ← logprob 最高
instances[1].resps = [(-4.56, False, "[{...}]")]  # " an insect injury"
instances[2].resps = [(-5.78, False, "[{...}]")]  # " a bee sting"
instances[3].resps = [(-6.12, False, "[{...}]")]  # " a broken bone"

# loglikelihood 是负数，越接近 0 = 模型认为越可能
# is_greedy = True 表示模型贪心解码恰好会生成这段文本

Stage 5 — process_results(doc, results) → 指标字典

框架自动汇总 4 个 Instance 的结果，计算 acc 和 acc_norm：

# results = [(-1.23, True, {}), (-4.56, False, {}), (-5.78, False, {}), (-6.12, False, {})]
lls = [-1.23, -4.56, -5.78, -6.12]

# acc: 取 argmax(lls) = 0
pred = 0
gold = 0  # doc_to_target 返回的正确索引
acc = 1.0 if pred == gold else 0.0  # → 1.0 ✓

# acc_norm: 除以选项 token 长度后再取 argmax
completion_len = [len("a viral infection"), len("an insect injury"), len("a bee sting"), len("a broken bone")]
#               = [17, 16, 10, 12]
norm_lls = [ll/length for ll, length in zip(lls, completion_len)]
#         = [-0.072, -0.285, -0.578, -0.510]
pred_norm = argmax(norm_lls) = 0
acc_norm = 1.0 if pred_norm == gold else 0.0  # → 1.0 ✓

# 返回：
{"acc": 1.0, "acc_norm": 1.0}

Stage 6 — aggregation → 最终分数

# 所有样本的指标取 mean：
{
    "arc_challenge_local": {
        "acc": 0.5623,
        "acc_stderr": 0.0145,
        "acc_norm": 0.5891,
        "acc_norm_stderr": 0.0144
    }
}

# 假设数据集中有这样一条文档：
doc = {
    "question": "What is the capital of France?",
    "choices": {"text": ["London", "Paris", "Berlin", "Madrid"], "label": ["A", "B", "C", "D"]},
    "answerKey": "B",
    "prompt": "Question: What is the capital of France?\nA. London\nB. Paris\nC. Berlin\nD. Madrid\nAnswer:"
}

# ═══════════════════════════════════════════════════
# 方式一：纯字段名
# ═══════════════════════════════════════════════════
# YAML: doc_to_text: prompt
# 框架执行: doc["prompt"]
# 输出 →
"Question: What is the capital of France?\nA. London\nB. Paris\nC. Berlin\nD. Madrid\nAnswer:"

# ═══════════════════════════════════════════════════
# 方式二：Jinja2 模板
# ═══════════════════════════════════════════════════
# YAML: doc_to_text: "Question: {{question}}\nAnswer:"
# 框架执行: jinja2.Template("Question: {{question}}\nAnswer:").render(doc)
# 输出 →
"Question: What is the capital of France?\nAnswer:"

# YAML: doc_to_target: "{{choices.label.index(answerKey)}}"
# 框架执行: jinja2 求值 choices.label.index("B") → 1
# 输出 → 1  （即选项 B 的索引）

# YAML: doc_to_choice: "{{choices.text}}"
# 框架执行: jinja2 求值 → ["London", "Paris", "Berlin", "Madrid"]

# ═══════════════════════════════════════════════════
# 方式三：!function 引用
# ═══════════════════════════════════════════════════
# YAML: doc_to_text: !function utils.doc_to_text
# 框架执行: utils.doc_to_text(doc)
# 输出 →（由 Python 函数自由定制）
"Question: What is the capital of France?\nA. London\nB. Paris\nC. Berlin\nD. Madrid\nAnswer: "

# 模型原始生成输出（generate_until 的返回值）
raw_output = "Let me solve this step by step.\n16 - 3 = 13 eggs eaten\n13 * 2 = 26 dollars at market\nThe answer is: 42."

# Instance.resps = [["Let me solve this step by step.\n16 - 3 = 13...\nThe answer is: 42."]]
# （外层列表对应 repeats，内层是单次输出）

# ═══════════════════════════════════════════════════
# filter pipeline 逐步执行：
# ═══════════════════════════════════════════════════

# Step 1: regex — regex_pattern: "The answer is: (.*)."
#   输入: ["Let me solve this step by step.\n...\nThe answer is: 42."]
#   匹配捕获组 (.*) → "42"
#   输出: ["42"]

# Step 2: regex — regex_pattern: "(\\-?[0-9\\.\\,]+)"
#   输入: ["42"]
#   匹配纯数字 → "42"
#   输出: ["42"]

# Step 3: take_first
#   输入: ["42"]  （列表）
#   取第一个元素
#   输出: "42"    （字符串）

# 最终：instance.filtered_resps["get-answer"] = "42"
# 与 doc_to_target 的 "42" 进行 exact_match 比较 → 1.0