List Experiments

This is a human/operator audit guide, not a live advisor or student skill. Run the script below to fetch all experiment PRs from the advisor branch and organize them into 3 files.

The BASE_BRANCH should be set to the advisor branch (e.g. noam). Check the $ADVISOR_BRANCH env var or the PR base branch.

Code

import subprocess, json, os, re
from datetime import datetime

BASE_BRANCH = os.environ.get("ADVISOR_BRANCH", "noam")

def get_repo_info():
    r = subprocess.run(["gh", "repo", "view", "--json", "owner,name"], capture_output=True, text=True)
    if r.returncode:
        raise RuntimeError(r.stderr)
    info = json.loads(r.stdout)
    return info["owner"]["login"], info["name"]

def fetch_issue_comments(owner, name, number):
    """Fetch all PR conversation comments for a PR, across every REST page."""
    comments = []
    page = 1
    while True:
        endpoint = f"repos/{owner}/{name}/issues/{number}/comments?per_page=100&page={page}"
        r = subprocess.run(["gh", "api", endpoint], capture_output=True, text=True)
        if r.returncode:
            raise RuntimeError(r.stderr)
        batch = json.loads(r.stdout)
        comments.extend((comment.get("body") or "") for comment in batch)
        if len(batch) < 100:
            return comments
        page += 1

def fetch():
    """Fetch PRs with complete comments via GraphQL plus REST overflow pages."""
    owner, name = get_repo_info()
    prs = []
    cursor = None
    while True:
        after = f', after: "{cursor}"' if cursor else ""
        query = f'''{{
          repository(owner: "{owner}", name: "{name}") {{
            pullRequests(first: 100, baseRefName: "{BASE_BRANCH}", states: [OPEN, CLOSED, MERGED]{after}) {{
              pageInfo {{ hasNextPage endCursor }}
              nodes {{
                number title body state mergedAt url
                labels(first: 10) {{ nodes {{ name }} }}
                comments(first: 100) {{
                  pageInfo {{ hasNextPage }}
                  nodes {{ body }}
                }}
              }}
            }}
          }}
        }}'''
        r = subprocess.run(["gh", "api", "graphql", "-f", f"query={query}"], capture_output=True, text=True)
        if r.returncode:
            raise RuntimeError(r.stderr)
        data = json.loads(r.stdout)["data"]["repository"]["pullRequests"]
        for node in data["nodes"]:
            first_comments = [c["body"] or "" for c in node["comments"]["nodes"]]
            comments = (
                fetch_issue_comments(owner, name, node["number"])
                if node["comments"]["pageInfo"]["hasNextPage"]
                else first_comments
            )
            prs.append({
                "number": node["number"],
                "title": node["title"],
                "body": node["body"] or "",
                "state": node["state"],
                "mergedAt": node["mergedAt"],
                "url": node["url"],
                "labels": [{"name": l["name"]} for l in node["labels"]["nodes"]],
                "comments": comments,
            })
        if not data["pageInfo"]["hasNextPage"]:
            break
        cursor = data["pageInfo"]["endCursor"]
    return prs

def extract_section(text, header):
    """Extract a markdown section (# or ##) by header name."""
    match = re.search(rf'(^#{{1,2}} {header}.*?)(?=^#{{1,2}} |\Z)', text, re.MULTILINE | re.DOTALL)
    return match.group(1).strip() if match else None

def find_results_text(pr):
    """Search body then comments (newest first) for the Results section."""
    body = pr.get("body", "") or ""
    section = extract_section(body, "Results")
    if section and "_To be filled" not in section:
        return section
    for comment in reversed(pr.get("comments", [])):
        section = extract_section(comment, "Results")
        if section and "_To be filled" not in section:
            return section
    return None

def extract_val_loss(text):
    if not text:
        return None
    patterns = [
        r'val[/_]loss[:\s]*\*?\*?(\d+\.\d+)',
        r'\*\*val/loss\*\*[:\s|]*\*?\*?(\d+\.\d+)',
        r'val/loss[:\s|]+(\d+\.\d+)',
    ]
    for p in patterns:
        m = re.search(p, text, re.IGNORECASE)
        if m:
            return float(m.group(1))
    return None

def has_results(pr):
    return find_results_text(pr) is not None

def labels_str(pr):
    return ", ".join(l["name"] for l in pr.get("labels", []))

def hypothesis_oneline(body):
    """Extract first sentence of hypothesis."""
    sec = extract_section(body or "", "Hypothesis")
    if not sec:
        return ""
    text = re.sub(r'^#{1,2} Hypothesis\n', '', sec).strip()
    m = re.match(r'(.+?[.!])\s', text)
    return m.group(1) if m else text[:120]

# --- Fetch ---
prs = fetch()
merged = sorted([p for p in prs if p.get("mergedAt")], key=lambda p: p["mergedAt"])
with_results = [p for p in prs if has_results(p) and not p.get("mergedAt")]
no_results = [p for p in prs if not has_results(p)]

ts = datetime.now().strftime("%Y-%m-%d_%H-%M")
outdir = "experiment_log"
os.makedirs(outdir, exist_ok=True)

# --- File 1: Merged winners (chronological) ---
merged_path = f"{outdir}/merged_{ts}.md"
with open(merged_path, "w") as f:
    f.write(f"# Merged Experiments ({len(merged)} winners)\n\n")
    f.write("These PRs beat baseline and were merged, in chronological order.\n\n")
    for pr in merged:
        body = pr.get("body", "") or ""
        hyp = extract_section(body, "Hypothesis") or ""
        results = find_results_text(pr) or ""
        val = extract_val_loss(results)
        val_str = f" | val_loss: {val}" if val else ""
        f.write(f"## #{pr['number']}: {pr['title']}{val_str}\n\n")
        if hyp:
            f.write(f"{hyp}\n\n")
        if results:
            f.write(f"{results}\n\n")
        f.write("---\n\n")

# --- File 2: Compact results table ---
table_path = f"{outdir}/results_table_{ts}.md"
with open(table_path, "w") as f:
    f.write(f"# Experiment Results Table\n\n")
    f.write(f"Total: {len(prs)} PRs | Merged: {len(merged)} | ")
    f.write(f"Ran (not merged): {len(with_results)} | Never ran: {len(no_results)}\n\n")

    f.write("## Merged\n\n")
    f.write("| PR | Title | val_loss |\n|---|---|---|\n")
    for pr in merged:
        results = find_results_text(pr)
        val = extract_val_loss(results)
        val_str = f"{val:.4f}" if val else "—"
        f.write(f"| #{pr['number']} | {pr['title']} | {val_str} |\n")

    f.write(f"\n## Ran but not merged ({len(with_results)})\n\n")
    f.write("| PR | Title | val_loss | Hypothesis (short) |\n|---|---|---|---|\n")
    for pr in sorted(with_results, key=lambda p: p["number"], reverse=True):
        body = pr.get("body", "") or ""
        results = find_results_text(pr)
        val = extract_val_loss(results)
        val_str = f"{val:.4f}" if val else "—"
        hyp = hypothesis_oneline(body)
        f.write(f"| #{pr['number']} | {pr['title']} | {val_str} | {hyp[:80]} |\n")

    f.write(f"\n## Never ran ({len(no_results)})\n\n")
    f.write("| PR | Title | Hypothesis (short) |\n|---|---|---|\n")
    for pr in sorted(no_results, key=lambda p: p["number"], reverse=True):
        hyp = hypothesis_oneline(pr.get("body", ""))
        f.write(f"| #{pr['number']} | {pr['title']} | {hyp[:80]} |\n")

# --- File 3: Full details (only PRs that ran) ---
full_path = f"{outdir}/full_details_{ts}.md"
with open(full_path, "w") as f:
    f.write(f"# Full Experiment Details ({len(merged) + len(with_results)} experiments)\n\n")
    for pr in merged + sorted(with_results, key=lambda p: p["number"], reverse=True):
        body = pr.get("body", "").strip() or "_No description._"
        state = "MERGED" if pr.get("mergedAt") else "CLOSED"
        f.write(f"# #{pr['number']}: {pr['title']} [{state}]\n\n{body}\n\n")
        results = find_results_text(pr)
        if results and results not in body:
            f.write(f"{results}\n\n")
        f.write("---\n\n")

print(f"Saved {len(prs)} PRs ({len(merged)} merged, {len(with_results)} ran, {len(no_results)} never ran):")
print(f"  {merged_path}         — merged winners with hypothesis + results")
print(f"  {table_path}   — compact table of all experiments")
print(f"  {full_path}   — full PR bodies (only experiments that ran)")