GitHub - hexo-ai/socrates: A multi-agent protocol pairing a tool-using Scientist with a question-only advisor — no tools, no answers, no directives — improves Kaggle test performance on 4 of 5 MLE-bench tasks

Note

The asciinema badge above is a placeholder. To record your own: bash scripts/record_demo.sh, then asciinema upload and paste the returned cast ID into this README in place of YOUR_CAST_ID (two occurrences).

# 1. Clone the repo
git clone https://github.com/hexo-ai/socrates.git
cd socrates

# 2. Create an isolated environment (conda or venv — pick one)
conda create -n socrates python=3.11 -y
conda activate socrates
#   or
python -m venv .venv && source .venv/bin/activate

# 3. Install dependencies
pip install -r requirements.txt
pip install --no-deps -r socratic-evolve/public-repo/requirements_base.txt
pip install --no-deps -r socratic-evolve/public-repo/requirements_ml.txt
pip install --no-deps -r socratic-evolve/public-repo/requirements_domain.txt

# 4. Set API keys
export ANTHROPIC_API_KEY="sk-ant-..."        # required
export OPENAI_API_KEY="sk-..."               # optional, only if you use OpenAI models

# 5. Create a local test config (gitignored)
cp socratic-evolve/test_config.yaml.example socratic-evolve/test_config.yaml
cp discover/test_config.yaml.example          discover/test_config.yaml
# Edit each to set dataset_dir and model.

# 6. Smoke-test the sequential scaffold
python discover/test_agent_locally.py

socrates/
├── discover/                 # Sequential scaffold (single agent, one experiment at a time)
│   ├── custom_agent.py       # Agent implementation
│   ├── base_agent.py         # Base class with webhook protocol
│   ├── models.py             # Message models
│   └── test_agent_locally.py # Local smoke test
│
├── socratic-evolve/          # Evolutionary scaffold (MLevolve + MCGS tree search)
│   ├── custom_agent.py       # Agent wrapper
│   ├── base_agent.py         # Base class
│   ├── models.py             # Message models
│   └── public-repo/          # MLevolve core
│       ├── run.py            # Main entry point for full experiments
│       ├── config/           # Default configuration
│       ├── engine/           # MCGS tree search, code execution
│       ├── agents/           # Multi-agent subsystem
│       │   ├── socrates/     # Socrates PI implementation
│       │   │   ├── prompts.py        # Question-only system prompt + [APPROVED] gate
│       │   │   ├── approval_loop.py  # Multi-round discussion loop
│       │   │   └── config.py         # Toggle flags
│       │   ├── evolution_agent.py    # Paradigm-shift mutations
│       │   └── fusion_agent.py       # Cross-branch solution merging
│       └── llm/              # LLM client wrappers
│
├── assets/
│   └── protocol.png          # Protocol diagram
├── scripts/
│   └── record_demo.sh        # Records the asciinema demo cast
├── conda.sh                  # Quick env activation helper
├── requirements.txt          # Top-level dependency manifest
├── LICENSE                   # MIT
└── README.md                 # This file

cd discover/
# Edit test_config.yaml to set:
#   AGENT_CONFIG.exp_id        -> the MLE-bench task id (e.g. "statoil-iceberg-classifier-challenge")
#   AGENT_CONFIG.dataset_dir   -> the local path you put the data in
#   AGENT_CONFIG.model         -> the LLM (default: claude-opus-4-6)
python test_agent_locally.py

cd socratic-evolve/public-repo/
python run.py \
  exp_id="statoil-iceberg-classifier-challenge" \
  agent.use_socrates_review=True \
  agent.steps=50

cd socratic-evolve/public-repo/
python collect_and_plot.py   # aggregates per-experiment logs into the paper's tables/plots
python dashboard.py          # optional live dashboard

# Sequential scaffold smoke test (no real run; mocks the LLM):
python discover/test_agent_locally.py --dry-run

# Evolutionary scaffold live test (requires API key):
cd socratic-evolve/public-repo/
pytest tests/test_socrates_live.py -k "test_socrates_basic"

@inproceedings{vrabac2026socrates,
  title     = {Socrates: Structured Questioning Unlocks Latent Knowledge in AI Research Agents},
  author    = {Vrabac, Damir and Hebbar, Prannay and Manawat, Yogendra and Palanimalai, Selvam and Verboomen, Samuel and Juneja, Gurusha and Bhatia, Kunal and Baskaran, Vignesh},
  booktitle = {Conference on Language Modeling (COLM)},
  year      = {2026}
}