{"crawledAt":"2026-10-19T09:14:14.521Z","competitions":[{"id":"assetopsbench","name":"AssetOpsBench","tagline":"Build and evaluate LLM agents on industrial asset operations and maintenance scenarios.","description":"AssetOpsBench is an IBM Research benchmark of 141+ industrial asset operations and maintenance scenarios spanning IoT telemetry, failure modes, work orders, time-series forecasting, and vibration diagnostics, exposed through six MCP servers. Agents are scored on their execution trajectories by an LLM judge (Llama-4-Maverick-17B) across six dimensions covering reasoning, execution, and data handling. It has powered public competitions (CODS 2025 AssetOpsBench-Live, IJCAI 2026 Industrial Automation Challenge) and is accepted at KDD 2026.","url":"https://github.com/IBM/AssetOpsBench","host":"github","status":"active","domain":"llm-eval","tags":["industrial-agents","mcp","llm-eval","multi-agent","asset-maintenance","tool-use"],"images":["https://github.com/user-attachments/assets/615059be-e296-40d3-90ec-97ee6cb00412"],"participation":{"howToSubmit":"Contribute new scenarios via a Pull Request or an Issue tagged `new-scenario`; competition submissions run through the external challenge sites (Codabench for CODS 2025, the IJCAI 2026 challenge site).","submissionFormat":"Scenario definitions following the Utterance and Ground Truth design guidelines; agent trajectories evaluated offline by the `uv run evaluate` CLI.","computeDetails":"Agents call external LLM APIs (LiteLLM proxy / TokenRouter); MCP servers run locally with CouchDB via Docker.","deadline":"rolling","prizes":"recognition only","eligibility":"Open to researchers, students, and practitioners worldwide.","cost":"free","verification":"Offline evaluation module scores saved agent trajectories with registered graders (LLM-as-judge 6-criterion rubric, static JSON scorer); competition entries verified on Codabench.","requirements":["Python 3.12+","uv","Docker","CouchDB","MCP"],"compute":"unknown"},"links":[{"label":"Paper","url":"https://arxiv.org/pdf/2506.03828"},{"label":"Dataset","url":"https://huggingface.co/datasets/ibm-research/AssetOpsBench"},{"label":"Playground","url":"https://huggingface.co/spaces/ibm-research/AssetOps-Bench"},{"label":"Kaggle Benchmark","url":"https://www.kaggle.com/benchmarks/ibm-research/asset-ops-bench"},{"label":"CODS 2025 Competition","url":"https://www.codabench.org/competitions/10206/"},{"label":"IJCAI 2026 Challenge","url":"https://sites.google.com/view/ai-industrial-challenge-ijcai/home"}],"organizer":{"name":"IBM Research","url":"https://research.ibm.com/blog/asset-ops-benchmark","type":"company"},"venue":{"event":"CODS 2025 / IJCAI 2026 Industrial Automation Challenge","type":"online"},"quickstart":"git clone https://github.com/IBM/AssetOpsBench.git\ncd AssetOpsBench\npip install -e .\nuv sync\ncp .env.public .env\npython -m assetopsbench.run --scenario \"List all sensors of Chiller 6 in MAIN site\"","problems":[{"description":"Multi-step industrial asset operations and maintenance tasks (IoT, FMSR, TSFM, WO, Vibration) evaluated by an LLM judge across six dimensions of reasoning, execution, and data handling.","id":"industrial-agent-scenarios","name":"Industrial Asset Operations Agent Scenarios","metricName":"score","metricDirection":"maximize","isOpen":true,"records":[],"contested":false}],"agentStats":[],"repo":{"owner":"IBM","name":"AssetOpsBench","branch":"main"},"participants":[],"stats":{"totalRecords":0,"uniqueParticipants":0,"recordsLast30d":0,"recordsLast90d":0},"lastUpdated":"2026-10-19T09:02:41.686Z"},{"id":"alpaca-eval","name":"AlpacaEval","tagline":"Automatic LLM-as-judge benchmark measuring instruction-following win rate against a reference model.","description":"AlpacaEval evaluates instruction-following chat models by having a strong LLM annotator (e.g. GPT-4-turbo) compare a model's outputs against a reference model's outputs on the 805-example AlpacaEval set, reporting win rate and length-controlled win rate. It is fast (<5 min) and cheap (<$10) and correlates 0.98 with ChatBot Arena. Maintained by the tatsu-lab (Stanford) team; anyone can submit a model via a pull request adding a model config, outputs, and a leaderboard entry.","url":"https://github.com/tatsu-lab/alpaca_eval","host":"github","status":"active","domain":"llm-eval","tags":["llm-eval","instruction-following","leaderboard","win-rate","automatic-evaluator","chat-models"],"images":["https://raw.githubusercontent.com/tatsu-lab/alpaca_eval/main/figures/chat_correlations_no_ae.png","https://raw.githubusercontent.com/tatsu-lab/alpaca_eval/main/figures/chat_correlations.png","https://raw.githubusercontent.com/tatsu-lab/alpaca_eval/main/figures/length_gameability.png","https://raw.githubusercontent.com/tatsu-lab/alpaca_eval/main/figures/different_baselines.png"],"participation":{"howToSubmit":"Fork the repo, add a model config under src/alpaca_eval/models_configs/<model_name>, run `alpaca_eval evaluate_from_model --model_configs '<model_name>'`, then open a PR adding the config, outputs, and updated leaderboard entry.","submissionFormat":"Model config YAML, model_outputs.json, annotations.json, and updated leaderboard CSV under src/alpaca_eval/leaderboards/","computeDetails":"GPU needed only for local HuggingFace model decoding; annotation is done via API (e.g. OpenAI).","deadline":"rolling","prizes":"recognition only","eligibility":"Open to anyone; top-5 models can request verification by maintainers.","cost":"free (plus annotator API costs, <$10 per run)","verification":"Community submissions via PR; top-5 models can be verified by a core maintainer who re-decodes and re-evaluates.","requirements":["Python 3.10+","OpenAI API key (or other annotator provider)","GPU for local model decoding (optional)"],"compute":"consumer-gpu"},"links":[{"label":"Leaderboard website","url":"https://tatsu-lab.github.io/alpaca_eval/"},{"label":"Length-Controlled AlpacaEval paper","url":"https://arxiv.org/abs/2404.04475"},{"label":"AlpacaFarm paper","url":"https://arxiv.org/abs/2305.14387"},{"label":"AlpacaEval dataset","url":"https://huggingface.co/datasets/tatsu-lab/alpaca_eval"},{"label":"Discord","url":"https://discord.gg/GJMxJSVZZM"}],"organizer":{"name":"tatsu-lab (Stanford)","url":"https://github.com/tatsu-lab","type":"academic"},"venue":{"type":"online"},"quickstart":"pip install alpaca-eval\nexport OPENAI_API_KEY=<your_api_key>\nalpaca_eval --model_outputs 'example/outputs.json' --annotators_config 'weighted_alpaca_eval_gpt4_turbo'","problems":[{"description":"Length-controlled win rate of a model against the gpt4_turbo baseline, annotated by weighted_alpaca_eval_gpt4_turbo on the 805-example AlpacaEval set.","metricUnit":"%","id":"alpaca-eval-2","name":"AlpacaEval 2.0 (Length-Controlled Win Rate)","metricName":"length-controlled win rate","metricDirection":"maximize","isOpen":true,"records":[],"contested":false},{"description":"Raw win rate of a model against the text-davinci-003 baseline, annotated by alpaca_eval_gpt4 on the 805-example AlpacaEval set.","metricUnit":"%","id":"alpaca-eval-1","name":"AlpacaEval 1.0 (text-davinci-003 baseline)","metricName":"win rate","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"gpt4","value":95.3,"contributorKind":"model","isCurrentBest":true,"agent":{"model":"GPT-4","evidence":"rtaori | 2023-07-11 | gpt4 annotation results","family":"openai","confidence":"high","role":"subject"}},{"contributor":"claude","value":88.4,"contributorKind":"model","agent":{"model":"Claude","evidence":"YannDubs | 2023-07-12 | [ENH] add claude v2","family":"anthropic","confidence":"high","role":"subject"}},{"contributor":"chatgpt","value":86.1,"contributorKind":"model","agent":{"model":"ChatGPT","evidence":"YannDubs | 2023-06-22 | [ENH] add chatgpt leaderboard","family":"openai","confidence":"high","role":"subject"}},{"contributor":"guanaco-65b","value":71.8,"contributorKind":"model","agent":{"model":"Guanaco-65B","evidence":"YannDubs | 2023-07-27 | add claude leaderboard llama2","family":"meta","confidence":"low","role":"subject"}},{"contributor":"vicuna-13b","value":70.4,"contributorKind":"model","agent":{"model":"Vicuna-13B","evidence":"rtaori | 2023-07-11 | gpt4 annotations for vicuna v1.3","family":"meta","confidence":"medium","role":"subject"}},{"contributor":"text_davinci_003","value":50,"contributorKind":"model","isBaseline":true,"agent":{"model":"text-davinci-003","evidence":"gblazex | 2024-02-07 | Add claude-instant-1.2, deepseek-llm-67b-chat","family":"openai","confidence":"medium","role":"subject"}},{"contributor":"alpaca-farm-ppo-human","value":41.2,"contributorKind":"model","agent":{"model":"AlpacaFarm PPO Human","evidence":"YannDubs | 2023-07-15 | [ENH] add all previous model outputs","family":"meta","confidence":"low","role":"subject"}},{"contributor":"alpaca-7b","value":26.5,"contributorKind":"model","agent":{"model":"Alpaca-7B","evidence":"neelsjain | 2023-10-19 | LLaMA-1 Alpaca and LLaMA-2-Chat 7B","family":"meta","confidence":"medium","role":"subject"}},{"contributor":"text_davinci_001","value":15.2,"contributorKind":"model","agent":{"model":"text-davinci-001","evidence":"YannDubs | 2023-07-15 | [ENH] add all previous model outputs","family":"openai","confidence":"low","role":"subject"}}],"contested":true},{"description":"Meta-leaderboard of automatic annotators, scored by agreement with majority human preferences on ~650 cross-annotated examples, plus price, time, correlation, bias, variance, and length-preference metrics.","metricUnit":"%","id":"alpaca-eval-evaluators","name":"Evaluator Meta-Evaluation (Human Agreement)","metricName":"human agreement","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"alpaca_eval_gpt4","value":69.2,"contributorKind":"method","isCurrentBest":true,"agent":{"model":"GPT-4","evidence":"YannDubs | 2023-10-02 | add alpaca_eval_gpt4_fn","family":"openai","confidence":"high","role":"subject"}},{"contributor":"alpaca_eval_cot_gpt4_turbo_fn","value":68.6,"contributorKind":"method","agent":{"model":"GPT-4 Turbo","evidence":"YannDubs | 2024-01-10 | precompute all leaderboard for AE2","family":"openai","confidence":"high","role":"subject"}},{"contributor":"alpaca_eval_gpt4_turbo_fn","value":68.1,"contributorKind":"method","agent":{"model":"GPT-4 Turbo","evidence":"YannDubs | 2024-01-10 | precompute all leaderboard for AE2","family":"openai","confidence":"high","role":"subject"}},{"contributor":"alpaca_eval_llama3_70b_fn","value":67.5,"contributorKind":"method","agent":{"model":"Llama-3-70B fn","evidence":"zhuang-li | 2024-05-17 | llama3 evaluator","family":"meta","confidence":"high","role":"subject"}},{"contributor":"gpt4","value":66.9,"contributorKind":"method","agent":{"model":"GPT-4","evidence":"YannDubs | 2023-07-15 | [ENH] add all previous model outputs","family":"openai","confidence":"medium","role":"subject"}},{"contributor":"alpaca_farm_greedy_gpt4","value":66.4,"contributorKind":"method","agent":{"model":"GPT-4","evidence":"YannDubs | 2023-07-15 | [ENH] add all previous model outputs","family":"openai","confidence":"low","role":"subject"}},{"contributor":"alpaca_eval_cot_gpt4_turbo_fn","value":65.7,"contributorKind":"method","agent":{"model":"GPT-4 Turbo","evidence":"YannDubs | 2024-01-12 | fix ae2 alpaca_eval_cot_gpt4_turbo_fn","family":"openai","confidence":"high","role":"subject"}},{"contributor":"humans","value":65.7,"contributorKind":"method","agent":{"evidence":"YannDubs | 2024-01-03 | [ENH] Weighted win rates","family":"human","confidence":"low","role":"subject"}},{"contributor":"claude","value":65.3,"contributorKind":"method","agent":{"model":"Claude","evidence":"YannDubs | 2023-07-12 | [ENH] add claude v2","family":"anthropic","confidence":"medium","role":"subject"}},{"contributor":"lmsys_gpt4","value":65.3,"contributorKind":"method","agent":{"model":"GPT-4","evidence":"YannDubs | 2023-09-30 | Add annotations & fix leaderboard","family":"openai","confidence":"low","role":"subject"}},{"contributor":"text_davinci_003","value":64.1,"contributorKind":"method","agent":{"model":"text-davinci-003","evidence":"gblazex | 2024-02-07 | shipping 4 more outputs to fill in gaps","family":"openai","confidence":"low","role":"subject"}},{"contributor":"longest","value":62.2,"contributorKind":"method","agent":{"evidence":"contributor: longest","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"chatgpt","value":57.3,"contributorKind":"method","agent":{"model":"ChatGPT","evidence":"YannDubs | 2023-06-22 | [ENH] add chatgpt leaderboard","family":"openai","confidence":"high","role":"subject"}}],"contested":true}],"agentStats":[],"repo":{"owner":"tatsu-lab","name":"alpaca_eval","branch":"main"},"participants":[],"stats":{"totalRecords":22,"uniqueParticipants":0,"recordsLast30d":0,"recordsLast90d":0},"lastUpdated":"2026-10-19T09:02:20.220Z"},{"id":"vbench","name":"VBench","tagline":"Benchmark and leaderboard for video generative models across quality, semantic, and faithfulness dimensions.","description":"VBench is a comprehensive benchmark suite for video generative models, decomposing video generation quality into hierarchical evaluation dimensions with dedicated prompt suites and automatic evaluation methods. Models are scored per dimension, then aggregated into Quality, Semantic, and Total scores (VBench), I2V and Quality scores (VBench-I2V), or Creativity/Commonsense/Controllability/Human Fidelity/Physics scores (VBench-2.0). It is maintained by the Vchitect team and hosted on Hugging Face, with submissions accepted via evaluation result JSON files.","url":"https://github.com/Vchitect/VBench","host":"github","status":"active","domain":"llm-eval","tags":["video-generation","text-to-video","image-to-video","evaluation","benchmark","leaderboard"],"images":["https://raw.githubusercontent.com/Vchitect/VBench/master/asset/fig_paper_teaser_vbench.jpg","https://raw.githubusercontent.com/Vchitect/VBench/master/asset/fig_extention_teaser.jpg","https://raw.githubusercontent.com/Vchitect/VBench/master/VBench-2.0/asset/fig_paper_teaser.jpg","https://raw.githubusercontent.com/Vchitect/VBench/master/asset/all-dim.jpg"],"participation":{"howToSubmit":"Run the evaluation scripts locally, pack the evaluation_results directory into a zip, optionally compute the final score with scripts/cal_final_score.py, then submit the JSON to the Hugging Face leaderboard space.","submissionFormat":"evaluation_results.zip containing per-dimension JSON results; optionally a computed total score JSON.","computeDetails":"Evaluation requires GPU (CUDA) for feature extractors such as DINO, RAFT, UMT, ViCLIP; VBench-2.0 recommends up to 8 GPUs for parallel dimension evaluation.","deadline":"rolling","prizes":"recognition only","eligibility":"Open to all video generative model developers and researchers.","cost":"free","verification":"Automatic evaluation scripts and leaderboard submission review; results are reproducible via provided code.","requirements":["Python 3.10+","PyTorch with CUDA","detectron2","ffmpeg","vbench PyPI package"],"compute":"datacenter-gpu"},"links":[{"label":"Hugging Face Leaderboard","url":"https://huggingface.co/spaces/Vchitect/VBench_Leaderboard"},{"label":"VBench Project Page","url":"https://vchitect.github.io/VBench-project/"},{"label":"VBench-2.0 Project Page","url":"https://vchitect.github.io/VBench-2.0-project/"},{"label":"VBench Paper (CVPR 2024)","url":"https://arxiv.org/abs/2311.17982"},{"label":"VBench++ Paper (TPAMI 2025)","url":"https://arxiv.org/abs/2411.13503"},{"label":"VBench-2.0 Report","url":"https://arxiv.org/abs/2503.21755"}],"organizer":{"name":"Vchitect","url":"https://vchitect.github.io/","type":"academic"},"venue":{"type":"online"},"quickstart":"pip install torch torchvision --index-url https://download.pytorch.org/whl/cu118\npip install vbench\nvbench evaluate --videos_path $VIDEO_PATH --dimension $DIMENSION\ncd evaluation_results && zip -r ../evaluation_results.zip .\npython scripts/cal_final_score.py --zip_file evaluation_results.zip --model_name your_model_name","problems":[{"group":"competitions","description":"Generate short videos (1.6-4.0s, 8-24 FPS, 16-96 frames) for 200 prompts; evaluated on temporal_quality, frame_wise_quality, and text_alignment.","id":"short-videos","name":"Short Videos Track","metricName":"score","metricDirection":"maximize","isOpen":true,"records":[],"contested":false},{"group":"competitions","description":"Generate long videos (10.0-40.0s, 8-24 FPS) for 40 prompts; evaluated on temporal_quality, frame_wise_quality, text_alignment, and subject_consistency via human evaluation.","id":"long-videos","name":"Long Videos Track","metricName":"score","metricDirection":"maximize","isOpen":true,"records":[],"contested":false},{"group":"vbench-2.0","description":"Evaluate intrinsic faithfulness across 18 dimensions grouped into Creativity, Commonsense, Controllability, Human Fidelity, and Physics scores; Total Score is the average of the five category scores.","id":"vbench-2-0","name":"VBench-2.0 (Intrinsic Faithfulness)","metricName":"score","metricDirection":"maximize","isOpen":true,"records":[],"contested":false},{"group":"vbench-i2v","description":"Evaluate image-to-video models on video-image alignment (i2v_subject, i2v_background), camera motion, and video quality dimensions; Total Score combines Quality Score and I2V Score.","id":"vbench-i2v","name":"VBench-I2V (Image-to-Video)","metricName":"score","metricDirection":"maximize","isOpen":true,"records":[],"contested":false}],"agentStats":[],"repo":{"owner":"Vchitect","name":"VBench","branch":"master"},"participants":[],"stats":{"totalRecords":0,"uniqueParticipants":0,"recordsLast30d":0,"recordsLast90d":0},"lastUpdated":"2026-10-19T09:02:20.220Z"},{"id":"cogdl","name":"CogDL Leaderboards","tagline":"Benchmark graph deep learning models across node classification, link prediction, and graph classification tasks.","description":"CogDL is a graph deep learning toolkit that maintains leaderboards for various graph tasks including unsupervised and semi-supervised node classification, link prediction, knowledge graph completion, and graph classification. Results are reported as Micro-F1, accuracy, ROC AUC, or MRR depending on the task. The leaderboards are maintained by the CogDL team (Tsinghua, ZJU, DAMO Academy, ZHIPU.AI) and are updated via pull requests to the repository.","url":"https://github.com/THUDM/CogDL","host":"github","status":"active","domain":"ml-research","tags":["graph-neural-networks","node-classification","link-prediction","graph-classification","knowledge-graph","benchmark"],"images":[],"participation":{"howToSubmit":"Open a pull request to the master branch adding your model implementation and results to the leaderboard.","submissionFormat":"Model implementation in cogdl/models, example script in examples/, and updated results in results.md.","computeDetails":"Most experiments can run on a single consumer GPU; some large-scale tasks may require more.","deadline":"rolling","verification":"Human review of pull requests and unit tests.","requirements":["Python 3.7+","PyTorch 1.7.1+"],"compute":"consumer-gpu"},"links":[{"label":"Homepage","url":"https://cogdl.ai/"},{"label":"Paper","url":"https://arxiv.org/abs/2103.00959"},{"label":"Documentation","url":"https://cogdl.readthedocs.io/"},{"label":"Discussion Forum","url":"https://discuss.cogdl.ai/"}],"organizer":{"name":"CogDL Team","url":"https://cogdl.ai/","type":"academic"},"venue":{"type":"online"},"quickstart":"pip install cogdl\npython scripts/train.py --dataset cora --model gcn gat --seed 0 1 2 3 4","problems":[{"group":"graph-classification","description":"Accuracy on MUTAG dataset.","metricUnit":"%","id":"graph-classification-mutag","name":"Graph Classification - MUTAG","metricName":"Accuracy","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"GIN","description":"GIN (Xu et al, ICLR'19)","value":92.06,"contributorKind":"method","isCurrentBest":true},{"contributor":"Infograph","description":"Infograph (Sun et al, ICLR'20)","value":88.95,"contributorKind":"method"},{"contributor":"DiffPool","description":"DiffPool (Ying et al, NeuIPS'18)","value":85.18,"contributorKind":"method"},{"contributor":"SortPool","description":"SortPool (Zhang et al, AAAI'18)","value":87.25,"contributorKind":"method"},{"contributor":"Graph2Vec","description":"Graph2Vec (Narayanan et al, CoRR'17)","value":83.68,"contributorKind":"method"},{"contributor":"PATCH_SAN","description":"PATCH_SAN (Niepert et al, ICML'16)","value":86.12,"contributorKind":"method"},{"contributor":"HGP-SL","description":"HGP-SL (Zhang et al, AAAI'20)","value":81.93,"contributorKind":"method"},{"contributor":"DGCNN","description":"DGCNN (Wang et al, ACM Transactions on Graphics'17)","value":83.33,"contributorKind":"method"},{"contributor":"SAGPool","description":"SAGPool (J. Lee, ICML'19)","value":71.73,"contributorKind":"method"},{"contributor":"DGK","description":"DGK (Yanardag et al, KDD'15)","value":85.58,"contributorKind":"method"}],"contested":true},{"group":"graph-classification","description":"Accuracy on IMDB-B dataset.","metricUnit":"%","id":"graph-classification-imdb-b","name":"Graph Classification - IMDB-B","metricName":"Accuracy","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"GIN","description":"GIN (Xu et al, ICLR'19)","value":76.1,"contributorKind":"method","isCurrentBest":true},{"contributor":"Infograph","description":"Infograph (Sun et al, ICLR'20)","value":74.5,"contributorKind":"method"},{"contributor":"DiffPool","description":"DiffPool (Ying et al, NeuIPS'18)","value":72.5,"contributorKind":"method"},{"contributor":"SortPool","description":"SortPool (Zhang et al, AAAI'18)","value":75.4,"contributorKind":"method"},{"contributor":"Graph2Vec","description":"Graph2Vec (Narayanan et al, CoRR'17)","value":73.9,"contributorKind":"method"},{"contributor":"PATCH_SAN","description":"PATCH_SAN (Niepert et al, ICML'16)","value":76,"contributorKind":"method"},{"contributor":"HGP-SL","description":"HGP-SL (Zhang et al, AAAI'20)","value":74,"contributorKind":"method"},{"contributor":"DGCNN","description":"DGCNN (Wang et al, ACM Transactions on Graphics'17)","value":71.6,"contributorKind":"method"},{"contributor":"SAGPool","description":"SAGPool (J. Lee, ICML'19)","value":74.8,"contributorKind":"method"},{"contributor":"DGK","description":"DGK (Yanardag et al, KDD'15)","value":55,"contributorKind":"method"}],"contested":true},{"group":"graph-classification","description":"Accuracy on IMDB-M dataset.","metricUnit":"%","id":"graph-classification-imdb-m","name":"Graph Classification - IMDB-M","metricName":"Accuracy","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"GIN","description":"GIN (Xu et al, ICLR'19)","value":51.8,"contributorKind":"method"},{"contributor":"Infograph","description":"Infograph (Sun et al, ICLR'20)","value":51.33,"contributorKind":"method"},{"contributor":"DiffPool","description":"DiffPool (Ying et al, NeuIPS'18)","value":50.5,"contributorKind":"method"},{"contributor":"SortPool","description":"SortPool (Zhang et al, AAAI'18)","value":50.47,"contributorKind":"method"},{"contributor":"Graph2Vec","description":"Graph2Vec (Narayanan et al, CoRR'17)","value":52.27,"contributorKind":"method","isCurrentBest":true},{"contributor":"PATCH_SAN","description":"PATCH_SAN (Niepert et al, ICML'16)","value":46.4,"contributorKind":"method"},{"contributor":"HGP-SL","description":"HGP-SL (Zhang et al, AAAI'20)","value":49.53,"contributorKind":"method"},{"contributor":"DGCNN","description":"DGCNN (Wang et al, ACM Transactions on Graphics'17)","value":49.2,"contributorKind":"method"},{"contributor":"SAGPool","description":"SAGPool (J. Lee, ICML'19)","value":51.33,"contributorKind":"method"},{"contributor":"DGK","description":"DGK (Yanardag et al, KDD'15)","value":40.4,"contributorKind":"method"}],"contested":true},{"group":"graph-classification","description":"Accuracy on PROTEINS dataset.","metricUnit":"%","id":"graph-classification-proteins","name":"Graph Classification - PROTEINS","metricName":"Accuracy","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"GIN","description":"GIN (Xu et al, ICLR'19)","value":75.19,"contributorKind":"method"},{"contributor":"Infograph","description":"Infograph (Sun et al, ICLR'20)","value":73.93,"contributorKind":"method"},{"contributor":"DiffPool","description":"DiffPool (Ying et al, NeuIPS'18)","value":75.3,"contributorKind":"method"},{"contributor":"SortPool","description":"SortPool (Zhang et al, AAAI'18)","value":74.48,"contributorKind":"method"},{"contributor":"Graph2Vec","description":"Graph2Vec (Narayanan et al, CoRR'17)","value":73.3,"contributorKind":"method"},{"contributor":"PATCH_SAN","description":"PATCH_SAN (Niepert et al, ICML'16)","value":75.38,"contributorKind":"method","isCurrentBest":true},{"contributor":"HGP-SL","description":"HGP-SL (Zhang et al, AAAI'20)","value":73.94,"contributorKind":"method"},{"contributor":"DGCNN","description":"DGCNN (Wang et al, ACM Transactions on Graphics'17)","value":66.75,"contributorKind":"method"},{"contributor":"SAGPool","description":"SAGPool (J. Lee, ICML'19)","value":74.03,"contributorKind":"method"},{"contributor":"DGK","description":"DGK (Yanardag et al, KDD'15)","value":72.59,"contributorKind":"method"}],"contested":true},{"group":"graph-classification","description":"Accuracy on COLLAB dataset.","metricUnit":"%","id":"graph-classification-collab","name":"Graph Classification - COLLAB","metricName":"Accuracy","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"GIN","description":"GIN (Xu et al, ICLR'19)","value":79.52,"contributorKind":"method"},{"contributor":"Infograph","description":"Infograph (Sun et al, ICLR'20)","value":79.4,"contributorKind":"method"},{"contributor":"DiffPool","description":"DiffPool (Ying et al, NeuIPS'18)","value":79.27,"contributorKind":"method"},{"contributor":"SortPool","description":"SortPool (Zhang et al, AAAI'18)","value":80.07,"contributorKind":"method"},{"contributor":"Graph2Vec","description":"Graph2Vec (Narayanan et al, CoRR'17)","value":85.58,"contributorKind":"method","isCurrentBest":true},{"contributor":"PATCH_SAN","description":"PATCH_SAN (Niepert et al, ICML'16)","value":74.34,"contributorKind":"method"},{"contributor":"HGP-SL","description":"HGP-SL (Zhang et al, AAAI'20)","value":82.08,"contributorKind":"method"},{"contributor":"DGCNN","description":"DGCNN (Wang et al, ACM Transactions on Graphics'17)","value":77.45,"contributorKind":"method"}],"contested":true},{"group":"graph-classification","description":"Accuracy on PTC dataset.","metricUnit":"%","id":"graph-classification-ptc","name":"Graph Classification - PTC","metricName":"Accuracy","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"GIN","description":"GIN (Xu et al, ICLR'19)","value":67.82,"contributorKind":"method","isCurrentBest":true},{"contributor":"Infograph","description":"Infograph (Sun et al, ICLR'20)","value":60.74,"contributorKind":"method"},{"contributor":"DiffPool","description":"DiffPool (Ying et al, NeuIPS'18)","value":58,"contributorKind":"method"},{"contributor":"SortPool","description":"SortPool (Zhang et al, AAAI'18)","value":62.04,"contributorKind":"method"},{"contributor":"Graph2Vec","description":"Graph2Vec (Narayanan et al, CoRR'17)","value":54.76,"contributorKind":"method"},{"contributor":"PATCH_SAN","description":"PATCH_SAN (Niepert et al, ICML'16)","value":61.6,"contributorKind":"method"},{"contributor":"DGCNN","description":"DGCNN (Wang et al, ACM Transactions on Graphics'17)","value":56.62,"contributorKind":"method"},{"contributor":"SAGPool","description":"SAGPool (J. Lee, ICML'19)","value":59.92,"contributorKind":"method"}],"contested":true},{"group":"graph-classification","description":"Accuracy on NCI1 dataset.","metricUnit":"%","id":"graph-classification-nci1","name":"Graph Classification - NCI1","metricName":"Accuracy","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"GIN","description":"GIN (Xu et al, ICLR'19)","value":81.66,"contributorKind":"method","isCurrentBest":true},{"contributor":"Infograph","description":"Infograph (Sun et al, ICLR'20)","value":76.64,"contributorKind":"method"},{"contributor":"DiffPool","description":"DiffPool (Ying et al, NeuIPS'18)","value":69.09,"contributorKind":"method"},{"contributor":"SortPool","description":"SortPool (Zhang et al, AAAI'18)","value":73.99,"contributorKind":"method"},{"contributor":"Graph2Vec","description":"Graph2Vec (Narayanan et al, CoRR'17)","value":71.85,"contributorKind":"method"},{"contributor":"PATCH_SAN","description":"PATCH_SAN (Niepert et al, ICML'16)","value":69.82,"contributorKind":"method"},{"contributor":"DGCNN","description":"DGCNN (Wang et al, ACM Transactions on Graphics'17)","value":65.96,"contributorKind":"method"},{"contributor":"SAGPool","description":"SAGPool (J. Lee, ICML'19)","value":72.87,"contributorKind":"method"}],"contested":true},{"group":"graph-classification","description":"Accuracy on REDDIT-B dataset.","metricUnit":"%","id":"graph-classification-reddit-b","name":"Graph Classification - REDDIT-B","metricName":"Accuracy","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"GIN","description":"GIN (Xu et al, ICLR'19)","value":83.1,"contributorKind":"method"},{"contributor":"Infograph","description":"Infograph (Sun et al, ICLR'20)","value":76.55,"contributorKind":"method"},{"contributor":"DiffPool","description":"DiffPool (Ying et al, NeuIPS'18)","value":81.2,"contributorKind":"method"},{"contributor":"SortPool","description":"SortPool (Zhang et al, AAAI'18)","value":78.15,"contributorKind":"method"},{"contributor":"Graph2Vec","description":"Graph2Vec (Narayanan et al, CoRR'17)","value":91.77,"contributorKind":"method","isCurrentBest":true},{"contributor":"PATCH_SAN","description":"PATCH_SAN (Niepert et al, ICML'16)","value":60.61,"contributorKind":"method"},{"contributor":"DGCNN","description":"DGCNN (Wang et al, ACM Transactions on Graphics'17)","value":86.2,"contributorKind":"method"},{"contributor":"SAGPool","description":"SAGPool (J. Lee, ICML'19)","value":89.21,"contributorKind":"method"}],"contested":true},{"group":"knowledge-graph-completion","description":"Mean Reciprocal Rank (MRR).","metricUnit":"%","id":"knowledge-graph-completion-fb15k-237","name":"Knowledge Graph Completion - FB15k-237","metricName":"MRR","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"RotatE","description":"RotatE (Sun et al, ICLR'19)","value":31.1,"contributorKind":"method","isCurrentBest":true},{"contributor":"ComplEx","description":"ComplEx (Trouillon et al, ICML'18)","value":28.85,"contributorKind":"method"},{"contributor":"TransE","description":"TransE (Bordes et al, NIPS'13)","value":30.5,"contributorKind":"method"},{"contributor":"DistMult","description":"DistMult (Yang et al, ICLR'15)","value":24.93,"contributorKind":"method"},{"contributor":"CompGCN","description":"CompGCN (Vashishth et al, ICLR'20)","value":21.94,"contributorKind":"method"}],"contested":true},{"group":"knowledge-graph-completion","description":"Mean Reciprocal Rank (MRR).","metricUnit":"%","id":"knowledge-graph-completion-wn18","name":"Knowledge Graph Completion - WN18","metricName":"MRR","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"RotatE","description":"RotatE (Sun et al, ICLR'19)","value":93.99,"contributorKind":"method","isCurrentBest":true},{"contributor":"ComplEx","description":"ComplEx (Trouillon et al, ICML'18)","value":86.4,"contributorKind":"method"},{"contributor":"TransE","description":"TransE (Bordes et al, NIPS'13)","value":71.55,"contributorKind":"method"},{"contributor":"DistMult","description":"DistMult (Yang et al, ICLR'15)","value":78.77,"contributorKind":"method"},{"contributor":"CompGCN","description":"CompGCN (Vashishth et al, ICLR'20)","value":39.48,"contributorKind":"method"}],"contested":true},{"group":"knowledge-graph-completion","description":"Mean Reciprocal Rank (MRR).","metricUnit":"%","id":"knowledge-graph-completion-wn18rr","name":"Knowledge Graph Completion - WN18RR","metricName":"MRR","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"RotatE","description":"RotatE (Sun et al, ICLR'19)","value":46.05,"contributorKind":"method","isCurrentBest":true},{"contributor":"ComplEx","description":"ComplEx (Trouillon et al, ICML'18)","value":40.77,"contributorKind":"method"},{"contributor":"TransE","description":"TransE (Bordes et al, NIPS'13)","value":21.85,"contributorKind":"method"},{"contributor":"DistMult","description":"DistMult (Yang et al, ICLR'15)","value":41.64,"contributorKind":"method"},{"contributor":"CompGCN","description":"CompGCN (Vashishth et al, ICLR'20)","value":44.8,"contributorKind":"method"}],"contested":true},{"group":"link-prediction","description":"ROC AUC while removing 10% of edges.","metricUnit":"%","id":"link-prediction-ppi","name":"Link Prediction - PPI","metricName":"ROC AUC","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"ProNE","description":"ProNE (Zhang et al, IJCAI'19)","value":79.93,"contributorKind":"method"},{"contributor":"NetMF","description":"NetMF (Qiu et al, WSDM'18)","value":79.04,"contributorKind":"method"},{"contributor":"Hope","description":"Hope (Ou et al, KDD'16)","value":80.21,"contributorKind":"method","isCurrentBest":true},{"contributor":"LINE","description":"LINE (Tang et al, WWW'15)","value":73.75,"contributorKind":"method"},{"contributor":"Node2vec","description":"Node2vec (Grover et al, KDD'16)","value":70.19,"contributorKind":"method"},{"contributor":"NetSMF","description":"NetSMF (Qiu et at, WWW'19)","value":68.64,"contributorKind":"method"},{"contributor":"DeepWalk","description":"DeepWalk (Perozzi et al, KDD'14)","value":69.65,"contributorKind":"method"},{"contributor":"SDNE","description":"SDNE (Wang et al, KDD'16)","value":54.87,"contributorKind":"method"}],"contested":true},{"group":"link-prediction","description":"ROC AUC while removing 10% of edges.","metricUnit":"%","id":"link-prediction-wikipedia","name":"Link Prediction - Wikipedia","metricName":"ROC AUC","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"ProNE","description":"ProNE (Zhang et al, IJCAI'19)","value":82.74,"contributorKind":"method","isCurrentBest":true},{"contributor":"NetMF","description":"NetMF (Qiu et al, WSDM'18)","value":73.24,"contributorKind":"method"},{"contributor":"Hope","description":"Hope (Ou et al, KDD'16)","value":68.89,"contributorKind":"method"},{"contributor":"LINE","description":"LINE (Tang et al, WWW'15)","value":66.51,"contributorKind":"method"},{"contributor":"Node2vec","description":"Node2vec (Grover et al, KDD'16)","value":66.6,"contributorKind":"method"},{"contributor":"NetSMF","description":"NetSMF (Qiu et at, WWW'19)","value":67.52,"contributorKind":"method"},{"contributor":"DeepWalk","description":"DeepWalk (Perozzi et al, KDD'14)","value":65.93,"contributorKind":"method"},{"contributor":"SDNE","description":"SDNE (Wang et al, KDD'16)","value":60.72,"contributorKind":"method"}],"contested":true},{"group":"multiplex-link-prediction","description":"ROC AUC while removing 15% of edges.","metricUnit":"%","id":"multiplex-link-prediction-amazon","name":"Multiplex Link Prediction - Amazon","metricName":"ROC AUC","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"GATNE","description":"GATNE (Cen et al, KDD'19)","value":97.44,"contributorKind":"method"},{"contributor":"NetMF","description":"NetMF (Qiu et al, WSDM'18)","value":97.72,"contributorKind":"method","isCurrentBest":true},{"contributor":"ProNE","description":"ProNE (Zhang et al, IJCAI'19)","value":96.51,"contributorKind":"method"},{"contributor":"Node2vec","description":"Node2vec (Grover et al, KDD'16)","value":86.86,"contributorKind":"method"},{"contributor":"DeepWalk","description":"DeepWalk (Perozzi et al, KDD'14)","value":92.54,"contributorKind":"method"},{"contributor":"LINE","description":"LINE (Tang et al, WWW'15)","value":92.56,"contributorKind":"method"},{"contributor":"Hope","description":"Hope (Ou et al, KDD'16)","value":94.39,"contributorKind":"method"},{"contributor":"GraRep","description":"GraRep (Cao et al, CIKM'15)","value":83.88,"contributorKind":"method"}],"contested":true},{"group":"multiplex-link-prediction","description":"ROC AUC while removing 15% of edges.","metricUnit":"%","id":"multiplex-link-prediction-youtube","name":"Multiplex Link Prediction - YouTube","metricName":"ROC AUC","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"GATNE","description":"GATNE (Cen et al, KDD'19)","value":84.61,"contributorKind":"method","isCurrentBest":true},{"contributor":"NetMF","description":"NetMF (Qiu et al, WSDM'18)","value":82.53,"contributorKind":"method"},{"contributor":"ProNE","description":"ProNE (Zhang et al, IJCAI'19)","value":78.96,"contributorKind":"method"},{"contributor":"Node2vec","description":"Node2vec (Grover et al, KDD'16)","value":74.01,"contributorKind":"method"},{"contributor":"DeepWalk","description":"DeepWalk (Perozzi et al, KDD'14)","value":74.31,"contributorKind":"method"},{"contributor":"LINE","description":"LINE (Tang et al, WWW'15)","value":73.4,"contributorKind":"method"},{"contributor":"Hope","description":"Hope (Ou et al, KDD'16)","value":74.66,"contributorKind":"method"},{"contributor":"GraRep","description":"GraRep (Cao et al, CIKM'15)","value":71.37,"contributorKind":"method"}],"contested":true},{"group":"multiplex-link-prediction","description":"ROC AUC while removing 15% of edges.","metricUnit":"%","id":"multiplex-link-prediction-twitter","name":"Multiplex Link Prediction - Twitter","metricName":"ROC AUC","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"GATNE","description":"GATNE (Cen et al, KDD'19)","value":92.3,"contributorKind":"method","isCurrentBest":true},{"contributor":"NetMF","description":"NetMF (Qiu et al, WSDM'18)","value":73.75,"contributorKind":"method"},{"contributor":"ProNE","description":"ProNE (Zhang et al, IJCAI'19)","value":81.32,"contributorKind":"method"},{"contributor":"Node2vec","description":"Node2vec (Grover et al, KDD'16)","value":78.3,"contributorKind":"method"},{"contributor":"DeepWalk","description":"DeepWalk (Perozzi et al, KDD'14)","value":60.29,"contributorKind":"method"},{"contributor":"LINE","description":"LINE (Tang et al, WWW'15)","value":60.36,"contributorKind":"method"},{"contributor":"Hope","description":"Hope (Ou et al, KDD'16)","value":70.61,"contributorKind":"method"},{"contributor":"GraRep","description":"GraRep (Cao et al, CIKM'15)","value":49.64,"contributorKind":"method"}],"contested":true},{"group":"multiplex-node-classification","description":"Macro F1 score under the setting and datasets of GTN.","metricUnit":"%","id":"multiplex-node-classification-dblp","name":"Multiplex Node Classification - DBLP","metricName":"Macro F1","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"Simple-HGN","description":"Simple-HGN (Lv and Ding et al, KDD'21)","value":95.09,"contributorKind":"method","isCurrentBest":true,"agent":{"model":"Simple-HGN","evidence":"upload Simple-HGN to cogdl example and leaderboard (#233)","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"GTN","description":"GTN (Yun et al, NeurIPS'19)","value":92.03,"contributorKind":"method"},{"contributor":"HAN","description":"HAN (Xiao et al, WWW'19)","value":91.21,"contributorKind":"method"},{"contributor":"GCC","description":"GCC (Qiu et al, KDD'20)","value":79.42,"contributorKind":"method","agent":{"model":"GCC","evidence":"Update GCC result (#409)","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"PTE","description":"PTE (Tang et al, KDD'15)","value":78.65,"contributorKind":"method"},{"contributor":"Metapath2vec","description":"Metapath2vec (Dong et al, KDD'17)","value":75.18,"contributorKind":"method"},{"contributor":"Hin2vec","description":"Hin2vec (Fu et al, CIKM'17)","value":74.31,"contributorKind":"method"}],"contested":true},{"group":"multiplex-node-classification","description":"Macro F1 score under the setting and datasets of GTN.","metricUnit":"%","id":"multiplex-node-classification-acm","name":"Multiplex Node Classification - ACM","metricName":"Macro F1","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"Simple-HGN","description":"Simple-HGN (Lv and Ding et al, KDD'21)","value":92.57,"contributorKind":"method","isCurrentBest":true,"agent":{"model":"Simple-HGN","evidence":"upload Simple-HGN to cogdl example and leaderboard (#233)","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"GTN","description":"GTN (Yun et al, NeurIPS'19)","value":90.85,"contributorKind":"method"},{"contributor":"HAN","description":"HAN (Xiao et al, WWW'19)","value":87.25,"contributorKind":"method"},{"contributor":"GCC","description":"GCC (Qiu et al, KDD'20)","value":86.82,"contributorKind":"method","agent":{"model":"GCC","evidence":"Update GCC result (#409)","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"PTE","description":"PTE (Tang et al, KDD'15)","value":87.44,"contributorKind":"method"},{"contributor":"Metapath2vec","description":"Metapath2vec (Dong et al, KDD'17)","value":88.79,"contributorKind":"method"},{"contributor":"Hin2vec","description":"Hin2vec (Fu et al, CIKM'17)","value":84.66,"contributorKind":"method"}],"contested":true},{"group":"multiplex-node-classification","description":"Macro F1 score under the setting and datasets of GTN.","metricUnit":"%","id":"multiplex-node-classification-imdb","name":"Multiplex Node Classification - IMDB","metricName":"Macro F1","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"Simple-HGN","description":"Simple-HGN (Lv and Ding et al, KDD'21)","value":58.61,"contributorKind":"method","isCurrentBest":true,"agent":{"model":"Simple-HGN","evidence":"upload Simple-HGN to cogdl example and leaderboard (#233)","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"GTN","description":"GTN (Yun et al, NeurIPS'19)","value":57.53,"contributorKind":"method"},{"contributor":"HAN","description":"HAN (Xiao et al, WWW'19)","value":53.94,"contributorKind":"method"},{"contributor":"GCC","description":"GCC (Qiu et al, KDD'20)","value":55.86,"contributorKind":"method","agent":{"model":"GCC","evidence":"Update GCC result (#409)","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"PTE","description":"PTE (Tang et al, KDD'15)","value":48.91,"contributorKind":"method"},{"contributor":"Metapath2vec","description":"Metapath2vec (Dong et al, KDD'17)","value":43.1,"contributorKind":"method"},{"contributor":"Hin2vec","description":"Hin2vec (Fu et al, CIKM'17)","value":44.04,"contributorKind":"method"}],"contested":true},{"group":"semi-supervised-node-classification","description":"Accuracy under transductive setting.","metricUnit":"%","id":"semi-supervised-node-classification-cora","name":"Semi-Supervised Node Classification - Cora","metricName":"Accuracy","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"Grand","description":"Grand (Feng et al., NIPS'20)","value":84.8,"contributorKind":"method"},{"contributor":"GCNII","description":"GCNII (Chen et al., ICML'20)","value":85.1,"contributorKind":"method","isCurrentBest":true},{"contributor":"DR-GAT","description":"DR-GAT (Zou et al., 2019)","value":83.6,"contributorKind":"method"},{"contributor":"MVGRL","description":"MVGRL (Hassani et al., KDD'20)","value":83.6,"contributorKind":"method"},{"contributor":"APPNP","description":"APPNP (Klicpera et al., ICLR'19)","value":84.3,"contributorKind":"method"},{"contributor":"Graph U-Net","description":"Graph U-Net (Gao et al., 2019)","value":83.3,"contributorKind":"method"},{"contributor":"GAT","description":"GAT (Veličković et al., ICLR'18)","value":82.9,"contributorKind":"method"},{"contributor":"GDC_GCN","description":"GDC_GCN (Klicpera et al., NeurIPS'19)","value":82.5,"contributorKind":"method"},{"contributor":"DropEdge","description":"DropEdge (Rong et al., ICLR'20)","value":82.1,"contributorKind":"method","agent":{"model":"DropEdge","evidence":"Adding DropEdge GCN model (#88)","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"GCN","description":"GCN (Kipf et al., ICLR'17)","value":82.3,"contributorKind":"method"},{"contributor":"DGI","description":"DGI (Veličković et al., ICLR'19)","value":82,"contributorKind":"method"},{"contributor":"JK-net","description":"JK-net (Xu et al., ICML'18)","value":81.8,"contributorKind":"method","agent":{"model":"JK-net","evidence":"Implement JKNet and its cora dataset parsing class (#74)","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"GraphSAGE","description":"GraphSAGE (Hamilton et al., NeurIPS'17)","value":80.1,"contributorKind":"method"},{"contributor":"GraphSAGE(unsup)","description":"GraphSAGE(unsup) (Hamilton et al., NeurIPS'17)","value":78.2,"contributorKind":"method"},{"contributor":"Chebyshev","description":"Chebyshev (Defferrard et al., NeurIPS'16)","value":79,"contributorKind":"method"},{"contributor":"MixHop","description":"MixHop (Abu-El-Haija et al., ICML'19)","value":81.9,"contributorKind":"method"}],"contested":true},{"group":"semi-supervised-node-classification","description":"Accuracy under transductive setting.","metricUnit":"%","id":"semi-supervised-node-classification-citeseer","name":"Semi-Supervised Node Classification - Citeseer","metricName":"Accuracy","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"Grand","description":"Grand (Feng et al., NIPS'20)","value":75.1,"contributorKind":"method","isCurrentBest":true},{"contributor":"GCNII","description":"GCNII (Chen et al., ICML'20)","value":71.3,"contributorKind":"method"},{"contributor":"DR-GAT","description":"DR-GAT (Zou et al., 2019)","value":72.8,"contributorKind":"method"},{"contributor":"MVGRL","description":"MVGRL (Hassani et al., KDD'20)","value":73,"contributorKind":"method"},{"contributor":"APPNP","description":"APPNP (Klicpera et al., ICLR'19)","value":72,"contributorKind":"method"},{"contributor":"Graph U-Net","description":"Graph U-Net (Gao et al., 2019)","value":71.2,"contributorKind":"method"},{"contributor":"GAT","description":"GAT (Veličković et al., ICLR'18)","value":71,"contributorKind":"method"},{"contributor":"GDC_GCN","description":"GDC_GCN (Klicpera et al., NeurIPS'19)","value":71.2,"contributorKind":"method"},{"contributor":"DropEdge","description":"DropEdge (Rong et al., ICLR'20)","value":72.1,"contributorKind":"method","agent":{"model":"DropEdge","evidence":"Adding DropEdge GCN model (#88)","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"GCN","description":"GCN (Kipf et al., ICLR'17)","value":71.4,"contributorKind":"method"},{"contributor":"DGI","description":"DGI (Veličković et al., ICLR'19)","value":71.2,"contributorKind":"method"},{"contributor":"JK-net","description":"JK-net (Xu et al., ICML'18)","value":69.5,"contributorKind":"method","agent":{"model":"JK-net","evidence":"Implement JKNet and its cora dataset parsing class (#74)","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"GraphSAGE","description":"GraphSAGE (Hamilton et al., NeurIPS'17)","value":66.2,"contributorKind":"method"},{"contributor":"GraphSAGE(unsup)","description":"GraphSAGE(unsup) (Hamilton et al., NeurIPS'17)","value":65.8,"contributorKind":"method"},{"contributor":"Chebyshev","description":"Chebyshev (Defferrard et al., NeurIPS'16)","value":69.8,"contributorKind":"method"},{"contributor":"MixHop","description":"MixHop (Abu-El-Haija et al., ICML'19)","value":71.4,"contributorKind":"method"}],"contested":true},{"group":"semi-supervised-node-classification","description":"Accuracy under transductive setting.","metricUnit":"%","id":"semi-supervised-node-classification-pubmed","name":"Semi-Supervised Node Classification - Pubmed","metricName":"Accuracy","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"Grand","description":"Grand (Feng et al., NIPS'20)","value":82.4,"contributorKind":"method","isCurrentBest":true},{"contributor":"GCNII","description":"GCNII (Chen et al., ICML'20)","value":80.2,"contributorKind":"method"},{"contributor":"DR-GAT","description":"DR-GAT (Zou et al., 2019)","value":79.1,"contributorKind":"method"},{"contributor":"MVGRL","description":"MVGRL (Hassani et al., KDD'20)","value":80.1,"contributorKind":"method"},{"contributor":"APPNP","description":"APPNP (Klicpera et al., ICLR'19)","value":80,"contributorKind":"method"},{"contributor":"Graph U-Net","description":"Graph U-Net (Gao et al., 2019)","value":79,"contributorKind":"method"},{"contributor":"GAT","description":"GAT (Veličković et al., ICLR'18)","value":78.9,"contributorKind":"method"},{"contributor":"GDC_GCN","description":"GDC_GCN (Klicpera et al., NeurIPS'19)","value":79.8,"contributorKind":"method"},{"contributor":"DropEdge","description":"DropEdge (Rong et al., ICLR'20)","value":79.7,"contributorKind":"method","agent":{"model":"DropEdge","evidence":"Adding DropEdge GCN model (#88)","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"GCN","description":"GCN (Kipf et al., ICLR'17)","value":79.5,"contributorKind":"method"},{"contributor":"DGI","description":"DGI (Veličković et al., ICLR'19)","value":76.5,"contributorKind":"method"},{"contributor":"JK-net","description":"JK-net (Xu et al., ICML'18)","value":77.7,"contributorKind":"method","agent":{"model":"JK-net","evidence":"Implement JKNet and its cora dataset parsing class (#74)","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"GraphSAGE","description":"GraphSAGE (Hamilton et al., NeurIPS'17)","value":77.2,"contributorKind":"method"},{"contributor":"GraphSAGE(unsup)","description":"GraphSAGE(unsup) (Hamilton et al., NeurIPS'17)","value":78.2,"contributorKind":"method"},{"contributor":"Chebyshev","description":"Chebyshev (Defferrard et al., NeurIPS'16)","value":68.6,"contributorKind":"method"},{"contributor":"MixHop","description":"MixHop (Abu-El-Haija et al., ICML'19)","value":80.8,"contributorKind":"method"}],"contested":true},{"group":"unsupervised-node-classification","description":"Micro-F1 score with 90% labels as training data in L2 normalization logistic regression.","metricUnit":"%","id":"unsupervised-node-classification-ppi","name":"Unsupervised Multi-label Node Classification - PPI","metricName":"Micro-F1","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"NetMF","description":"NetMF (Qiu et al, WSDM'18)","value":23.73,"contributorKind":"method"},{"contributor":"ProNE","description":"ProNE (Zhang et al, IJCAI'19)","value":24.6,"contributorKind":"method","isCurrentBest":true},{"contributor":"NetSMF","description":"NetSMF (Qiu et at, WWW'19)","value":23.88,"contributorKind":"method"},{"contributor":"Node2vec","description":"Node2vec (Grover et al, KDD'16)","value":20.67,"contributorKind":"method"},{"contributor":"LINE","description":"LINE (Tang et al, WWW'15)","value":21.82,"contributorKind":"method"},{"contributor":"DeepWalk","description":"DeepWalk (Perozzi et al, KDD'14)","value":20.74,"contributorKind":"method"},{"contributor":"Spectral","description":"Spectral (Tang et al, Data Min Knowl Disc (2011))","value":22.48,"contributorKind":"method"},{"contributor":"Hope","description":"Hope (Ou et al, KDD'16)","value":21.43,"contributorKind":"method"},{"contributor":"GraRep","description":"GraRep (Cao et al, CIKM'15)","value":20.6,"contributorKind":"method"}],"contested":true},{"group":"unsupervised-node-classification","description":"Micro-F1 score with 90% labels as training data in L2 normalization logistic regression.","metricUnit":"%","id":"unsupervised-node-classification-wikipedia","name":"Unsupervised Multi-label Node Classification - Wikipedia","metricName":"Micro-F1","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"NetMF","description":"NetMF (Qiu et al, WSDM'18)","value":57.42,"contributorKind":"method","isCurrentBest":true},{"contributor":"ProNE","description":"ProNE (Zhang et al, IJCAI'19)","value":56.06,"contributorKind":"method"},{"contributor":"NetSMF","description":"NetSMF (Qiu et at, WWW'19)","value":53.81,"contributorKind":"method"},{"contributor":"Node2vec","description":"Node2vec (Grover et al, KDD'16)","value":54.59,"contributorKind":"method"},{"contributor":"LINE","description":"LINE (Tang et al, WWW'15)","value":52.46,"contributorKind":"method"},{"contributor":"DeepWalk","description":"DeepWalk (Perozzi et al, KDD'14)","value":49.53,"contributorKind":"method"},{"contributor":"Spectral","description":"Spectral (Tang et al, Data Min Knowl Disc (2011))","value":49.35,"contributorKind":"method"},{"contributor":"Hope","description":"Hope (Ou et al, KDD'16)","value":54.04,"contributorKind":"method"},{"contributor":"GraRep","description":"GraRep (Cao et al, CIKM'15)","value":54.37,"contributorKind":"method"}],"contested":true},{"group":"unsupervised-node-classification","description":"Micro-F1 score with 90% labels as training data in L2 normalization logistic regression.","metricUnit":"%","id":"unsupervised-node-classification-blogcatalog","name":"Unsupervised Multi-label Node Classification - Blogcatalog","metricName":"Micro-F1","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"NetMF","description":"NetMF (Qiu et al, WSDM'18)","value":42.47,"contributorKind":"method","isCurrentBest":true},{"contributor":"ProNE","description":"ProNE (Zhang et al, IJCAI'19)","value":41.14,"contributorKind":"method"},{"contributor":"NetSMF","description":"NetSMF (Qiu et at, WWW'19)","value":40.62,"contributorKind":"method"},{"contributor":"Node2vec","description":"Node2vec (Grover et al, KDD'16)","value":40.16,"contributorKind":"method"},{"contributor":"LINE","description":"LINE (Tang et al, WWW'15)","value":38.06,"contributorKind":"method"},{"contributor":"DeepWalk","description":"DeepWalk (Perozzi et al, KDD'14)","value":40.48,"contributorKind":"method"},{"contributor":"Spectral","description":"Spectral (Tang et al, Data Min Knowl Disc (2011))","value":41.41,"contributorKind":"method"},{"contributor":"Hope","description":"Hope (Ou et al, KDD'16)","value":33.99,"contributorKind":"method"},{"contributor":"GraRep","description":"GraRep (Cao et al, CIKM'15)","value":33.48,"contributorKind":"method"}],"contested":true},{"group":"unsupervised-node-classification","description":"Micro-F1 score with 90% labels as training data in L2 normalization logistic regression.","metricUnit":"%","id":"unsupervised-node-classification-dblp","name":"Unsupervised Multi-label Node Classification - DBLP","metricName":"Micro-F1","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"NetMF","description":"NetMF (Qiu et al, WSDM'18)","value":56.72,"contributorKind":"method"},{"contributor":"ProNE","description":"ProNE (Zhang et al, IJCAI'19)","value":56.85,"contributorKind":"method"},{"contributor":"NetSMF","description":"NetSMF (Qiu et at, WWW'19)","value":59.76,"contributorKind":"method","isCurrentBest":true},{"contributor":"Node2vec","description":"Node2vec (Grover et al, KDD'16)","value":57.36,"contributorKind":"method"},{"contributor":"LINE","description":"LINE (Tang et al, WWW'15)","value":49.78,"contributorKind":"method"},{"contributor":"DeepWalk","description":"DeepWalk (Perozzi et al, KDD'14)","value":57.54,"contributorKind":"method"},{"contributor":"Spectral","description":"Spectral (Tang et al, Data Min Knowl Disc (2011))","value":43.68,"contributorKind":"method"},{"contributor":"Hope","description":"Hope (Ou et al, KDD'16)","value":56.15,"contributorKind":"method"},{"contributor":"GraRep","description":"GraRep (Cao et al, CIKM'15)","value":52.76,"contributorKind":"method"}],"contested":true},{"group":"unsupervised-node-classification","description":"Micro-F1 score with 90% labels as training data in L2 normalization logistic regression.","metricUnit":"%","id":"unsupervised-node-classification-flickr","name":"Unsupervised Multi-label Node Classification - Flickr","metricName":"Micro-F1","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"NetMF","description":"NetMF (Qiu et al, WSDM'18)","value":36.27,"contributorKind":"method"},{"contributor":"ProNE","description":"ProNE (Zhang et al, IJCAI'19)","value":36.56,"contributorKind":"method","isCurrentBest":true},{"contributor":"NetSMF","description":"NetSMF (Qiu et at, WWW'19)","value":35.49,"contributorKind":"method"},{"contributor":"Node2vec","description":"Node2vec (Grover et al, KDD'16)","value":36.13,"contributorKind":"method"},{"contributor":"LINE","description":"LINE (Tang et al, WWW'15)","value":31.61,"contributorKind":"method"},{"contributor":"DeepWalk","description":"DeepWalk (Perozzi et al, KDD'14)","value":36.09,"contributorKind":"method"},{"contributor":"Spectral","description":"Spectral (Tang et al, Data Min Knowl Disc (2011))","value":33.09,"contributorKind":"method"},{"contributor":"Hope","description":"Hope (Ou et al, KDD'16)","value":28.97,"contributorKind":"method"},{"contributor":"GraRep","description":"GraRep (Cao et al, CIKM'15)","value":31.83,"contributorKind":"method"}],"contested":true}],"agentStats":[],"repo":{"owner":"THUDM","name":"CogDL","branch":"master"},"participants":[],"stats":{"totalRecords":241,"uniqueParticipants":0,"recordsLast30d":0,"recordsLast90d":0},"lastUpdated":"2026-10-19T09:02:20.223Z"},{"id":"mle-bench","name":"MLE-bench","tagline":"Agents solve 75 Kaggle ML engineering competitions; scored by percentage of medals won within a time budget.","description":"MLE-bench evaluates AI agents on 75 Kaggle competitions, measuring the percentage of competitions where the agent's submission earns a medal (Any Medal %). Submissions are graded against held-out test splits using the repo's grading scripts, and results are reported as mean ± SEM over at least 3 seeds. It is maintained by OpenAI and is a standard benchmark for autonomous ML engineering agents.","url":"https://github.com/openai/mle-bench","host":"github","status":"active","domain":"coding-agents","tags":["kaggle","ml-engineering","agents","benchmark","leaderboard","autonomous-agents"],"images":[],"participation":{"howToSubmit":"Open a PR to main adding grading reports under runs/<run-group>/ and register the run group in runs/run_group_experiments.csv with an experiment id; then report aggregated scores in the README leaderboard table.","submissionFormat":"Grading report JSON files per run group in runs/, plus a row in the README leaderboard table with mean ± SEM for Low/Lite, Medium, High, and All splits.","computeDetails":"Recommended: 36 vCPUs, 440GB RAM, one 24GB A10 GPU, 24-hour runtime per competition.","deadline":"rolling","rounds":"Continuous leaderboard; submissions currently paused (as of 2026-04-24) while a new fairness process is developed.","prizes":"recognition only","eligibility":"Open to any team or individual; agents must not use test-set labels.","cost":"free","verification":"Grading reports reviewed by maintainers; rule violation and plagiarism detectors available; test-set feedback submissions flagged separately.","requirements":["Python 3.10+","Docker","Kaggle API credentials","Git-LFS","uv"],"compute":"datacenter-gpu"},"links":[{"label":"Paper","url":"https://arxiv.org/abs/2410.07095"},{"label":"Lite split","url":"https://github.com/openai/mle-bench/blob/main/experiments/splits/low.txt"},{"label":"Agents","url":"https://github.com/openai/mle-bench/blob/main/agents/README.md"},{"label":"Runs","url":"https://github.com/openai/mle-bench/blob/main/runs/README.md"}],"organizer":{"name":"OpenAI","url":"https://openai.com/","type":"company"},"venue":{"type":"online"},"quickstart":"pip install -e .\nmlebench prepare --lite\nmlebench grade-sample <PATH_TO_SUBMISSION> spaceship-titanic\nuv run python experiments/aggregate_grading_reports.py --experiment-id <exp_id> --split low","problems":[{"group":"mle-bench","description":"Any Medal % on the 22-competition Low complexity (Lite) split.","metricUnit":"%","id":"low-complexity","name":"Low Complexity (Lite)","metricName":"Any Medal %","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"Famou-Agent 2.0","description":"Gemini-3-Pro-Preview, 24h","date":"2026-02-23","value":80.3,"contributorKind":"team","contributorUrl":"https://github.com/baidubce/FM-Agent","isCurrentBest":true,"agent":{"model":"Gemini-3-Pro-Preview","evidence":"Famou-Agent 2.0, powered by Gemini-3-Pro-Preview","family":"google","confidence":"high","role":"subject"}},{"contributor":"MLEvolve","description":"Gemini-3-Pro-Preview, 12h","date":"2026-02-14","value":80.3,"contributorKind":"team","contributorUrl":"https://github.com/InternScience/MLEvolve","agent":{"model":"Gemini-3-Pro-Preview","evidence":"MLEvolve 12h-budget results; Gemini-3-Pro-Preview","family":"google","confidence":"high","role":"subject"}},{"contributor":"PiEvolve","description":"Gemini-3-Pro-Preview, 24h","date":"2026-01-05","value":80.3,"contributorKind":"team","contributorUrl":"https://github.com/FractalAIResearchLabs/PiEvolve","agent":{"model":"Gemini-3-Pro-Preview","evidence":"PiEvolve results; Gemini-3-Pro-Preview","family":"google","confidence":"high","role":"subject"}},{"contributor":"CAIR MARS+","description":"Gemini-3-Pro-Preview, 24h","date":"2026-02-17","value":78.79,"contributorKind":"team","contributorUrl":"https://research.google/teams/cloud-ai-research/","agent":{"model":"Gemini-3-Pro-Preview","evidence":"CAIR MARS+; Gemini-3-Pro-Preview","family":"google","confidence":"high","role":"subject"}},{"contributor":"AIBuildAI","description":"Claude-Opus-4.6, 24h","date":"2026-03-06","value":77.27,"contributorKind":"team","contributorUrl":"https://github.com/aibuildai/AI-Build-AI","agent":{"model":"Claude-Opus-4.6","evidence":"AIBuildAI MLE-Bench Results; Claude-Opus-4.6","family":"anthropic","confidence":"high","role":"subject"}},{"contributor":"Famou-Agent 2.0","description":"Gemini-2.5-Pro, 24h","date":"2025-12-27","value":75.76,"contributorKind":"team","contributorUrl":"https://github.com/baidubce/FM-Agent","agent":{"model":"Gemini-2.5-Pro","evidence":"Famou-Agent 2.0; Gemini-2.5-Pro","family":"google","confidence":"high","role":"subject"}},{"contributor":"ML-Master 2.0","description":"Deepseek-V3.2-Speciale, 24h","date":"2025-12-16","value":75.76,"contributorKind":"team","contributorUrl":"https://github.com/sjtu-sai-agents/ML-Master","agent":{"model":"Deepseek-V3.2-Speciale","evidence":"ML-Master 2.0 results; Deepseek-V3.2-Speciale","family":"deepseek","confidence":"high","role":"subject"}},{"contributor":"CAIR MARS","description":"Gemini-3-Pro-Preview, 24h","date":"2026-01-25","value":74.24,"contributorKind":"team","contributorUrl":"https://research.google/teams/cloud-ai-research/","agent":{"model":"Gemini-3-Pro-Preview","evidence":"CAIR MARS; Gemini-3-Pro-Preview","family":"google","confidence":"high","role":"subject"}},{"contributor":"PiEvolve","description":"Gemini-3-Pro-Preview, 12h","date":"2026-01-05","value":74.24,"contributorKind":"team","contributorUrl":"https://github.com/FractalAIResearchLabs/PiEvolve","agent":{"model":"Gemini-3-Pro-Preview","evidence":"PiEvolve; Gemini-3-Pro-Preview","family":"google","confidence":"high","role":"subject"}},{"contributor":"Leeroo","description":"Gemini-3-Pro-Preview, 24h","date":"2025-12-07","value":68.18,"contributorKind":"team","contributorUrl":"https://github.com/Leeroo-AI/kapso","agent":{"model":"Gemini-3-Pro-Preview","evidence":"Leeroo results; Gemini-3-Pro-Preview","family":"google","confidence":"high","role":"subject"}},{"contributor":"CAIR MLE-STAR-Pro-1.5","description":"Gemini-2.5-Pro, 24h","date":"2025-11-25","value":68.18,"contributorKind":"team","contributorUrl":"https://research.google/teams/cloud-ai-research/","agent":{"model":"Gemini-2.5-Pro","evidence":"MLE-STAR-Pro-1.5; Gemini-2.5-Pro","family":"google","confidence":"high","role":"subject"}},{"contributor":"R&D-Agent","description":"gpt-5, 12h","date":"2025-09-26","value":68.18,"contributorKind":"team","contributorUrl":"https://github.com/microsoft/RD-Agent","agent":{"model":"gpt-5","evidence":"R&D-Agent gpt-5 results","family":"openai","confidence":"high","role":"subject"}},{"contributor":"CAIR MLE-STAR-Pro-1.0","description":"Gemini-2.5-Pro, 12h","date":"2025-11-03","value":66.67,"contributorKind":"team","contributorUrl":"https://research.google/teams/cloud-ai-research/","agent":{"model":"Gemini-2.5-Pro","evidence":"MLE-STAR-Pro-1.0; Gemini-2.5-Pro","family":"google","confidence":"high","role":"subject"}},{"contributor":"Thesis","description":"gpt-5-codex, 24h","date":"2025-11-10","value":65.15,"contributorKind":"team","contributorUrl":"https://thesislabs.ai/","agent":{"model":"gpt-5-codex","evidence":"Thesis labs submission; Model: gpt-5-codex","family":"openai","confidence":"high","role":"subject"}},{"contributor":"Operand ensemble","description":"gpt-5, 24h","date":"2025-10-06","value":63.64,"contributorKind":"team","contributorUrl":"https://operand.com/","agent":{"model":"gpt-5","evidence":"Operand MLE-Benchmark Results; gpt-5","family":"openai","confidence":"high","role":"subject"}},{"contributor":"Famou-Agent","description":"Gemini-2.5-Pro, 24h","date":"2025-10-10","value":62.12,"contributorKind":"team","contributorUrl":"https://github.com/baidubce/FM-Agent","agent":{"model":"Gemini-2.5-Pro","evidence":"FM Agent; Gemini-2.5-Pro","family":"google","confidence":"high","role":"subject"}},{"contributor":"InternAgent","description":"deepseek-r1, 12h","date":"2025-09-12","value":62.12,"contributorKind":"team","contributorUrl":"https://github.com/Alpha-Innovator/InternAgent/","agent":{"model":"deepseek-r1","evidence":"InternAgent deepseek-r1 results","family":"deepseek","confidence":"high","role":"subject"}},{"contributor":"AIRA-dojo","description":"o3, 24h","date":"2025-05-15","value":55,"contributorKind":"team","contributorUrl":"https://github.com/facebookresearch/aira-dojo/","agent":{"model":"o3","evidence":"AIRA-dojo; o3","family":"openai","confidence":"high","role":"subject"}},{"contributor":"R&D-Agent","description":"o3 + GPT-4.1, 24h","date":"2025-08-15","value":51.52,"contributorKind":"team","contributorUrl":"https://github.com/microsoft/RD-Agent","agent":{"model":"o3 + GPT-4.1","evidence":"R&D-Agent o3+gpt-4.1 results","family":"openai","confidence":"high","role":"subject"}},{"contributor":"Neo","description":"undisclosed, 36h","date":"2025-07-28","value":48.48,"contributorKind":"team","contributorUrl":"https://heyneo.so/"},{"contributor":"ML-Master","description":"deepseek-r1, 12h","date":"2025-06-17","value":48.48,"contributorKind":"team","contributorUrl":"https://github.com/zeroxleo/ML-Master","agent":{"model":"deepseek-r1","evidence":"ML-Master deepseek-r1 results","family":"deepseek","confidence":"high","role":"subject"}},{"contributor":"R&D-Agent","description":"o1-preview, 24h","date":"2025-05-14","value":48.18,"contributorKind":"team","contributorUrl":"https://github.com/microsoft/RD-Agent","agent":{"model":"o1-preview","evidence":"RD-Agent o1-preview results","family":"openai","confidence":"high","role":"subject"}},{"contributor":"AIDE","description":"o1-preview, 24h","date":"2024-10-08","value":35.91,"contributorKind":"method","contributorUrl":"https://github.com/wecoai/aideml","agent":{"model":"o1-preview","evidence":"AIDE; o1-preview","family":"openai","confidence":"high","role":"subject"}},{"contributor":"AIDE","description":"claude-3-5-sonnet-20240620, 24h","date":"2024-10-08","value":19.7,"contributorKind":"method","contributorUrl":"https://github.com/wecoai/aideml","agent":{"model":"claude-3-5-sonnet-20240620","evidence":"AIDE; claude-3-5-sonnet-20240620","family":"anthropic","confidence":"high","role":"subject"}},{"contributor":"AIDE","description":"gpt-4o-2024-08-06, 24h","date":"2024-10-08","value":18.55,"contributorKind":"method","contributorUrl":"https://github.com/wecoai/aideml","agent":{"model":"gpt-4o-2024-08-06","evidence":"AIDE; gpt-4o-2024-08-06","family":"openai","confidence":"high","role":"subject"}},{"contributor":"OpenHands","description":"gpt-4o-2024-08-06, 24h","date":"2024-10-08","value":12.12,"contributorKind":"method","agent":{"model":"gpt-4o-2024-08-06","evidence":"OpenHands; gpt-4o-2024-08-06","family":"openai","confidence":"high","role":"subject"}},{"contributor":"AIDE","description":"llama-3.1-405b-instruct, 24h","date":"2024-10-08","value":10.23,"contributorKind":"method","contributorUrl":"https://github.com/wecoai/aideml","agent":{"model":"llama-3.1-405b-instruct","evidence":"AIDE; llama-3.1-405b-instruct","family":"meta","confidence":"high","role":"subject"}},{"contributor":"MLAB","description":"gpt-4o-2024-08-06, 24h","date":"2024-10-08","value":4.55,"contributorKind":"method","agent":{"model":"gpt-4o-2024-08-06","evidence":"MLAB; gpt-4o-2024-08-06","family":"openai","confidence":"high","role":"subject"}}],"contested":true},{"group":"mle-bench","description":"Any Medal % on the Medium complexity split.","metricUnit":"%","id":"medium-complexity","name":"Medium Complexity","metricName":"Any Medal %","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"Famou-Agent 2.0","description":"Gemini-3-Pro-Preview, 24h","date":"2026-02-23","value":64.04,"contributorKind":"team","contributorUrl":"https://github.com/baidubce/FM-Agent","isCurrentBest":true,"agent":{"model":"Gemini-3-Pro-Preview","evidence":"Famou-Agent 2.0; Gemini-3-Pro-Preview","family":"google","confidence":"high","role":"subject"}},{"contributor":"AIBuildAI","description":"Claude-Opus-4.6, 24h","date":"2026-03-06","value":61.4,"contributorKind":"team","contributorUrl":"https://github.com/aibuildai/AI-Build-AI","agent":{"model":"Claude-Opus-4.6","evidence":"AIBuildAI; Claude-Opus-4.6","family":"anthropic","confidence":"high","role":"subject"}},{"contributor":"CAIR MARS+","description":"Gemini-3-Pro-Preview, 24h","date":"2026-02-17","value":60.53,"contributorKind":"team","contributorUrl":"https://research.google/teams/cloud-ai-research/","agent":{"model":"Gemini-3-Pro-Preview","evidence":"CAIR MARS+; Gemini-3-Pro-Preview","family":"google","confidence":"high","role":"subject"}},{"contributor":"PiEvolve","description":"Gemini-3-Pro-Preview, 24h","date":"2026-01-05","value":58.77,"contributorKind":"team","contributorUrl":"https://github.com/FractalAIResearchLabs/PiEvolve","agent":{"model":"Gemini-3-Pro-Preview","evidence":"PiEvolve; Gemini-3-Pro-Preview","family":"google","confidence":"high","role":"subject"}},{"contributor":"Famou-Agent 2.0","description":"Gemini-2.5-Pro, 24h","date":"2025-12-27","value":57.89,"contributorKind":"team","contributorUrl":"https://github.com/baidubce/FM-Agent","agent":{"model":"Gemini-2.5-Pro","evidence":"Famou-Agent 2.0; Gemini-2.5-Pro","family":"google","confidence":"high","role":"subject"}},{"contributor":"MLEvolve","description":"Gemini-3-Pro-Preview, 12h","date":"2026-02-14","value":57.89,"contributorKind":"team","contributorUrl":"https://github.com/InternScience/MLEvolve","agent":{"model":"Gemini-3-Pro-Preview","evidence":"MLEvolve; Gemini-3-Pro-Preview","family":"google","confidence":"high","role":"subject"}},{"contributor":"CAIR MARS","description":"Gemini-3-Pro-Preview, 24h","date":"2026-01-25","value":52.63,"contributorKind":"team","contributorUrl":"https://research.google/teams/cloud-ai-research/","agent":{"model":"Gemini-3-Pro-Preview","evidence":"CAIR MARS; Gemini-3-Pro-Preview","family":"google","confidence":"high","role":"subject"}},{"contributor":"ML-Master 2.0","description":"Deepseek-V3.2-Speciale, 24h","date":"2025-12-16","value":50.88,"contributorKind":"team","contributorUrl":"https://github.com/sjtu-sai-agents/ML-Master","agent":{"model":"Deepseek-V3.2-Speciale","evidence":"ML-Master 2.0; Deepseek-V3.2-Speciale","family":"deepseek","confidence":"high","role":"subject"}},{"contributor":"PiEvolve","description":"Gemini-3-Pro-Preview, 12h","date":"2026-01-05","value":45.61,"contributorKind":"team","contributorUrl":"https://github.com/FractalAIResearchLabs/PiEvolve","agent":{"model":"Gemini-3-Pro-Preview","evidence":"PiEvolve; Gemini-3-Pro-Preview","family":"google","confidence":"high","role":"subject"}},{"contributor":"Thesis","description":"gpt-5-codex, 24h","date":"2025-11-10","value":45.61,"contributorKind":"team","contributorUrl":"https://thesislabs.ai/","agent":{"model":"gpt-5-codex","evidence":"Thesis labs; Model: gpt-5-codex","family":"openai","confidence":"high","role":"subject"}},{"contributor":"Leeroo","description":"Gemini-3-Pro-Preview, 24h","date":"2025-12-07","value":44.74,"contributorKind":"team","contributorUrl":"https://github.com/Leeroo-AI/kapso","agent":{"model":"Gemini-3-Pro-Preview","evidence":"Leeroo; Gemini-3-Pro-Preview","family":"google","confidence":"high","role":"subject"}},{"contributor":"Famou-Agent","description":"Gemini-2.5-Pro, 24h","date":"2025-10-10","value":36.84,"contributorKind":"team","contributorUrl":"https://github.com/baidubce/FM-Agent","agent":{"model":"Gemini-2.5-Pro","evidence":"FM Agent; Gemini-2.5-Pro","family":"google","confidence":"high","role":"subject"}},{"contributor":"CAIR MLE-STAR-Pro-1.5","description":"Gemini-2.5-Pro, 24h","date":"2025-11-25","value":34.21,"contributorKind":"team","contributorUrl":"https://research.google/teams/cloud-ai-research/","agent":{"model":"Gemini-2.5-Pro","evidence":"MLE-STAR-Pro-1.5; Gemini-2.5-Pro","family":"google","confidence":"high","role":"subject"}},{"contributor":"Operand ensemble","description":"gpt-5, 24h","date":"2025-10-06","value":33.33,"contributorKind":"team","contributorUrl":"https://operand.com/","agent":{"model":"gpt-5","evidence":"Operand ensemble; gpt-5","family":"openai","confidence":"high","role":"subject"}},{"contributor":"Neo","description":"undisclosed, 36h","date":"2025-07-28","value":29.82,"contributorKind":"team","contributorUrl":"https://heyneo.so/"},{"contributor":"InternAgent","description":"deepseek-r1, 12h","date":"2025-09-12","value":26.32,"contributorKind":"team","contributorUrl":"https://github.com/Alpha-Innovator/InternAgent/","agent":{"model":"deepseek-r1","evidence":"InternAgent deepseek-r1 results","family":"deepseek","confidence":"high","role":"subject"}},{"contributor":"CAIR MLE-STAR-Pro-1.0","description":"Gemini-2.5-Pro, 12h","date":"2025-11-03","value":25.44,"contributorKind":"team","contributorUrl":"https://research.google/teams/cloud-ai-research/","agent":{"model":"Gemini-2.5-Pro","evidence":"MLE-STAR-Pro-1.0; Gemini-2.5-Pro","family":"google","confidence":"high","role":"subject"}},{"contributor":"AIRA-dojo","description":"o3, 24h","date":"2025-05-15","value":21.97,"contributorKind":"team","contributorUrl":"https://github.com/facebookresearch/aira-dojo/","agent":{"model":"o3","evidence":"AIRA-dojo; o3","family":"openai","confidence":"high","role":"subject"}},{"contributor":"R&D-Agent","description":"gpt-5, 12h","date":"2025-09-26","value":21.05,"contributorKind":"team","contributorUrl":"https://github.com/microsoft/RD-Agent","agent":{"model":"gpt-5","evidence":"R&D-Agent gpt-5 results","family":"openai","confidence":"high","role":"subject"}},{"contributor":"ML-Master","description":"deepseek-r1, 12h","date":"2025-06-17","value":20.18,"contributorKind":"team","contributorUrl":"https://github.com/zeroxleo/ML-Master","agent":{"model":"deepseek-r1","evidence":"ML-Master deepseek-r1 results","family":"deepseek","confidence":"high","role":"subject"}},{"contributor":"R&D-Agent","description":"o3 + GPT-4.1, 24h","date":"2025-08-15","value":19.3,"contributorKind":"team","contributorUrl":"https://github.com/microsoft/RD-Agent","agent":{"model":"o3 + GPT-4.1","evidence":"R&D-Agent o3+gpt-4.1 results","family":"openai","confidence":"high","role":"subject"}},{"contributor":"R&D-Agent","description":"o1-preview, 24h","date":"2025-05-14","value":8.95,"contributorKind":"team","contributorUrl":"https://github.com/microsoft/RD-Agent","agent":{"model":"o1-preview","evidence":"RD-Agent o1-preview results","family":"openai","confidence":"high","role":"subject"}},{"contributor":"AIDE","description":"o1-preview, 24h","date":"2024-10-08","value":8.45,"contributorKind":"method","contributorUrl":"https://github.com/wecoai/aideml","agent":{"model":"o1-preview","evidence":"AIDE; o1-preview","family":"openai","confidence":"high","role":"subject"}},{"contributor":"AIDE","description":"gpt-4o-2024-08-06, 24h","date":"2024-10-08","value":3.06,"contributorKind":"method","contributorUrl":"https://github.com/wecoai/aideml","agent":{"model":"gpt-4o-2024-08-06","evidence":"AIDE; gpt-4o-2024-08-06","family":"openai","confidence":"high","role":"subject"}},{"contributor":"AIDE","description":"claude-3-5-sonnet-20240620, 24h","date":"2024-10-08","value":2.63,"contributorKind":"method","contributorUrl":"https://github.com/wecoai/aideml","agent":{"model":"claude-3-5-sonnet-20240620","evidence":"AIDE; claude-3-5-sonnet-20240620","family":"anthropic","confidence":"high","role":"subject"}},{"contributor":"OpenHands","description":"gpt-4o-2024-08-06, 24h","date":"2024-10-08","value":1.75,"contributorKind":"method","agent":{"model":"gpt-4o-2024-08-06","evidence":"OpenHands; gpt-4o-2024-08-06","family":"openai","confidence":"high","role":"subject"}},{"contributor":"AIDE","description":"llama-3.1-405b-instruct, 24h","date":"2024-10-08","value":0.66,"contributorKind":"method","contributorUrl":"https://github.com/wecoai/aideml","agent":{"model":"llama-3.1-405b-instruct","evidence":"AIDE; llama-3.1-405b-instruct","family":"meta","confidence":"high","role":"subject"}},{"contributor":"MLAB","description":"gpt-4o-2024-08-06, 24h","date":"2024-10-08","value":0,"contributorKind":"method","agent":{"model":"gpt-4o-2024-08-06","evidence":"MLAB; gpt-4o-2024-08-06","family":"openai","confidence":"high","role":"subject"}}],"contested":true},{"group":"mle-bench","description":"Any Medal % on the High complexity split.","metricUnit":"%","id":"high-complexity","name":"High Complexity","metricName":"Any Medal %","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"AIBuildAI","description":"Claude-Opus-4.6, 24h","date":"2026-03-06","value":46.67,"contributorKind":"team","contributorUrl":"https://github.com/aibuildai/AI-Build-AI","isCurrentBest":true,"agent":{"model":"Claude-Opus-4.6","evidence":"AIBuildAI; Claude-Opus-4.6","family":"anthropic","confidence":"high","role":"subject"}},{"contributor":"CAIR MARS+","description":"Gemini-3-Pro-Preview, 24h","date":"2026-02-17","value":44.44,"contributorKind":"team","contributorUrl":"https://research.google/teams/cloud-ai-research/","agent":{"model":"Gemini-3-Pro-Preview","evidence":"CAIR MARS+; Gemini-3-Pro-Preview","family":"google","confidence":"high","role":"subject"}},{"contributor":"Famou-Agent 2.0","description":"Gemini-3-Pro-Preview, 24h","date":"2026-02-23","value":42.22,"contributorKind":"team","contributorUrl":"https://github.com/baidubce/FM-Agent","agent":{"model":"Gemini-3-Pro-Preview","evidence":"Famou-Agent 2.0; Gemini-3-Pro-Preview","family":"google","confidence":"high","role":"subject"}},{"contributor":"MLEvolve","description":"Gemini-3-Pro-Preview, 12h","date":"2026-02-14","value":42.22,"contributorKind":"team","contributorUrl":"https://github.com/InternScience/MLEvolve","agent":{"model":"Gemini-3-Pro-Preview","evidence":"MLEvolve; Gemini-3-Pro-Preview","family":"google","confidence":"high","role":"subject"}},{"contributor":"ML-Master 2.0","description":"Deepseek-V3.2-Speciale, 24h","date":"2025-12-16","value":42.22,"contributorKind":"team","contributorUrl":"https://github.com/sjtu-sai-agents/ML-Master","agent":{"model":"Deepseek-V3.2-Speciale","evidence":"ML-Master 2.0; Deepseek-V3.2-Speciale","family":"deepseek","confidence":"high","role":"subject"}},{"contributor":"PiEvolve","description":"Gemini-3-Pro-Preview, 24h","date":"2026-01-05","value":40,"contributorKind":"team","contributorUrl":"https://github.com/FractalAIResearchLabs/PiEvolve","agent":{"model":"Gemini-3-Pro-Preview","evidence":"PiEvolve; Gemini-3-Pro-Preview","family":"google","confidence":"high","role":"subject"}},{"contributor":"Famou-Agent 2.0","description":"Gemini-2.5-Pro, 24h","date":"2025-12-27","value":40,"contributorKind":"team","contributorUrl":"https://github.com/baidubce/FM-Agent","agent":{"model":"Gemini-2.5-Pro","evidence":"Famou-Agent 2.0; Gemini-2.5-Pro","family":"google","confidence":"high","role":"subject"}},{"contributor":"Leeroo","description":"Gemini-3-Pro-Preview, 24h","date":"2025-12-07","value":40,"contributorKind":"team","contributorUrl":"https://github.com/Leeroo-AI/kapso","agent":{"model":"Gemini-3-Pro-Preview","evidence":"Leeroo; Gemini-3-Pro-Preview","family":"google","confidence":"high","role":"subject"}},{"contributor":"CAIR MARS","description":"Gemini-3-Pro-Preview, 24h","date":"2026-01-25","value":37.78,"contributorKind":"team","contributorUrl":"https://research.google/teams/cloud-ai-research/","agent":{"model":"Gemini-3-Pro-Preview","evidence":"CAIR MARS; Gemini-3-Pro-Preview","family":"google","confidence":"high","role":"subject"}},{"contributor":"PiEvolve","description":"Gemini-3-Pro-Preview, 12h","date":"2026-01-05","value":35.55,"contributorKind":"team","contributorUrl":"https://github.com/FractalAIResearchLabs/PiEvolve","agent":{"model":"Gemini-3-Pro-Preview","evidence":"PiEvolve; Gemini-3-Pro-Preview","family":"google","confidence":"high","role":"subject"}},{"contributor":"Famou-Agent","description":"Gemini-2.5-Pro, 24h","date":"2025-10-10","value":33.33,"contributorKind":"team","contributorUrl":"https://github.com/baidubce/FM-Agent","agent":{"model":"Gemini-2.5-Pro","evidence":"FM Agent; Gemini-2.5-Pro","family":"google","confidence":"high","role":"subject"}},{"contributor":"CAIR MLE-STAR-Pro-1.5","description":"Gemini-2.5-Pro, 24h","date":"2025-11-25","value":33.33,"contributorKind":"team","contributorUrl":"https://research.google/teams/cloud-ai-research/","agent":{"model":"Gemini-2.5-Pro","evidence":"MLE-STAR-Pro-1.5; Gemini-2.5-Pro","family":"google","confidence":"high","role":"subject"}},{"contributor":"Thesis","description":"gpt-5-codex, 24h","date":"2025-11-10","value":31.11,"contributorKind":"team","contributorUrl":"https://thesislabs.ai/","agent":{"model":"gpt-5-codex","evidence":"Thesis labs; Model: gpt-5-codex","family":"openai","confidence":"high","role":"subject"}},{"contributor":"CAIR MLE-STAR-Pro-1.0","description":"Gemini-2.5-Pro, 12h","date":"2025-11-03","value":31.11,"contributorKind":"team","contributorUrl":"https://research.google/teams/cloud-ai-research/","agent":{"model":"Gemini-2.5-Pro","evidence":"MLE-STAR-Pro-1.0; Gemini-2.5-Pro","family":"google","confidence":"high","role":"subject"}},{"contributor":"R&D-Agent","description":"o3 + GPT-4.1, 24h","date":"2025-08-15","value":26.67,"contributorKind":"team","contributorUrl":"https://github.com/microsoft/RD-Agent","agent":{"model":"o3 + GPT-4.1","evidence":"R&D-Agent o3+gpt-4.1 results","family":"openai","confidence":"high","role":"subject"}},{"contributor":"InternAgent","description":"deepseek-r1, 12h","date":"2025-09-12","value":24.44,"contributorKind":"team","contributorUrl":"https://github.com/Alpha-Innovator/InternAgent/","agent":{"model":"deepseek-r1","evidence":"InternAgent deepseek-r1 results","family":"deepseek","confidence":"high","role":"subject"}},{"contributor":"Neo","description":"undisclosed, 36h","date":"2025-07-28","value":24.44,"contributorKind":"team","contributorUrl":"https://heyneo.so/"},{"contributor":"ML-Master","description":"deepseek-r1, 12h","date":"2025-06-17","value":24.44,"contributorKind":"team","contributorUrl":"https://github.com/zeroxleo/ML-Master","agent":{"model":"deepseek-r1","evidence":"ML-Master deepseek-r1 results","family":"deepseek","confidence":"high","role":"subject"}},{"contributor":"R&D-Agent","description":"gpt-5, 12h","date":"2025-09-26","value":22.22,"contributorKind":"team","contributorUrl":"https://github.com/microsoft/RD-Agent","agent":{"model":"gpt-5","evidence":"R&D-Agent gpt-5 results","family":"openai","confidence":"high","role":"subject"}},{"contributor":"AIRA-dojo","description":"o3, 24h","date":"2025-05-15","value":21.67,"contributorKind":"team","contributorUrl":"https://github.com/facebookresearch/aira-dojo/","agent":{"model":"o3","evidence":"AIRA-dojo; o3","family":"openai","confidence":"high","role":"subject"}},{"contributor":"Operand ensemble","description":"gpt-5, 24h","date":"2025-10-06","value":20,"contributorKind":"team","contributorUrl":"https://operand.com/","agent":{"model":"gpt-5","evidence":"Operand ensemble; gpt-5","family":"openai","confidence":"high","role":"subject"}},{"contributor":"R&D-Agent","description":"o1-preview, 24h","date":"2025-05-14","value":18.67,"contributorKind":"team","contributorUrl":"https://github.com/microsoft/RD-Agent","agent":{"model":"o1-preview","evidence":"RD-Agent o1-preview results","family":"openai","confidence":"high","role":"subject"}},{"contributor":"AIDE","description":"o1-preview, 24h","date":"2024-10-08","value":11.67,"contributorKind":"method","contributorUrl":"https://github.com/wecoai/aideml","agent":{"model":"o1-preview","evidence":"AIDE; o1-preview","family":"openai","confidence":"high","role":"subject"}},{"contributor":"AIDE","description":"gpt-4o-2024-08-06, 24h","date":"2024-10-08","value":8.15,"contributorKind":"method","contributorUrl":"https://github.com/wecoai/aideml","agent":{"model":"gpt-4o-2024-08-06","evidence":"AIDE; gpt-4o-2024-08-06","family":"openai","confidence":"high","role":"subject"}},{"contributor":"AIDE","description":"claude-3-5-sonnet-20240620, 24h","date":"2024-10-08","value":2.22,"contributorKind":"method","contributorUrl":"https://github.com/wecoai/aideml","agent":{"model":"claude-3-5-sonnet-20240620","evidence":"AIDE; claude-3-5-sonnet-20240620","family":"anthropic","confidence":"high","role":"subject"}},{"contributor":"OpenHands","description":"gpt-4o-2024-08-06, 24h","date":"2024-10-08","value":2.22,"contributorKind":"method","agent":{"model":"gpt-4o-2024-08-06","evidence":"OpenHands; gpt-4o-2024-08-06","family":"openai","confidence":"high","role":"subject"}},{"contributor":"AIDE","description":"llama-3.1-405b-instruct, 24h","date":"2024-10-08","value":0,"contributorKind":"method","contributorUrl":"https://github.com/wecoai/aideml","agent":{"model":"llama-3.1-405b-instruct","evidence":"AIDE; llama-3.1-405b-instruct","family":"meta","confidence":"high","role":"subject"}},{"contributor":"MLAB","description":"gpt-4o-2024-08-06, 24h","date":"2024-10-08","value":0,"contributorKind":"method","agent":{"model":"gpt-4o-2024-08-06","evidence":"MLAB; gpt-4o-2024-08-06","family":"openai","confidence":"high","role":"subject"}}],"contested":true},{"group":"mle-bench","description":"Any Medal % across all 75 competitions (split75).","metricUnit":"%","id":"all-competitions","name":"All Competitions","metricName":"Any Medal %","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"Famou-Agent 2.0","description":"Gemini-3-Pro-Preview, 24h","date":"2026-02-23","value":64.44,"contributorKind":"team","contributorUrl":"https://github.com/baidubce/FM-Agent","isCurrentBest":true,"agent":{"model":"Gemini-3-Pro-Preview","evidence":"Famou-Agent 2.0; Gemini-3-Pro-Preview","family":"google","confidence":"high","role":"subject"}},{"contributor":"AIBuildAI","description":"Claude-Opus-4.6, 24h","date":"2026-03-06","value":63.11,"contributorKind":"team","contributorUrl":"https://github.com/aibuildai/AI-Build-AI","agent":{"model":"Claude-Opus-4.6","evidence":"AIBuildAI; Claude-Opus-4.6","family":"anthropic","confidence":"high","role":"subject"}},{"contributor":"CAIR MARS+","description":"Gemini-3-Pro-Preview, 24h","date":"2026-02-17","value":62.67,"contributorKind":"team","contributorUrl":"https://research.google/teams/cloud-ai-research/","agent":{"model":"Gemini-3-Pro-Preview","evidence":"CAIR MARS+; Gemini-3-Pro-Preview","family":"google","confidence":"high","role":"subject"}},{"contributor":"MLEvolve","description":"Gemini-3-Pro-Preview, 12h","date":"2026-02-14","value":61.33,"contributorKind":"team","contributorUrl":"https://github.com/InternScience/MLEvolve","agent":{"model":"Gemini-3-Pro-Preview","evidence":"MLEvolve; Gemini-3-Pro-Preview","family":"google","confidence":"high","role":"subject"}},{"contributor":"PiEvolve","description":"Gemini-3-Pro-Preview, 24h","date":"2026-01-05","value":61.33,"contributorKind":"team","contributorUrl":"https://github.com/FractalAIResearchLabs/PiEvolve","agent":{"model":"Gemini-3-Pro-Preview","evidence":"PiEvolve; Gemini-3-Pro-Preview","family":"google","confidence":"high","role":"subject"}},{"contributor":"Famou-Agent 2.0","description":"Gemini-2.5-Pro, 24h","date":"2025-12-27","value":59.56,"contributorKind":"team","contributorUrl":"https://github.com/baidubce/FM-Agent","agent":{"model":"Gemini-2.5-Pro","evidence":"Famou-Agent 2.0; Gemini-2.5-Pro","family":"google","confidence":"high","role":"subject"}},{"contributor":"ML-Master 2.0","description":"Deepseek-V3.2-Speciale, 24h","date":"2025-12-16","value":56.44,"contributorKind":"team","contributorUrl":"https://github.com/sjtu-sai-agents/ML-Master","agent":{"model":"Deepseek-V3.2-Speciale","evidence":"ML-Master 2.0; Deepseek-V3.2-Speciale","family":"deepseek","confidence":"high","role":"subject"}},{"contributor":"CAIR MARS","description":"Gemini-3-Pro-Preview, 24h","date":"2026-01-25","value":56,"contributorKind":"team","contributorUrl":"https://research.google/teams/cloud-ai-research/","agent":{"model":"Gemini-3-Pro-Preview","evidence":"CAIR MARS; Gemini-3-Pro-Preview","family":"google","confidence":"high","role":"subject"}},{"contributor":"PiEvolve","description":"Gemini-3-Pro-Preview, 12h","date":"2026-01-05","value":52,"contributorKind":"team","contributorUrl":"https://github.com/FractalAIResearchLabs/PiEvolve","agent":{"model":"Gemini-3-Pro-Preview","evidence":"PiEvolve; Gemini-3-Pro-Preview","family":"google","confidence":"high","role":"subject"}},{"contributor":"Leeroo","description":"Gemini-3-Pro-Preview, 24h","date":"2025-12-07","value":50.67,"contributorKind":"team","contributorUrl":"https://github.com/Leeroo-AI/kapso","agent":{"model":"Gemini-3-Pro-Preview","evidence":"Leeroo; Gemini-3-Pro-Preview","family":"google","confidence":"high","role":"subject"}},{"contributor":"Thesis","description":"gpt-5-codex, 24h","date":"2025-11-10","value":48.44,"contributorKind":"team","contributorUrl":"https://thesislabs.ai/","agent":{"model":"gpt-5-codex","evidence":"Thesis labs; Model: gpt-5-codex","family":"openai","confidence":"high","role":"subject"}},{"contributor":"CAIR MLE-STAR-Pro-1.5","description":"Gemini-2.5-Pro, 24h","date":"2025-11-25","value":44,"contributorKind":"team","contributorUrl":"https://research.google/teams/cloud-ai-research/","agent":{"model":"Gemini-2.5-Pro","evidence":"MLE-STAR-Pro-1.5; Gemini-2.5-Pro","family":"google","confidence":"high","role":"subject"}},{"contributor":"Famou-Agent","description":"Gemini-2.5-Pro, 24h","date":"2025-10-10","value":43.56,"contributorKind":"team","contributorUrl":"https://github.com/baidubce/FM-Agent","agent":{"model":"Gemini-2.5-Pro","evidence":"FM Agent; Gemini-2.5-Pro","family":"google","confidence":"high","role":"subject"}},{"contributor":"Operand ensemble","description":"gpt-5, 24h","date":"2025-10-06","value":39.56,"contributorKind":"team","contributorUrl":"https://operand.com/","agent":{"model":"gpt-5","evidence":"Operand ensemble; gpt-5","family":"openai","confidence":"high","role":"subject"}},{"contributor":"CAIR MLE-STAR-Pro-1.0","description":"Gemini-2.5-Pro, 12h","date":"2025-11-03","value":38.67,"contributorKind":"team","contributorUrl":"https://research.google/teams/cloud-ai-research/","agent":{"model":"Gemini-2.5-Pro","evidence":"MLE-STAR-Pro-1.0; Gemini-2.5-Pro","family":"google","confidence":"high","role":"subject"}},{"contributor":"InternAgent","description":"deepseek-r1, 12h","date":"2025-09-12","value":36.44,"contributorKind":"team","contributorUrl":"https://github.com/Alpha-Innovator/InternAgent/","agent":{"model":"deepseek-r1","evidence":"InternAgent deepseek-r1 results","family":"deepseek","confidence":"high","role":"subject"}},{"contributor":"R&D-Agent","description":"gpt-5, 12h","date":"2025-09-26","value":35.11,"contributorKind":"team","contributorUrl":"https://github.com/microsoft/RD-Agent","agent":{"model":"gpt-5","evidence":"R&D-Agent gpt-5 results","family":"openai","confidence":"high","role":"subject"}},{"contributor":"Neo","description":"undisclosed, 36h","date":"2025-07-28","value":34.22,"contributorKind":"team","contributorUrl":"https://heyneo.so/"},{"contributor":"AIRA-dojo","description":"o3, 24h","date":"2025-05-15","value":31.6,"contributorKind":"team","contributorUrl":"https://github.com/facebookresearch/aira-dojo/","agent":{"model":"o3","evidence":"AIRA-dojo; o3","family":"openai","confidence":"high","role":"subject"}},{"contributor":"R&D-Agent","description":"o3 + GPT-4.1, 24h","date":"2025-08-15","value":30.22,"contributorKind":"team","contributorUrl":"https://github.com/microsoft/RD-Agent","agent":{"model":"o3 + GPT-4.1","evidence":"R&D-Agent o3+gpt-4.1 results","family":"openai","confidence":"high","role":"subject"}},{"contributor":"ML-Master","description":"deepseek-r1, 12h","date":"2025-06-17","value":29.33,"contributorKind":"team","contributorUrl":"https://github.com/zeroxleo/ML-Master","agent":{"model":"deepseek-r1","evidence":"ML-Master deepseek-r1 results","family":"deepseek","confidence":"high","role":"subject"}},{"contributor":"R&D-Agent","description":"o1-preview, 24h","date":"2025-05-14","value":22.4,"contributorKind":"team","contributorUrl":"https://github.com/microsoft/RD-Agent","agent":{"model":"o1-preview","evidence":"RD-Agent o1-preview results","family":"openai","confidence":"high","role":"subject"}},{"contributor":"AIDE","description":"o1-preview, 24h","date":"2024-10-08","value":17.12,"contributorKind":"method","contributorUrl":"https://github.com/wecoai/aideml","agent":{"model":"o1-preview","evidence":"AIDE; o1-preview","family":"openai","confidence":"high","role":"subject"}},{"contributor":"AIDE","description":"gpt-4o-2024-08-06, 24h","date":"2024-10-08","value":8.63,"contributorKind":"method","contributorUrl":"https://github.com/wecoai/aideml","agent":{"model":"gpt-4o-2024-08-06","evidence":"AIDE; gpt-4o-2024-08-06","family":"openai","confidence":"high","role":"subject"}},{"contributor":"AIDE","description":"claude-3-5-sonnet-20240620, 24h","date":"2024-10-08","value":7.56,"contributorKind":"method","contributorUrl":"https://github.com/wecoai/aideml","agent":{"model":"claude-3-5-sonnet-20240620","evidence":"AIDE; claude-3-5-sonnet-20240620","family":"anthropic","confidence":"high","role":"subject"}},{"contributor":"OpenHands","description":"gpt-4o-2024-08-06, 24h","date":"2024-10-08","value":4.89,"contributorKind":"method","agent":{"model":"gpt-4o-2024-08-06","evidence":"OpenHands; gpt-4o-2024-08-06","family":"openai","confidence":"high","role":"subject"}},{"contributor":"AIDE","description":"llama-3.1-405b-instruct, 24h","date":"2024-10-08","value":3.33,"contributorKind":"method","contributorUrl":"https://github.com/wecoai/aideml","agent":{"model":"llama-3.1-405b-instruct","evidence":"AIDE; llama-3.1-405b-instruct","family":"meta","confidence":"high","role":"subject"}},{"contributor":"MLAB","description":"gpt-4o-2024-08-06, 24h","date":"2024-10-08","value":1.6,"contributorKind":"method","agent":{"model":"gpt-4o-2024-08-06","evidence":"MLAB; gpt-4o-2024-08-06","family":"openai","confidence":"high","role":"subject"}}],"contested":true}],"agentStats":[],"repo":{"owner":"openai","name":"mle-bench","branch":"main"},"participants":[{"name":"Famou-Agent 2.0","url":"https://github.com/baidubce/FM-Agent","submissions":8,"problems":["low-complexity","medium-complexity","high-complexity","all-competitions"],"bestRank":1,"lastActive":"2026-02-23"},{"name":"AIBuildAI","url":"https://github.com/aibuildai/AI-Build-AI","submissions":4,"problems":["low-complexity","medium-complexity","high-complexity","all-competitions"],"lastActive":"2026-03-06","bestRank":1},{"name":"R","url":"https://github.com/R","submissions":12,"problems":["low-complexity","medium-complexity","high-complexity","all-competitions"],"lastActive":"2025-09-26"},{"name":"D-Agent","url":"https://github.com/D-Agent","submissions":12,"problems":["low-complexity","medium-complexity","high-complexity","all-competitions"],"lastActive":"2025-09-26"},{"name":"PiEvolve","url":"https://github.com/FractalAIResearchLabs/PiEvolve","submissions":8,"problems":["low-complexity","medium-complexity","high-complexity","all-competitions"],"lastActive":"2026-01-05"},{"name":"MLEvolve","url":"https://github.com/InternScience/MLEvolve","submissions":4,"problems":["low-complexity","medium-complexity","high-complexity","all-competitions"],"lastActive":"2026-02-14"},{"name":"CAIR MARS+","url":"https://research.google/teams/cloud-ai-research/","submissions":4,"problems":["low-complexity","medium-complexity","high-complexity","all-competitions"],"lastActive":"2026-02-17"},{"name":"ML-Master 2.0","url":"https://github.com/sjtu-sai-agents/ML-Master","submissions":4,"problems":["low-complexity","medium-complexity","high-complexity","all-competitions"],"lastActive":"2025-12-16"},{"name":"CAIR MARS","url":"https://research.google/teams/cloud-ai-research/","submissions":4,"problems":["low-complexity","medium-complexity","high-complexity","all-competitions"],"lastActive":"2026-01-25"},{"name":"Leeroo","url":"https://github.com/Leeroo-AI/kapso","submissions":4,"problems":["low-complexity","medium-complexity","high-complexity","all-competitions"],"lastActive":"2025-12-07"},{"name":"CAIR MLE-STAR-Pro-1.5","url":"https://research.google/teams/cloud-ai-research/","submissions":4,"problems":["low-complexity","medium-complexity","high-complexity","all-competitions"],"lastActive":"2025-11-25"},{"name":"CAIR MLE-STAR-Pro-1.0","url":"https://research.google/teams/cloud-ai-research/","submissions":4,"problems":["low-complexity","medium-complexity","high-complexity","all-competitions"],"lastActive":"2025-11-03"},{"name":"Thesis","url":"https://thesislabs.ai/","submissions":4,"problems":["low-complexity","medium-complexity","high-complexity","all-competitions"],"lastActive":"2025-11-10"},{"name":"Operand ensemble","url":"https://operand.com/","submissions":4,"problems":["low-complexity","medium-complexity","high-complexity","all-competitions"],"lastActive":"2025-10-06"},{"name":"Famou-Agent","url":"https://github.com/baidubce/FM-Agent","submissions":4,"problems":["low-complexity","medium-complexity","high-complexity","all-competitions"],"lastActive":"2025-10-10"},{"name":"InternAgent","url":"https://github.com/Alpha-Innovator/InternAgent/","submissions":4,"problems":["low-complexity","medium-complexity","high-complexity","all-competitions"],"lastActive":"2025-09-12"},{"name":"AIRA-dojo","url":"https://github.com/facebookresearch/aira-dojo/","submissions":4,"problems":["low-complexity","medium-complexity","high-complexity","all-competitions"],"lastActive":"2025-05-15"},{"name":"Neo","url":"https://heyneo.so/","submissions":4,"problems":["low-complexity","medium-complexity","high-complexity","all-competitions"],"lastActive":"2025-07-28"},{"name":"ML-Master","url":"https://github.com/zeroxleo/ML-Master","submissions":4,"problems":["low-complexity","medium-complexity","high-complexity","all-competitions"],"lastActive":"2025-06-17"}],"stats":{"totalRecords":112,"uniqueParticipants":19,"firstSubmission":"2024-10-08","lastSubmission":"2026-03-06","recordsLast30d":0,"recordsLast90d":0},"lastUpdated":"2026-10-19T09:02:20.225Z"},{"id":"chess-challenge","name":"Chess Coding Challenge","tagline":"Write a tiny C# chess bot under a 1024-token limit that wins games against other bots.","description":"Participants implement a chess bot in a single MyBot.cs file using the provided C# framework, with code size capped at 1024 tokens and only a restricted set of namespaces allowed. Submissions were entered via a Google Form and then played in a large Swiss tournament followed by a knockout playoff to determine the best bot. The challenge is run by Sebastian Lague and drew 636 submitted bots; results are published in a separate results repository.","url":"https://github.com/SebLague/Chess-Challenge","host":"github","status":"ended","domain":"coding-agents","tags":["chess","game-playing","csharp","code-golf","tournament"],"images":[],"participation":{"howToSubmit":"Fill out the Google Form submission page with your MyBot.cs entry; entries could be edited until submissions closed.","submissionFormat":"A single MyBot.cs file containing the MyBot struct and Think function, within the 1024-token bot brain capacity.","computeDetails":"Bots run locally for testing; tournament games are played by the organizers.","deadline":"closed","rounds":"Large Swiss tournament followed by a knockout tournament for top bots.","prizes":"recognition only","eligibility":"Open to individuals or groups of any size; maximum two entries per participant.","cost":"free","verification":"Organizers run submitted bots in the official tournament; illegal moves or timeouts lose the game.","requirements":[".NET 6.0","C#","Visual Studio or another IDE"],"compute":"cpu"},"links":[{"label":"Submission Page","url":"https://forms.gle/6jjj8jxNQ5Ln53ie6"},{"label":"Documentation","url":"https://seblague.github.io/chess-coding-challenge/documentation/"},{"label":"Results Repository","url":"https://github.com/SebLague/Tiny-Chess-Bot-Challenge-Results"},{"label":"Results Video","url":"https://youtu.be/Ne40a5LkK6A"},{"label":"Unofficial Live Leaderboard","url":"https://chess.stjo.dev/"}],"organizer":{"name":"Sebastian Lague","url":"https://github.com/SebLague","type":"individual"},"venue":{"type":"online"},"problems":[{"description":"Create a C# chess bot within a 1024-token code limit that plays legal chess and wins games in the tournament.","id":"chess-bot","name":"Tiny Chess Bot","metricName":"score","metricDirection":"maximize","isOpen":true,"records":[],"contested":false}],"agentStats":[],"repo":{"owner":"SebLague","name":"Chess-Challenge","branch":"main"},"participants":[],"stats":{"totalRecords":0,"uniqueParticipants":0,"recordsLast30d":0,"recordsLast90d":0},"lastUpdated":"2026-10-19T09:02:20.223Z"},{"id":"polymarket-paper-trader","name":"Polymarket Paper Trader — Forecast Arena & AI Trader Leaderboard","tagline":"Forecast real Polymarket questions and paper-trade them; scored by Brier score and ROI against the crowd.","description":"A zero-risk gym and benchmark for AI agents on real Polymarket prediction markets. The Forecast Arena has AI models and naive baselines publish one YES probability per soon-resolving binary market before it resolves, scored by Brier score, calibration error and alpha vs the crowd. A separate paper-trading leaderboard ranks agents by ROI% on a fixed $10,000 paper balance with real order books and official fees. Both are run by the agent-next project and hosted at polymarket-leaderboard.com.","url":"https://github.com/agent-next/polymarket-paper-trader","host":"github","status":"active","domain":"llm-eval","tags":["prediction-markets","forecasting","paper-trading","llm-eval","brier-score","agents"],"images":[],"participation":{"howToSubmit":"Forecast Arena: add an entrant block to benchmark/arena.yaml (id, label, kind, model) and open a PR. Paper-trading leaderboard: generate your entry with the `leaderboard_entry` MCP tool or `pm-trader leaderboard` CLI, then open an issue titled `[SUBMIT] {agent_name} — ROI: {roi}%` with the JSON output.","submissionFormat":"arena.yaml entrant block (id, label, kind ai|baseline, model litellm id, optional api_base/api_key_env, web_access, cutoff) for the Arena; JSON leaderboard_entry output pasted into a GitHub issue for the trading leaderboard.","computeDetails":"Forecasting is a single-shot LLM call per market; paper trading uses live public Polymarket APIs. No GPU required.","deadline":"rolling","rounds":"Daily — one forecast per entrant per market, resolved as markets close.","prizes":"recognition only","eligibility":"Open to any model provider or agent author; AI entrants and deterministic baselines both accepted.","cost":"free","verification":"Append-only JSONL data on the arena-data branch; forecasts recorded before resolution and scored against real outcomes. Trading leaderboard entries are self-reported via issue and reviewed by maintainers.","requirements":["Python 3.10+","polymarket-paper-trader (pip)","litellm-compatible model API key (Arena AI entrants)"],"compute":"none"},"links":[{"label":"Forecast Arena leaderboard","url":"https://polymarket-leaderboard.com/"},{"label":"Arena entry guide","url":"https://github.com/agent-next/polymarket-paper-trader/blob/main/docs/arena.md"},{"label":"Benchmark README","url":"https://github.com/agent-next/polymarket-paper-trader/blob/main/benchmark/README.md"},{"label":"Arena raw data branch","url":"https://github.com/agent-next/polymarket-paper-trader/tree/arena-data"},{"label":"Leaderboard server README","url":"https://github.com/agent-next/polymarket-paper-trader/blob/main/leaderboard-server/README.md"}],"organizer":{"name":"agent-next","url":"https://github.com/agent-next","type":"community"},"venue":{"location":"https://polymarket-leaderboard.com","type":"online"},"quickstart":"pip install -e \"benchmark[dev]\"\npolymarket-benchmark arena predict --data /tmp/arena-data\npolymarket-benchmark arena resolve --data /tmp/arena-data\npolymarket-benchmark arena build --data /tmp/arena-data --out site/","problems":[{"description":"AI entrants and naive baselines each publish one YES probability per soon-resolving Polymarket binary market before it resolves, with no access to the market price. Scored by Brier score, calibration error (ECE) and alpha vs the crowd once outcomes land.","id":"forecast-arena","name":"Forecast Arena","metricName":"Brier score","metricDirection":"minimize","isOpen":true,"records":[],"contested":false},{"description":"AI agents paper-trade Polymarket with a fixed $10,000 starting balance against live order books and official fees. Ranked by ROI% with Sharpe ratio as tiebreaker; minimum 10 trades to qualify.","metricUnit":"%","id":"paper-trading","name":"Paper Trading Leaderboard","metricName":"ROI","metricDirection":"maximize","isOpen":true,"records":[],"contested":false}],"agentStats":[],"repo":{"owner":"agent-next","name":"polymarket-paper-trader","branch":"main"},"participants":[],"stats":{"totalRecords":0,"uniqueParticipants":0,"recordsLast30d":0,"recordsLast90d":0},"lastUpdated":"2026-10-19T09:04:33.245Z"},{"id":"skab","name":"Skoltech Anomaly Benchmark (SKAB)","tagline":"Benchmark for outlier and changepoint detection on multivariate time series from a water pump testbed.","description":"SKAB provides a benchmark of 34 datasets with collective anomalies from a physical industrial testbed. It evaluates algorithms on two tasks: outlier detection (point anomalies) and changepoint detection (collective anomalies). The leaderboard ranks algorithms by F1 for outlier detection and NAB score for changepoint detection. Submissions are made via Kaggle, where the benchmark is hosted.","url":"https://github.com/waico/SKAB","host":"github","status":"active","domain":"other","tags":["anomaly-detection","changepoint-detection","time-series","benchmark","leaderboard"],"images":[],"participation":{"howToSubmit":"Submit your algorithm and results on the Kaggle competition page.","submissionFormat":"Algorithm predictions and evaluation on Kaggle.","computeDetails":"The benchmark datasets are relatively small; no GPU is required.","deadline":"rolling","cost":"free","verification":"Submissions are evaluated on Kaggle using the provided evaluation metrics.","requirements":["Python 3.10+","poetry","LightGBM"],"compute":"cpu"},"links":[{"label":"Kaggle Leaderboard","url":"https://www.kaggle.com/yuriykatser/skoltech-anomaly-benchmark-skab"},{"label":"Paper with Code","url":"https://paperswithcode.com/dataset/skab"}],"organizer":{"name":"Skoltech","url":"https://www.skoltech.ru/en"},"venue":{"type":"online"},"quickstart":"pip install poetry\npoetry install\npoetry run python -m skab --help","problems":[{"group":"changepoint-detection","description":"Detect collective anomalies (changepoints) in multivariate time series. Evaluated with NAB score and counts of missed/false positive changepoints.","metricUnit":"","id":"changepoint-detection","name":"Changepoint Detection","metricName":"NAB (standard)","metricDirection":"maximize","baseline":0,"isOpen":false,"records":[{"contributor":"Perfect detector","value":100,"contributorKind":"method","isBaseline":false,"isCurrentBest":true},{"contributor":"MSCRED","value":32.42,"contributorKind":"method","isBaseline":false,"isCurrentBest":false},{"contributor":"Isolation forest","value":26.16,"contributorKind":"method","isBaseline":false,"isCurrentBest":false},{"contributor":"T-squared+Q (PCA-based)","value":25.35,"contributorKind":"method","isBaseline":false,"isCurrentBest":false},{"contributor":"Conv-AE","value":23.61,"contributorKind":"method","isBaseline":false,"isCurrentBest":false},{"contributor":"LSTM-AE","value":23.51,"contributorKind":"method","isBaseline":false,"isCurrentBest":false},{"contributor":"T-squared","value":19.54,"contributorKind":"method","isBaseline":false,"isCurrentBest":false},{"contributor":"MSET","value":13.84,"contributorKind":"method","isBaseline":false,"isCurrentBest":false},{"contributor":"Vanilla AE","value":11.41,"contributorKind":"method","isBaseline":false,"isCurrentBest":false},{"contributor":"Vanilla LSTM","value":11.31,"contributorKind":"method","isBaseline":false,"isCurrentBest":false},{"contributor":"ArimaFD","value":-0.09,"contributorKind":"method","isBaseline":false,"isCurrentBest":false},{"contributor":"Null detector","value":0,"contributorKind":"method","isBaseline":true,"isCurrentBest":false}],"contested":true},{"group":"outlier-detection","description":"Detect point anomalies in multivariate time series. Evaluated as a binary classification problem with F1, FAR, and MAR metrics.","metricUnit":"","id":"outlier-detection","name":"Outlier Detection","metricName":"F1","metricDirection":"maximize","baseline":0,"isOpen":false,"records":[{"contributor":"Perfect detector","value":1,"contributorKind":"method","isBaseline":false,"isCurrentBest":true},{"contributor":"Conv-AE","value":0.78,"contributorKind":"method","isBaseline":false,"isCurrentBest":false},{"contributor":"MSET","value":0.78,"contributorKind":"method","isBaseline":false,"isCurrentBest":false},{"contributor":"T-squared+Q (PCA-based)","value":0.76,"contributorKind":"method","isBaseline":false,"isCurrentBest":false},{"contributor":"LSTM-AE","value":0.74,"contributorKind":"method","isBaseline":false,"isCurrentBest":false},{"contributor":"T-squared","value":0.66,"contributorKind":"method","isBaseline":false,"isCurrentBest":false},{"contributor":"LSTM-VAE","value":0.56,"contributorKind":"method","isBaseline":false,"isCurrentBest":false},{"contributor":"Vanilla LSTM","value":0.54,"contributorKind":"method","isBaseline":false,"isCurrentBest":false},{"contributor":"MSCRED","value":0.36,"contributorKind":"method","isBaseline":false,"isCurrentBest":false},{"contributor":"Vanilla AE","value":0.39,"contributorKind":"method","isBaseline":false,"isCurrentBest":false},{"contributor":"Isolation forest","value":0.29,"contributorKind":"method","isBaseline":false,"isCurrentBest":false},{"contributor":"Null detector","value":0,"contributorKind":"method","isBaseline":true,"isCurrentBest":false}],"contested":true}],"agentStats":[],"repo":{"owner":"waico","name":"SKAB","branch":"master"},"participants":[],"stats":{"totalRecords":24,"uniqueParticipants":0,"recordsLast30d":0,"recordsLast90d":0},"lastUpdated":"2026-10-19T09:02:20.225Z"},{"id":"llm-leaderboard","name":"LLM Leaderboard","tagline":"Compare large language models across standard benchmarks like MMLU, HellaSwag, HumanEval, and Chatbot Arena Elo.","description":"A joint community effort to create one central leaderboard for LLMs, aggregating published benchmark results from papers and model authors. Models are compared across multiple benchmarks including Chatbot Arena Elo, HellaSwag, HumanEval-Python, LAMBADA, MMLU, TriviaQA, and WinoGrande with different shot settings. Contributions are welcome via pull requests to fill missing entries, add new models, or add new benchmarks.","url":"https://github.com/LudwigStumpp/llm-leaderboard","host":"github","status":"active","domain":"llm-eval","tags":["llm","evaluation","benchmark","leaderboard","nlp"],"images":[],"participation":{"howToSubmit":"Open a PR to main adding a new model row to the leaderboard table (keeping alphabetic order) and add benchmark results with source links.","submissionFormat":"Modify the README.md leaderboard table; add model name, publisher, open status, and benchmark scores with source links.","cost":"free","requirements":["GitHub account","Markdown"],"compute":"none"},"links":[{"label":"Interactive Dashboard","url":"https://llm-leaderboard.streamlit.app/"},{"label":"Hugging Face Space","url":"https://huggingface.co/spaces/ludwigstumpp/llm-leaderboard"},{"label":"README","url":"https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"}],"problems":[{"group":"chatbot-arena-elo","description":"Elo rating from Chatbot Arena, a crowdsourced anonymous battle platform for LLMs.","id":"chatbot-arena-elo","name":"Chatbot Arena Elo","metricName":"Elo rating","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"vicuna-13b","value":1169,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"isCurrentBest":true,"agent":{"model":"Vicuna-13B","evidence":"contributor vicuna-13b is a model name","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"koala-13b","value":1082,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"Koala-13B","evidence":"contributor koala-13b is a model name","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"oasst-pythia-12b","value":1065,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"OASST-Pythia-12B","evidence":"contributor oasst-pythia-12b is a model name","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"alpaca-13b","value":1008,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"Alpaca-13B","evidence":"contributor alpaca-13b is a model name","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"chatglm-6b","value":985,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"ChatGLM-6B","evidence":"contributor chatglm-6b is a model name","family":"zhipu","confidence":"high","role":"subject"}},{"contributor":"fastchat-t5-3b","value":951,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"FastChat-T5-3B","evidence":"contributor fastchat-t5-3b is a model name","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"dolly-v2-12b","value":944,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"Dolly-v2-12B","evidence":"contributor dolly-v2-12b is a model name","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"llama-13b","value":932,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"LLaMA-13B","evidence":"contributor llama-13b is a model name","family":"meta","confidence":"high","role":"subject"}},{"contributor":"stablelm-tuned-alpha-7b","value":858,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"StableLM-Tuned-Alpha-7B","evidence":"contributor stablelm-tuned-alpha-7b is a model name","family":"other-ai","confidence":"high","role":"subject"}}],"contested":true},{"group":"hellaswag-few-shot","description":"Accuracy on HellaSwag commonsense NLI benchmark with few-shot prompting.","metricUnit":"%","id":"hellaswag-few-shot","name":"HellaSwag (few-shot)","metricName":"accuracy","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"gpt-4","value":0.953,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"isCurrentBest":true,"agent":{"model":"GPT-4","evidence":"contributor gpt-4 is a model name","family":"openai","confidence":"high","role":"subject"}},{"contributor":"llama-2-70b","value":0.873,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"Llama-2-70B","evidence":"contributor llama-2-70b is a model name","family":"meta","confidence":"high","role":"subject"}},{"contributor":"falcon-40b","value":0.853,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"Falcon-40B","evidence":"contributor falcon-40b is a model name","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"palm-540b","value":0.838,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"PaLM-540B","evidence":"contributor palm-540b is a model name","family":"google","confidence":"high","role":"subject"}},{"contributor":"gpt-3.5-175b / text-davinci-003","value":0.822,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"GPT-3.5-175B / text-davinci-003","evidence":"contributor gpt-3.5-175b / text-davinci-003 is a model name","family":"openai","confidence":"high","role":"subject"}},{"contributor":"gpt-3-175b / davinci","value":0.793,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"GPT-3-175B / davinci","evidence":"contributor gpt-3-175b / davinci is a model name","family":"openai","confidence":"high","role":"subject"}},{"contributor":"falcon-7b","value":0.781,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"Falcon-7B","evidence":"contributor falcon-7b is a model name","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"opt-175b","value":0.791,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"OPT-175B","evidence":"contributor opt-175b is a model name","family":"meta","confidence":"high","role":"subject"}},{"contributor":"opt-66b","value":0.745,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"OPT-66B","evidence":"contributor opt-66b is a model name","family":"meta","confidence":"high","role":"subject"}},{"contributor":"bloom-176b","value":0.744,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"BLOOM-176B","evidence":"contributor bloom-176b is a model name","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"gpt-neox-20b","value":0.718,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"GPT-NeoX-20B","evidence":"contributor gpt-neox-20b is a model name","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"gpt-3-7b / curie","value":0.682,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"GPT-3-7B / curie","evidence":"contributor gpt-3-7b / curie is a model name","family":"openai","confidence":"high","role":"subject"}},{"contributor":"gpt-j-6b","value":0.663,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"GPT-J-6B","evidence":"contributor gpt-j-6b is a model name","family":"other-ai","confidence":"high","role":"subject"}}],"contested":true},{"group":"hellaswag-one-shot","description":"Accuracy on HellaSwag commonsense NLI benchmark with one-shot prompting.","metricUnit":"%","id":"hellaswag-one-shot","name":"HellaSwag (one-shot)","metricName":"accuracy","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"palm-2-l","value":0.868,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"isCurrentBest":true,"agent":{"model":"PaLM-2-L","evidence":"contributor palm-2-l is a model name","family":"google","confidence":"high","role":"subject"}},{"contributor":"palm-2-m","value":0.84,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"PaLM-2-M","evidence":"contributor palm-2-m is a model name","family":"google","confidence":"high","role":"subject"}},{"contributor":"palm-540b","value":0.836,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"PaLM-540B","evidence":"contributor palm-540b is a model name","family":"google","confidence":"high","role":"subject"}},{"contributor":"palm-2-s","value":0.82,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"PaLM-2-S","evidence":"contributor palm-2-s is a model name","family":"google","confidence":"high","role":"subject"}}],"contested":true},{"group":"hellaswag-zero-shot","description":"Accuracy on HellaSwag commonsense NLI benchmark with zero-shot prompting.","metricUnit":"%","id":"hellaswag-zero-shot","name":"HellaSwag (zero-shot)","metricName":"accuracy","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"llama-65b","value":0.842,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"isCurrentBest":true,"agent":{"model":"Llama-65B","evidence":"contributor llama-65b is a model name","family":"meta","confidence":"high","role":"subject"}},{"contributor":"palm-540b","value":0.834,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"PaLM-540B","evidence":"contributor palm-540b is a model name","family":"google","confidence":"high","role":"subject"}},{"contributor":"gpt-3.5-175b / text-davinci-003","value":0.834,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"GPT-3.5-175B / text-davinci-003","evidence":"contributor gpt-3.5-175b / text-davinci-003 is a model name","family":"openai","confidence":"high","role":"subject"}},{"contributor":"llama-33b","value":0.828,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"Llama-33B","evidence":"contributor llama-33b is a model name","family":"meta","confidence":"high","role":"subject"}},{"contributor":"chinchilla-70b","value":0.808,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"Chinchilla-70B","evidence":"contributor chinchilla-70b is a model name","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"llama-13b","value":0.792,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"Llama-13B","evidence":"contributor llama-13b is a model name","family":"meta","confidence":"high","role":"subject"}},{"contributor":"gpt-3-175b / davinci","value":0.789,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"GPT-3-175B / davinci","evidence":"contributor gpt-3-175b / davinci is a model name","family":"openai","confidence":"high","role":"subject"}},{"contributor":"mpt-7b","value":0.761,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"MPT-7B","evidence":"contributor mpt-7b is a model name","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"gpt4all-13b-snoozy","value":0.75,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"GPT4All-13B-snoozy","evidence":"contributor gpt4all-13b-snoozy is a model name","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"alpaca-7b","value":0.739,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"Alpaca-7B","evidence":"contributor alpaca-7b is a model name","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"llama-7b","value":0.738,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"Llama-7B","evidence":"contributor llama-7b is a model name","family":"meta","confidence":"high","role":"subject"}},{"contributor":"koala-13b","value":0.726,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"Koala-13B","evidence":"contributor koala-13b is a model name","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"gpt-neox-20b","value":0.719,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"GPT-NeoX-20B","evidence":"contributor gpt-neox-20b is a model name","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"dolly-v2-12b","value":0.71,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"Dolly-v2-12B","evidence":"contributor dolly-v2-12b is a model name","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"eleuther-pythia-12b","value":0.704,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"Eleuther-Pythia-12B","evidence":"contributor eleuther-pythia-12b is a model name","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"opt-13b","value":0.692,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"OPT-13B","evidence":"contributor opt-13b is a model name","family":"meta","confidence":"high","role":"subject"}},{"contributor":"gpt-j-6b","value":0.683,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"GPT-J-6B","evidence":"contributor gpt-j-6b is a model name","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"oasst-pythia-12b","value":0.681,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"OASST-Pythia-12B","evidence":"contributor oasst-pythia-12b is a model name","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"opt-7b","value":0.677,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"OPT-7B","evidence":"contributor opt-7b is a model name","family":"meta","confidence":"high","role":"subject"}},{"contributor":"eleuther-pythia-7b","value":0.667,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"Eleuther-Pythia-7B","evidence":"contributor eleuther-pythia-7b is a model name","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"cerebras-gpt-7b","value":0.636,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"Cerebras-GPT-7B","evidence":"contributor cerebras-gpt-7b is a model name","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"cerebras-gpt-13b","value":0.635,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"Cerebras-GPT-13B","evidence":"contributor cerebras-gpt-13b is a model name","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"stablelm-tuned-alpha-7b","value":0.536,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"StableLM-Tuned-Alpha-7B","evidence":"contributor stablelm-tuned-alpha-7b is a model name","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"stablelm-base-alpha-7b","value":0.412,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"StableLM-Base-Alpha-7B","evidence":"contributor stablelm-base-alpha-7b is a model name","family":"other-ai","confidence":"high","role":"subject"}}],"contested":true},{"group":"humaneval-python-pass1","description":"Functional correctness on HumanEval Python code generation benchmark, pass@1.","metricUnit":"%","id":"humaneval-python-pass1","name":"HumanEval-Python (pass@1)","metricName":"pass@1","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"gpt-4","value":0.67,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"isCurrentBest":true,"agent":{"model":"GPT-4","evidence":"contributor gpt-4 is a model name","family":"openai","confidence":"high","role":"subject"}},{"contributor":"gpt-3.5-175b / text-davinci-003","value":0.481,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"GPT-3.5-175B / text-davinci-003","evidence":"contributor gpt-3.5-175b / text-davinci-003 is a model name","family":"openai","confidence":"high","role":"subject"}},{"contributor":"gpt-3.5-175b / code-davinci-002","value":0.463,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"GPT-3.5-175B / code-davinci-002","evidence":"contributor gpt-3.5-175b / code-davinci-002 is a model name","family":"openai","confidence":"high","role":"subject"}},{"contributor":"palm-2-s*","value":0.376,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"PaLM-2-S*","evidence":"contributor palm-2-s* is a model name","family":"google","confidence":"high","role":"subject"}},{"contributor":"palm-coder-540b","value":0.359,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"PaLM-Coder-540B","evidence":"contributor palm-coder-540b is a model name","family":"google","confidence":"high","role":"subject"}},{"contributor":"starcoder-16b","value":0.336,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"StarCoder-16B","evidence":"contributor starcoder-16b is a model name","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"codex-12b / code-cushman-001","value":0.317,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"Codex-12B / code-cushman-001","evidence":"contributor codex-12b / code-cushman-001 is a model name","family":"openai","confidence":"high","role":"subject"}},{"contributor":"starcoder-base-16b","value":0.304,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"StarCoder-Base-16B","evidence":"contributor starcoder-base-16b is a model name","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"codegen-16B-mono","value":0.293,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"CodeGen-16B-mono","evidence":"contributor codegen-16B-mono is a model name","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"palm-540b","value":0.262,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"PaLM-540B","evidence":"contributor palm-540b is a model name","family":"google","confidence":"high","role":"subject"}},{"contributor":"llama-65b","value":0.237,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"Llama-65B","evidence":"contributor llama-65b is a model name","family":"meta","confidence":"high","role":"subject"}},{"contributor":"codegx-13b","value":0.229,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"CodeGX-13B","evidence":"contributor codegx-13b is a model name","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"replit-code-v1-3b","value":0.219,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"Replit-Code-v1-3B","evidence":"contributor replit-code-v1-3b is a model name","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"llama-33b","value":0.217,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"Llama-33B","evidence":"contributor llama-33b is a model name","family":"meta","confidence":"high","role":"subject"}},{"contributor":"codegen-16B-multi","value":0.183,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"CodeGen-16B-multi","evidence":"contributor codegen-16B-multi is a model name","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"llama-13b","value":0.158,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"Llama-13B","evidence":"contributor llama-13b is a model name","family":"meta","confidence":"high","role":"subject"}},{"contributor":"bloom-176b","value":0.155,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"BLOOM-176B","evidence":"contributor bloom-176b is a model name","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"llama-7b","value":0.105,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"Llama-7B","evidence":"contributor llama-7b is a model name","family":"meta","confidence":"high","role":"subject"}}],"contested":true},{"group":"lambada-one-shot","description":"Accuracy on LAMBADA word prediction benchmark with one-shot prompting.","metricUnit":"%","id":"lambada-one-shot","name":"LAMBADA (one-shot)","metricName":"accuracy","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"palm-2-l","value":0.869,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"isCurrentBest":true,"agent":{"model":"PaLM-2-L","evidence":"contributor palm-2-l is a model name","family":"google","confidence":"high","role":"subject"}},{"contributor":"palm-2-m","value":0.837,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"PaLM-2-M","evidence":"contributor palm-2-m is a model name","family":"google","confidence":"high","role":"subject"}},{"contributor":"palm-540b","value":0.818,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"PaLM-540B","evidence":"contributor palm-540b is a model name","family":"google","confidence":"high","role":"subject"}},{"contributor":"palm-2-s","value":0.807,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"PaLM-2-S","evidence":"contributor palm-2-s is a model name","family":"google","confidence":"high","role":"subject"}}],"contested":true},{"group":"lambada-zero-shot","description":"Accuracy on LAMBADA word prediction benchmark with zero-shot prompting.","metricUnit":"%","id":"lambada-zero-shot","name":"LAMBADA (zero-shot)","metricName":"accuracy","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"palm-540b","value":0.779,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"isCurrentBest":true,"agent":{"model":"PaLM-540B","evidence":"contributor palm-540b is a model name","family":"google","confidence":"high","role":"subject"}},{"contributor":"chinchilla-70b","value":0.774,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"Chinchilla-70B","evidence":"contributor chinchilla-70b is a model name","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"gpt-3.5-175b / text-davinci-003","value":0.762,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"GPT-3.5-175B / text-davinci-003","evidence":"contributor gpt-3.5-175b / text-davinci-003 is a model name","family":"openai","confidence":"high","role":"subject"}},{"contributor":"llama-7b","value":0.738,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"Llama-7B","evidence":"contributor llama-7b is a model name","family":"meta","confidence":"high","role":"subject"}},{"contributor":"gpt-neox-20b","value":0.719,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"GPT-NeoX-20B","evidence":"contributor gpt-neox-20b is a model name","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"mpt-7b","value":0.702,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"MPT-7B","evidence":"contributor mpt-7b is a model name","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"opt-13b","value":0.692,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"OPT-13B","evidence":"contributor opt-13b is a model name","family":"meta","confidence":"high","role":"subject"}},{"contributor":"gpt-j-6b","value":0.683,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"GPT-J-6B","evidence":"contributor gpt-j-6b is a model name","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"opt-7b","value":0.677,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"OPT-7B","evidence":"contributor opt-7b is a model name","family":"meta","confidence":"high","role":"subject"}},{"contributor":"eleuther-pythia-7b","value":0.667,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"Eleuther-Pythia-7B","evidence":"contributor eleuther-pythia-7b is a model name","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"cerebras-gpt-7b","value":0.636,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"Cerebras-GPT-7B","evidence":"contributor cerebras-gpt-7b is a model name","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"cerebras-gpt-13b","value":0.635,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"Cerebras-GPT-13B","evidence":"contributor cerebras-gpt-13b is a model name","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"stablelm-base-alpha-7b","value":0.533,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"StableLM-Base-Alpha-7B","evidence":"contributor stablelm-base-alpha-7b is a model name","family":"other-ai","confidence":"high","role":"subject"}}],"contested":true},{"group":"mmlu-few-shot","description":"Accuracy on MMLU multi-task language understanding benchmark with few-shot prompting.","metricUnit":"%","id":"mmlu-few-shot","name":"MMLU (few-shot)","metricName":"accuracy","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"gpt-4","value":0.864,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"isCurrentBest":true,"agent":{"model":"GPT-4","evidence":"contributor gpt-4 is a model name","family":"openai","confidence":"high","role":"subject"}},{"contributor":"llama-2-70b","value":0.698,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"Llama-2-70B","evidence":"contributor llama-2-70b is a model name","family":"meta","confidence":"high","role":"subject"}},{"contributor":"palm-540b","value":0.693,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"PaLM-540B","evidence":"contributor palm-540b is a model name","family":"google","confidence":"high","role":"subject"}},{"contributor":"chinchilla-70b","value":0.675,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"Chinchilla-70B","evidence":"contributor chinchilla-70b is a model name","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"llama-65b","value":0.634,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"Llama-65B","evidence":"contributor llama-65b is a model name","family":"meta","confidence":"high","role":"subject"}},{"contributor":"gpt-3.5-175b / text-davinci-003","value":0.569,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"GPT-3.5-175B / text-davinci-003","evidence":"contributor gpt-3.5-175b / text-davinci-003 is a model name","family":"openai","confidence":"high","role":"subject"}},{"contributor":"falcon-40b","value":0.527,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"Falcon-40B","evidence":"contributor falcon-40b is a model name","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"falcon-7b","value":0.35,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"Falcon-7B","evidence":"contributor falcon-7b is a model name","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"opt-175b","value":0.318,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"OPT-175B","evidence":"contributor opt-175b is a model name","family":"meta","confidence":"high","role":"subject"}},{"contributor":"opt-66b","value":0.276,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"OPT-66B","evidence":"contributor opt-66b is a model name","family":"meta","confidence":"high","role":"subject"}},{"contributor":"gpt-neox-20b","value":0.276,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"GPT-NeoX-20B","evidence":"contributor gpt-neox-20b is a model name","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"gpt-j-6b","value":0.249,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"GPT-J-6B","evidence":"contributor gpt-j-6b is a model name","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"gpt-3-7b / curie","value":0.243,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"GPT-3-7B / curie","evidence":"contributor gpt-3-7b / curie is a model name","family":"openai","confidence":"high","role":"subject"}},{"contributor":"gpt-3-175b / davinci","value":0.439,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"GPT-3-175B / davinci","evidence":"contributor gpt-3-175b / davinci is a model name","family":"openai","confidence":"high","role":"subject"}}],"contested":true},{"group":"mmlu-zero-shot","description":"Accuracy on MMLU multi-task language understanding benchmark with zero-shot prompting.","metricUnit":"%","id":"mmlu-zero-shot","name":"MMLU (zero-shot)","metricName":"accuracy","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"gal-120b","value":0.526,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"isCurrentBest":true,"agent":{"model":"GAL-120B","evidence":"contributor gal-120b is a model name","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"palm-coder-540b","value":0.359,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"PaLM-Coder-540B","evidence":"contributor palm-coder-540b is a model name","family":"google","confidence":"high","role":"subject"}},{"contributor":"llama-7b","value":0.302,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"Llama-7B","evidence":"contributor llama-7b is a model name","family":"meta","confidence":"high","role":"subject"}},{"contributor":"bloom-176b","value":0.299,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"BLOOM-176B","evidence":"contributor bloom-176b is a model name","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"mpt-7b","value":0.296,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"MPT-7B","evidence":"contributor mpt-7b is a model name","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"gpt-neox-20b","value":0.269,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"GPT-NeoX-20B","evidence":"contributor gpt-neox-20b is a model name","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"eleuther-pythia-7b","value":0.265,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"Eleuther-Pythia-7B","evidence":"contributor eleuther-pythia-7b is a model name","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"gpt-j-6b","value":0.261,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"GPT-J-6B","evidence":"contributor gpt-j-6b is a model name","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"cerebras-gpt-7b","value":0.259,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"Cerebras-GPT-7B","evidence":"contributor cerebras-gpt-7b is a model name","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"cerebras-gpt-13b","value":0.258,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"Cerebras-GPT-13B","evidence":"contributor cerebras-gpt-13b is a model name","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"opt-13b","value":0.257,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"OPT-13B","evidence":"contributor opt-13b is a model name","family":"meta","confidence":"high","role":"subject"}},{"contributor":"eleuther-pythia-12b","value":0.253,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"Eleuther-Pythia-12B","evidence":"contributor eleuther-pythia-12b is a model name","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"opt-7b","value":0.251,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"OPT-7B","evidence":"contributor opt-7b is a model name","family":"meta","confidence":"high","role":"subject"}},{"contributor":"stablelm-base-alpha-7b","value":0.251,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"StableLM-Base-Alpha-7B","evidence":"contributor stablelm-base-alpha-7b is a model name","family":"other-ai","confidence":"high","role":"subject"}}],"contested":true},{"group":"triviaqa-one-shot","description":"Accuracy on TriviaQA reading comprehension benchmark with one-shot prompting.","metricUnit":"%","id":"triviaqa-one-shot","name":"TriviaQA (one-shot)","metricName":"accuracy","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"palm-2-l","value":0.83,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"isCurrentBest":true,"agent":{"model":"PaLM-2-L","evidence":"contributor palm-2-l is a model name","family":"google","confidence":"high","role":"subject"}},{"contributor":"palm-2-m","value":0.792,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"PaLM-2-M","evidence":"contributor palm-2-m is a model name","family":"google","confidence":"high","role":"subject"}},{"contributor":"palm-2-s","value":0.779,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"PaLM-2-S","evidence":"contributor palm-2-s is a model name","family":"google","confidence":"high","role":"subject"}}],"contested":true},{"group":"triviaqa-zero-shot","description":"Accuracy on TriviaQA reading comprehension benchmark with zero-shot prompting.","metricUnit":"%","id":"triviaqa-zero-shot","name":"TriviaQA (zero-shot)","metricName":"accuracy","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"gpt-neox-20b","value":0.347,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"isCurrentBest":true,"agent":{"model":"GPT-NeoX-20B","evidence":"contributor gpt-neox-20b is a model name","family":"other-ai","confidence":"high","role":"subject"}}],"contested":false},{"group":"winogrande-few-shot","description":"Accuracy on WinoGrande pronoun resolution benchmark with few-shot prompting.","metricUnit":"%","id":"winogrande-few-shot","name":"WinoGrande (few-shot)","metricName":"accuracy","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"gpt-4","value":0.875,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"isCurrentBest":true,"agent":{"model":"GPT-4","evidence":"contributor gpt-4 is a model name","family":"openai","confidence":"high","role":"subject"}}],"contested":false},{"group":"winogrande-one-shot","description":"Accuracy on WinoGrande pronoun resolution benchmark with one-shot prompting.","metricUnit":"%","id":"winogrande-one-shot","name":"WinoGrande (one-shot)","metricName":"accuracy","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"palm-2-l-instruct","value":0.909,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"isCurrentBest":true,"agent":{"model":"PaLM-2-L-Instruct","evidence":"contributor palm-2-l-instruct is a model name","family":"google","confidence":"high","role":"subject"}},{"contributor":"palm-540b","value":0.811,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"PaLM-540B","evidence":"contributor palm-540b is a model name","family":"google","confidence":"high","role":"subject"}}],"contested":true},{"group":"winogrande-zero-shot","description":"Accuracy on WinoGrande pronoun resolution benchmark with zero-shot prompting.","metricUnit":"%","id":"winogrande-zero-shot","name":"WinoGrande (zero-shot)","metricName":"accuracy","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"llama-65b","value":0.77,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"isCurrentBest":true,"agent":{"model":"Llama-65B","evidence":"contributor llama-65b is a model name","family":"meta","confidence":"high","role":"subject"}},{"contributor":"llama-33b","value":0.76,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"Llama-33B","evidence":"contributor llama-33b is a model name","family":"meta","confidence":"high","role":"subject"}},{"contributor":"gpt-3.5-175b / text-davinci-003","value":0.758,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"GPT-3.5-175B / text-davinci-003","evidence":"contributor gpt-3.5-175b / text-davinci-003 is a model name","family":"openai","confidence":"high","role":"subject"}},{"contributor":"gpt4all-13b-snoozy","value":0.713,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"GPT4All-13B-snoozy","evidence":"contributor gpt4all-13b-snoozy is a model name","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"koala-13b","value":0.688,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"Koala-13B","evidence":"contributor koala-13b is a model name","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"alpaca-7b","value":0.661,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"Alpaca-7B","evidence":"contributor alpaca-7b is a model name","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"eleuther-pythia-7b","value":0.661,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"Eleuther-Pythia-7B","evidence":"contributor eleuther-pythia-7b is a model name","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"oasst-pythia-12b","value":0.65,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"OASST-Pythia-12B","evidence":"contributor oasst-pythia-12b is a model name","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"eleuther-pythia-12b","value":0.638,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"Eleuther-Pythia-12B","evidence":"contributor eleuther-pythia-12b is a model name","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"dolly-v2-12b","value":0.622,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"Dolly-v2-12B","evidence":"contributor dolly-v2-12b is a model name","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"stablelm-tuned-alpha-7b","value":0.548,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"StableLM-Tuned-Alpha-7B","evidence":"contributor stablelm-tuned-alpha-7b is a model name","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"stablelm-base-alpha-7b","value":0.501,"contributorKind":"model","submissionUrls":["https://github.com/LudwigStumpp/llm-leaderboard/blob/main/README.md"],"agent":{"model":"StableLM-Base-Alpha-7B","evidence":"contributor stablelm-base-alpha-7b is a model name","family":"other-ai","confidence":"high","role":"subject"}}],"contested":true}],"agentStats":[],"repo":{"owner":"LudwigStumpp","name":"llm-leaderboard","branch":"main"},"participants":[],"stats":{"totalRecords":132,"uniqueParticipants":0,"recordsLast30d":0,"recordsLast90d":0},"lastUpdated":"2026-10-19T09:02:20.227Z"},{"id":"medmcqa","name":"MedMCQA","tagline":"Answer medical entrance exam multiple-choice questions; scored by test-set accuracy via a blind submission form.","description":"MedMCQA is a large-scale multiple-choice QA dataset of 194k AIIMS & NEET PG medical entrance exam questions across 21 subjects. Models are evaluated on a held-out test set whose ground truth is not public; participants submit a CSV of question-id to predicted option to the organizers via a Google Form, and accuracy is reported on the external leaderboard. It matters as a domain-specific reasoning benchmark for medical QA.","url":"https://github.com/medmcqa/medmcqa","host":"github","status":"active","domain":"llm-eval","tags":["medical-qa","multiple-choice","question-answering","nlp","benchmark","dataset"],"images":["https://raw.githubusercontent.com/medmcqa/medmcqa/main/Extra/medmcqa.png"],"participation":{"howToSubmit":"Evaluate your model on the test set and submit a CSV of predictions via the Google Form: https://forms.gle/xLJHNbuvaRa2FXbD8","submissionFormat":"CSV with exactly two columns: `id` and `Prediction (correct option)`; answer index starts at 1 and must be in [1,2,3,4].","computeDetails":"Fine-tuning BERT-base on 182k training questions; a single consumer GPU is sufficient.","deadline":"rolling","eligibility":"Open to all; test ground truth withheld, submissions only via the Google Form.","cost":"free","verification":"Organizers score submitted CSVs against the private test-set ground truth.","requirements":["Python 3","PyTorch or TensorFlow","BERT-style model"],"compute":"consumer-gpu"},"links":[{"label":"Homepage","url":"https://medmcqa.github.io/"},{"label":"Paper (arXiv)","url":"https://arxiv.org/abs/2203.14371"},{"label":"PMLR proceedings","url":"https://proceedings.mlr.press/v174/pal22a.html"},{"label":"Leaderboard (Papers with Code)","url":"https://paperswithcode.com/dataset/medmcqa"},{"label":"Submission form","url":"https://forms.gle/xLJHNbuvaRa2FXbD8"}],"organizer":{"name":"MedMCQA (Ankit Pal, Logesh Kumar Umapathi, Malaikannan Sankarasubbu)","url":"https://medmcqa.github.io/","type":"academic"},"venue":{"type":"online"},"quickstart":"pip3 install -r requirements.txt\n# download data from the Google Drive link in the README, unzip it\npython3 train.py --model bert-base-uncased --dataset_folder_name \"/content/medmcqa_data/\"\n# then write predictions to CSV with columns: id, Prediction (correct option)\n# and submit via https://forms.gle/xLJHNbuvaRa2FXbD8","problems":[{"description":"Predict the correct option (1-4) for each held-out AIIMS PG test question; scored by accuracy on the private test set.","metricUnit":"%","id":"medmcqa-test-accuracy","name":"MedMCQA Test Accuracy","metricName":"accuracy","metricDirection":"maximize","isOpen":true,"records":[],"contested":false}],"agentStats":[],"repo":{"owner":"medmcqa","name":"medmcqa","branch":"main"},"participants":[],"stats":{"totalRecords":0,"uniqueParticipants":0,"recordsLast30d":0,"recordsLast90d":0},"lastUpdated":"2026-10-19T09:02:20.227Z"},{"id":"fstandhartinger-jevbench-2acd10345790c8d1","name":"JevBench","tagline":"Benchmark for AI decision models: typed decision accuracy, calibration, latency and cost.","description":"JevBench by Benchmark Heaven evaluates AI decision models and Jev-compatible systems on typed decisions: a state, a rubric and a bounded label set, answered with one typed answer and a probability. Systems are scored on Intelligence (chance-corrected accuracy), Calibration, Speed and Cost, combined into a JevBench Score (geometric mean, 25% each, with a near-chance penalty). It is maintained by Benchmark Heaven independently of TypeSafe AI; the canonical board is at benchmarkheaven.com/jev-models. Entrants are measured by the organizers through each author's published interface, and new systems are added on request.","url":"https://github.com/fstandhartinger/jevbench","host":"github","status":"active","domain":"llm-eval","tags":["decision-models","calibration","llm-eval","latency","cost","typed-decisions"],"images":["https://raw.githubusercontent.com/fstandhartinger/jevbench/main/results/v1.2/charts/main-score.png","https://raw.githubusercontent.com/fstandhartinger/jevbench/main/results/v1.2/charts/axes.png","https://raw.githubusercontent.com/fstandhartinger/jevbench/main/results/v1.2/charts/hard-tier.png","https://raw.githubusercontent.com/fstandhartinger/jevbench/main/results/v1.2/charts/calibration.png"],"participation":{"howToSubmit":"Request a run by opening an issue (e.g. issue #6, #9) or replying to the launch posts; the organizers run your system through its author's published interface and add a row. Mappings, endpoint conditions and cost bases are pre-registered in docs/ before the run.","submissionFormat":"A reachable endpoint or public weights plus serving code; the organizers supply the adapter. Pre-registration documents the mapping and cost basis.","computeDetails":"Organizers run entrants on their own CPU, rented RunPod/lium.io GPUs (RTX 3090, A6000, RTX PRO 4500, A40, H100, H200) or the entrant's production API.","deadline":"rolling","rounds":"Versioned releases (v1.0, v1.1, v1.1.3, v1.2.x, v1.3.0, v1.4.x, v1.5, v1.6.1); new systems are added as revisions on the same frozen items.","prizes":"recognition only","cost":"free","verification":"Organizers run the frozen task set serially through the entrant's own interface; scoring code is public (jevbench/composite_v13.py, composite_v14.py) and artifacts are rebuilt from frozen measurements.","requirements":["Python 3.10+","A reachable HTTP endpoint or public weights","TypeSafe-compatible /v1/systemone or an adapter"],"compute":"unknown"},"links":[{"label":"Live board","url":"https://benchmarkheaven.com/jev-models"},{"label":"Hosted API board","url":"https://benchmarkheaven.com/jev-models/api"},{"label":"Methodology","url":"https://benchmarkheaven.com/jev-models/methodology"},{"label":"v1.6.1 results JSON","url":"https://benchmarkheaven.com/api/decision-results/jevbench/v1.6.1/json"},{"label":"v1.2 results","url":"https://github.com/fstandhartinger/jevbench/blob/main/RESULTS-v1.2.md"},{"label":"Combination study","url":"https://github.com/fstandhartinger/jevbench/blob/main/RESULTS-COMBINATIONS.md"}],"organizer":{"name":"Benchmark Heaven","url":"https://benchmarkheaven.com/jev-models","type":"company"},"venue":{"type":"online"},"quickstart":"python -m unittest discover -s tests -v\n\npython -m jevbench.cli run --tasks datasets/public/original.jsonl \\\n  --adapter typesafe --model jev-latest --key-env TYPESAFE_API_KEY \\\n  --price-in-per-m 0.042 --price-out-per-m 0 \\\n  --results RUN/results.jsonl --raw-dir RUN/raw \\\n  --ledger RUN/ledger.jsonl --cap-usd 15 --manifest RUN/manifest.json","problems":[{"description":"Historical release: v1.4.2.2 adds Imajev-4B to the v1.4.2.1 result on the unchanged v1.4.2 scorer. 95 systems, 91 ranked.","id":"jevbench-v1.4.2.2","name":"JevBench v1.4.2.2","metricName":"JevBench Score","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"Imajev-4B","value":67.37,"contributorKind":"model","isCurrentBest":true,"agent":{"model":"Imajev-4B","evidence":"Record contributor is a model name: Imajev-4B","family":"other-ai","confidence":"low","role":"subject"}},{"contributor":"Plumb-4B (crh225, JevK5 v0.2 + LoRA)","value":65.84,"contributorKind":"model","agent":{"model":"Plumb-4B","evidence":"Release JevBench v1.4.2.1 with Plumb-4B","family":"other-ai","confidence":"medium","role":"subject"}},{"contributor":"decider-4b v2 (Mapika)","value":64.13,"contributorKind":"model","agent":{"model":"decider-4b v2","evidence":"JevBench v1.2.8: add decider-35b-a3b, reflex-27b, reflex 4B, decider-2b","family":"other-ai","confidence":"medium","role":"subject"}},{"contributor":"Jev 1.13.0 (TypeSafe AI)","value":63.29,"contributorKind":"model","agent":{"model":"Jev 1.13.0","evidence":"Jev 1.13.0 (TypeSafe AI): 96.3% over 242 scorable decisions","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"JevK5 v0.2.0","value":62.04,"contributorKind":"model","agent":{"model":"JevK5 v0.2.0","evidence":"Plumb-4B (crh225, JevK5 v0.2 + LoRA)","family":"other-ai","confidence":"low","role":"subject"}}],"contested":true},{"description":"Previous release: adds Plumb-4B to the live v1.4.2 baseline. 94 systems, 90 ranked. Score blends 20% chance-corrected Intelligence from 308 fresh sealed decisions with 80% of the v1.3.0 Intelligence axis.","id":"jevbench-v1.4.2.1","name":"JevBench v1.4.2.1","metricName":"JevBench Score","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"Plumb-4B (crh225, JevK5 v0.2 + LoRA)","value":65.84,"contributorKind":"model","isCurrentBest":true,"agent":{"model":"Plumb-4B","evidence":"Release JevBench v1.4.2.1 with Plumb-4B","family":"other-ai","confidence":"medium","role":"subject"}},{"contributor":"decider-4b v2 (Mapika)","value":64.13,"contributorKind":"model","agent":{"model":"decider-4b v2","evidence":"JevBench v1.2.8: add decider-35b-a3b, reflex-27b, reflex 4B, decider-2b","family":"other-ai","confidence":"medium","role":"subject"}},{"contributor":"Jev 1.13.0 (TypeSafe AI)","value":63.29,"contributorKind":"model","agent":{"model":"Jev 1.13.0","evidence":"Jev 1.13.0 (TypeSafe AI): 96.3% over 242 scorable decisions","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"JevK5 v0.2.0","value":62.04,"contributorKind":"model","agent":{"model":"JevK5 v0.2.0","evidence":"Plumb-4B (crh225, JevK5 v0.2 + LoRA)","family":"other-ai","confidence":"low","role":"subject"}},{"contributor":"Cygnet (blockbrain, frozen Gemma-4-12B-it)","value":61.76,"contributorKind":"model","agent":{"model":"Gemma-4-12B-it","evidence":"Cygnet (blockbrain, frozen Gemma-4-12B-it)","family":"google","confidence":"high","role":"subject"}}],"contested":true},{"description":"The v1.2 task set: 534 decisions per system (220 hard, 111 public / 109 held out; plus easy, standard and judge tiers). JevBench Score = geometric mean of Intelligence, Calibration, Speed and Cost, 25% each, with a near-chance penalty below 50 Intelligence.","id":"jevbench-v1.3.0","name":"JevBench v1.3.0 (v1.2 results)","metricName":"JevBench Score","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"Jev 1.13.0","value":74.4,"contributorKind":"model","isCurrentBest":true,"agent":{"model":"Jev 1.13.0","evidence":"Jev 1.13.0 (TypeSafe AI): 96.3% over 242 scorable decisions","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"SemIf (Qwen3.5-4B)","value":73.1,"contributorKind":"model","agent":{"model":"Qwen3.5-4B","evidence":"SemIf (Qwen3.5-4B)","family":"alibaba","confidence":"high","role":"subject"}},{"contributor":"djev (Maisa, diffusion-gemma)","value":73,"contributorKind":"model","agent":{"model":"diffusion-gemma","evidence":"JevBench v1.2.1: add djev (Maisa, diffusion-gemma)","family":"google","confidence":"high","role":"subject"}},{"contributor":"Winnow-12B Q8","value":71.2,"contributorKind":"model","agent":{"model":"Gemma 3 12B","family":"google","confidence":"high","role":"subject"}},{"contributor":"reflex 4B (kshetrajna12)","value":70.3,"contributorKind":"model","agent":{"model":"Qwen3.5-4B","family":"alibaba","confidence":"high","role":"subject"}},{"contributor":"jqv (Qwen3-32B zero-shot)","value":68.6,"contributorKind":"model","agent":{"model":"Qwen3-32B","evidence":"jqv (Qwen3-32B zero-shot)","family":"alibaba","confidence":"high","role":"subject"}},{"contributor":"decision-machine-1 (milliseconds.ai)","value":68.3,"contributorKind":"model","agent":{"model":"decision-machine-1","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"decider-35b-a3b (Mapika)","value":67.6,"contributorKind":"model","agent":{"model":"decider-35b-a3b","evidence":"JevBench v1.2.8: add decider-35b-a3b, reflex-27b, reflex 4B, decider-2b","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"open-alternative-jev (Qwen3.5-4B, IkerMoel)","value":67,"contributorKind":"model","agent":{"model":"Qwen3.5-4B","evidence":"open-alternative-jev (Qwen3.5-4B, IkerMoel)","family":"alibaba","confidence":"high","role":"subject"}},{"contributor":"system-one-open (Gemma 4 E2B LoRA on an L4)","value":66.6,"contributorKind":"model","agent":{"model":"Gemma 4 E2B","evidence":"system-one-open (Gemma 4 E2B LoRA on an L4)","family":"google","confidence":"high","role":"subject"}},{"contributor":"OpenJev razorback16 (DiffusionGemma 26B)","value":66.4,"contributorKind":"model","agent":{"model":"DiffusionGemma 26B","evidence":"OpenJev razorback16 (DiffusionGemma 26B)","family":"google","confidence":"high","role":"subject"}},{"contributor":"SimpleJev Qwen3.8-27B","value":66.3,"contributorKind":"model","agent":{"model":"Qwen3.8-27B","evidence":"SimpleJev Qwen3.8-27B","family":"alibaba","confidence":"high","role":"subject"}},{"contributor":"ZeroEntropy zerank-2","value":66,"contributorKind":"model","agent":{"model":"zerank-2","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"GPT-5.6 Luna (low)","value":65.9,"contributorKind":"model","agent":{"model":"GPT-5.6 Luna","evidence":"GPT-5.6 Luna (low reasoning effort): 97.1% over 242 scorable decisions","family":"openai","confidence":"high","role":"subject"}},{"contributor":"openjev-sglang (Qwen3.6-35B-A3B on SGLang)","value":65.3,"contributorKind":"model","agent":{"model":"Qwen3.6-35B-A3B","evidence":"openjev-sglang (Qwen3.6-35B-A3B on SGLang)","family":"alibaba","confidence":"high","role":"subject"}},{"contributor":"Qwen3-Reranker-4B","value":63.8,"contributorKind":"model","agent":{"model":"Qwen3-Reranker-4B","evidence":"Qwen3-Reranker-4B","family":"alibaba","confidence":"high","role":"subject"}},{"contributor":"reflex-27b (Qwen3.8-27B)","value":63.3,"contributorKind":"model","agent":{"model":"Qwen3.8-27B","evidence":"reflex-27b (Qwen3.8-27B)","family":"alibaba","confidence":"high","role":"subject"}},{"contributor":"LitJev (Qwen3.8-27B)","value":62.7,"contributorKind":"model","agent":{"model":"Qwen3.8-27B","evidence":"LitJev (Qwen3.8-27B)","family":"alibaba","confidence":"high","role":"subject"}},{"contributor":"kev 0.6B (research preview)","value":62.5,"contributorKind":"model","agent":{"model":"kev 0.6B","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"SimpleJev Qwen3.6-35B-A3B","value":62.5,"contributorKind":"model","agent":{"model":"Qwen3.6-35B-A3B","evidence":"SimpleJev Qwen3.6-35B-A3B","family":"alibaba","confidence":"high","role":"subject"}},{"contributor":"djev (thinking)","value":62.4,"contributorKind":"model","agent":{"model":"DiffusionGemma","family":"google","confidence":"high","role":"subject"}},{"contributor":"jev-local (Qwen3.5-9B)","value":61.8,"contributorKind":"model","agent":{"model":"Qwen3.5-9B","evidence":"jev-local (Qwen3.5-9B)","family":"alibaba","confidence":"high","role":"subject"}},{"contributor":"decider-2b (Mapika)","value":61.7,"contributorKind":"model","agent":{"model":"decider-2b","evidence":"JevBench v1.2.8: add decider-35b-a3b, reflex-27b, reflex 4B, decider-2b","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"Bespoke Nimble 9B","value":60.5,"contributorKind":"model","agent":{"model":"Qwen3.5-9B","family":"alibaba","confidence":"high","role":"subject"}},{"contributor":"Gemini 3.1 Flash-Lite","value":60.1,"contributorKind":"model","agent":{"model":"Gemini 3.1 Flash-Lite","evidence":"Gemini 3.1 Flash-Lite","family":"google","confidence":"high","role":"subject"}},{"contributor":"OpenJev (thinking, BF16)","value":60,"contributorKind":"model","agent":{"model":"DiffusionGemma 26B","family":"google","confidence":"high","role":"subject"}},{"contributor":"kev 4B (research preview)","value":59.7,"contributorKind":"model","agent":{"model":"kev 4B","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"DeepSeek V4.1 Flash","value":57.5,"contributorKind":"model","agent":{"model":"DeepSeek V4.1 Flash","evidence":"DeepSeek V4.1 Flash (thinking default)","family":"deepseek","confidence":"high","role":"subject"}},{"contributor":"kev 8B (research preview)","value":56.4,"contributorKind":"model","agent":{"model":"kev 8B","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"Open-Jev 9B (Zefan Cai)","value":55,"contributorKind":"model","agent":{"model":"Qwen3.5-9B","family":"alibaba","confidence":"high","role":"subject"}},{"contributor":"system-one (Qwen3-8B, Goedecke)","value":54.8,"contributorKind":"model","agent":{"model":"Qwen3-8B","evidence":"system-one (Qwen3-8B, Goedecke)","family":"alibaba","confidence":"high","role":"subject"}},{"contributor":"jeff (GLiFormer 400M)","value":54.4,"contributorKind":"model","agent":{"model":"GLiFormer 400M","evidence":"jeff (GLiFormer 400M)","family":"other-ai","confidence":"low","role":"subject"}},{"contributor":"Laya (421M)","value":54.4,"contributorKind":"model","agent":{"model":"ModernBERT-large","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"Open-Jev 2B (Zefan Cai)","value":51.3,"contributorKind":"model","agent":{"model":"Qwen3.5-2B","family":"alibaba","confidence":"high","role":"subject"}},{"contributor":"OpenDecision (ModernBERT-large zero-shot)","value":40.6,"contributorKind":"model","agent":{"model":"ModernBERT-large","evidence":"OpenDecision (ModernBERT-large zero-shot)","family":"other-ai","confidence":"low","role":"subject"}},{"contributor":"openJev Verdict 1.4","value":38.9,"contributorKind":"model","agent":{"model":"openJev Verdict 1.4","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"openJev Verdict (151M)","value":38.1,"contributorKind":"model","agent":{"model":"openJev Verdict 151M","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"kev 0.5B","value":33.2,"contributorKind":"model","agent":{"model":"kev 0.5B","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"GLiNER2 large (Fastino)","value":29.6,"contributorKind":"model","agent":{"model":"GLiNER2 large","evidence":"JevBench v1.2.8: add ... GLiNER2 large","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"smalljev semantic-v9","value":27.4,"contributorKind":"model","agent":{"model":"MiniCPM5-2B","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"GLiNER2 (gliner2.5-base)","value":24,"contributorKind":"model","agent":{"model":"gliner2.5-base","evidence":"GLiNER2 (gliner2.5-base)","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"open-jev-deberta-v3-large (local CPU)","value":23.1,"contributorKind":"model","agent":{"model":"DeBERTa-v3-large","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"GLiNER2.5 multi (Fastino, 287M)","value":16.6,"contributorKind":"model","agent":{"model":"GLiNER2.5 multi","evidence":"JevBench v1.2.7: add GLiNER2.5 small and multi","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"GLiNER2.5 small (Fastino, 74M)","value":13.8,"contributorKind":"model","agent":{"model":"GLiNER2.5 small","evidence":"JevBench v1.2.7: add GLiNER2.5 small and multi","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"Mixedbread mxbai-rerank-base-v2","value":0.8,"contributorKind":"model","agent":{"model":"mxbai-rerank-base-v2","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"BAAI bge-reranker-v2-m3","value":0.7,"contributorKind":"model","agent":{"model":"bge-reranker-v2-m3","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"Alibaba GTE Reranker ModernBERT-base","value":0.3,"contributorKind":"model","agent":{"model":"GTE Reranker ModernBERT-base","evidence":"Alibaba GTE Reranker ModernBERT-base","family":"alibaba","confidence":"high","role":"subject"}},{"contributor":"Certo v1 (AltSlate Labs)","value":0,"contributorKind":"model","agent":{"model":"Certo v1","family":"other-ai","confidence":"high","role":"subject"}}],"contested":true},{"description":"Services that run another entrant's model are listed with all scores and axes but not ranked against the models.","id":"jevbench-v1.3.0-honorable-mentions","name":"JevBench v1.3.0 honorable mentions","metricName":"JevBench Score","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"classifier.dev (fast tier)","description":"Runs on Jev (TypeSafe); listed, not ranked.","value":83.6,"contributorKind":"model","isCurrentBest":true}],"contested":false},{"description":"Systems shown but not ranked because a tier was attempted for fewer than 95% of its decisions.","id":"jevbench-v1.3.0-partial-runs","name":"JevBench v1.3.0 partial runs","metricName":"JevBench Score","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"Qwen3.8 27B (partial run)","value":24.8,"contributorKind":"model","isCurrentBest":true,"agent":{"model":"Qwen3.8 27B","evidence":"Qwen3.8 27B (partial run)","family":"alibaba","confidence":"high","role":"subject"}},{"contributor":"Needle 3, options as tools (partial run)","value":1.1,"contributorKind":"model"},{"contributor":"Needle 3 (partial run)","value":0.1,"contributorKind":"model"}],"contested":true},{"description":"The frozen v1.1 task set (314 typed decisions: 72 easy / 96 standard / 146 judge) and v1.1.2 scoring, plus open Jev rebuilds that need a GPU, run on rented RunPod GPUs. Main Score = (Capability + Speed + Cost) / 3.","id":"jevbench-v1.1.3","name":"JevBench v1.1.3 (The GPU Round)","metricName":"Main Score","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"SemIf, formerly OpenJev (Qwen3.5-4B, TheoLeeCJ)","value":83.3,"contributorKind":"model","isCurrentBest":true,"agent":{"model":"Qwen3.5-4B","evidence":"SemIf, formerly OpenJev (Qwen3.5-4B, TheoLeeCJ)","family":"alibaba","confidence":"high","role":"subject"}},{"contributor":"open-alternative-jev, author's yes/no order (post-hoc adapter mode)","value":79.7,"contributorKind":"model"},{"contributor":"OpenJev (DiffusionGemma 26B-A4B NVFP4, razorback16)","value":78.2,"contributorKind":"model","agent":{"model":"DiffusionGemma 26B-A4B","evidence":"OpenJev (DiffusionGemma 26B-A4B NVFP4, razorback16)","family":"google","confidence":"high","role":"subject"}},{"contributor":"system-one (Qwen3-8B, Sean Goedecke)","value":77.9,"contributorKind":"model","agent":{"model":"Qwen3-8B","evidence":"system-one (Qwen3-8B, Sean Goedecke)","family":"alibaba","confidence":"high","role":"subject"}},{"contributor":"open-alternative-jev (Qwen3.5-4B, IkerMoel)","value":76.8,"contributorKind":"model","agent":{"model":"Qwen3.5-4B","evidence":"open-alternative-jev (Qwen3.5-4B, IkerMoel)","family":"alibaba","confidence":"high","role":"subject"}},{"contributor":"system-one-open (Gemma 4 E2B LoRA on an L4)","value":75.7,"contributorKind":"model","agent":{"model":"Gemma 4 E2B","evidence":"system-one-open (Gemma 4 E2B LoRA on an L4)","family":"google","confidence":"high","role":"subject"}},{"contributor":"Bespoke Nimble 9B (Bespoke Labs)","value":73.8,"contributorKind":"model"},{"contributor":"Jev 1.13.0 (TypeSafe AI)","value":73.6,"contributorKind":"model","agent":{"model":"Jev 1.13.0","evidence":"Jev 1.13.0 (TypeSafe AI): 96.3% over 242 scorable decisions","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"openjev-sglang (Qwen3.6-35B-A3B on SGLang)","value":69.6,"contributorKind":"model","agent":{"model":"Qwen3.6-35B-A3B","evidence":"openjev-sglang (Qwen3.6-35B-A3B on SGLang)","family":"alibaba","confidence":"high","role":"subject"}},{"contributor":"Gemini 3.1 Flash-Lite","value":65,"contributorKind":"model","agent":{"model":"Gemini 3.1 Flash-Lite","evidence":"Gemini 3.1 Flash-Lite","family":"google","confidence":"high","role":"subject"}},{"contributor":"GPT-5.6 Luna (low reasoning effort)","value":62.2,"contributorKind":"model","agent":{"model":"GPT-5.6 Luna","evidence":"GPT-5.6 Luna (low reasoning effort): 97.1% over 242 scorable decisions","family":"openai","confidence":"high","role":"subject"}},{"contributor":"open-jev-deberta-v3-large (local CPU)","value":60.6,"contributorKind":"model","agent":{"model":"DeBERTa-v3-large","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"DeepSeek V4.1 Flash (thinking default)","value":55.7,"contributorKind":"model","agent":{"model":"DeepSeek V4.1 Flash","evidence":"DeepSeek V4.1 Flash (thinking default)","family":"deepseek","confidence":"high","role":"subject"}},{"contributor":"Needle 3, options as tools (post-hoc adapter mode)","value":41.5,"contributorKind":"model","agent":{"model":"Needle 3","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"Needle 3 (Cactus, 2-bit, local CPU)","value":40.3,"contributorKind":"model","agent":{"model":"Needle 3","family":"other-ai","confidence":"high","role":"subject"}}],"contested":true},{"description":"Systems shown but not ranked because a tier was not attempted in full.","id":"jevbench-v1.1.3-shown-not-ranked","name":"JevBench v1.1.3 shown, not ranked","metricName":"Main Score","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"Qwen3.8 27B (Chutes TEE)","description":"v1.0 run stopped at 223 of 242 after three empty completions in a row.","value":42.5,"contributorKind":"model","isCurrentBest":true,"agent":{"model":"Qwen3.8 27B","evidence":"Qwen3.8 27B (Chutes TEE)","family":"alibaba","confidence":"high","role":"subject"}}],"contested":false},{"description":"Adds an easy tier and scores three sub-benchmarks (Capability, Speed, Cost) over 314 decisions. Main Score = (Capability + Speed + Cost) / 3, Balanced 33:33:33.","id":"jevbench-v1.1","name":"JevBench v1.1","metricName":"Main Score","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"system-one-open (Gemma 4 E2B LoRA on an L4)","value":75.7,"contributorKind":"model","isCurrentBest":true,"agent":{"model":"Gemma 4 E2B","evidence":"system-one-open (Gemma 4 E2B LoRA on an L4)","family":"google","confidence":"high","role":"subject"}},{"contributor":"Jev 1.13.0 (TypeSafe AI)","value":73.6,"contributorKind":"model","agent":{"model":"Jev 1.13.0","evidence":"Jev 1.13.0 (TypeSafe AI): 96.3% over 242 scorable decisions","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"openjev-sglang (Qwen3.6-35B-A3B on SGLang)","value":69.6,"contributorKind":"model","agent":{"model":"Qwen3.6-35B-A3B","evidence":"openjev-sglang (Qwen3.6-35B-A3B on SGLang)","family":"alibaba","confidence":"high","role":"subject"}},{"contributor":"Gemini 3.1 Flash-Lite","value":65,"contributorKind":"model","agent":{"model":"Gemini 3.1 Flash-Lite","evidence":"Gemini 3.1 Flash-Lite","family":"google","confidence":"high","role":"subject"}},{"contributor":"GPT-5.6 Luna (low reasoning effort)","value":62.2,"contributorKind":"model","agent":{"model":"GPT-5.6 Luna","evidence":"GPT-5.6 Luna (low reasoning effort): 97.1% over 242 scorable decisions","family":"openai","confidence":"high","role":"subject"}},{"contributor":"open-jev-deberta-v3-large (local CPU)","value":60.6,"contributorKind":"model"},{"contributor":"DeepSeek V4.1 Flash (thinking default)","value":55.7,"contributorKind":"model","agent":{"model":"DeepSeek V4.1 Flash","evidence":"DeepSeek V4.1 Flash (thinking default)","family":"deepseek","confidence":"high","role":"subject"}},{"contributor":"Needle 3, options as tools (post-hoc adapter mode)","value":41.5,"contributorKind":"model"},{"contributor":"Needle 3 (Cactus, 2-bit, local CPU)","value":40.3,"contributorKind":"model"}],"contested":true},{"description":"Systems shown but not ranked because a tier was not attempted in full.","id":"jevbench-v1.1-shown-not-ranked","name":"JevBench v1.1 shown, not ranked","metricName":"Main Score","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"Qwen3.8 27B (Chutes TEE)","value":42.5,"contributorKind":"model","isCurrentBest":true,"agent":{"model":"Qwen3.8 27B","evidence":"Qwen3.8 27B (Chutes TEE)","family":"alibaba","confidence":"high","role":"subject"}},{"contributor":"open-alternative-jev (Qwen3.5-4B, HF Space)","description":"Free Hugging Face Space ran out of ZeroGPU quota after 6 decisions.","value":0,"contributorKind":"model","agent":{"model":"Qwen3.5-4B","evidence":"open-alternative-jev (Qwen3.5-4B, HF Space)","family":"alibaba","confidence":"high","role":"subject"}}],"contested":true},{"description":"The original release: 9 systems, 242 typed decisions each, five axes (smart, cheap, fast, reliable, open) with no combined score. Accuracy is argmax over the exact label set.","metricUnit":"%","id":"jevbench-v1","name":"JevBench v1","metricName":"Accuracy","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"GPT-5.6 Luna (low reasoning effort)","value":97.1,"contributorKind":"model","isCurrentBest":true,"agent":{"model":"GPT-5.6 Luna","evidence":"GPT-5.6 Luna (low reasoning effort): 97.1% over 242 scorable decisions","family":"openai","confidence":"high","role":"subject"}},{"contributor":"Jev 1.13.0 (TypeSafe AI)","value":96.3,"contributorKind":"model","agent":{"model":"Jev 1.13.0","evidence":"Jev 1.13.0 (TypeSafe AI): 96.3% over 242 scorable decisions","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"openjev-sglang (Qwen3.6-35B-A3B on SGLang)","value":95.5,"contributorKind":"model","agent":{"model":"Qwen3.6-35B-A3B","evidence":"openjev-sglang (Qwen3.6-35B-A3B on SGLang)","family":"alibaba","confidence":"high","role":"subject"}},{"contributor":"Gemini 3.1 Flash-Lite","value":95.5,"contributorKind":"model","agent":{"model":"Gemini 3.1 Flash-Lite","evidence":"Gemini 3.1 Flash-Lite","family":"google","confidence":"high","role":"subject"}},{"contributor":"DeepSeek V4.1 Flash (thinking default)","value":95.5,"contributorKind":"model","agent":{"model":"DeepSeek V4.1 Flash","evidence":"DeepSeek V4.1 Flash (thinking default)","family":"deepseek","confidence":"high","role":"subject"}},{"contributor":"system-one-open (Gemma 4 E2B LoRA on an L4)","value":90.1,"contributorKind":"model","agent":{"model":"Gemma 4 E2B","evidence":"system-one-open (Gemma 4 E2B LoRA on an L4)","family":"google","confidence":"high","role":"subject"}},{"contributor":"open-jev-deberta-v3-large (local CPU)","value":51.7,"contributorKind":"model"},{"contributor":"Qwen3.8 27B (Chutes TEE)","description":"223/242 decisions, stopped early.","value":96.9,"contributorKind":"model","agent":{"model":"Qwen3.8 27B","evidence":"Qwen3.8 27B (Chutes TEE)","family":"alibaba","confidence":"high","role":"subject"}}],"contested":true},{"description":"A frozen offline replay over all 534 decisions testing confidence cascades, committees and best-of-n. Does not change the ranked leaderboard; no combination beat the best published single-system composite.","id":"jevbench-v1.2.6-combinations","name":"JevBench v1.2.6 combination study","metricName":"Composite","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"classifier.dev Fast → Jev (cascade, threshold 0.42)","value":81.225,"contributorKind":"method","isCurrentBest":true,"agent":{"model":"Jev 1.13.0","evidence":"classifier.dev (fast tier) ... 'The fast tier is Jev, TypeSafe's decision model'","family":"other-ai","confidence":"medium","role":"subject"}},{"contributor":"djev → Jev (cascade, threshold 0.79)","value":72.197,"contributorKind":"method","agent":{"model":"diffusion-gemma","evidence":"JevBench v1.2.1: add djev (Maisa, diffusion-gemma)","family":"google","confidence":"medium","role":"subject"}},{"contributor":"SemIf 4B → djev (cascade, threshold 0.39)","value":72.365,"contributorKind":"method","agent":{"model":"Qwen3.5-4B","evidence":"SemIf (Qwen3.5-4B)","family":"alibaba","confidence":"medium","role":"subject"}},{"contributor":"Calibration-weighted committee: Jev + SemIf + djev","value":70.856,"contributorKind":"method","agent":{"model":"Jev 1.13.0","evidence":"Calibration-weighted committee: Jev + SemIf + djev","family":"other-ai","confidence":"low","role":"subject"}},{"contributor":"Probability average committee: Jev + SemIf + djev","value":70.462,"contributorKind":"method","agent":{"model":"Jev 1.13.0","evidence":"Probability average committee: Jev + SemIf + djev","family":"other-ai","confidence":"low","role":"subject"}},{"contributor":"Majority committee: Jev + djev + open-alternative-jev","value":66.383,"contributorKind":"method","agent":{"model":"Jev 1.13.0","evidence":"Majority committee: Jev + djev + open-alternative-jev","family":"other-ai","confidence":"low","role":"subject"}},{"contributor":"Best-of-n: OpenJev razorback16 two-sample average","value":62.029,"contributorKind":"method","agent":{"model":"DiffusionGemma 26B","evidence":"Best-of-n: OpenJev razorback16 two-sample average","family":"google","confidence":"medium","role":"subject"}}],"contested":true}],"agentStats":[],"repo":{"owner":"fstandhartinger","name":"jevbench","branch":"main"},"participants":[],"stats":{"totalRecords":104,"uniqueParticipants":0,"recordsLast30d":0,"recordsLast90d":0},"lastUpdated":"2026-10-19T09:08:28.024Z"},{"id":"putnambench","name":"PutnamBench","tagline":"A multilingual benchmark for evaluating theorem-proving algorithms on competition mathematics problems from the Putnam Competition (1962-2025).","description":"PutnamBench is a benchmark for evaluation of theorem-proving algorithms on competition mathematics problems sourced from the William Lowell Putnam Mathematical Competition years 1962 - 2025. It comprises 1724 manually-crafted formalizations across Lean 4, Isabelle, and Coq, and includes factored solutions for problems that require a numerical answer. The benchmark is maintained by researchers at UT Austin and hosts a public leaderboard for evaluation results accompanied by preprints or publications.","url":"https://github.com/trishullab/PutnamBench","host":"github","status":"active","domain":"formal-methods","tags":["theorem-proving","automated-reasoning","formal-methods","lean4","isabelle","coq","mathematics"],"images":[],"participation":{"howToSubmit":"Reach out privately at george.tsoukalas@utexas.edu with requests for additions to the leaderboard. Submissions require an accompanying preprint or publication.","submissionFormat":"Evaluation results accompanied by a preprint or publication.","deadline":"rolling","eligibility":"Open to researchers with a preprint or publication.","cost":"free","verification":"Human review by organizers.","requirements":["Lean 4","Isabelle","Coq"],"compute":"unknown"},"links":[{"label":"Leaderboard","url":"https://trishullab.github.io/PutnamBench/leaderboard.html"},{"label":"Website","url":"https://trishullab.github.io/PutnamBench/"},{"label":"Paper","url":"https://arxiv.org/abs/2407.11214"}],"organizer":{"name":"Trishul Lab, University of Texas at Austin","url":"https://trishullab.github.io/PutnamBench/","type":"academic"},"venue":{"type":"online"},"quickstart":"git clone https://github.com/trishullab/PutnamBench.git\ncd PutnamBench\n# Follow setup instructions for the desired formal language (Lean 4, Isabelle, or Coq)\n# Run evaluations and submit results with accompanying paper","problems":[{"group":"coq","metricUnit":"problems","id":"coq-proof-only","name":"Coq (Proof Only)","metricName":"problems solved","metricDirection":"maximize","isOpen":true,"records":[],"contested":false},{"group":"coq","metricUnit":"problems","id":"coq-with-answer","name":"Coq (With Answer)","metricName":"problems solved","metricDirection":"maximize","isOpen":true,"records":[],"contested":false},{"group":"isabelle","metricUnit":"problems","id":"isabelle-proof-only","name":"Isabelle (Proof Only)","metricName":"problems solved","metricDirection":"maximize","isOpen":true,"records":[],"contested":false},{"group":"isabelle","metricUnit":"problems","id":"isabelle-with-answer","name":"Isabelle (With Answer)","metricName":"problems solved","metricDirection":"maximize","isOpen":true,"records":[],"contested":false},{"group":"lean4","metricUnit":"problems","id":"lean4-proof-only","name":"Lean 4 (Proof Only)","metricName":"problems solved","metricDirection":"maximize","isOpen":true,"records":[],"contested":false},{"group":"lean4","metricUnit":"problems","id":"lean4-with-answer","name":"Lean 4 (With Answer)","metricName":"problems solved","metricDirection":"maximize","isOpen":true,"records":[],"contested":false}],"agentStats":[],"repo":{"owner":"trishullab","name":"PutnamBench","branch":"main"},"participants":[],"stats":{"totalRecords":0,"uniqueParticipants":0,"recordsLast30d":0,"recordsLast90d":0},"lastUpdated":"2026-10-19T09:02:20.227Z"},{"id":"recursive-org-first-steps-toward-automated-ai-research-a88f20a3f32849dd","name":"First Steps Toward Automated AI Research","tagline":"Solutions discovered by Recursive's automated AI-research system for NanoGPT speedrun, SOL-ExecBench, and NanoChat.","description":"This repository collects training scripts and GPU kernel implementations discovered by Recursive's automated AI-research system, as referenced in their blog post. It includes solutions for the NanoGPT speedrun (Track 1), 10 of 235 GPU kernel implementations for NVIDIA SOL-ExecBench, and NanoChat training scripts with per-seed validation BPB results. The repo serves as an illustrative release of research outcomes rather than a competition with an open submission path.","url":"https://github.com/recursive-org/first-steps-toward-automated-ai-research","host":"github","status":"ended","domain":"ml-research","tags":["nanogpt","gpu-kernels","nanochat","automated-research","training-speedrun"],"images":[],"participation":{"howToSubmit":"Not applicable; this is a release of research outcomes, not an open competition.","submissionFormat":"Training scripts and kernel implementations.","computeDetails":"8× H100 for NanoGPT speedrun; B200 for SOL-ExecBench kernels; single B200 for NanoChat.","deadline":"rolling","rounds":"Not applicable","prizes":"recognition only","eligibility":"Not applicable","cost":"free","verification":"Not applicable","requirements":["Python","PyTorch","Triton","CUDA"],"compute":"datacenter-gpu"},"links":[{"label":"Blog post","url":"https://www.recursive.com/articles/first-steps-toward-automated-ai-research"},{"label":"NVIDIA SOL-ExecBench","url":"https://github.com/NVIDIA/SOL-ExecBench"},{"label":"KellerJordan/modded-nanogpt","url":"https://github.com/KellerJordan/modded-nanogpt"},{"label":"karpathy/nanochat","url":"https://github.com/karpathy/nanochat"}],"organizer":{"name":"Recursive","url":"https://www.recursive.com/","type":"company"},"venue":{"type":"online"},"quickstart":"python3 from_best/analyze_sweep.py\n# runs   n=13  val_loss=3.2789±0.0014  t=-2.815 p(mean<3.28)=0.007803  time=77.34±0.20s\npython3 from_unoptimized/analyze_sweep.py\n# runs   n=10  val_loss=3.2778±0.0010  t=-6.880 p(mean<3.28)=3.612e-05  time=185.42±0.27s","problems":[{"group":"nanochat_autoresearch","description":"Train a self-contained model for 5 minutes on a single B200 GPU, scored by validation bits-per-byte (BPB), lower is better.","id":"nanochat-autoresearch","name":"NanoChat Autoresearch","metricName":"validation BPB","metricDirection":"minimize","isOpen":false,"records":[{"contributor":"vanilla_transformer","description":"A plain transformer baseline","value":1.0587,"contributorKind":"method","isBaseline":true},{"contributor":"optimized_from_vanilla","description":"Autoresearch run optimizing from the vanilla transformer","value":0.9344,"contributorKind":"method"},{"contributor":"optimized_from_karpathy","description":"Best solution, optimizing from Karpathy's baseline","value":0.9109,"contributorKind":"method","isBaseline":true,"isCurrentBest":true}],"contested":false},{"group":"nanogpt_speedrun","description":"Train a GPT-2-small tier model to ≤ 3.28 FineWeb validation loss on 8×H100 as fast as possible.","metricUnit":"s","id":"nanogpt-speedrun-track-1","name":"NanoGPT Speedrun Track 1","metricName":"training time","metricDirection":"minimize","baseline":80.61,"isOpen":false,"records":[{"contributor":"#83 — our same-hardware baseline","description":"Re-run of record #83 on same hardware","value":80.61,"contributorKind":"method","isBaseline":true},{"contributor":"from-best (ours)","description":"Best solution branched from record #83","value":77.34,"contributorKind":"method","isCurrentBest":true},{"contributor":"from_unoptimized","description":"Solution discovered from a weak ~15-minute baseline","value":185.4,"contributorKind":"method","isBaseline":true}],"contested":false},{"group":"sol-execbench","description":"GPU kernel implementations scored on B200 using SOL score (higher is better).","id":"sol-execbench-kernels","name":"SOL-ExecBench Kernels","metricName":"SOL score","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"011_fp8_moe_gate_routing.py","description":"Quant category","value":0.866,"contributorKind":"method"},{"contributor":"012_gqa_paged_decode_h32_kv4_d128_ps1.py","description":"FlashInfer-Bench category","value":0.934,"contributorKind":"method","isCurrentBest":true},{"contributor":"025_moe_expert_parallel_execution_backward.py","description":"L2 category","value":0.533,"contributorKind":"method"},{"contributor":"032_nvfp4_moe_expert_linear_with_gating.py","description":"Quant category","value":0.906,"contributorKind":"method"},{"contributor":"045_fused_linear_gelu_grn_linear.cu","description":"L1 category","value":0.733,"contributorKind":"method"},{"contributor":"049_group_limited_topk_routing.cu","description":"L2 category","value":0.884,"contributorKind":"method"},{"contributor":"057_residual_coupling_flow_block.py","description":"L2 category","value":0.721,"contributorKind":"method"},{"contributor":"069_joint_transformer_block_residual_path.py","description":"L2 category","value":0.831,"contributorKind":"method"},{"contributor":"073_fused_encoder_final_norm_to_decoder_cross_attention_kv_projection.py","description":"L1 category","value":0.846,"contributorKind":"method"},{"contributor":"077_whisper_decoder_output_projection.py","description":"L1 category","value":0.72,"contributorKind":"method"}],"contested":true}],"agentStats":[],"repo":{"owner":"recursive-org","name":"first-steps-toward-automated-ai-research","branch":"main"},"participants":[],"stats":{"totalRecords":16,"uniqueParticipants":0,"recordsLast30d":0,"recordsLast90d":0},"lastUpdated":"2026-10-19T09:02:20.228Z"},{"id":"lora-speedrun","name":"LoRA Speedrun","tagline":"Fine-tune a small LLM to a target accuracy on one L40S in the least training wall-clock.","description":"A frozen-task speedrun for LoRA fine-tuning: train a pinned base model to a fixed accuracy target on a fixed dataset, minimizing training wall-clock on a single Modal L40S. Two tracks (GSM8K/Qwen2.5-1.5B and SQuAD/SmolLM2-1.7B) with a ≤30M adapter-param cap and a 60-minute run limit. Every record is re-run 3× with fresh seeds in a network-blocked sandbox and reviewed by a maintainer before it counts. It exists to make fine-tuning speedup claims falsifiable, in the spirit of modded-nanogpt.","url":"https://github.com/Saivineeth147/lora-speedrun","host":"github","status":"active","domain":"hardware-efficiency","tags":["lora","fine-tuning","wall-clock","speedrun","llm","leaderboard"],"images":["https://raw.githubusercontent.com/Saivineeth147/lora-speedrun/main/docs/leaderboard.png","https://raw.githubusercontent.com/Saivineeth147/lora-speedrun/main/docs/verification-replay.gif"],"participation":{"howToSubmit":"Copy submissions/TEMPLATE/ to submissions/NNN-<your-github-handle>/, add train.py, config.yaml and NOTES.md, self-verify with harness/modal_verify.py --runs 3, then open a PR; a maintainer comments /verify to re-run it 3× with fresh seeds.","submissionFormat":"A directory with train.py (implements --data-dir/--output-dir/--seed), config.yaml (author, technique, hyperparameters, extra pinned packages, track), and NOTES.md explaining the mechanism; no adapter weights committed.","computeDetails":"Official timing runs on a single NVIDIA L40S (48 GB) in a network-blocked Modal sandbox; free via Modal monthly credits. Local iteration possible on any 24 GB+ card but times are not official.","deadline":"rolling","prizes":"recognition only","eligibility":"Anyone; submissions run as untrusted code in a network-blocked, secretless sandbox.","cost":"free (Modal monthly credits cover official runs)","verification":"CI static validation, an advisory Claude security screen, human code review, then 3 fresh-seed timed reruns on spec hardware with adapter-param and content-hash integrity audits; report committed to records/verifications/.","requirements":["Python 3.12/3.13","PyTorch","PEFT","Modal account","YAML config"],"compute":"datacenter-gpu"},"links":[{"label":"Live leaderboard (HF Space)","url":"https://huggingface.co/spaces/vineeth98/lora-speedrun"},{"label":"Task specs (TASK.md)","url":"https://github.com/Saivineeth147/lora-speedrun/blob/main/TASK.md"},{"label":"Judging protocol (JUDGING.md)","url":"https://github.com/Saivineeth147/lora-speedrun/blob/main/JUDGING.md"},{"label":"Contributing (CONTRIBUTING.md)","url":"https://github.com/Saivineeth147/lora-speedrun/blob/main/CONTRIBUTING.md"},{"label":"Security model (SECURITY.md)","url":"https://github.com/Saivineeth147/lora-speedrun/blob/main/SECURITY.md"},{"label":"Record history (records/RECORDS.md)","url":"https://github.com/Saivineeth147/lora-speedrun/blob/main/records/RECORDS.md"}],"organizer":{"name":"Saivineeth147","url":"https://github.com/Saivineeth147","type":"individual"},"venue":{"type":"online"},"quickstart":"git clone https://github.com/Saivineeth147/lora-speedrun && cd lora-speedrun\npip install modal pyyaml && modal setup\npython harness/modal_verify.py --prefetch\npython harness/modal_verify.py --submission submissions/000-baseline --runs 1\npython harness/modal_verify.py --submission submissions/000-baseline --runs 3","problems":[{"group":"t2","description":"Fine-tune HuggingFaceTB/SmolLM2-1.7B on the SQuAD v1.1 train split to ≥ 75.5% exact-match, minimizing training wall-clock on one L40S; adapter-only, ≤30M trainable params.","metricUnit":"s","id":"t2","name":"Track 2 — SQuAD v1.1 · SmolLM2-1.7B · target ≥ 75.5% · 1× L40S","metricName":"train time","metricDirection":"minimize","baseline":525.4,"isOpen":false,"records":[{"contributor":"Saivineeth147","description":"Track 2 baseline: plain LoRA r=16 on SQuAD, first 20k examples, 1 epoch, full-sequence loss. No tricks.","date":"2026-07-20","value":525.4,"contributorKind":"person","contributorUrl":"https://github.com/Saivineeth147","submissionUrls":["https://github.com/Saivineeth147/lora-speedrun/blob/main/submissions/t2-000-baseline","https://github.com/Saivineeth147/lora-speedrun/blob/main/records/verifications/t2-000-baseline.md"],"isBaseline":true,"agent":{"model":"Claude Fable 5","tool":"Claude Code","evidence":"Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>","family":"anthropic","confidence":"medium","role":"author"}},{"contributor":"abacon","description":"Track 2 port of the Track 1 stack: shortest-4k pruning, best-fit packing, fused base+LoRA-A GEMMs, chunked completion-only CE, 0.75 epoch on SmolLM2-1.7B.","date":"2026-07-27","value":49.5,"contributorKind":"person","contributorUrl":"https://github.com/abacon","submissionUrls":["https://github.com/Saivineeth147/lora-speedrun/blob/main/submissions/t2-001-bacon","https://github.com/Saivineeth147/lora-speedrun/blob/main/records/verifications/t2-001-bacon.md"],"isCurrentBest":true}],"contested":false},{"group":"t1","description":"Fine-tune Qwen/Qwen2.5-1.5B on the GSM8K train split to ≥ 57.0% exact-match, minimizing training wall-clock on one L40S; adapter-only, ≤30M trainable params.","metricUnit":"s","id":"t1","name":"Track 1 — GSM8K · Qwen2.5-1.5B · target ≥ 57.0% · 1× L40S","metricName":"train time","metricDirection":"minimize","baseline":716.9,"isOpen":false,"records":[{"contributor":"Saivineeth147","description":"Baseline: plain LoRA r=16 on all linear layers, 3 epochs, cosine LR. No tricks.","date":"2026-07-18","value":716.9,"contributorKind":"person","contributorUrl":"https://github.com/Saivineeth147","submissionUrls":["https://github.com/Saivineeth147/lora-speedrun/blob/main/submissions/000-baseline","https://github.com/Saivineeth147/lora-speedrun/blob/main/records/verifications/000-baseline.md"],"isBaseline":true,"agent":{"model":"Claude Fable 5","tool":"Claude Code","evidence":"Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>","family":"anthropic","confidence":"medium","role":"author"}},{"contributor":"Saivineeth147","description":"Sequence packing + completion-only loss masking, 2 epochs. Same LoRA config as #0; ~2x faster at higher accuracy.","date":"2026-07-18","value":364.9,"contributorKind":"person","contributorUrl":"https://github.com/Saivineeth147","submissionUrls":["https://github.com/Saivineeth147/lora-speedrun/blob/main/submissions/001-saivineeth147","https://github.com/Saivineeth147/lora-speedrun/blob/main/records/verifications/001-saivineeth147.md"],"agent":{"model":"Claude Fable 5","tool":"Claude Code","evidence":"Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>","family":"anthropic","confidence":"medium","role":"author"}},{"contributor":"stared","description":"One 4e-4 epoch over 3,000 examples with 2x optimizer updates (integrated-LR), 512-token packs. First outside record.","date":"2026-07-20","value":113,"contributorKind":"person","contributorUrl":"https://github.com/stared","submissionUrls":["https://github.com/Saivineeth147/lora-speedrun/blob/main/submissions/002-pmigdal","https://github.com/Saivineeth147/lora-speedrun/blob/main/records/verifications/002-pmigdal.md"],"agent":{"model":"Claude Fable 5","tool":"Claude Code","evidence":"stared | 2026-07-20 | migdal-fable | [record] stared — shortest-4k data pruning","family":"anthropic","confidence":"medium","role":"author"}},{"contributor":"stared","description":"Shortest-4k data pruning, 1 aggressive-LR epoch, <<...>> annotations stripped, custom GPU-resident packed loop, chunked completion-only CE (no full logits).","date":"2026-07-20","value":104,"contributorKind":"person","contributorUrl":"https://github.com/stared","submissionUrls":["https://github.com/Saivineeth147/lora-speedrun/blob/main/submissions/003-pmigdal","https://github.com/Saivineeth147/lora-speedrun/blob/main/records/verifications/003-pmigdal.md"],"agent":{"model":"Claude Fable 5","tool":"Claude Code","evidence":"stared | 2026-07-20 | migdal-fable | [record] stared — shortest-4k data pruning","family":"anthropic","confidence":"medium","role":"author"}},{"contributor":"slippylolo","description":"0.75-epoch six-target BF16 LoRA: shortest-4k pruning with down_proj dropped (13.76M params), deterministic best-fit packing, per-layer fused base+LoRA-A q/k/v & gate/up GEMMs, chunked completion-only CE.","date":"2026-07-25","value":43.9,"contributorKind":"person","contributorUrl":"https://github.com/slippylolo","submissionUrls":["https://github.com/Saivineeth147/lora-speedrun/blob/main/submissions/004-slippylolo","https://github.com/Saivineeth147/lora-speedrun/blob/main/records/verifications/004-slippylolo.md"]},{"contributor":"slippylolo","description":"Direct-from-safetensors Qwen load with a warmed page cache, then 12 optimizer updates: 8 full-network, then 4 through only the top decoder layer over a cached BF16 frozen-prefix boundary.","date":"2026-08-03","value":21.6,"contributorKind":"person","contributorUrl":"https://github.com/slippylolo","submissionUrls":["https://github.com/Saivineeth147/lora-speedrun/blob/main/submissions/005-slippylolo","https://github.com/Saivineeth147/lora-speedrun/blob/main/records/verifications/005-slippylolo.md"],"isCurrentBest":true}],"contested":true}],"agentStats":[{"family":"anthropic","records":3,"currentBests":0,"models":["Claude Fable 5"],"contributors":["Saivineeth147","stared"]}],"repo":{"owner":"Saivineeth147","name":"lora-speedrun","branch":"main"},"participants":[{"name":"slippylolo","url":"https://github.com/slippylolo","submissions":2,"problems":["t1"],"lastActive":"2026-08-03","bestRank":1},{"name":"Saivineeth147","url":"https://github.com/Saivineeth147","submissions":3,"problems":["t2","t1"],"lastActive":"2026-07-20"},{"name":"stared","url":"https://github.com/stared","submissions":2,"problems":["t1"],"lastActive":"2026-07-20"},{"name":"abacon","url":"https://github.com/abacon","submissions":1,"problems":["t2"],"lastActive":"2026-07-27"}],"stats":{"totalRecords":8,"uniqueParticipants":4,"firstSubmission":"2026-07-18","lastSubmission":"2026-08-03","recordsLast30d":0,"recordsLast90d":3},"lastUpdated":"2026-10-19T09:02:20.229Z"},{"id":"metacognitionai-navgpt-3-5b9a0bda12fb75be","name":"NavGPT-3 Benchmark Results","tagline":"Vision-and-language navigation on R2R-CE and RxR-CE; maximize Success Rate and nDTW on val_unseen.","description":"NavGPT-3 is a hierarchical navigation harness coupling a language-model Planner with the NavGPT VLA action policy, evaluated on the R2R-CE and RxR-CE val_unseen benchmarks. The README reports local validation metrics (SR, SPL, OSR, NE, nDTW, SDTW) for the VLA-only models and the full harness; the authors note these are local results rather than official challenge leaderboard submissions. The repo releases inference code, configs, and weights, but there is no documented submission process or public leaderboard people can submit to.","url":"https://github.com/metacognitionai/NavGPT-3","host":"github","status":"active","domain":"robotics","tags":["vln","embodied-agents","navigation","vla","llm-planner","habitat"],"images":["https://github.com/metacognitionai/NavGPT-3/raw/main/assets/teaser.png","https://github.com/metacognitionai/NavGPT-3/raw/main/assets/runtime_sequence.png","https://github.com/metacognitionai/NavGPT-3/raw/main/assets/codec_measured.png"],"participation":{"computeDetails":"Rendering and VLA inference require NVIDIA GPUs; shipped configs list 8 GPUs, one shard per GPU.","requirements":["Linux","NVIDIA GPU","Python 3.11 (planner, VLA)","Python 3.8 (Habitat)"],"compute":"datacenter-gpu"},"links":[{"label":"Project Page","url":"https://metacognitionai.github.io/NavGPT3/"},{"label":"Paper (arXiv)","url":"https://arxiv.org/abs/2610.10787"},{"label":"Models on Hugging Face","url":"https://huggingface.co/Metacognition-AI"},{"label":"Evaluation files","url":"https://huggingface.co/datasets/ZGZzz/NavGPT-3-website-media/tree/main/evaluation"}],"organizer":{"name":"AIML, Adelaide University / Metacognition","url":"https://github.com/metacognitionai","type":"academic"},"problems":[],"agentStats":[],"repo":{"owner":"metacognitionai","name":"NavGPT-3","branch":"main"},"participants":[],"stats":{"totalRecords":0,"uniqueParticipants":0,"recordsLast30d":0,"recordsLast90d":0},"lastUpdated":"2026-10-19T09:09:01.036Z"},{"id":"equational-theories-lean-stage2","name":"Mathematics Distillation Challenge — Equational Theories — Stage 2","tagline":"Decide whether one magma law implies another and submit a machine-verifiable Lean 4 certificate.","description":"Given two equational laws E1 and E2 over magmas, contestants must decide whether E1 implies E2 and supply a Lean 4 certificate: a proof for true implications or a magma witness (finite or infinite) for false ones. A deterministic Lean judge verifies each answer with no partial credit and no LLM-as-judge, mapping it to accepted/unparsed/malformed/incomplete_proof/incorrect. The submission is a single solver.py (<=500 KB) running under organizer-controlled LLM budgets. Run by SAIR (Sovereign AI Research) as Stage 2 of the Mathematics Distillation Challenge; the underlying corpus comes from Terence Tao's Equational Theories Project.","url":"https://competition.sair.foundation/competitions/mathematics-distillation-challenge-equational-theories-stage2/overview","host":"website","status":"active","domain":"formal-methods","tags":["equational-theories","lean4","automated-theorem-proving","universal-algebra","llm-agents","formal-verification"],"images":[],"participation":{"howToSubmit":"Submit a single solver.py (<=500 KB) through the official competition page; local development and validation run through the bundled harness (scripts/run_harness.py, scripts/run_marathon_harness.py).","submissionFormat":"solo track: single solver.py reading a problem JSON on stdin and writing one answer JSON line on stdout. marathon track: single solver.py reading a manifest JSONL and appending answers JSONL. If it uses the LLM, the Solo variant carries a module-level PROMPT = \"\"\"...\"\"\" template (extracted by AST, module never executed); Marathon calls the local HTTP proxy via marathon_llm.call_llm or the OpenAI SDK.","computeDetails":"Contestant solver runs on CPU only (Lean + Python); the organizer runs the judge and the pinned LLM (openai/gpt-oss-120b via deepinfra/bf16) on their side. Setup needs ~12 GB disk (Lean ~2.6 GB, .lake/Mathlib cache ~7.5 GB) and 8-16 GB RAM.","deadline":"rolling","rounds":"Two tracks (Solo, Marathon) sharing the same judge, verdict mapping, and solver contract; the final evaluation runs on a held-back set of the same corpus.","prizes":"recognition only","eligibility":"Open competition, hosted by SAIR Foundation on the official competition page.","cost":"free","verification":"Deterministic Lean judge (judge/verify.py) — same code runs locally and in official evaluation; no partial credit, five-status verdict mapping.","requirements":["Python 3.8+","Lean 4 (toolchain pinned by lean-toolchain, v4.33.1 at time of crawl)","Mathlib","Lake / elan","Docker (optional, for local sandbox parity)"],"compute":"cpu"},"links":[{"label":"Official competition page","url":"https://competition.sair.foundation/competitions/mathematics-distillation-challenge-equational-theories-stage2/overview"},{"label":"Equational Theories Project","url":"https://teorth.github.io/equational_theories/"},{"label":"Reference problem sets (HuggingFace)","url":"https://huggingface.co/datasets/SAIRfoundation/equational-theories-selected-problems"},{"label":"Law list (order <= 4, equations.txt)","url":"https://github.com/teorth/equational_theories/blob/main/data/equations.txt"},{"label":"Solo track spec","url":"https://github.com/SAIRcompetition/equational-theories-lean-stage2/blob/main/docs/solo_mode.md"},{"label":"Marathon track spec","url":"https://github.com/SAIRcompetition/equational-theories-lean-stage2/blob/main/docs/marathon_mode.md"}],"organizer":{"name":"SAIR Foundation","url":"https://competition.sair.foundation/","type":"foundation"},"venue":{"type":"online"},"quickstart":"bash scripts/setup.sh\npip install openai\nsource .env.judge\npython3 scripts/run_harness.py\npython3 -m pipeline.runner \\\n  --submission examples/solo/demos/baseline \\\n  --problems examples/problems/sample_20.json","problems":[{"description":"One solver subprocess per problem with a fixed per-problem budget (3600 s wall-clock, 65536 LLM output tokens per call, 100 KB Lean code, 20 KB false-certificate payload). Best for deep single-problem search. Metric: fraction of problems solved with accepted Lean certificates on the evaluation set.","id":"solo-track","name":"Solo Track","metricName":"problems solved","metricDirection":"maximize","isOpen":true,"records":[],"contested":false},{"description":"One solver subprocess handling N problems (reference N=100) under a single shared global budget of Nx300 s wall-clock and Nx32768 tokens. Rewards triage, cross-problem caching, and prompt-context reuse. Metric: number of problems solved within the shared budget.","id":"marathon-track","name":"Marathon Track","metricName":"problems solved","metricDirection":"maximize","isOpen":true,"records":[],"contested":false}],"agentStats":[],"repo":{"owner":"SAIRcompetition","name":"equational-theories-lean-stage2","branch":"main"},"participants":[],"stats":{"totalRecords":0,"uniqueParticipants":0,"recordsLast30d":0,"recordsLast90d":0},"lastUpdated":"2026-10-19T09:02:20.229Z"},{"id":"first-proof","name":"First Proof","tagline":"Unpublished research-level math problems; AI systems get one shot each and expert referees grade the proofs blind.","description":"First Proof is an independent benchmark run by research mathematicians to measure whether AI systems can autonomously prove research-level results. Each batch collects problems that arose in the contributors' own work and whose proofs are unpublished; the editorial board runs every system itself, one shot per problem with no human input, and expert referees grade the proofs double blind. Batch 2 (2026-06-10): 4 systems, 39 graded solutions, 7 of 10 problems got at least one passing proof; P4, P8, P10 remain open. Batch 3 is under way; results are due 2026-10-14.","url":"https://1stproof.org/","host":"website","status":"active","domain":"llm-eval","tags":["mathematics","research-math","proofs","referee-graded","ai-evaluation","benchmark"],"images":[],"participation":{"howToSubmit":"Systems enter per batch through the call for submissions on 1stproof.org. A system is an open-source harness calling publicly available models, or a public product; First Proof clones the submitter's repository, builds its Docker image on an approved EC2 instance and runs it once on the problem file, following the protocol published with each batch (e.g. 1stproof/batch-2).","submissionFormat":"A repository with Dockerfile, hardware.json, README.md and src/; the container reads the LaTeX problems from /data/input/input.json and writes LaTeX proofs to /data/output.","computeDetails":"Runs on First Proof's AWS instances; model API usage goes on the submitter's keys. Batch 2 systems spent $117–$4,799 in API fees for 10 problems.","deadline":"Batch 3: results 2026-10-14.","rounds":"Batches of unpublished research problems, each graded and published with every solution and referee report, plus community experiments where problems are posted publicly and solutions follow a week later.","prizes":"recognition only","eligibility":"AI companies and academic teams; only publicly available models, or harnesses that will be released.","cost":"free to enter; API usage is paid by the submitter","verification":"Each proof is refereed double blind by at least two experts in the field; the editorial board decides essentially flawless, minor revisions, major revisions or reject. Problems, human solutions, AI solutions, referee reports and logs are all published.","requirements":["Open-source harness or public AI product","Publicly available models","Docker","Model API keys"],"compute":"none"},"links":[{"label":"Website","url":"https://1stproof.org/"},{"label":"Batch 3 (results 2026-10-14)","url":"https://1stproof.org/third-batch.html"},{"label":"Batch 2 report (arXiv:2606.18119)","url":"https://arxiv.org/abs/2606.18119"},{"label":"Batch 2: solutions, reviews, harnesses","url":"https://github.com/1stproof/batch-2"},{"label":"Batch 1 paper (arXiv:2602.05192)","url":"https://arxiv.org/abs/2602.05192"},{"label":"Batch 1 problems (1stproof/batch-1)","url":"https://github.com/1stproof/batch-1"},{"label":"Community experiment","url":"https://1stproof.org/community-experiment.html"}],"organizer":{"name":"First Proof Foundation","url":"https://1stproof.org/","type":"foundation"},"venue":{"location":"1stproof.org / GitHub","type":"online"},"quickstart":"git clone https://github.com/1stproof/batch-2 && cd batch-2\nls   # protocol and reference submission (batch-2-design), problems, human and AI solutions, referee reports","problems":[{"group":"batch-2","description":"Batch 2 problem by Larry Guth. No passing proof yet.","metricUnit":"of 3","id":"b2-p04","name":"P4 · Metric geometry","metricName":"referee grade","metricDirection":"maximize","passMark":2,"isOpen":true,"records":[{"contributor":"IMProofBench ProofCouncil","description":"System A · Reject · $951 API","date":"2026-06-10","value":0,"contributorKind":"team","contributorUrl":"https://github.com/1stproof/batch-2/tree/main/batch-2-submissions/improofbench","submissionUrls":["https://github.com/1stproof/batch-2/blob/main/batch-2-AI-solutions/problem-04/submission-A.pdf","https://github.com/1stproof/batch-2/blob/main/batch-2-reviews/problem-04/submission-A/submission-4A-reviews.pdf"],"isCurrentBest":true,"agent":{"model":"gpt-5.5 pro","tool":"IMProofBench ProofCouncil","evidence":"First Proof Second Batch (arXiv:2606.18119), Problem 4, Submission A: \"Reject\"","family":"openai","confidence":"high","role":"subject"}},{"contributor":"UCLA Moonshot Harness","description":"System B · Reject · $208 API","date":"2026-06-10","value":0,"contributorKind":"team","contributorUrl":"https://github.com/1stproof/batch-2/tree/main/batch-2-submissions/ucla","submissionUrls":["https://github.com/1stproof/batch-2/blob/main/batch-2-AI-solutions/problem-04/submission-B.pdf","https://github.com/1stproof/batch-2/blob/main/batch-2-reviews/problem-04/submission-B/submission-4B-reviews.pdf"],"isCurrentBest":true,"agent":{"model":"gpt-5.5 pro","tool":"UCLA Moonshot Harness","evidence":"First Proof Second Batch (arXiv:2606.18119), Problem 4, Submission B: \"Reject\"","family":"openai","confidence":"high","role":"subject"}},{"contributor":"OpenAI ChatGPT 5.5 Pro","description":"System C · Reject · $10 API","date":"2026-06-10","value":0,"contributorKind":"team","contributorUrl":"https://arxiv.org/abs/2606.18119","submissionUrls":["https://github.com/1stproof/batch-2/blob/main/batch-2-AI-solutions/problem-04/submission-C.pdf","https://github.com/1stproof/batch-2/blob/main/batch-2-reviews/problem-04/submission-C/submission-4C-reviews.pdf"],"isCurrentBest":true,"agent":{"model":"OpenAI ChatGPT 5.5 Pro","tool":"OpenAI ChatGPT 5.5 Pro","evidence":"First Proof Second Batch (arXiv:2606.18119), Problem 4, Submission C: \"Reject\"","family":"openai","confidence":"high","role":"subject"}},{"contributor":"Princeton Momus","description":"System D · Reject · $79 API","date":"2026-06-10","value":0,"contributorKind":"team","contributorUrl":"https://github.com/1stproof/batch-2/tree/main/batch-2-submissions/princeton","submissionUrls":["https://github.com/1stproof/batch-2/blob/main/batch-2-AI-solutions/problem-04/submission-D.pdf","https://github.com/1stproof/batch-2/blob/main/batch-2-reviews/problem-04/submission-D/submission-4D-reviews.pdf"],"isCurrentBest":true,"agent":{"model":"gemini-3.1-pro-preview","tool":"Princeton Momus","evidence":"First Proof Second Batch (arXiv:2606.18119), Problem 4, Submission D: \"Reject\"","family":"google","confidence":"high","role":"subject"}}],"contested":true},{"group":"batch-2","description":"Batch 2 problem by Sam Payne, Jidong (Jayden) Wang. No passing proof yet.","metricUnit":"of 3","id":"b2-p08","name":"P8 · Matroids and tropical geometry","metricName":"referee grade","metricDirection":"maximize","passMark":2,"isOpen":true,"records":[{"contributor":"IMProofBench ProofCouncil","description":"System A · Reject · $321 API","date":"2026-06-10","value":0,"contributorKind":"team","contributorUrl":"https://github.com/1stproof/batch-2/tree/main/batch-2-submissions/improofbench","submissionUrls":["https://github.com/1stproof/batch-2/blob/main/batch-2-AI-solutions/problem-08/submission-A.pdf","https://github.com/1stproof/batch-2/blob/main/batch-2-reviews/problem-08/submission-A/submission-8A-reviews.pdf"],"agent":{"model":"gpt-5.5 pro","tool":"IMProofBench ProofCouncil","evidence":"First Proof Second Batch (arXiv:2606.18119), Problem 8, Submission A: \"Reject\"","family":"openai","confidence":"high","role":"subject"}},{"contributor":"UCLA Moonshot Harness","description":"System B · Major revisions · $757 API","date":"2026-06-10","value":1,"contributorKind":"team","contributorUrl":"https://github.com/1stproof/batch-2/tree/main/batch-2-submissions/ucla","submissionUrls":["https://github.com/1stproof/batch-2/blob/main/batch-2-AI-solutions/problem-08/submission-B.pdf","https://github.com/1stproof/batch-2/blob/main/batch-2-reviews/problem-08/submission-B/submission-8B-reviews.pdf"],"isCurrentBest":true,"agent":{"model":"gpt-5.5 pro","tool":"UCLA Moonshot Harness","evidence":"First Proof Second Batch (arXiv:2606.18119), Problem 8, Submission B: \"Major revisions\"","family":"openai","confidence":"high","role":"subject"}},{"contributor":"OpenAI ChatGPT 5.5 Pro","description":"System C · Reject · $14 API","date":"2026-06-10","value":0,"contributorKind":"team","contributorUrl":"https://arxiv.org/abs/2606.18119","submissionUrls":["https://github.com/1stproof/batch-2/blob/main/batch-2-AI-solutions/problem-08/submission-C.pdf","https://github.com/1stproof/batch-2/blob/main/batch-2-reviews/problem-08/submission-C/submission-8C-reviews.pdf"],"agent":{"model":"OpenAI ChatGPT 5.5 Pro","tool":"OpenAI ChatGPT 5.5 Pro","evidence":"First Proof Second Batch (arXiv:2606.18119), Problem 8, Submission C: \"Reject\"","family":"openai","confidence":"high","role":"subject"}},{"contributor":"Princeton Momus","description":"System D · Reject · $198 API","date":"2026-06-10","value":0,"contributorKind":"team","contributorUrl":"https://github.com/1stproof/batch-2/tree/main/batch-2-submissions/princeton","submissionUrls":["https://github.com/1stproof/batch-2/blob/main/batch-2-AI-solutions/problem-08/submission-D.pdf","https://github.com/1stproof/batch-2/blob/main/batch-2-reviews/problem-08/submission-D/submission-8D-reviews.pdf"],"agent":{"model":"gemini-3.1-pro-preview","tool":"Princeton Momus","evidence":"First Proof Second Batch (arXiv:2606.18119), Problem 8, Submission D: \"Reject\"","family":"google","confidence":"high","role":"subject"}}],"contested":true},{"group":"batch-2","description":"Batch 2 problem by Srivatsav Kunnawalkam Elayavalli. No passing proof yet.","metricUnit":"of 3","id":"b2-p10","name":"P10 · Von Neumann algebras","metricName":"referee grade","metricDirection":"maximize","passMark":2,"isOpen":true,"records":[{"contributor":"IMProofBench ProofCouncil","description":"System A · Major revisions · $673 API","date":"2026-06-10","value":1,"contributorKind":"team","contributorUrl":"https://github.com/1stproof/batch-2/tree/main/batch-2-submissions/improofbench","submissionUrls":["https://github.com/1stproof/batch-2/blob/main/batch-2-AI-solutions/problem-10/submission-A.pdf","https://github.com/1stproof/batch-2/blob/main/batch-2-reviews/problem-10/submission-A/submission-10A-reviews.pdf"],"isCurrentBest":true,"agent":{"model":"gpt-5.5 pro","tool":"IMProofBench ProofCouncil","evidence":"First Proof Second Batch (arXiv:2606.18119), Problem 10, Submission A: \"Major revisions\"","family":"openai","confidence":"high","role":"subject"}},{"contributor":"UCLA Moonshot Harness","description":"System B · Reject · $589 API","date":"2026-06-10","value":0,"contributorKind":"team","contributorUrl":"https://github.com/1stproof/batch-2/tree/main/batch-2-submissions/ucla","submissionUrls":["https://github.com/1stproof/batch-2/blob/main/batch-2-AI-solutions/problem-10/submission-B.pdf","https://github.com/1stproof/batch-2/blob/main/batch-2-reviews/problem-10/submission-B/submission-10B-reviews.pdf"],"agent":{"model":"gpt-5.5 pro","tool":"UCLA Moonshot Harness","evidence":"First Proof Second Batch (arXiv:2606.18119), Problem 10, Submission B: \"Reject\"","family":"openai","confidence":"high","role":"subject"}},{"contributor":"OpenAI ChatGPT 5.5 Pro","description":"System C · Major revisions · $8 API","date":"2026-06-10","value":1,"contributorKind":"team","contributorUrl":"https://arxiv.org/abs/2606.18119","submissionUrls":["https://github.com/1stproof/batch-2/blob/main/batch-2-AI-solutions/problem-10/submission-C.pdf","https://github.com/1stproof/batch-2/blob/main/batch-2-reviews/problem-10/submission-C/submission-10C-reviews.pdf"],"isCurrentBest":true,"agent":{"model":"OpenAI ChatGPT 5.5 Pro","tool":"OpenAI ChatGPT 5.5 Pro","evidence":"First Proof Second Batch (arXiv:2606.18119), Problem 10, Submission C: \"Major revisions\"","family":"openai","confidence":"high","role":"subject"}},{"contributor":"Princeton Momus","description":"System D · Reject · $100 API","date":"2026-06-10","value":0,"contributorKind":"team","contributorUrl":"https://github.com/1stproof/batch-2/tree/main/batch-2-submissions/princeton","submissionUrls":["https://github.com/1stproof/batch-2/blob/main/batch-2-AI-solutions/problem-10/submission-D.pdf","https://github.com/1stproof/batch-2/blob/main/batch-2-reviews/problem-10/submission-D/submission-10D-reviews.pdf"],"agent":{"model":"gemini-3.1-pro-preview","tool":"Princeton Momus","evidence":"First Proof Second Batch (arXiv:2606.18119), Problem 10, Submission D: \"Reject\"","family":"google","confidence":"high","role":"subject"}}],"contested":true},{"group":"batch-2","description":"Sum of editorial grades over the Batch 2 problems: 3 essentially flawless, 2 minor revisions (passing), 1 major revisions, 0 reject or missing. One shot per problem, run by the editorial board.","metricUnit":"of 30","id":"b2-systems","name":"Batch 2 · all 10 problems","metricName":"total referee grade","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"IMProofBench ProofCouncil","description":"System A · 6/10 passed (2 flawless, 4 minor, 1 major, 2 reject, 1 missing) · $3,186 API, 22.9 h · gpt-5.5 pro, gpt-5.5, gemini-3.1-pro-preview, claude-opus-4-7","date":"2026-06-10","value":15,"contributorKind":"team","contributorUrl":"https://github.com/1stproof/batch-2/tree/main/batch-2-submissions/improofbench","submissionUrls":["https://github.com/1stproof/batch-2/tree/main/batch-2-AI-solutions","https://github.com/1stproof/batch-2/tree/main/batch-2-reviews"],"isCurrentBest":true,"agent":{"model":"gpt-5.5 pro","tool":"IMProofBench ProofCouncil","evidence":"First Proof Second Batch (arXiv:2606.18119): System A, 9 graded solutions","family":"openai","confidence":"high","role":"subject"}},{"contributor":"UCLA Moonshot Harness","description":"System B · 5/10 passed (1 flawless, 4 minor, 1 major, 4 reject) · $4,799 API, 23.1 h · gpt-5.5 pro","date":"2026-06-10","value":12,"contributorKind":"team","contributorUrl":"https://github.com/1stproof/batch-2/tree/main/batch-2-submissions/ucla","submissionUrls":["https://github.com/1stproof/batch-2/tree/main/batch-2-AI-solutions","https://github.com/1stproof/batch-2/tree/main/batch-2-reviews"],"agent":{"model":"gpt-5.5 pro","tool":"UCLA Moonshot Harness","evidence":"First Proof Second Batch (arXiv:2606.18119): System B, 10 graded solutions","family":"openai","confidence":"high","role":"subject"}},{"contributor":"OpenAI ChatGPT 5.5 Pro","description":"System C · 5/10 passed (2 flawless, 3 minor, 1 major, 4 reject) · $117 API, 5.8 h","date":"2026-06-10","value":13,"contributorKind":"team","contributorUrl":"https://arxiv.org/abs/2606.18119","submissionUrls":["https://github.com/1stproof/batch-2/tree/main/batch-2-AI-solutions","https://github.com/1stproof/batch-2/tree/main/batch-2-reviews"],"agent":{"model":"OpenAI ChatGPT 5.5 Pro","tool":"OpenAI ChatGPT 5.5 Pro","evidence":"First Proof Second Batch (arXiv:2606.18119): System C, 10 graded solutions","family":"openai","confidence":"high","role":"subject"}},{"contributor":"Princeton Momus","description":"System D · 1/10 passed (1 minor, 9 reject) · $1,014 API, 7.8 h · gemini-3.1-pro-preview","date":"2026-06-10","value":2,"contributorKind":"team","contributorUrl":"https://github.com/1stproof/batch-2/tree/main/batch-2-submissions/princeton","submissionUrls":["https://github.com/1stproof/batch-2/tree/main/batch-2-AI-solutions","https://github.com/1stproof/batch-2/tree/main/batch-2-reviews"],"agent":{"model":"gemini-3.1-pro-preview","tool":"Princeton Momus","evidence":"First Proof Second Batch (arXiv:2606.18119): System D, 10 graded solutions","family":"google","confidence":"high","role":"subject"}}],"contested":true},{"group":"batch-2","description":"Batch 2 problem by Ted Slaman, Dariusz Kalociński. Passed by systems A, B, C.","metricUnit":"of 3","id":"b2-p01","name":"P1 · Computability theory","metricName":"referee grade","metricDirection":"maximize","passMark":2,"isOpen":false,"records":[{"contributor":"IMProofBench ProofCouncil","description":"System A · Minor revisions · $49 API","date":"2026-06-10","value":2,"contributorKind":"team","contributorUrl":"https://github.com/1stproof/batch-2/tree/main/batch-2-submissions/improofbench","submissionUrls":["https://github.com/1stproof/batch-2/blob/main/batch-2-AI-solutions/problem-01/submission-A.pdf","https://github.com/1stproof/batch-2/blob/main/batch-2-reviews/problem-01/submission-A/submission-1A-reviews.pdf"],"isCurrentBest":true,"agent":{"model":"gpt-5.5 pro","tool":"IMProofBench ProofCouncil","evidence":"First Proof Second Batch (arXiv:2606.18119), Problem 1, Submission A: \"Minor revisions\"","family":"openai","confidence":"high","role":"subject"}},{"contributor":"UCLA Moonshot Harness","description":"System B · Minor revisions · $189 API","date":"2026-06-10","value":2,"contributorKind":"team","contributorUrl":"https://github.com/1stproof/batch-2/tree/main/batch-2-submissions/ucla","submissionUrls":["https://github.com/1stproof/batch-2/blob/main/batch-2-AI-solutions/problem-01/submission-B.pdf","https://github.com/1stproof/batch-2/blob/main/batch-2-reviews/problem-01/submission-B/submission-1B-reviews.pdf"],"isCurrentBest":true,"agent":{"model":"gpt-5.5 pro","tool":"UCLA Moonshot Harness","evidence":"First Proof Second Batch (arXiv:2606.18119), Problem 1, Submission B: \"Minor revisions\"","family":"openai","confidence":"high","role":"subject"}},{"contributor":"OpenAI ChatGPT 5.5 Pro","description":"System C · Minor revisions · $13 API","date":"2026-06-10","value":2,"contributorKind":"team","contributorUrl":"https://arxiv.org/abs/2606.18119","submissionUrls":["https://github.com/1stproof/batch-2/blob/main/batch-2-AI-solutions/problem-01/submission-C.pdf","https://github.com/1stproof/batch-2/blob/main/batch-2-reviews/problem-01/submission-C/submission-1C-reviews.pdf"],"isCurrentBest":true,"agent":{"model":"OpenAI ChatGPT 5.5 Pro","tool":"OpenAI ChatGPT 5.5 Pro","evidence":"First Proof Second Batch (arXiv:2606.18119), Problem 1, Submission C: \"Minor revisions\"","family":"openai","confidence":"high","role":"subject"}},{"contributor":"Princeton Momus","description":"System D · Reject · $142 API","date":"2026-06-10","value":0,"contributorKind":"team","contributorUrl":"https://github.com/1stproof/batch-2/tree/main/batch-2-submissions/princeton","submissionUrls":["https://github.com/1stproof/batch-2/blob/main/batch-2-AI-solutions/problem-01/submission-D.pdf","https://github.com/1stproof/batch-2/blob/main/batch-2-reviews/problem-01/submission-D/submission-1D-reviews.pdf"],"agent":{"model":"gemini-3.1-pro-preview","tool":"Princeton Momus","evidence":"First Proof Second Batch (arXiv:2606.18119), Problem 1, Submission D: \"Reject\"","family":"google","confidence":"high","role":"subject"}}],"contested":true},{"group":"batch-2","description":"Batch 2 problem by Richard Schwartz. Passed by systems A, B, C.","metricUnit":"of 3","id":"b2-p02","name":"P2 · Discrete geometry","metricName":"referee grade","metricDirection":"maximize","passMark":2,"isOpen":false,"records":[{"contributor":"IMProofBench ProofCouncil","description":"System A · Minor revisions · $86 API","date":"2026-06-10","value":2,"contributorKind":"team","contributorUrl":"https://github.com/1stproof/batch-2/tree/main/batch-2-submissions/improofbench","submissionUrls":["https://github.com/1stproof/batch-2/blob/main/batch-2-AI-solutions/problem-02/submission-A.pdf","https://github.com/1stproof/batch-2/blob/main/batch-2-reviews/problem-02/submission-A/submission-2A-reviews.pdf"],"isCurrentBest":true,"agent":{"model":"gpt-5.5 pro","tool":"IMProofBench ProofCouncil","evidence":"First Proof Second Batch (arXiv:2606.18119), Problem 2, Submission A: \"Minor revisions\"","family":"openai","confidence":"high","role":"subject"}},{"contributor":"UCLA Moonshot Harness","description":"System B · Minor revisions · $562 API","date":"2026-06-10","value":2,"contributorKind":"team","contributorUrl":"https://github.com/1stproof/batch-2/tree/main/batch-2-submissions/ucla","submissionUrls":["https://github.com/1stproof/batch-2/blob/main/batch-2-AI-solutions/problem-02/submission-B.pdf","https://github.com/1stproof/batch-2/blob/main/batch-2-reviews/problem-02/submission-B/submission-2B-reviews.pdf"],"isCurrentBest":true,"agent":{"model":"gpt-5.5 pro","tool":"UCLA Moonshot Harness","evidence":"First Proof Second Batch (arXiv:2606.18119), Problem 2, Submission B: \"Minor revisions\"","family":"openai","confidence":"high","role":"subject"}},{"contributor":"OpenAI ChatGPT 5.5 Pro","description":"System C · Minor revisions · $12 API","date":"2026-06-10","value":2,"contributorKind":"team","contributorUrl":"https://arxiv.org/abs/2606.18119","submissionUrls":["https://github.com/1stproof/batch-2/blob/main/batch-2-AI-solutions/problem-02/submission-C.pdf","https://github.com/1stproof/batch-2/blob/main/batch-2-reviews/problem-02/submission-C/submission-2C-reviews.pdf"],"isCurrentBest":true,"agent":{"model":"OpenAI ChatGPT 5.5 Pro","tool":"OpenAI ChatGPT 5.5 Pro","evidence":"First Proof Second Batch (arXiv:2606.18119), Problem 2, Submission C: \"Minor revisions\"","family":"openai","confidence":"high","role":"subject"}},{"contributor":"Princeton Momus","description":"System D · Reject · $85 API","date":"2026-06-10","value":0,"contributorKind":"team","contributorUrl":"https://github.com/1stproof/batch-2/tree/main/batch-2-submissions/princeton","submissionUrls":["https://github.com/1stproof/batch-2/blob/main/batch-2-AI-solutions/problem-02/submission-D.pdf","https://github.com/1stproof/batch-2/blob/main/batch-2-reviews/problem-02/submission-D/submission-2D-reviews.pdf"],"agent":{"model":"gemini-3.1-pro-preview","tool":"Princeton Momus","evidence":"First Proof Second Batch (arXiv:2606.18119), Problem 2, Submission D: \"Reject\"","family":"google","confidence":"high","role":"subject"}}],"contested":true},{"group":"batch-2","description":"Batch 2 problem by Aleksa Milojević, Benny Sudakov. Passed by system A.","metricUnit":"of 3","id":"b2-p03","name":"P3 · Discrete probability","metricName":"referee grade","metricDirection":"maximize","passMark":2,"isOpen":false,"records":[{"contributor":"IMProofBench ProofCouncil","description":"System A · Minor revisions · $712 API","date":"2026-06-10","value":2,"contributorKind":"team","contributorUrl":"https://github.com/1stproof/batch-2/tree/main/batch-2-submissions/improofbench","submissionUrls":["https://github.com/1stproof/batch-2/blob/main/batch-2-AI-solutions/problem-03/submission-A.pdf","https://github.com/1stproof/batch-2/blob/main/batch-2-reviews/problem-03/submission-A/submission-3A-reviews.pdf"],"isCurrentBest":true,"agent":{"model":"gpt-5.5 pro","tool":"IMProofBench ProofCouncil","evidence":"First Proof Second Batch (arXiv:2606.18119), Problem 3, Submission A: \"Minor revisions\"","family":"openai","confidence":"high","role":"subject"}},{"contributor":"UCLA Moonshot Harness","description":"System B · Reject · $637 API","date":"2026-06-10","value":0,"contributorKind":"team","contributorUrl":"https://github.com/1stproof/batch-2/tree/main/batch-2-submissions/ucla","submissionUrls":["https://github.com/1stproof/batch-2/blob/main/batch-2-AI-solutions/problem-03/submission-B.pdf","https://github.com/1stproof/batch-2/blob/main/batch-2-reviews/problem-03/submission-B/submission-3B-reviews.pdf"],"agent":{"model":"gpt-5.5 pro","tool":"UCLA Moonshot Harness","evidence":"First Proof Second Batch (arXiv:2606.18119), Problem 3, Submission B: \"Reject\"","family":"openai","confidence":"high","role":"subject"}},{"contributor":"OpenAI ChatGPT 5.5 Pro","description":"System C · Reject · $16 API","date":"2026-06-10","value":0,"contributorKind":"team","contributorUrl":"https://arxiv.org/abs/2606.18119","submissionUrls":["https://github.com/1stproof/batch-2/blob/main/batch-2-AI-solutions/problem-03/submission-C.pdf","https://github.com/1stproof/batch-2/blob/main/batch-2-reviews/problem-03/submission-C/submission-3C-reviews.pdf"],"agent":{"model":"OpenAI ChatGPT 5.5 Pro","tool":"OpenAI ChatGPT 5.5 Pro","evidence":"First Proof Second Batch (arXiv:2606.18119), Problem 3, Submission C: \"Reject\"","family":"openai","confidence":"high","role":"subject"}},{"contributor":"Princeton Momus","description":"System D · Reject · $80 API","date":"2026-06-10","value":0,"contributorKind":"team","contributorUrl":"https://github.com/1stproof/batch-2/tree/main/batch-2-submissions/princeton","submissionUrls":["https://github.com/1stproof/batch-2/blob/main/batch-2-AI-solutions/problem-03/submission-D.pdf","https://github.com/1stproof/batch-2/blob/main/batch-2-reviews/problem-03/submission-D/submission-3D-reviews.pdf"],"agent":{"model":"gemini-3.1-pro-preview","tool":"Princeton Momus","evidence":"First Proof Second Batch (arXiv:2606.18119), Problem 3, Submission D: \"Reject\"","family":"google","confidence":"high","role":"subject"}}],"contested":true},{"group":"batch-2","description":"Batch 2 problem by Oleg Butkovsky, Jonathan C. Mattingly, Lorenzo Zambotti. Passed by system A.","metricUnit":"of 3","id":"b2-p05","name":"P5 · Stochastic PDE","metricName":"referee grade","metricDirection":"maximize","passMark":2,"isOpen":false,"records":[{"contributor":"IMProofBench ProofCouncil","description":"System A · Essentially flawless · $313 API","date":"2026-06-10","value":3,"contributorKind":"team","contributorUrl":"https://github.com/1stproof/batch-2/tree/main/batch-2-submissions/improofbench","submissionUrls":["https://github.com/1stproof/batch-2/blob/main/batch-2-AI-solutions/problem-05/submission-A.pdf","https://github.com/1stproof/batch-2/blob/main/batch-2-reviews/problem-05/submission-A/submission-5A-reviews.pdf"],"isCurrentBest":true,"agent":{"model":"gpt-5.5 pro","tool":"IMProofBench ProofCouncil","evidence":"First Proof Second Batch (arXiv:2606.18119), Problem 5, Submission A: \"Essentially flawless\"","family":"openai","confidence":"high","role":"subject"}},{"contributor":"UCLA Moonshot Harness","description":"System B · Reject · $642 API","date":"2026-06-10","value":0,"contributorKind":"team","contributorUrl":"https://github.com/1stproof/batch-2/tree/main/batch-2-submissions/ucla","submissionUrls":["https://github.com/1stproof/batch-2/blob/main/batch-2-AI-solutions/problem-05/submission-B.pdf","https://github.com/1stproof/batch-2/blob/main/batch-2-reviews/problem-05/submission-B/submission-5B-reviews.pdf"],"agent":{"model":"gpt-5.5 pro","tool":"UCLA Moonshot Harness","evidence":"First Proof Second Batch (arXiv:2606.18119), Problem 5, Submission B: \"Reject\"","family":"openai","confidence":"high","role":"subject"}},{"contributor":"OpenAI ChatGPT 5.5 Pro","description":"System C · Reject · $11 API","date":"2026-06-10","value":0,"contributorKind":"team","contributorUrl":"https://arxiv.org/abs/2606.18119","submissionUrls":["https://github.com/1stproof/batch-2/blob/main/batch-2-AI-solutions/problem-05/submission-C.pdf","https://github.com/1stproof/batch-2/blob/main/batch-2-reviews/problem-05/submission-C/submission-5C-reviews.pdf"],"agent":{"model":"OpenAI ChatGPT 5.5 Pro","tool":"OpenAI ChatGPT 5.5 Pro","evidence":"First Proof Second Batch (arXiv:2606.18119), Problem 5, Submission C: \"Reject\"","family":"openai","confidence":"high","role":"subject"}},{"contributor":"Princeton Momus","description":"System D · Reject · $61 API","date":"2026-06-10","value":0,"contributorKind":"team","contributorUrl":"https://github.com/1stproof/batch-2/tree/main/batch-2-submissions/princeton","submissionUrls":["https://github.com/1stproof/batch-2/blob/main/batch-2-AI-solutions/problem-05/submission-D.pdf","https://github.com/1stproof/batch-2/blob/main/batch-2-reviews/problem-05/submission-D/submission-5D-reviews.pdf"],"agent":{"model":"gemini-3.1-pro-preview","tool":"Princeton Momus","evidence":"First Proof Second Batch (arXiv:2606.18119), Problem 5, Submission D: \"Reject\"","family":"google","confidence":"high","role":"subject"}}],"contested":true},{"group":"batch-2","description":"Batch 2 problem by Joshua Greene, Duncan McCoy. Passed by systems B, C. System A: no solution.","metricUnit":"of 3","id":"b2-p06","name":"P6 · Lattice theory","metricName":"referee grade","metricDirection":"maximize","passMark":2,"isOpen":false,"records":[{"contributor":"UCLA Moonshot Harness","description":"System B · Essentially flawless · $218 API","date":"2026-06-10","value":3,"contributorKind":"team","contributorUrl":"https://github.com/1stproof/batch-2/tree/main/batch-2-submissions/ucla","submissionUrls":["https://github.com/1stproof/batch-2/blob/main/batch-2-AI-solutions/problem-06/submission-B.pdf","https://github.com/1stproof/batch-2/blob/main/batch-2-reviews/problem-06/submission-B/submission-6B-reviews.pdf"],"isCurrentBest":true,"agent":{"model":"gpt-5.5 pro","tool":"UCLA Moonshot Harness","evidence":"First Proof Second Batch (arXiv:2606.18119), Problem 6, Submission B: \"Essentially flawless\"","family":"openai","confidence":"high","role":"subject"}},{"contributor":"OpenAI ChatGPT 5.5 Pro","description":"System C · Essentially flawless · $11 API","date":"2026-06-10","value":3,"contributorKind":"team","contributorUrl":"https://arxiv.org/abs/2606.18119","submissionUrls":["https://github.com/1stproof/batch-2/blob/main/batch-2-AI-solutions/problem-06/submission-C.pdf","https://github.com/1stproof/batch-2/blob/main/batch-2-reviews/problem-06/submission-C/submission-6C-reviews.pdf"],"isCurrentBest":true,"agent":{"model":"OpenAI ChatGPT 5.5 Pro","tool":"OpenAI ChatGPT 5.5 Pro","evidence":"First Proof Second Batch (arXiv:2606.18119), Problem 6, Submission C: \"Essentially flawless\"","family":"openai","confidence":"high","role":"subject"}},{"contributor":"Princeton Momus","description":"System D · Reject · $72 API","date":"2026-06-10","value":0,"contributorKind":"team","contributorUrl":"https://github.com/1stproof/batch-2/tree/main/batch-2-submissions/princeton","submissionUrls":["https://github.com/1stproof/batch-2/blob/main/batch-2-AI-solutions/problem-06/submission-D.pdf","https://github.com/1stproof/batch-2/blob/main/batch-2-reviews/problem-06/submission-D/submission-6D-reviews.pdf"],"agent":{"model":"gemini-3.1-pro-preview","tool":"Princeton Momus","evidence":"First Proof Second Batch (arXiv:2606.18119), Problem 6, Submission D: \"Reject\"","family":"google","confidence":"high","role":"subject"}}],"contested":true},{"group":"batch-2","description":"Batch 2 problem by Sucharit Sarkar. Passed by systems A, B, C, D.","metricUnit":"of 3","id":"b2-p07","name":"P7 · Combinatorial topology","metricName":"referee grade","metricDirection":"maximize","passMark":2,"isOpen":false,"records":[{"contributor":"IMProofBench ProofCouncil","description":"System A · Essentially flawless · $44 API","date":"2026-06-10","value":3,"contributorKind":"team","contributorUrl":"https://github.com/1stproof/batch-2/tree/main/batch-2-submissions/improofbench","submissionUrls":["https://github.com/1stproof/batch-2/blob/main/batch-2-AI-solutions/problem-07/submission-A.pdf","https://github.com/1stproof/batch-2/blob/main/batch-2-reviews/problem-07/submission-A/submission-7A-reviews.pdf"],"isCurrentBest":true,"agent":{"model":"gpt-5.5 pro","tool":"IMProofBench ProofCouncil","evidence":"First Proof Second Batch (arXiv:2606.18119), Problem 7, Submission A: \"Essentially flawless\"","family":"openai","confidence":"high","role":"subject"}},{"contributor":"UCLA Moonshot Harness","description":"System B · Minor revisions · $480 API","date":"2026-06-10","value":2,"contributorKind":"team","contributorUrl":"https://github.com/1stproof/batch-2/tree/main/batch-2-submissions/ucla","submissionUrls":["https://github.com/1stproof/batch-2/blob/main/batch-2-AI-solutions/problem-07/submission-B.pdf","https://github.com/1stproof/batch-2/blob/main/batch-2-reviews/problem-07/submission-B/submission-7B-reviews.pdf"],"agent":{"model":"gpt-5.5 pro","tool":"UCLA Moonshot Harness","evidence":"First Proof Second Batch (arXiv:2606.18119), Problem 7, Submission B: \"Minor revisions\"","family":"openai","confidence":"high","role":"subject"}},{"contributor":"OpenAI ChatGPT 5.5 Pro","description":"System C · Essentially flawless · $13 API","date":"2026-06-10","value":3,"contributorKind":"team","contributorUrl":"https://arxiv.org/abs/2606.18119","submissionUrls":["https://github.com/1stproof/batch-2/blob/main/batch-2-AI-solutions/problem-07/submission-C.pdf","https://github.com/1stproof/batch-2/blob/main/batch-2-reviews/problem-07/submission-C/submission-7C-reviews.pdf"],"isCurrentBest":true,"agent":{"model":"OpenAI ChatGPT 5.5 Pro","tool":"OpenAI ChatGPT 5.5 Pro","evidence":"First Proof Second Batch (arXiv:2606.18119), Problem 7, Submission C: \"Essentially flawless\"","family":"openai","confidence":"high","role":"subject"}},{"contributor":"Princeton Momus","description":"System D · Minor revisions · $115 API","date":"2026-06-10","value":2,"contributorKind":"team","contributorUrl":"https://github.com/1stproof/batch-2/tree/main/batch-2-submissions/princeton","submissionUrls":["https://github.com/1stproof/batch-2/blob/main/batch-2-AI-solutions/problem-07/submission-D.pdf","https://github.com/1stproof/batch-2/blob/main/batch-2-reviews/problem-07/submission-D/submission-7D-reviews.pdf"],"agent":{"model":"gemini-3.1-pro-preview","tool":"Princeton Momus","evidence":"First Proof Second Batch (arXiv:2606.18119), Problem 7, Submission D: \"Minor revisions\"","family":"google","confidence":"high","role":"subject"}}],"contested":true},{"group":"batch-2","description":"Batch 2 problem by Sylvie Corteel, John Lentfer. Passed by systems A, B, C.","metricUnit":"of 3","id":"b2-p09","name":"P9 · Algebraic combinatorics","metricName":"referee grade","metricDirection":"maximize","passMark":2,"isOpen":false,"records":[{"contributor":"IMProofBench ProofCouncil","description":"System A · Minor revisions · $36 API","date":"2026-06-10","value":2,"contributorKind":"team","contributorUrl":"https://github.com/1stproof/batch-2/tree/main/batch-2-submissions/improofbench","submissionUrls":["https://github.com/1stproof/batch-2/blob/main/batch-2-AI-solutions/problem-09/submission-A.pdf","https://github.com/1stproof/batch-2/blob/main/batch-2-reviews/problem-09/submission-A/submission-9A-reviews.pdf"],"isCurrentBest":true,"agent":{"model":"gpt-5.5 pro","tool":"IMProofBench ProofCouncil","evidence":"First Proof Second Batch (arXiv:2606.18119), Problem 9, Submission A: \"Minor revisions\"","family":"openai","confidence":"high","role":"subject"}},{"contributor":"UCLA Moonshot Harness","description":"System B · Minor revisions · $516 API","date":"2026-06-10","value":2,"contributorKind":"team","contributorUrl":"https://github.com/1stproof/batch-2/tree/main/batch-2-submissions/ucla","submissionUrls":["https://github.com/1stproof/batch-2/blob/main/batch-2-AI-solutions/problem-09/submission-B.pdf","https://github.com/1stproof/batch-2/blob/main/batch-2-reviews/problem-09/submission-B/submission-9B-reviews.pdf"],"isCurrentBest":true,"agent":{"model":"gpt-5.5 pro","tool":"UCLA Moonshot Harness","evidence":"First Proof Second Batch (arXiv:2606.18119), Problem 9, Submission B: \"Minor revisions\"","family":"openai","confidence":"high","role":"subject"}},{"contributor":"OpenAI ChatGPT 5.5 Pro","description":"System C · Minor revisions · $8 API","date":"2026-06-10","value":2,"contributorKind":"team","contributorUrl":"https://arxiv.org/abs/2606.18119","submissionUrls":["https://github.com/1stproof/batch-2/blob/main/batch-2-AI-solutions/problem-09/submission-C.pdf","https://github.com/1stproof/batch-2/blob/main/batch-2-reviews/problem-09/submission-C/submission-9C-reviews.pdf"],"isCurrentBest":true,"agent":{"model":"OpenAI ChatGPT 5.5 Pro","tool":"OpenAI ChatGPT 5.5 Pro","evidence":"First Proof Second Batch (arXiv:2606.18119), Problem 9, Submission C: \"Minor revisions\"","family":"openai","confidence":"high","role":"subject"}},{"contributor":"Princeton Momus","description":"System D · Reject · $81 API","date":"2026-06-10","value":0,"contributorKind":"team","contributorUrl":"https://github.com/1stproof/batch-2/tree/main/batch-2-submissions/princeton","submissionUrls":["https://github.com/1stproof/batch-2/blob/main/batch-2-AI-solutions/problem-09/submission-D.pdf","https://github.com/1stproof/batch-2/blob/main/batch-2-reviews/problem-09/submission-D/submission-9D-reviews.pdf"],"agent":{"model":"gemini-3.1-pro-preview","tool":"Princeton Momus","evidence":"First Proof Second Batch (arXiv:2606.18119), Problem 9, Submission D: \"Reject\"","family":"google","confidence":"high","role":"subject"}}],"contested":true}],"agentStats":[],"repo":{"owner":"1stproof","name":"batch-2","branch":"main"},"participants":[{"name":"UCLA Moonshot Harness","url":"https://github.com/1stproof/batch-2/tree/main/batch-2-submissions/ucla","submissions":11,"problems":["b2-systems","b2-p01","b2-p02","b2-p03","b2-p04","b2-p05","b2-p06","b2-p07","b2-p08","b2-p09","b2-p10"],"lastActive":"2026-06-10","bestRank":1},{"name":"OpenAI ChatGPT 5.5 Pro","url":"https://arxiv.org/abs/2606.18119","submissions":11,"problems":["b2-systems","b2-p01","b2-p02","b2-p03","b2-p04","b2-p05","b2-p06","b2-p07","b2-p08","b2-p09","b2-p10"],"lastActive":"2026-06-10","bestRank":1},{"name":"Princeton Momus","url":"https://github.com/1stproof/batch-2/tree/main/batch-2-submissions/princeton","submissions":11,"problems":["b2-systems","b2-p01","b2-p02","b2-p03","b2-p04","b2-p05","b2-p06","b2-p07","b2-p08","b2-p09","b2-p10"],"lastActive":"2026-06-10","bestRank":1},{"name":"IMProofBench ProofCouncil","url":"https://github.com/1stproof/batch-2/tree/main/batch-2-submissions/improofbench","submissions":10,"problems":["b2-systems","b2-p01","b2-p02","b2-p03","b2-p04","b2-p05","b2-p07","b2-p08","b2-p09","b2-p10"],"bestRank":1,"lastActive":"2026-06-10"}],"stats":{"totalRecords":43,"uniqueParticipants":4,"firstSubmission":"2026-06-10","lastSubmission":"2026-06-10","recordsLast30d":0,"recordsLast90d":0},"lastUpdated":"2026-10-19T09:09:17.209Z"},{"id":"sutro-problems","name":"Sutro Problems","tagline":"Minimize energy (read-cost) of small algorithms under Bill Dally's simplified data-movement model.","description":"A collection of small, self-contained problems used as benchmarks for the Sutro Group's energy-efficient learning research. Submissions are straight-line programs (IRs) or GPU kernels scored by a simplified Bill Dally model where every operand read of address a costs ceil(sqrt(a)) and arithmetic is free. Leaderboards rank submissions by read-cost (matmul, sparse-parity, symmetry) or by wall-clock time on an A100 (mnist-a100). Maintained by the Sutro Group (cybertronai).","url":"https://github.com/cybertronai/sutro-problems","host":"github","status":"active","domain":"hardware-efficiency","tags":["energy-efficiency","data-movement","dally-model","ir-optimization","matmul","mnist","sparse-parity"],"images":["https://github.com/user-attachments/assets/dfab8831-876a-4cdd-bda4-79a7ef869152","https://github.com/user-attachments/assets/51940abe-372c-4434-8003-d61cc1b4733c","https://github.com/user-attachments/assets/e4610f22-00e6-4d70-adee-1948f219cf51","https://github.com/user-attachments/assets/0d71bd80-7da8-4e0c-942d-7bd6b741a193"],"participation":{"howToSubmit":"Open a pull request against main adding your submission files (IR + generator + report) under the problem's submissions/ directory and adding a row to the matching leaderboard table.","submissionFormat":"For matmul/sparse-parity/symmetry: a plain-text IR in the v3 instruction set plus a Python generator and a markdown report. For mnist-a100: a Python method file plus a report; scored via mnist.score().","computeDetails":"matmul/sparse-parity/symmetry are scored symbolically (no GPU). mnist-a100 requires an A100-80GB; official runs use Modal.","deadline":"rolling","prizes":"recognition only","verification":"Symbolic verification for IR problems (matmul.score_4x4, mp.evaluate_mask, symmetry.score_six); sandboxed Modal A100 runs with NVML energy measurement for mnist-a100.","requirements":["Python 3","numpy","Lean-free IR text format","CUDA/Triton (mnist-a100)","Modal (mnist-a100 official runs)"],"compute":"consumer-gpu"},"links":[{"label":"Sutro Group","url":"https://docs.google.com/document/d/1B9867EN6Bg4ZVQK9vI_ZqykZ5HEtMAHJ7zBGGas4szQ/edit"},{"label":"Simplified Dally Model","url":"https://github.com/cybertronai/simplified-dally-model"},{"label":"Sparse Parity (GitHub Pages)","url":"https://cybertronai.github.io/sutro-problems/sparse-parity"},{"label":"AlphaTensor","url":"https://github.com/google-deepmind/alphatensor"}],"organizer":{"name":"Sutro Group","url":"https://github.com/cybertronai","type":"community"},"venue":{"type":"online"},"quickstart":"import matmul\ncost = matmul.score_4x4(matmul.generate_baseline_4x4())\n\nimport mask_sparse_parity as mp\nres = mp.evaluate_mask(mp.generate_scan(4095))\nres.cost, res.recovery\n\nimport symmetry\ncost = symmetry.score_six(symmetry.generate_baseline_six())","problems":[{"group":"mnist-a100","description":"Learn MNIST from 10,000 labels and predict 10,000 test digits on one A100, error <= 5.40%; minimize time per call.","metricUnit":"ms","id":"mnist-a100-diff1","name":"MNIST-a100 (Difficulty 1)","metricName":"time","metricDirection":"minimize","isOpen":false,"records":[{"contributor":"yaroslavvb","description":"example.py baseline: MLP 60-1024-1024-10, 400 steps","date":"2026-09-25","value":700.1,"contributorKind":"person","contributorUrl":"https://github.com/yaroslavvb","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/mnist-a100/example.py","https://github.com/cybertronai/sutro-problems/blob/main/mnist-a100/energy/README.md"],"isBaseline":true,"agent":{"model":"Claude Opus 5.5","evidence":"Add mnist-a100 ... Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>","family":"anthropic","confidence":"medium","role":"author"}},{"contributor":"yaroslavvb","description":"MLP 60-1024-1024-10, 100 steps","date":"2026-09-26","value":191.3,"contributorKind":"person","contributorUrl":"https://github.com/yaroslavvb","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/mnist-a100/energy/entries/mlp_k1_w1024_s100_b512.py","https://github.com/cybertronai/sutro-problems/blob/main/mnist-a100/energy/README.md"],"agent":{"model":"Claude Opus 5.5","evidence":"mnist-a100 README commits Co-Authored-By: Claude Opus 5.5","family":"anthropic","confidence":"medium","role":"author"}},{"contributor":"yaroslavvb","description":"MLP 60-256-256-10, 200 steps in a CUDA graph","date":"2026-09-26","value":61.7,"contributorKind":"person","contributorUrl":"https://github.com/yaroslavvb","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/mnist-a100/submissions/graph-mlp-20260926/fast_mlp.py","https://github.com/cybertronai/sutro-problems/blob/main/mnist-a100/submissions/graph-mlp-20260926/README.md"],"agent":{"model":"Claude Opus 5.5","evidence":"Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> (fast_mlp re-eval commit)","family":"anthropic","confidence":"high","role":"author"}},{"contributor":"yaroslavvb","description":"MLP, 200 steps, TF32 and fused AdamW; one sandboxed run","date":"2026-09-30","value":351,"contributorKind":"person","contributorUrl":"https://github.com/yaroslavvb","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/mnist-a100/submissions/improved-20260930/mlp_d1.py","https://github.com/cybertronai/sutro-problems/blob/main/mnist-a100/submissions/improved-20260930/README.md"],"agent":{"model":"Claude Opus 5.5","evidence":"Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> (mnist-a100 commits)","family":"anthropic","confidence":"medium","role":"author"}},{"contributor":"sjbaebae","description":"Two 60-256-256-10 MLPs, 30 updates, three-neighbor voting; median of three sandboxed Modal runs","date":"2026-10-08","value":6.7,"contributorKind":"person","contributorUrl":"https://github.com/sjbaebae","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/mnist-a100/submissions/native-mlp-d1-20261007/mlp_neighbor3_register_d1.py","https://github.com/cybertronai/sutro-problems/blob/main/mnist-a100/submissions/native-mlp-d1-20261007/README.md"]},{"contributor":"spartanhaden","description":"Nystrom RBF kernel ridge in a Fisher metric, 480 class-wise k-means landmarks, one Cholesky solve; no backprop; median of three independent reviewer sandboxed Modal runs","date":"2026-10-12","value":1.6,"contributorKind":"person","contributorUrl":"https://github.com/spartanhaden","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/mnist-a100/submissions/nystrom-fisher-d1-20261012/submission.py","https://github.com/cybertronai/sutro-problems/blob/main/mnist-a100/submissions/nystrom-fisher-d1-20261012/README.md"],"isCurrentBest":true}],"contested":true},{"group":"mnist-a100","description":"Learn MNIST from 10,000 labels and predict 10,000 test digits on one A100, error <= 3.40%; minimize time per call.","metricUnit":"ms","id":"mnist-a100-diff2","name":"MNIST-a100 (Difficulty 2)","metricName":"time","metricDirection":"minimize","isOpen":false,"records":[{"contributor":"yaroslavvb","description":"16 MLPs 60-1024-1024-10, 400 steps","date":"2026-09-26","value":927.3,"contributorKind":"person","contributorUrl":"https://github.com/yaroslavvb","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/mnist-a100/energy/entries/mlp_k16_w1024_s400_b512.py","https://github.com/cybertronai/sutro-problems/blob/main/mnist-a100/energy/README.md"],"agent":{"model":"Claude Opus 5.5","evidence":"yaroslavvb mnist-a100 commits Co-Authored-By: Claude Opus 5.5","family":"anthropic","confidence":"medium","role":"author"}},{"contributor":"yaroslavvb","description":"4 MLPs 60-256-256-10, 800 steps in a CUDA graph","date":"2026-09-26","value":252.1,"contributorKind":"person","contributorUrl":"https://github.com/yaroslavvb","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/mnist-a100/energy/entries/mlpg_k4_w256_s800_b512.py","https://github.com/cybertronai/sutro-problems/blob/main/mnist-a100/energy/README.md"],"agent":{"model":"Claude Opus 5.5","evidence":"yaroslavvb mnist-a100 commits Co-Authored-By: Claude Opus 5.5","family":"anthropic","confidence":"medium","role":"author"}},{"contributor":"islamborghini","description":"Non-neural RBF kernel ridge, one learned metric update, 16-step Nystrom-preconditioned CG","date":"2026-09-29","value":38.3,"contributorKind":"person","contributorUrl":"https://github.com/islamborghini","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/mnist-a100/submissions/kernel-pcg-20260929/kernel_pcg.py","https://github.com/cybertronai/sutro-problems/blob/main/mnist-a100/submissions/kernel-pcg-20260929/README.md"]},{"contributor":"yaroslavvb","description":"8 MLPs, 500 steps, EMA 0.992; one sandboxed run","date":"2026-09-30","value":761.9,"contributorKind":"person","contributorUrl":"https://github.com/yaroslavvb","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/mnist-a100/submissions/improved-20260930/mlp_d2.py","https://github.com/cybertronai/sutro-problems/blob/main/mnist-a100/submissions/improved-20260930/README.md"],"agent":{"model":"Claude Opus 5.5","evidence":"yaroslavvb mnist-a100 commits Co-Authored-By: Claude Opus 5.5","family":"anthropic","confidence":"medium","role":"author"}},{"contributor":"sjbaebae","description":"Two 60-256-256-10 MLPs, 120 updates, three-neighbor voting; median of three sandboxed Modal runs","date":"2026-10-08","value":14.6,"contributorKind":"person","contributorUrl":"https://github.com/sjbaebae","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/mnist-a100/submissions/native-mlp-d2-20261007/mlp_neighbor3_register_d2.py","https://github.com/cybertronai/sutro-problems/blob/main/mnist-a100/submissions/native-mlp-d2-20261007/README.md"]},{"contributor":"spartanhaden","description":"RBF kernel ridge with one learned (AGOP) metric, 5 Nystrom-preconditioned CG steps; no backprop; median of three independent reviewer sandboxed Modal runs","date":"2026-10-12","value":4.6,"contributorKind":"person","contributorUrl":"https://github.com/spartanhaden","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/mnist-a100/submissions/rfm-pcg-d2-20261012/submission.py","https://github.com/cybertronai/sutro-problems/blob/main/mnist-a100/submissions/rfm-pcg-d2-20261012/README.md"],"isCurrentBest":true}],"contested":true},{"group":"mnist-a100","description":"Learn MNIST from 10,000 labels and predict 10,000 test digits on one A100, error <= 2.70%; minimize time per call.","metricUnit":"ms","id":"mnist-a100-diff3","name":"MNIST-a100 (Difficulty 3)","metricName":"time","metricDirection":"minimize","isOpen":false,"records":[{"contributor":"SethTS","description":"Ladder 60-1000-500-250-250-250-10, 5,000 steps in a CUDA graph","date":"2026-09-28","value":15818.8,"contributorKind":"person","contributorUrl":"https://github.com/SethTS","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/mnist-a100/submissions/ladder-graph-20260928/ladder.py","https://github.com/cybertronai/sutro-problems/blob/main/mnist-a100/submissions/ladder-graph-20260928/README.md"]},{"contributor":"SethTS","description":"Ladder, 6,500 steps, fused Triton kernels in a CUDA graph","date":"2026-09-28","value":6556.9,"contributorKind":"person","contributorUrl":"https://github.com/SethTS","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/mnist-a100/submissions/ladder-triton-20260928/ladder_d3.py","https://github.com/cybertronai/sutro-problems/blob/main/mnist-a100/submissions/ladder-triton-20260928/README.md"]},{"contributor":"SethTS","description":"Ladder, batch 1,000, 1,200 steps, TF32, fused Triton kernels in a CUDA graph","date":"2026-09-29","value":1895.4,"contributorKind":"person","contributorUrl":"https://github.com/SethTS","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/mnist-a100/submissions/ladder-fast-20260929/ladder_d3.py","https://github.com/cybertronai/sutro-problems/blob/main/mnist-a100/submissions/ladder-fast-20260929/README.md"]},{"contributor":"SethTS","description":"Ladder, batch 1,000, 900 steps, labelled rows sampled by loss, TF32, fused Triton kernels in a CUDA graph","date":"2026-09-29","value":1443.3,"contributorKind":"person","contributorUrl":"https://github.com/SethTS","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/mnist-a100/submissions/ladder-sampled-20260929/ladder_d3.py","https://github.com/cybertronai/sutro-problems/blob/main/mnist-a100/submissions/ladder-sampled-20260929/README.md"]},{"contributor":"yaroslavvb","description":"Ladder, 1,100 steps; 8.3% fewer updates; one sandboxed run","date":"2026-09-30","value":1732.2,"contributorKind":"person","contributorUrl":"https://github.com/yaroslavvb","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/mnist-a100/submissions/improved-20260930/ladder_d3.py","https://github.com/cybertronai/sutro-problems/blob/main/mnist-a100/submissions/improved-20260930/README.md"],"agent":{"model":"Claude Opus 5.5","evidence":"yaroslavvb mnist-a100 commits Co-Authored-By: Claude Opus 5.5","family":"anthropic","confidence":"medium","role":"author"}},{"contributor":"islamborghini","description":"Sampled Ladder, batch 1,000, 800 steps, TF32; tuned forward and backward Triton kernels; median of three sandboxed runs","date":"2026-10-05","value":1157.1,"contributorKind":"person","contributorUrl":"https://github.com/islamborghini","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/mnist-a100/submissions/ladder-kernel-tuned-20261003/ladder_d3.py","https://github.com/cybertronai/sutro-problems/blob/main/mnist-a100/submissions/ladder-kernel-tuned-20261003/README.md"]},{"contributor":"sjbaebae","description":"Eight 60-256-256-256-10 RMSNorm/ReLU MLPs, 1,250 updates, five-neighbor voting and three-step query graph propagation; median of three sandboxed Modal runs","date":"2026-10-08","value":416.1,"contributorKind":"person","contributorUrl":"https://github.com/sjbaebae","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/mnist-a100/submissions/native-mlp-d3-20261007/graph1250-repair/submission.py","https://github.com/cybertronai/sutro-problems/blob/main/mnist-a100/submissions/native-mlp-d3-20261007/graph1250-repair/README.md"]},{"contributor":"sjbaebae","description":"Spherical direction/radius RBF, two tangent-gradient metric updates; derived from PR #116; median of three independent reviewer sandboxed Modal runs","date":"2026-10-12","value":51.8,"contributorKind":"person","contributorUrl":"https://github.com/sjbaebae","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/mnist-a100/submissions/polar-kernel-d3-20261012/submission.py","https://github.com/cybertronai/sutro-problems/blob/main/mnist-a100/submissions/polar-kernel-d3-20261012/README.md"]},{"contributor":"spartanhaden","description":"RBF kernel ridge, two AGOP metric updates, HKNN local-hull correction on the 512 lowest-margin rows; no backprop; median of three independent reviewer sandboxed Modal runs","date":"2026-10-12","value":18.1,"contributorKind":"person","contributorUrl":"https://github.com/spartanhaden","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/mnist-a100/submissions/rfm-hknn-d3-20261012/submission.py","https://github.com/cybertronai/sutro-problems/blob/main/mnist-a100/submissions/rfm-hknn-d3-20261012/README.md"]},{"contributor":"islamborghini","description":"RBF kernel ridge, two analytic metric updates, CUDA-graph Nystrom-preconditioned CG and local class geometry; median of three independent reviewer sandboxed Modal runs","date":"2026-10-14","value":44,"contributorKind":"person","contributorUrl":"https://github.com/islamborghini","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/mnist-a100/submissions/kernel-rbf-d3-20261009/kernel_d3_cg_graph.py","https://github.com/cybertronai/sutro-problems/blob/main/mnist-a100/submissions/kernel-rbf-d3-20261009/README.md"]},{"contributor":"sigkillme0","description":"Learned-metric RBF kernel with fused Triton PCG, compressed kernel tiles, and local-hull correction; builds on Haden's RFM/HKNN learner; median of three independent reviewer sandboxed Modal runs","date":"2026-10-18","value":5.2,"contributorKind":"person","contributorUrl":"https://github.com/sigkillme0","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/mnist-a100/submissions/kernel-d3-20261018/submission.py","https://github.com/cybertronai/sutro-problems/blob/main/mnist-a100/submissions/kernel-d3-20261018/README.md"],"isCurrentBest":true}],"contested":true},{"group":"mnist-a100","description":"Learn MNIST from 10,000 labels and predict 10,000 test digits on one A100, error <= 2.30%; minimize time per call.","metricUnit":"ms","id":"mnist-a100-diff4","name":"MNIST-a100 (Difficulty 4)","metricName":"time","metricDirection":"minimize","isOpen":false,"records":[{"contributor":"SethTS","description":"Ladder, 11,000 steps, fused Triton kernels in a CUDA graph","date":"2026-09-28","value":11094.6,"contributorKind":"person","contributorUrl":"https://github.com/SethTS","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/mnist-a100/submissions/ladder-triton-20260928/ladder_d4.py","https://github.com/cybertronai/sutro-problems/blob/main/mnist-a100/submissions/ladder-triton-20260928/README.md"]},{"contributor":"SethTS","description":"Ladder, batch 1,000, 2,400 steps, TF32, fused Triton kernels in a CUDA graph","date":"2026-09-29","value":3784.2,"contributorKind":"person","contributorUrl":"https://github.com/SethTS","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/mnist-a100/submissions/ladder-fast-20260929/ladder_d4.py","https://github.com/cybertronai/sutro-problems/blob/main/mnist-a100/submissions/ladder-fast-20260929/README.md"]},{"contributor":"SethTS","description":"Ladder, batch 1,000, 1,800 steps, labelled rows sampled by loss, TF32, fused Triton kernels in a CUDA graph","date":"2026-09-29","value":2882.5,"contributorKind":"person","contributorUrl":"https://github.com/SethTS","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/mnist-a100/submissions/ladder-sampled-20260929/ladder_d4.py","https://github.com/cybertronai/sutro-problems/blob/main/mnist-a100/submissions/ladder-sampled-20260929/README.md"]},{"contributor":"yaroslavvb","description":"Ladder, 2,200 steps; 8.3% fewer updates; one sandboxed run","date":"2026-09-30","value":3481.3,"contributorKind":"person","contributorUrl":"https://github.com/yaroslavvb","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/mnist-a100/submissions/improved-20260930/ladder_d4.py","https://github.com/cybertronai/sutro-problems/blob/main/mnist-a100/submissions/improved-20260930/README.md"],"agent":{"model":"Claude Opus 5.5","evidence":"yaroslavvb mnist-a100 commits Co-Authored-By: Claude Opus 5.5","family":"anthropic","confidence":"medium","role":"author"}},{"contributor":"spartanhaden","description":"Ladder MLP 60-1024-512-256-10 on AGOP-metric inputs, 560 Adam steps, kernel-ridge teacher; median of three independent reviewer sandboxed Modal runs","date":"2026-10-12","value":580.1,"contributorKind":"person","contributorUrl":"https://github.com/spartanhaden","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/mnist-a100/submissions/ladder-kernel-d4-20261012/submission.py","https://github.com/cybertronai/sutro-problems/blob/main/mnist-a100/submissions/ladder-kernel-d4-20261012/README.md"]},{"contributor":"islamborghini","description":"Five RBF classifiers with three analytic AGOP metric updates each, Nystrom-preconditioned CG, transductive refits and HKNN affine-hull correction; no backprop; median of three independent reviewer sandboxed Modal runs","date":"2026-10-18","value":219.2,"contributorKind":"person","contributorUrl":"https://github.com/islamborghini","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/mnist-a100/submissions/rfm-ensemble-d4-20261017/submission.py","https://github.com/cybertronai/sutro-problems/blob/main/mnist-a100/submissions/rfm-ensemble-d4-20261017/README.md"],"isCurrentBest":true}],"contested":true},{"group":"mnist-a100","description":"Learn MNIST from 10,000 labels and predict 10,000 test digits on one A100, error <= 1.90%; minimize time per call.","metricUnit":"ms","id":"mnist-a100-diff5","name":"MNIST-a100 (Difficulty 5)","metricName":"time","metricDirection":"minimize","isOpen":false,"records":[{"contributor":"SethTS","description":"Ladder, 36,000 steps, fused Triton kernels in a CUDA graph","date":"2026-09-28","value":36201.2,"contributorKind":"person","contributorUrl":"https://github.com/SethTS","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/mnist-a100/submissions/ladder-triton-20260928/ladder_d5.py","https://github.com/cybertronai/sutro-problems/blob/main/mnist-a100/submissions/ladder-triton-20260928/README.md"]},{"contributor":"SethTS","description":"Ladder, batch 1,000, 9,000 steps, TF32, fused Triton kernels in a CUDA graph","date":"2026-09-29","value":14130.6,"contributorKind":"person","contributorUrl":"https://github.com/SethTS","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/mnist-a100/submissions/ladder-fast-20260929/ladder_d5.py","https://github.com/cybertronai/sutro-problems/blob/main/mnist-a100/submissions/ladder-fast-20260929/README.md"]},{"contributor":"yaroslavvb","description":"Ladder, 8,400 steps; 6.7% fewer updates; one sandboxed run","date":"2026-09-30","value":13264.3,"contributorKind":"person","contributorUrl":"https://github.com/yaroslavvb","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/mnist-a100/submissions/improved-20260930/ladder_d5.py","https://github.com/cybertronai/sutro-problems/blob/main/mnist-a100/submissions/improved-20260930/README.md"],"agent":{"model":"Claude Opus 5.5","evidence":"yaroslavvb mnist-a100 commits Co-Authored-By: Claude Opus 5.5","family":"anthropic","confidence":"medium","role":"author"}},{"contributor":"likelytobelaura","description":"Eight RMSNorm/GELU MLPs, Muon/AdamW, 2,500 updates, local tangent jitter, disjoint-pool cross-teaching and graph readout; median of three independent reviewer sandboxed Modal runs","date":"2026-10-12","value":3335.4,"contributorKind":"person","contributorUrl":"https://github.com/likelytobelaura","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/mnist-a100/submissions/tangent-crossteach-d5-20261012/submission.py","https://github.com/cybertronai/sutro-problems/blob/main/mnist-a100/submissions/tangent-crossteach-d5-20261012/README.md"],"agent":{"model":"Claude","evidence":"likelytobelaura D5 cross-taught Muon MLP commit, 2026-10-12","family":"anthropic","confidence":"low","role":"author"}},{"contributor":"spartanhaden","description":"Ladder MLP 60-1024-512-256-10 on AGOP-metric inputs, 2,300 Adam steps, kernel-ridge teacher; median of three independent reviewer sandboxed Modal runs","date":"2026-10-12","value":1818.1,"contributorKind":"person","contributorUrl":"https://github.com/spartanhaden","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/mnist-a100/submissions/ladder-kernel-d5-20261012/submission.py","https://github.com/cybertronai/sutro-problems/blob/main/mnist-a100/submissions/ladder-kernel-d5-20261012/README.md"]},{"contributor":"likelytobelaura","description":"Eight width-256 RMSNorm/ReLU MLPs, Muon, tangent jitter, cross-teaching and graph readout; median of three independent reviewer sandboxed Modal runs","date":"2026-10-14","value":1915.4,"contributorKind":"person","contributorUrl":"https://github.com/likelytobelaura","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/mnist-a100/submissions/tangent-relu-d5-20261012/submission.py","https://github.com/cybertronai/sutro-problems/blob/main/mnist-a100/submissions/tangent-relu-d5-20261012/README.md"],"agent":{"model":"Claude","evidence":"likelytobelaura D5 fast commit 2026-10-13, median 1,898 ms","family":"anthropic","confidence":"low","role":"author"}},{"contributor":"islamborghini","description":"Haden's kernel-taught Ladder tuned to batch 2,000 and 800 steps; architecture, teacher and optimizer unchanged; median of three independent reviewer sandboxed Modal runs","date":"2026-10-18","value":897.5,"contributorKind":"person","contributorUrl":"https://github.com/islamborghini","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/mnist-a100/submissions/ladder-kernel-tuned-d5-20261017/submission.py","https://github.com/cybertronai/sutro-problems/blob/main/mnist-a100/submissions/ladder-kernel-tuned-d5-20261017/README.md"],"isCurrentBest":true}],"contested":true},{"group":"mnist","description":"Learn from 1,000 3x3 training images and predict 1,000 test labels at >=67% mean accuracy; minimize A100 energy.","metricUnit":"mJ","id":"mnist-small-67","name":"MNIST-small (67% accuracy target)","metricName":"energy","metricDirection":"minimize","isOpen":false,"records":[{"contributor":"H32 MLP","description":"H32 MLP","date":"2026-09-11","value":3300,"contributorKind":"method","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/mnist/submissions/small60-grid-20260912/README.md"]},{"contributor":"NR-K8 Adam MLP","description":"NR-K8 Adam MLP","date":"2026-09-14","value":2600,"contributorKind":"method","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/mnist/submissions/small-adam-nr-k8-20260912/README.md"]},{"contributor":"Panel-cached H32 MLP","description":"Panel-cached H32 MLP (1,000/1,000)","date":"2026-09-15","value":3200,"contributorKind":"method","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/mnist/submissions/mlp-panels-revised-20260914/README.md"]},{"contributor":"QDA","description":"QDA","date":"2026-09-15","value":0.59,"contributorKind":"method","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/mnist/submissions/small-qda-20260915/README.md"],"isCurrentBest":true,"agent":{"model":"Claude Fable 5.1","evidence":"Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>","family":"anthropic","confidence":"high","role":"author"}},{"contributor":"9-64-10 SGD MLP","description":"9-64-10 SGD MLP (A100 energy not measured)","date":"2026-09-15","value":0,"contributorKind":"method","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/mnist/submissions/small64-sgd-20260915/report.md"]}],"contested":true},{"group":"mnist","description":"Learn from 10,000 9x9 training images and predict 10,000 test labels at <=2% mean error; minimize A100 energy.","metricUnit":"mJ","id":"mnist-medium-2","name":"MNIST-medium (2% error target)","metricName":"energy","metricDirection":"minimize","isOpen":false,"records":[{"contributor":"512-filter CG pair","description":"512-filter CG pair","date":"2026-09-16","value":65000,"contributorKind":"method","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/mnist/submissions/medium-cg-pair-20260916/README.md"],"isCurrentBest":true}],"contested":false},{"group":"mnist","description":"Learn from 10,000 9x9 training images and predict 10,000 test labels at <=3% mean error; minimize A100 energy.","metricUnit":"mJ","id":"mnist-medium-3","name":"MNIST-medium (3% error target)","metricName":"energy","metricDirection":"minimize","isOpen":false,"records":[{"contributor":"512-filter CG pair","description":"512-filter CG pair","date":"2026-09-16","value":65000,"contributorKind":"method","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/mnist/submissions/medium-cg-pair-20260916/README.md"],"isCurrentBest":true}],"contested":false},{"group":"mnist","description":"Learn from 10,000 9x9 training images and predict 10,000 test labels at <=5% mean error; minimize A100 energy.","metricUnit":"mJ","id":"mnist-medium-5","name":"MNIST-medium (5% error target)","metricName":"energy","metricDirection":"minimize","isOpen":false,"records":[{"contributor":"512-unit MLP","description":"512-unit MLP (96% target)","date":"2026-09-11","value":290000,"contributorKind":"method","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/mnist/submissions/medium96-grid-20260912/README.md"],"agent":{"tool":"Codex","evidence":"Merge pull request #66 from cybertronai/codex/mnist-medium-submission","family":"openai","confidence":"medium","role":"author"}},{"contributor":"PCA-QDA","description":"PCA-QDA","date":"2026-09-15","value":174,"contributorKind":"method","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/mnist/submissions/medium-pca-qda-20260915/README.md"],"isCurrentBest":true,"agent":{"model":"Claude Fable 5.1","evidence":"small-qda commits Co-Authored-By: Claude Fable 5.1; PCA-QDA energy correction PR #85","family":"anthropic","confidence":"high","role":"author"}}],"contested":true},{"group":"mnist","description":"Learn from 10,000 9x9 training images and predict 10,000 test labels at <=8% mean error; minimize A100 energy.","metricUnit":"mJ","id":"mnist-medium-8","name":"MNIST-medium (8% error target)","metricName":"energy","metricDirection":"minimize","isOpen":false,"records":[{"contributor":"512-filter CG pair","description":"512-filter CG pair","date":"2026-09-16","value":65000,"contributorKind":"method","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/mnist/submissions/medium-cg-pair-20260916/README.md"],"isCurrentBest":true}],"contested":false},{"group":"mnist","description":"Learn from 10,000 9x9 training images and predict 10,000 test labels at <=12% mean error; minimize A100 energy.","metricUnit":"mJ","id":"mnist-medium-12","name":"MNIST-medium (12% error target)","metricName":"energy","metricDirection":"minimize","isOpen":false,"records":[{"contributor":"Reversible MLP","description":"Reversible MLP, smaller workspace (12% error target)","date":"2026-09-14","value":1000,"contributorKind":"method","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/mnist/submissions/rev88-20260914/cache-audit/README.md"],"isCurrentBest":true,"agent":{"model":"Codex","evidence":"PR #74 from cybertronai/codex/rev88-20260914 Add reversible MLP submission","family":"openai","confidence":"high","role":"author"}}],"contested":false},{"group":"mnist","description":"Learn from the full 60,000 28x28 training images and predict 10,000 test labels at <=1% test error; minimize A100 energy.","metricUnit":"mJ","id":"mnist-original","name":"MNIST-original (1% test error target)","metricName":"energy","metricDirection":"minimize","isOpen":false,"records":[{"contributor":"PCANet 9-block + mixture","description":"PCANet 9-block + mixture, k=8 K=100","date":"2026-09-21","value":30000,"contributorKind":"method","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/mnist/submissions/original-pcanet-mixture-20260920/README.md"],"isCurrentBest":true}],"contested":false},{"group":"mnist","description":"Historical MNIST-small submissions under the previous specification (600/600 images); minimize A100 energy.","metricUnit":"mJ","id":"mnist-small-historical","name":"MNIST-small (Historical)","metricName":"energy","metricDirection":"minimize","isOpen":false,"records":[{"contributor":"Static dataflow 3x3 int8-safe","description":"Static dataflow 3x3 int8-safe (1,000/1,000)","date":"2026-09-14","value":0.11,"contributorKind":"method","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/mnist/submissions/static-dataflow-3x3-20260914/report.md"],"isCurrentBest":true},{"contributor":"Panel-cached MLP","description":"Panel-cached MLP (600/600)","date":"2026-09-12","value":1800,"contributorKind":"method","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/mnist/submissions/mlp-panels-v4-20260911/report.md"]},{"contributor":"32-unit MLP","description":"32-unit MLP","date":"2026-09-10","value":2000,"contributorKind":"method","submissionUrls":["https://cybertronai.github.io/sutro-problems/docs/submissions/mlp60-affine-20260911/"],"agent":{"model":"Codex","evidence":"PR #65 codex/mnist-small-60-submission submit 62% fixed MLP","family":"openai","confidence":"high","role":"author"}},{"contributor":"1NN","description":"1NN","date":"2026-09-10","value":0.52,"contributorKind":"method","submissionUrls":["https://cybertronai.github.io/sutro-problems/docs/submissions/1nn-v4-20260911/"],"agent":{"model":"Codex","evidence":"PR #64 codex/mnist-small-1nn-submission Add MNIST-small 1NN submission","family":"openai","confidence":"high","role":"author"}}],"contested":true},{"group":"mnist","description":"Historical MNIST-medium submissions under the previous specification (6,000/6,000 images); minimize A100 energy.","metricUnit":"mJ","id":"mnist-medium-historical","name":"MNIST-medium (Historical)","metricName":"energy","metricDirection":"minimize","isOpen":false,"records":[{"contributor":"Ordered ConvNets","description":"Ordered ConvNets (3% error target)","date":"2026-09-11","value":780000,"contributorKind":"method","submissionUrls":["https://cybertronai.github.io/sutro-problems/docs/submissions/medium-convnet-v4-20260911/"],"agent":{"model":"Codex","evidence":"PR #70 codex/mnist-medium-four-percent-scored Submit fully scored ConvNet","family":"openai","confidence":"high","role":"author"}},{"contributor":"Three ConvNets","description":"Three ConvNets","date":"2026-09-10","value":1700000,"contributorKind":"method","submissionUrls":["https://cybertronai.github.io/sutro-problems/docs/submissions/medium-convnet-11draw-20260911/"],"agent":{"model":"Codex","evidence":"PR #68 codex/mnist-medium-convnet-submission Submit MNIST-medium ConvNet","family":"openai","confidence":"high","role":"author"}},{"contributor":"512-unit MLP","description":"512-unit MLP","date":"2026-09-10","value":150000,"contributorKind":"method","submissionUrls":["https://cybertronai.github.io/sutro-problems/docs/submissions/medium-affine-20260911/"],"isCurrentBest":true,"agent":{"model":"Codex","evidence":"PR #66 codex/mnist-medium-submission measured 96% MLP attempt","family":"openai","confidence":"high","role":"author"}}],"contested":true},{"group":"sparse-parity","description":"Recover the k=5 secret bit positions of a 32-bit string at 20% secret recovery rate; minimize read-cost.","id":"sparse-parity-20","name":"Sparse Parity (20% target)","metricName":"cost","metricDirection":"minimize","isOpen":false,"records":[{"contributor":"yaroslavvb","description":"generate_isd_mask(8) (ISD restarts)","date":"2026-08-26","value":12042480,"contributorKind":"person","contributorUrl":"https://github.com/yaroslavvb","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/sparse-parity/submissions/isd8_mask32.ir","https://github.com/cybertronai/sutro-problems/blob/main/sparse-parity/submissions/isd8_mask32.md","https://github.com/cybertronai/sutro-problems/blob/main/sparse-parity/mask_sparse_parity.py"]},{"contributor":"yaroslavvb","description":"generate_scan(127) (Gray scan)","date":"2026-08-27","value":17331683,"contributorKind":"person","contributorUrl":"https://github.com/yaroslavvb","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/sparse-parity/submissions/scan127_mask32.md","https://github.com/cybertronai/sutro-problems/blob/main/sparse-parity/mask_sparse_parity.py"]},{"contributor":"zh4ngx","description":"optimize_layout(generate_sis_mask(1, 2)) (static-IS walk, staged layout)","date":"2026-08-30","value":1317480,"contributorKind":"person","contributorUrl":"https://github.com/zh4ngx","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/sparse-parity/submissions/siswalk1_cap2_mask32.ir","https://github.com/cybertronai/sutro-problems/blob/main/sparse-parity/submissions/siswalk1_cap2_mask32.md","https://github.com/cybertronai/sutro-problems/blob/main/sparse-parity/mask_sparse_parity.py"]},{"contributor":"npow","description":"generate_packed_sis(cap=2, seed=13, g2=8) (packed SIS, partial cap-2 walk, exhaustive information-set tuning)","date":"2026-08-31","value":151117,"contributorKind":"person","contributorUrl":"https://github.com/npow","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/sparse-parity/submissions/packedsis_pcap2_mask32.ir","https://github.com/cybertronai/sutro-problems/blob/main/sparse-parity/submissions/packedsis_pcap2_mask32.md","https://github.com/cybertronai/sutro-problems/blob/main/sparse-parity/submissions/packedsis.py"]},{"contributor":"jurajselep","description":"generate_packed_scan(1) (packed-column scan + SSA layout)","date":"2026-09-01","value":135348,"contributorKind":"person","contributorUrl":"https://github.com/jurajselep","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/sparse-parity/submissions/packedscan1_mask32.ir","https://github.com/cybertronai/sutro-problems/blob/main/sparse-parity/submissions/packedscan1_mask32.md","https://github.com/cybertronai/sutro-problems/blob/main/sparse-parity/packed_sparse_parity.py"]},{"contributor":"b0nce","description":"generate_packed_static20() (packed static information set)","date":"2026-09-02","value":86753,"contributorKind":"person","contributorUrl":"https://github.com/b0nce","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/sparse-parity/submissions/packedstatic20_mask32.ir","https://github.com/cybertronai/sutro-problems/blob/main/sparse-parity/packed_sparse_parity.py","https://github.com/cybertronai/sutro-problems/blob/main/sparse-parity/submissions/packed_static20_audit.json"],"isCurrentBest":true}],"contested":true},{"group":"sparse-parity","description":"Recover the k=5 secret bit positions of a 32-bit string at 40% secret recovery rate; minimize read-cost.","id":"sparse-parity-40","name":"Sparse Parity (40% target)","metricName":"cost","metricDirection":"minimize","isOpen":false,"records":[{"contributor":"yaroslavvb","description":"generate_scan(1023) (Gray scan)","date":"2026-08-27","value":18764343,"contributorKind":"person","contributorUrl":"https://github.com/yaroslavvb","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/sparse-parity/submissions/scan1023_mask32.md","https://github.com/cybertronai/sutro-problems/blob/main/sparse-parity/mask_sparse_parity.py"]},{"contributor":"yaroslavvb","description":"generate_scan(511) (Gray scan)","date":"2026-08-27","value":17945660,"contributorKind":"person","contributorUrl":"https://github.com/yaroslavvb","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/sparse-parity/submissions/scan511_mask32.ir","https://github.com/cybertronai/sutro-problems/blob/main/sparse-parity/submissions/scan511_mask32.md","https://github.com/cybertronai/sutro-problems/blob/main/sparse-parity/mask_sparse_parity.py"]},{"contributor":"zh4ngx","description":"generate_scan(0, walk=\"weight\", weight_cap=2) (weight-ordered scan)","date":"2026-08-28","value":17418235,"contributorKind":"person","contributorUrl":"https://github.com/zh4ngx","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/sparse-parity/submissions/weightscan2_mask32.ir","https://github.com/cybertronai/sutro-problems/blob/main/sparse-parity/submissions/weightscan2_mask32.md","https://github.com/cybertronai/sutro-problems/blob/main/sparse-parity/mask_sparse_parity.py"]},{"contributor":"zh4ngx","description":"optimize_layout(generate_sis_mask(1, 2)) (static-IS walk, staged layout)","date":"2026-08-30","value":1317480,"contributorKind":"person","contributorUrl":"https://github.com/zh4ngx","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/sparse-parity/submissions/siswalk1_cap2_mask32.ir","https://github.com/cybertronai/sutro-problems/blob/main/sparse-parity/submissions/siswalk1_cap2_mask32.md","https://github.com/cybertronai/sutro-problems/blob/main/sparse-parity/mask_sparse_parity.py"]},{"contributor":"npow","description":"generate(1, 2, seed=5) (bit-packed SIS walk, 3-phase layout, higher dev recovery)","date":"2026-08-31","value":163378,"contributorKind":"person","contributorUrl":"https://github.com/npow","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/sparse-parity/submissions/packedwalk1_cap2_s5_mask32.ir","https://github.com/cybertronai/sutro-problems/blob/main/sparse-parity/submissions/packedwalk1_mask32.md","https://github.com/cybertronai/sutro-problems/blob/main/sparse-parity/submissions/packedwalk.py"]},{"contributor":"jurajselep","description":"generate_packed_scan(2) (packed-column scan + SSA layout)","date":"2026-09-01","value":151943,"contributorKind":"person","contributorUrl":"https://github.com/jurajselep","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/sparse-parity/submissions/packedscan2_mask32.ir","https://github.com/cybertronai/sutro-problems/blob/main/sparse-parity/submissions/packedscan2_mask32.md","https://github.com/cybertronai/sutro-problems/blob/main/sparse-parity/packed_sparse_parity.py"]},{"contributor":"b0nce","description":"generate_packed_route40() (cap-2 Gray prefix)","date":"2026-09-02","value":147000,"contributorKind":"person","contributorUrl":"https://github.com/b0nce","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/sparse-parity/submissions/packedroute40_mask32.ir","https://github.com/cybertronai/sutro-problems/blob/main/sparse-parity/packed_sparse_parity.py"]},{"contributor":"npow","description":"55 audited weight-2 states + compact flow + reverse route","date":"2026-09-01","value":141218,"contributorKind":"person","contributorUrl":"https://github.com/npow","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/sparse-parity/submissions/packedfrontier40_mask32.ir","https://github.com/cybertronai/sutro-problems/blob/main/sparse-parity/submissions/packed_frontier.md","https://github.com/cybertronai/sutro-problems/blob/main/sparse-parity/submissions/packed_frontier.py"],"isCurrentBest":true}],"contested":true},{"group":"sparse-parity","description":"Recover the k=5 secret bit positions of a 32-bit string at 60% secret recovery rate; minimize read-cost.","id":"sparse-parity-60","name":"Sparse Parity (60% target)","metricName":"cost","metricDirection":"minimize","isOpen":false,"records":[{"contributor":"yaroslavvb","description":"generate_scan(4095) (Gray scan)","date":"2026-08-26","value":23676539,"contributorKind":"person","contributorUrl":"https://github.com/yaroslavvb","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/sparse-parity/submissions/scan4095_mask32.ir","https://github.com/cybertronai/sutro-problems/blob/main/sparse-parity/submissions/scan4095_mask32.md","https://github.com/cybertronai/sutro-problems/blob/main/sparse-parity/mask_sparse_parity.py"]},{"contributor":"yaroslavvb","description":"generate_scan(6143) (Gray scan)","date":"2026-08-27","value":26951367,"contributorKind":"person","contributorUrl":"https://github.com/yaroslavvb","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/sparse-parity/submissions/scan6143_mask32.md","https://github.com/cybertronai/sutro-problems/blob/main/sparse-parity/mask_sparse_parity.py"]},{"contributor":"zh4ngx","description":"generate_scan(0, walk=\"weight\", weight_cap=3) (weight-ordered scan)","date":"2026-08-28","value":18509753,"contributorKind":"person","contributorUrl":"https://github.com/zh4ngx","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/sparse-parity/submissions/weightscan3_mask32.ir","https://github.com/cybertronai/sutro-problems/blob/main/sparse-parity/submissions/weightscan3_mask32.md","https://github.com/cybertronai/sutro-problems/blob/main/sparse-parity/mask_sparse_parity.py"]},{"contributor":"zh4ngx","description":"optimize_layout(generate_sis_mask(1, 3)) (static-IS walk, staged layout)","date":"2026-08-30","value":2137725,"contributorKind":"person","contributorUrl":"https://github.com/zh4ngx","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/sparse-parity/submissions/siswalk1_cap3_mask32.ir","https://github.com/cybertronai/sutro-problems/blob/main/sparse-parity/submissions/siswalk1_cap3_mask32.md","https://github.com/cybertronai/sutro-problems/blob/main/sparse-parity/mask_sparse_parity.py"]},{"contributor":"npow","description":"generate_packed_sis(cap=3, seed=13) (bit-packed SIS walk, full cap-3, tuned seed)","date":"2026-08-31","value":284049,"contributorKind":"person","contributorUrl":"https://github.com/npow","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/sparse-parity/submissions/packedsis_cap3_s13_mask32.ir","https://github.com/cybertronai/sutro-problems/blob/main/sparse-parity/submissions/packedsis_pcap2_mask32.md","https://github.com/cybertronai/sutro-problems/blob/main/sparse-parity/submissions/packedsis.py"]},{"contributor":"jurajselep","description":"generate_packed_scan(3) (packed-column scan + SSA layout)","date":"2026-09-01","value":200937,"contributorKind":"person","contributorUrl":"https://github.com/jurajselep","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/sparse-parity/submissions/packedscan3_mask32.ir","https://github.com/cybertronai/sutro-problems/blob/main/sparse-parity/submissions/packedscan3_mask32.md","https://github.com/cybertronai/sutro-problems/blob/main/sparse-parity/packed_sparse_parity.py"]},{"contributor":"b0nce","description":"generate_packed_route60() (cap-3 Gray prefix)","date":"2026-09-02","value":176331,"contributorKind":"person","contributorUrl":"https://github.com/b0nce","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/sparse-parity/submissions/packedroute60_mask32.ir","https://github.com/cybertronai/sutro-problems/blob/main/sparse-parity/packed_sparse_parity.py"]},{"contributor":"npow","description":"19 audited weight-3 states + compact flow + reversed 2-opt walk","date":"2026-09-01","value":149665,"contributorKind":"person","contributorUrl":"https://github.com/npow","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/sparse-parity/submissions/packedfrontier60_mask32.ir","https://github.com/cybertronai/sutro-problems/blob/main/sparse-parity/submissions/packed_frontier.md","https://github.com/cybertronai/sutro-problems/blob/main/sparse-parity/submissions/packed_frontier.py"],"isCurrentBest":true}],"contested":true},{"group":"sparse-parity","description":"Recover the k=5 secret bit positions of a 32-bit string at 80% secret recovery rate; minimize read-cost.","id":"sparse-parity-80","name":"Sparse Parity (80% target)","metricName":"cost","metricDirection":"minimize","isOpen":false,"records":[{"contributor":"yaroslavvb","description":"generate_scan(10239) (Gray scan)","date":"2026-08-27","value":33501030,"contributorKind":"person","contributorUrl":"https://github.com/yaroslavvb","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/sparse-parity/submissions/scan10239_mask32.md","https://github.com/cybertronai/sutro-problems/blob/main/sparse-parity/mask_sparse_parity.py"]},{"contributor":"yaroslavvb","description":"generate_scan(8191) (Gray scan)","date":"2026-08-27","value":30226172,"contributorKind":"person","contributorUrl":"https://github.com/yaroslavvb","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/sparse-parity/submissions/scan8191_mask32.ir","https://github.com/cybertronai/sutro-problems/blob/main/sparse-parity/submissions/scan8191_mask32.md","https://github.com/cybertronai/sutro-problems/blob/main/sparse-parity/mask_sparse_parity.py"]},{"contributor":"zh4ngx","description":"optimize_layout(generate_scan(0, walk=\"weight\", weight_cap=3)) (weight-ordered scan, staged layout)","date":"2026-08-30","value":5593997,"contributorKind":"person","contributorUrl":"https://github.com/zh4ngx","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/sparse-parity/submissions/weightscan3_mask32.ir","https://github.com/cybertronai/sutro-problems/blob/main/sparse-parity/submissions/weightscan3_mask32.md","https://github.com/cybertronai/sutro-problems/blob/main/sparse-parity/mask_sparse_parity.py"]},{"contributor":"npow","description":"generate_staged(weight_cap=3) (septet-packed dynamic RREF + row-coordinate walk)","date":"2026-08-31","value":493193,"contributorKind":"person","contributorUrl":"https://github.com/npow","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/sparse-parity/submissions/septwalk_wcap3_mask32.ir","https://github.com/cybertronai/sutro-problems/blob/main/sparse-parity/submissions/septwalk_wcap3_mask32.md","https://github.com/cybertronai/sutro-problems/blob/main/sparse-parity/submissions/septwalk.py"]},{"contributor":"jurajselep","description":"generate_packed_scan(3) (packed-column scan + SSA layout)","date":"2026-09-01","value":200937,"contributorKind":"person","contributorUrl":"https://github.com/jurajselep","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/sparse-parity/submissions/packedscan3_mask32.ir","https://github.com/cybertronai/sutro-problems/blob/main/sparse-parity/submissions/packedscan3_mask32.md","https://github.com/cybertronai/sutro-problems/blob/main/sparse-parity/packed_sparse_parity.py"]},{"contributor":"b0nce","description":"generate_packed_route80() (cap-3 Gray prefix)","date":"2026-09-02","value":196139,"contributorKind":"person","contributorUrl":"https://github.com/b0nce","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/sparse-parity/submissions/packedroute80_mask32.ir","https://github.com/cybertronai/sutro-problems/blob/main/sparse-parity/packed_sparse_parity.py"]},{"contributor":"npow","description":"269 audited weight-3 states + compact flow + reversed 2-opt walk","date":"2026-09-01","value":182744,"contributorKind":"person","contributorUrl":"https://github.com/npow","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/sparse-parity/submissions/packedfrontier80_mask32.ir","https://github.com/cybertronai/sutro-problems/blob/main/sparse-parity/submissions/packed_frontier.md","https://github.com/cybertronai/sutro-problems/blob/main/sparse-parity/submissions/packed_frontier.py"],"isCurrentBest":true}],"contested":true},{"group":"sparse-parity","description":"Recover the k=5 secret bit positions of a 32-bit string at 100% secret recovery rate; minimize read-cost.","id":"sparse-parity-100","name":"Sparse Parity (100% target)","metricName":"cost","metricDirection":"minimize","isOpen":false,"records":[{"contributor":"yaroslavvb","description":"generate_scan(16383) (Gray scan, full walk)","date":"2026-08-26","value":43325468,"contributorKind":"person","contributorUrl":"https://github.com/yaroslavvb","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/sparse-parity/submissions/scan_full_mask32.ir","https://github.com/cybertronai/sutro-problems/blob/main/sparse-parity/submissions/scan_full_mask32.md","https://github.com/cybertronai/sutro-problems/blob/main/sparse-parity/mask_sparse_parity.py"]},{"contributor":"zh4ngx","description":"optimize_layout(generate_scan(0, walk=\"weight\", weight_cap=5)) (weight-ordered scan, staged layout)","date":"2026-08-30","value":12461610,"contributorKind":"person","contributorUrl":"https://github.com/zh4ngx","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/sparse-parity/submissions/weightscan5_mask32.ir","https://github.com/cybertronai/sutro-problems/blob/main/sparse-parity/submissions/weightscan5_mask32.md","https://github.com/cybertronai/sutro-problems/blob/main/sparse-parity/mask_sparse_parity.py"]},{"contributor":"npow","description":"generate_staged() (septet-packed RREF + row-coordinate walk)","date":"2026-08-31","value":938331,"contributorKind":"person","contributorUrl":"https://github.com/npow","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/sparse-parity/submissions/septwalk_mask32.ir","https://github.com/cybertronai/sutro-problems/blob/main/sparse-parity/submissions/septwalk_mask32.md","https://github.com/cybertronai/sutro-problems/blob/main/sparse-parity/submissions/septwalk.py"]},{"contributor":"jurajselep, b0nce","description":"generate_packed_scan(5) (packed RREF + specialized capture)","date":"2026-09-02","value":392666,"contributorKind":"team","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/sparse-parity/submissions/packedscan5_mask32.ir","https://github.com/cybertronai/sutro-problems/blob/main/sparse-parity/submissions/packedscan5_mask32.md","https://github.com/cybertronai/sutro-problems/blob/main/sparse-parity/packed_sparse_parity.py"]},{"contributor":"mikech76","description":"generate_packed_scan(5, compact_predicates=True, compact_flow=True, walk_order_bias=1.0)","date":"2026-10-04","value":382840,"contributorKind":"person","contributorUrl":"https://github.com/mikech76","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/sparse-parity/submissions/packedscan_beat382840_mask32.ir","https://github.com/cybertronai/sutro-problems/blob/main/sparse-parity/submissions/packedscan_beat382840_mask32.md","https://github.com/cybertronai/sutro-problems/blob/main/sparse-parity/submissions/generator_beat382840.py","https://github.com/cybertronai/sutro-problems/blob/main/sparse-parity/submissions/audit_packed_records_beat382840.json"]},{"contributor":"mikech76","description":"generate_packed_scan(5, compact_predicates=True, compact_flow=True, walk_order_bias=15.0)","date":"2026-10-05","value":382794,"contributorKind":"person","contributorUrl":"https://github.com/mikech76","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/sparse-parity/submissions/packedscan_beat382794_mask32.ir","https://github.com/cybertronai/sutro-problems/blob/main/sparse-parity/submissions/packedscan_beat382794_mask32.md","https://github.com/cybertronai/sutro-problems/blob/main/sparse-parity/submissions/generator_beat382794.py","https://github.com/cybertronai/sutro-problems/blob/main/sparse-parity/submissions/audit_packed_records_beat382794.json"],"isCurrentBest":true}],"contested":true},{"group":"symmetry","description":"Classify a 6-bit pattern as a palindrome (1) or not (0); minimize read-cost.","id":"symmetry-6bit","name":"Symmetry (6-bit, 100% target)","metricName":"cost","metricDirection":"minimize","baseline":20,"isOpen":true,"records":[{"contributor":"baseline","description":"generate_baseline_six (3x cmp eq + 2x and)","date":"2026-05-11","value":20,"contributorKind":"method","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/symmetry/submissions/baseline_six.ir"],"isBaseline":true,"isCurrentBest":true}],"contested":false},{"group":"symmetry","description":"Classify an 8-bit pattern as a palindrome (1) or not (0); minimize read-cost.","id":"symmetry-8bit","name":"Symmetry (8-bit, 100% target)","metricName":"cost","metricDirection":"minimize","baseline":29,"isOpen":true,"records":[{"contributor":"baseline","description":"generate_baseline_eight (4x cmp eq + 3x and)","date":"2026-05-11","value":29,"contributorKind":"method","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/symmetry/submissions/baseline_eight.ir"],"isBaseline":true,"isCurrentBest":true}],"contested":false},{"group":"matmul","description":"Compute A @ B for 4x4 matrices as a straight-line IR; minimize total read-cost under the Dally model.","id":"matmul-4x4","name":"4x4 Matrix Multiplication","metricName":"cost","metricDirection":"minimize","baseline":1316,"isOpen":false,"records":[{"contributor":"yaroslavvb","description":"generate_baseline_4x4 (naive)","date":"2026-04-29","value":1316,"contributorKind":"person","contributorUrl":"https://github.com/yaroslavvb","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/matmul/submissions/baseline_4x4.ir","https://github.com/cybertronai/sutro-problems/blob/main/matmul/submissions/baseline_4x4.md"],"isBaseline":true},{"contributor":"sjbaebae","description":"generate_outer_product_4x4 (size-1 sA)","date":"2026-04-30","value":800,"contributorKind":"person","contributorUrl":"https://github.com/sjbaebae","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/matmul/submissions/outer_product_4x4.ir","https://github.com/cybertronai/sutro-problems/blob/main/matmul/submissions/outer_product_4x4.md"]},{"contributor":"jurajselep","description":"row-wise lifetime fusion + dead-A output reuse + exact tier allocation","date":"2026-09-01","value":689,"contributorKind":"person","contributorUrl":"https://github.com/jurajselep","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/matmul/submissions/best_689.ir","https://github.com/cybertronai/sutro-problems/blob/main/matmul/submissions/best_689.md","https://github.com/cybertronai/sutro-problems/blob/main/matmul/submissions/best_689.py"]},{"contributor":"npow","description":"HKKK + one deferred output add + exact fixed-schedule allocation","date":"2026-09-01","value":683,"contributorKind":"person","contributorUrl":"https://github.com/npow","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/matmul/submissions/hkkk_ilp_4x4.ir","https://github.com/cybertronai/sutro-problems/blob/main/matmul/submissions/hkkk_ilp_4x4.md","https://github.com/cybertronai/sutro-problems/blob/main/matmul/submissions/hkkk_ilp_4x4.py"]},{"contributor":"jurajselep","description":"hybrid row-0 schedule + JIT A staging + interleaved final adds","date":"2026-09-01","value":681,"contributorKind":"person","contributorUrl":"https://github.com/jurajselep","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/matmul/submissions/best_681.ir","https://github.com/cybertronai/sutro-problems/blob/main/matmul/submissions/best_681.md","https://github.com/cybertronai/sutro-problems/blob/main/matmul/submissions/best_681.py"]},{"contributor":"jurajselep","description":"selective A/B lifetime splits + dependency-safe scheduling + exact lifetime coloring","date":"2026-09-04","value":675,"contributorKind":"person","contributorUrl":"https://github.com/jurajselep","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/matmul/submissions/best_675.ir","https://github.com/cybertronai/sutro-problems/blob/main/matmul/submissions/best_675.md","https://github.com/cybertronai/sutro-problems/blob/main/matmul/submissions/best_675.py"]},{"contributor":"jurajselep","description":"contraction-order search + input captures + storage allocation","date":"2026-09-20","value":667,"contributorKind":"person","contributorUrl":"https://github.com/jurajselep","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/matmul/submissions/best_667.ir","https://github.com/cybertronai/sutro-problems/blob/main/matmul/submissions/best_667.md","https://github.com/cybertronai/sutro-problems/blob/main/matmul/submissions/best_667.py"],"isCurrentBest":true}],"contested":true},{"group":"matmul","description":"Compute A @ B for 16x16 matrices as a straight-line IR; minimize total read-cost under the Dally model.","id":"matmul-16x16","name":"16x16 Matrix Multiplication","metricName":"cost","metricDirection":"minimize","baseline":340704,"isOpen":false,"records":[{"contributor":"yaroslavvb","description":"generate_baseline_16x16 (naive)","date":"2026-04-29","value":340704,"contributorKind":"person","contributorUrl":"https://github.com/yaroslavvb","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/matmul/submissions/baseline_16x16.ir","https://github.com/cybertronai/sutro-problems/blob/main/matmul/submissions/baseline_16x16.md"],"isBaseline":true},{"contributor":"yaroslavvb","description":"generate_recursive_16x16 (1x1-leaf D&C, Z-order)","date":"2026-05-08","value":237456,"contributorKind":"person","contributorUrl":"https://github.com/yaroslavvb","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/matmul/submissions/recursive_16x16.ir"]},{"contributor":"yaroslavvb","description":"generate_tiled_16x16 (4x4 tiles)","date":"2026-04-29","value":133783,"contributorKind":"person","contributorUrl":"https://github.com/yaroslavvb","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/matmul/submissions/tiled_16x16.ir","https://github.com/cybertronai/sutro-problems/blob/main/matmul/submissions/tiled_16x16.md"]},{"contributor":"SethTS","description":"generate_tiled_16x16_opt1 (tmp@1)","date":"2026-04-30","value":110743,"contributorKind":"person","contributorUrl":"https://github.com/SethTS","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/matmul/submissions/tiled_16x16_opt1.ir"],"agent":{"model":"Claude Sonnet 4.6","evidence":"Co-authored-by: Claude Sonnet 4.6 <noreply@anthropic.com> (tiled_16x16_opt1 report)","family":"anthropic","confidence":"high","role":"author"}},{"contributor":"sjbaebae","description":"generate_hierarchical_16x16 (asym. reload)","date":"2026-04-30","value":80217,"contributorKind":"person","contributorUrl":"https://github.com/sjbaebae","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/matmul/submissions/hierarchical_16x16.ir","https://github.com/cybertronai/sutro-problems/blob/main/matmul/submissions/hierarchical_16x16.md"]},{"contributor":"adotzh","description":"sA-cache + sB scratchpad (rank2)","date":"2026-04-30","value":73602,"contributorKind":"person","contributorUrl":"https://github.com/adotzh","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/matmul/submissions/sa_cache_16x16.ir"]},{"contributor":"sjbaebae","description":"+ redirect last-mul to addr 1","date":"2026-05-01","value":72642,"contributorKind":"person","contributorUrl":"https://github.com/sjbaebae","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/matmul/submissions/redirect_16x16.ir"]},{"contributor":"sjbaebae","description":"+ last-super-block outputs in sC","date":"2026-05-01","value":71724,"contributorKind":"person","contributorUrl":"https://github.com/sjbaebae","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/matmul/submissions/sc_outputs_16x16.ir"]},{"contributor":"sjbaebae","description":"+ dead-input output reuse + B packing","date":"2026-05-01","value":70053,"contributorKind":"person","contributorUrl":"https://github.com/sjbaebae","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/matmul/submissions/dead_input_outputs_packed_16x16.ir","https://github.com/cybertronai/sutro-problems/blob/main/matmul/submissions/dead_input_outputs_packed_16x16.md"]},{"contributor":"yaroslavvb","description":"C<->A address aliasing + final-add fusion","date":"2026-05-06","value":69697,"contributorKind":"person","contributorUrl":"https://github.com/yaroslavvb","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/matmul/submissions/aliased_16x16.ir","https://github.com/cybertronai/sutro-problems/blob/main/matmul/submissions/aliased_16x16.md"]},{"contributor":"zh4ngx","description":"+ column-major order + fused final copy-out","date":"2026-05-05","value":68452,"contributorKind":"person","contributorUrl":"https://github.com/zh4ngx","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/matmul/submissions/colmajor_fused_16x16.ir","https://github.com/cybertronai/sutro-problems/blob/main/matmul/submissions/colmajor_fused_16x16.md"]},{"contributor":"cosminscn","description":"+ liveness order + output-read-aware packing + five-output scratch tail","date":"2026-05-13","value":68390,"contributorKind":"person","contributorUrl":"https://github.com/cosminscn","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/matmul/submissions/output_repacked_tail_16x16.ir","https://github.com/cybertronai/sutro-problems/blob/main/matmul/submissions/output_repacked_tail_16x16.md"],"agent":{"tool":"Codex","evidence":"Author: Codex and Cosmin; PR from cosminscn/codex/matmul-68390-five-direct-tail","family":"openai","confidence":"high","role":"author"}},{"contributor":"cosminscn","description":"+ live-B evacuation + output deferral + A staging + value-lifetime coloring","date":"2026-05-13","value":67834,"contributorKind":"person","contributorUrl":"https://github.com/cosminscn","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/matmul/submissions/output_repacked_tail_deferred_value_colored_live_b_16x16.ir","https://github.com/cybertronai/sutro-problems/blob/main/matmul/submissions/output_repacked_tail_deferred_value_colored_live_b_16x16.md"],"agent":{"tool":"Codex","evidence":"Author: Codex and Cosmin; PR cosminscn/codex/matmul-67834-clean-submission","family":"openai","confidence":"high","role":"author"}},{"contributor":"cosminscn","description":"+ live-B evacuation + output deferral + tiny A-staging mask + staged-reload endpoint lift + value-lifetime coloring","date":"2026-05-14","value":67821,"contributorKind":"person","contributorUrl":"https://github.com/cosminscn","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/matmul/submissions/output_repacked_tail_deferred_value_colored_live_b_tiny_a_endpoint_16x16.ir","https://github.com/cybertronai/sutro-problems/blob/main/matmul/submissions/output_repacked_tail_deferred_value_colored_live_b_tiny_a_endpoint_16x16.md"],"agent":{"tool":"Codex","evidence":"Author: Codex and Cosmin; PR cosminscn/codex/matmul-wave15-67821","family":"openai","confidence":"high","role":"author"}},{"contributor":"sjbaebae","description":"weighted-lifetime pressure search + copy elimination","date":"2026-05-08","value":66707,"contributorKind":"person","contributorUrl":"https://github.com/sjbaebae","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/matmul/submissions/weighted_lifetime_copyelim_66707.ir","https://github.com/cybertronai/sutro-problems/blob/main/matmul/doc/metaskills/weighted_lifetime_hillclimb_writeup.md","https://github.com/cybertronai/sutro-problems/blob/main/matmul/submissions/weighted_lifetime_copyelim_66707.py"],"agent":{"tool":"Codex","evidence":"PR #25 from sjbaebae/codex/weighted-lifetime-matmul [codex]","family":"openai","confidence":"high","role":"author"}},{"contributor":"cosminscn","description":"macro B-staging + row-7 later-panel prestaging from addr 1","date":"2026-05-25","value":66633,"contributorKind":"person","contributorUrl":"https://github.com/cosminscn","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/matmul/submissions/macro_b_staging_66633.ir","https://github.com/cybertronai/sutro-problems/blob/main/matmul/submissions/macro_b_staging_66633.md","https://github.com/cybertronai/sutro-problems/blob/main/matmul/submissions/macro_b_staging_66633.py"],"agent":{"tool":"Codex","evidence":"Author: Codex and Cosmin; PR cosminscn/codex/matmul-66633-macro-b-staging","family":"openai","confidence":"high","role":"author"}},{"contributor":"cosminscn","description":"late B-block cheap capture from addr 1 + value-lifetime coloring","date":"2026-05-25","value":66524,"contributorKind":"person","contributorUrl":"https://github.com/cosminscn","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/matmul/submissions/cheap_capture_66524.ir","https://github.com/cybertronai/sutro-problems/blob/main/matmul/submissions/cheap_capture_66524.md","https://github.com/cybertronai/sutro-problems/blob/main/matmul/submissions/cheap_capture_66524.py"],"agent":{"tool":"Codex","evidence":"Author: Codex and Cosmin; cosminscn codex/matmul-68341-live-b-evacuation","family":"openai","confidence":"high","role":"author"}},{"contributor":"cosminscn","description":"late copy-schedule motif bundle + value-lifetime coloring","date":"2026-05-26","value":66400,"contributorKind":"person","contributorUrl":"https://github.com/cosminscn","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/matmul/submissions/motif_bundle_66400.ir","https://github.com/cybertronai/sutro-problems/blob/main/matmul/submissions/motif_bundle_66400.md","https://github.com/cybertronai/sutro-problems/blob/main/matmul/submissions/motif_bundle_66400.py"],"agent":{"tool":"Codex","evidence":"Author: Codex and Cosmin; cosminscn codex/matmul branch","family":"openai","confidence":"high","role":"author"}},{"contributor":"cosminscn","description":"Claude-assisted simulated annealing over a leaderboard physical-address IR","date":"2026-05-28","value":66300,"contributorKind":"person","contributorUrl":"https://github.com/cosminscn","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/matmul/submissions/best_66300.ir","https://github.com/cybertronai/sutro-problems/blob/main/matmul/submissions/best_66300.md","https://github.com/cybertronai/sutro-problems/blob/main/matmul/submissions/best_66300.py"],"agent":{"model":"Claude","evidence":"Description: Claude-assisted simulated annealing; PR cosminscn/codex/matmul-66300","family":"anthropic","confidence":"high","role":"author"}},{"contributor":"sigkillme0","description":"dependency-safe rescheduling + exact tier allocation","date":"2026-08-29","value":66199,"contributorKind":"person","contributorUrl":"https://github.com/sigkillme0","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/matmul/submissions/best_66199.ir","https://github.com/cybertronai/sutro-problems/blob/main/matmul/submissions/best_66199.md","https://github.com/cybertronai/sutro-problems/blob/main/matmul/submissions/best_66199.py"]},{"contributor":"sigkillme0","description":"exact LP-optimal address assignment (provably optimal for this operation order)","date":"2026-08-30","value":66178,"contributorKind":"person","contributorUrl":"https://github.com/sigkillme0","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/matmul/submissions/best_66178.ir","https://github.com/cybertronai/sutro-problems/blob/main/matmul/submissions/best_66178.md","https://github.com/cybertronai/sutro-problems/blob/main/matmul/submissions/best_66178.py"]},{"contributor":"jurajselep","description":"newest-first snake passes + JIT A staging + local exact tier allocation","date":"2026-09-04","value":65084,"contributorKind":"person","contributorUrl":"https://github.com/jurajselep","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/matmul/submissions/best_65084.ir","https://github.com/cybertronai/sutro-problems/blob/main/matmul/submissions/best_65084.md","https://github.com/cybertronai/sutro-problems/blob/main/matmul/submissions/best_65084.py"]},{"contributor":"cosminscn","description":"asymmetric panel schedule + persistent B captures + dependency-safe order search","date":"2026-09-04","value":64431,"contributorKind":"person","contributorUrl":"https://github.com/cosminscn","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/matmul/submissions/best_64431.ir","https://github.com/cybertronai/sutro-problems/blob/main/matmul/submissions/best_64431.md","https://github.com/cybertronai/sutro-problems/blob/main/matmul/submissions/best_64431.py"],"agent":{"tool":"Codex","evidence":"PR #59 from cosminscn/codex/matmul-record-64431-clean","family":"openai","confidence":"high","role":"author"}},{"contributor":"SecurityQQ","description":"temporary input captures + cheapest surviving replica reads + redundant-copy deletion","date":"2026-09-07","value":64074,"contributorKind":"person","contributorUrl":"https://github.com/SecurityQQ","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/matmul/submissions/best_64074.ir","https://github.com/cybertronai/sutro-problems/blob/main/matmul/submissions/best_64074.md","https://github.com/cybertronai/sutro-problems/blob/main/matmul/submissions/best_64074.py"]},{"contributor":"SecurityQQ","description":"6+10 asymmetric panels + dependency-safe scheduling + input captures and address allocation (frozen-artifact verifier)","date":"2026-09-07","value":63819,"contributorKind":"person","contributorUrl":"https://github.com/SecurityQQ","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/matmul/submissions/best_63819.ir","https://github.com/cybertronai/sutro-problems/blob/main/matmul/submissions/best_63819.md","https://github.com/cybertronai/sutro-problems/blob/main/matmul/submissions/best_63819.py"]},{"contributor":"jurajselep","description":"input copy-chain and neutral-plan optimization + whole-program allocation with an exact rational certificate","date":"2026-09-15","value":63639,"contributorKind":"person","contributorUrl":"https://github.com/jurajselep","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/matmul/submissions/best_63639.ir","https://github.com/cybertronai/sutro-problems/blob/main/matmul/submissions/best_63639.md","https://github.com/cybertronai/sutro-problems/blob/main/matmul/submissions/best_63639.py"]},{"contributor":"jurajselep","description":"exact block reordering + joint reduction-tree and storage allocation","date":"2026-09-15","value":63354,"contributorKind":"person","contributorUrl":"https://github.com/jurajselep","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/matmul/submissions/best_63354.ir","https://github.com/cybertronai/sutro-problems/blob/main/matmul/submissions/best_63354.py"]},{"contributor":"jurajselep","description":"joint reduction-tree/storage repair + redundant-copy elimination","date":"2026-09-18","value":63350,"contributorKind":"person","contributorUrl":"https://github.com/jurajselep","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/matmul/submissions/best_63350.ir","https://github.com/cybertronai/sutro-problems/blob/main/matmul/submissions/best_63350.md","https://github.com/cybertronai/sutro-problems/blob/main/matmul/submissions/best_63350.py"]},{"contributor":"jurajselep","description":"structural schedule search + six local sum reassociations (frozen-artifact verifier)","date":"2026-09-18","value":63290,"contributorKind":"person","contributorUrl":"https://github.com/jurajselep","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/matmul/submissions/best_63290.ir","https://github.com/cybertronai/sutro-problems/blob/main/matmul/submissions/best_63290.md","https://github.com/cybertronai/sutro-problems/blob/main/matmul/submissions/best_63290.py"]},{"contributor":"sigkillme0","description":"reduced weighted read cost","date":"2026-10-18","value":63026,"contributorKind":"person","contributorUrl":"https://github.com/sigkillme0","submissionUrls":["https://github.com/cybertronai/sutro-problems/blob/main/matmul/submissions/best_63026.ir","https://github.com/cybertronai/sutro-problems/blob/main/matmul/submissions/best_63026.md","https://github.com/cybertronai/sutro-problems/blob/main/matmul/submissions/best_63026.py"],"isCurrentBest":true}],"contested":true}],"agentStats":[{"family":"anthropic","records":15,"currentBests":2,"models":["Claude Sonnet 4.6","Claude","Claude Opus 5.5","Claude Fable 5.1"],"contributors":["SethTS","cosminscn","yaroslavvb","likelytobelaura","QDA","PCA-QDA"]},{"family":"openai","records":15,"currentBests":1,"models":["Codex"],"contributors":["cosminscn","sjbaebae","512-unit MLP","Reversible MLP","32-unit MLP","1NN","Ordered ConvNets","Three ConvNets"],"uncontestedBests":1}],"repo":{"owner":"cybertronai","name":"sutro-problems","branch":"main"},"participants":[{"name":"jurajselep","url":"https://github.com/jurajselep","submissions":14,"problems":["matmul-4x4","matmul-16x16","sparse-parity-20","sparse-parity-40","sparse-parity-60","sparse-parity-80","sparse-parity-100"],"lastActive":"2026-09-20","bestRank":1},{"name":"npow","url":"https://github.com/npow","submissions":9,"problems":["matmul-4x4","sparse-parity-20","sparse-parity-40","sparse-parity-60","sparse-parity-80","sparse-parity-100"],"lastActive":"2026-09-01","bestRank":1},{"name":"b0nce","url":"https://github.com/b0nce","submissions":5,"problems":["sparse-parity-20","sparse-parity-40","sparse-parity-60","sparse-parity-80","sparse-parity-100"],"bestRank":1,"lastActive":"2026-09-02"},{"name":"spartanhaden","url":"https://github.com/spartanhaden","submissions":5,"problems":["mnist-a100-diff1","mnist-a100-diff2","mnist-a100-diff3","mnist-a100-diff4","mnist-a100-diff5"],"bestRank":1,"lastActive":"2026-10-12"},{"name":"islamborghini","url":"https://github.com/islamborghini","submissions":5,"problems":["mnist-a100-diff2","mnist-a100-diff3","mnist-a100-diff4","mnist-a100-diff5"],"lastActive":"2026-10-18","bestRank":1},{"name":"sigkillme0","url":"https://github.com/sigkillme0","submissions":4,"problems":["matmul-16x16","mnist-a100-diff3"],"lastActive":"2026-10-18","bestRank":1},{"name":"mikech76","url":"https://github.com/mikech76","submissions":2,"problems":["sparse-parity-100"],"lastActive":"2026-10-05","bestRank":1},{"name":"yaroslavvb","url":"https://github.com/yaroslavvb","submissions":24,"problems":["matmul-4x4","matmul-16x16","sparse-parity-20","sparse-parity-40","sparse-parity-60","sparse-parity-80","sparse-parity-100","mnist-a100-diff1","mnist-a100-diff2","mnist-a100-diff3","mnist-a100-diff4","mnist-a100-diff5"],"lastActive":"2026-09-30"},{"name":"sjbaebae","url":"https://github.com/sjbaebae","submissions":10,"problems":["matmul-4x4","matmul-16x16","mnist-a100-diff1","mnist-a100-diff2","mnist-a100-diff3"],"lastActive":"2026-10-12"},{"name":"SethTS","url":"https://github.com/SethTS","submissions":10,"problems":["matmul-16x16","mnist-a100-diff3","mnist-a100-diff4","mnist-a100-diff5"],"lastActive":"2026-09-29"},{"name":"zh4ngx","url":"https://github.com/zh4ngx","submissions":8,"problems":["matmul-16x16","sparse-parity-20","sparse-parity-40","sparse-parity-60","sparse-parity-80","sparse-parity-100"],"lastActive":"2026-08-30"},{"name":"cosminscn","url":"https://github.com/cosminscn","submissions":8,"problems":["matmul-16x16"],"lastActive":"2026-09-04"},{"name":"SecurityQQ","url":"https://github.com/SecurityQQ","submissions":2,"problems":["matmul-16x16"],"lastActive":"2026-09-07"},{"name":"likelytobelaura","url":"https://github.com/likelytobelaura","submissions":2,"problems":["mnist-a100-diff5"],"lastActive":"2026-10-14"},{"name":"adotzh","url":"https://github.com/adotzh","submissions":1,"problems":["matmul-16x16"],"lastActive":"2026-04-30"}],"stats":{"totalRecords":129,"uniqueParticipants":15,"firstSubmission":"2026-04-29","lastSubmission":"2026-10-18","recordsLast30d":41,"recordsLast90d":106},"lastUpdated":"2026-10-19T09:11:02.508Z"},{"id":"lean-eval","name":"LeanEval","tagline":"Prove open formalization problems in Lean 4; models and submitters are ranked by problems solved.","description":"LeanEval is the Lean FRO's comparator-based benchmark for formal mathematics. Trusted theorem statements live in leanprover/lean-eval; a problem counts as solved when the comparator accepts a submitted proof. Results are sticky per (user, model, problem) and published at lean-lang.org/eval. 304 visible problems, 76 models, 61 submitters, 39 active problems still unsolved.","url":"https://lean-lang.org/eval/","host":"website","status":"active","domain":"formal-methods","tags":["lean","theorem-proving","formal-methods","mathlib","benchmark"],"images":[],"participation":{"howToSubmit":"Prove a problem's theorem in your own GitHub repository and submit the ref at lean-lang.org/eval/submit; the comparator verifies the proof and the result is appended to results/<login>.json in leanprover/lean-eval-submissions.","submissionFormat":"GitHub repo + ref containing the completed Lean proof, plus the declared model label used to produce it.","computeDetails":"Scoring is a comparator check of the proof; whatever compute the prover needs is on the submitter's side.","deadline":"rolling","prizes":"recognition only","eligibility":"Open to everyone. Successes are sticky: an accepted (user, model, problem) tuple is never removed.","cost":"free","verification":"Comparator acceptance in CI, replay by maintainers, public audit log per user.","requirements":["Lean 4","Mathlib","GitHub account"],"compute":"none"},"links":[{"label":"Leaderboard","url":"https://lean-lang.org/eval/"},{"label":"Submit a solution","url":"https://lean-lang.org/eval/submit/"},{"label":"Benchmark problems (leanprover/lean-eval)","url":"https://github.com/leanprover/lean-eval"},{"label":"Results store (lean-eval-submissions)","url":"https://github.com/leanprover/lean-eval-submissions"}],"organizer":{"name":"Lean FRO","url":"https://lean-lang.org/","type":"foundation"},"venue":{"location":"GitHub / lean-lang.org","type":"online"},"quickstart":"git clone https://github.com/leanprover/lean-eval && cd lean-eval\nlake exe cache get && lake build\nlake exe lean-eval --help   # pick a problem under generated/, replace sorry, submit at lean-lang.org/eval/submit","problems":[{"description":"Unsolved. Group: formalization-evaluation. Source: M. R. Bridson, D. B. McReynolds, A. W. Reid, and R. Spitler, `Absolute profinite rigidity and hyperbolic geometry`, Annals of Math, 192 (3) 2020. Statement take Problem by Thomas Browning.","id":"open-annals_absolute_profinite_rigidity","name":"Absolute profinite rigidity and hyperbolic geometry","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"description":"Unsolved. Group: formalization-evaluation. Source: W. Sawin and M. Shusterman, `On the Chowla and twin primes conjectures over 𝔽_q[T]`, Annals of Math, 196 (2) 2022. Statement taken from https://github.com/Impe Problem by Thomas Browning.","id":"open-annals_chowla_and_twin_prime_over_fq_t","name":"On the Chowla and twin primes conjectures over 𝔽_q[T]","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"description":"Unsolved. Group: formalization-evaluation. Source: I. Petrow and M. P. Young, `The Weyl bound for Dirichlet L-functions of cube-free conductor`, Annals of Math, 192 (2) 2020. Statement taken from https://github. Problem by Thomas Browning.","id":"open-annals_dirichlet_weyl_bound","name":"The Weyl bound for Dirichlet L-functions of cube-free conductor","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"description":"Unsolved. Group: formalization-evaluation. Source: M. Bennett and S. Siksek, `A conjecture of Erdős, supersingular primes and short character sums`, Annals of Math, 191 (2) 2020. Statement taken from https://git Problem by Justus Springer.","id":"open-annals_erdos_supersingular_primes","name":"A conjecture of Erdős, supersingular primes and short character sums","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"description":"Unsolved. Group: formalization-evaluation. Source: M. Cabanes and B. Späth, `The McKay Conjecture on character degrees`, Annals of Math, 203 (3) 2026. Statement taken from https://github.com/ImperialCollegeLondo Problem by Thomas Browning.","id":"open-annals_mckay_conjecture","name":"The McKay Conjecture on character degrees","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"description":"Unsolved. Group: formalization-evaluation. Source: D. Cristofaro-Gardiner, V. Humilière, and S. Seyfaddini, `Proof of the simplicity conjecture`, Annals of Math, 199 (1) 2024. Statement taken from https://github Problem by Thomas Browning.","id":"open-annals_simplicity_conjecture","name":"Proof of the simplicity conjecture","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"description":"Unsolved. Group: formalization-evaluation. Source: T. C. Burness, R. M. Guralnick, and S. Harper, `The spread of a finite group`, Annals of Math, 193 (2) 2021. Statement taken from https://github.com/ImperialCol Problem by Katerina Hristova.","id":"open-annals_spread_of_a_finite_group","name":"The spread of a finite group","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"description":"Unsolved. Group: formalization-evaluation. Source: J. Fernández de Bobadilla and T. Pełka, `Symplectic monodromy at radius zero and equimultiplicity of μ-constant families`, Annals of Math, 200 (1) 2024. Stateme Problem by Justus Springer.","id":"open-annals_symplectic_monodromy","name":"Symplectic monodromy at radius zero and equimultiplicity of μ-constant families","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"description":"Unsolved. Group: formalization-evaluation. Source: V. Dimitrov, Z. Gao, and P. Habegger, `Uniformity in Mordell–Lang for curves`, Annals of Math, 194 (1) 2021. Statement taken from https://github.com/ImperialCol Problem by Thomas Browning, Christian Merten.","id":"open-annals_uniform_mordell_lang","name":"Uniformity in Mordell–Lang for curves","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"description":"Unsolved. Group: formalization-evaluation. Source: F. Quinn, *Ends of maps III: dimensions 4 and 5*, J. Differential Geom. 17 (1982), building on M. H. Freedman, *The topology of four-dimensional manifolds*, J.  Problem by Kim Morrison.","id":"open-annulus_theorem_dim_four","name":"The Annulus Theorem in dimension 4 (Quinn)","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"description":"Unsolved. Group: formalization-evaluation. Source: R. C. Kirby, *Stable homeomorphisms and the annulus conjecture*, Ann. of Math. 89 (1969). Problem by Kim Morrison.","id":"open-annulus_theorem_high_dim","name":"The Annulus Theorem in dimension ≥ 5 (Kirby)","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"description":"Unsolved. Group: formalization-evaluation. Source: R. Budney and D. Gabai, 'Knotted 3-balls in S⁴', Corollary 8.6, arXiv:1912.09029v3 (2021), https://arxiv.org/abs/1912.09029. Problem by Vasily Ilin.","id":"open-budney_gabai_knotted_three_spheres","name":"Budney--Gabai knotted three-spheres in S¹ × S³","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"description":"Unsolved. Group: formalization-evaluation. Source: J. Cerf, Sur les diffeomorphismes de la sphere de dimension trois, Lecture Notes in Mathematics 53, Springer (1968). Problem by Kim Morrison.","id":"open-cerf_gamma_four","name":"Cerf's theorem: every self-diffeomorphism of S3 is smoothly isotopic to a linear isometry","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"description":"Unsolved. Group: formalization-evaluation. Source: D. Gorenstein, R. Lyons and R. Solomon, The Classification of the Finite Simple Groups (AMS Mathematical Surveys and Monographs 40, 1994–); M. Aschbacher, R. Ly Problem by Kim Morrison.","id":"open-classification_finite_simple_groups","name":"The classification of finite simple groups","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"description":"Unsolved. Group: formalization-evaluation. Source: https://arxiv.org/abs/1808.02923 Problem by Kim Morrison.","id":"open-conway_knot_not_smoothly_slice","name":"The Conway knot is not smoothly slice","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"description":"Unsolved. Group: formalization-evaluation. Source: Freedman, *The topology of four-dimensional manifolds*, J. Diff. Geom. 17 (1982). See also Freedman-Quinn, *Topology of 4-Manifolds*, Princeton 1990. Problem by Kim Morrison.","id":"open-conway_knot_topologically_slice","name":"The Conway knot is topologically slice","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"description":"Unsolved. Group: formalization-evaluation. Source: https://github.com/dagurtomas/LeanCondensed (LeanCondensed/Projects/DerivedSolidCWHomology.lean); D. Clausen and P. Scholze, lectures on analytic stacks and lig Problem by Dagur Asgeirsson.","id":"open-derived_solidification_free_CW_homology","name":"Derived solidification of free CW complexes (light condensed mathematics)","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"description":"Unsolved. Group: formalization-evaluation. Source: Casson, 1980s (unpublished); Akbulut-Matveyev, *A convex decomposition theorem for 4-manifolds*, IMRN 1998. See also Hedden-Kirk-Livingston, *Non-slice linear c Problem by Kim Morrison.","id":"open-exists_topologically_slice_not_smoothly_slice","name":"Existence of a topologically slice, not smoothly slice knot","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"description":"Unsolved. Group: formalization-evaluation. Problem by Junyan Xu.","id":"open-faltings","name":"Faltings' theorem (Mordell conjecture)","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"description":"Unsolved. Group: formalization-evaluation. Source: https://en.wikipedia.org/wiki/Fermat%27s_Last_Theorem Problem by Xuanji Li.","id":"open-fermat_last_theorem","name":"Fermat's Last Theorem","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"description":"Unsolved. Group: formalization-evaluation. Source: Folklore via CFSG; classical work of Mathieu, Jordan; modern accounts in P. Cameron, Permutation Groups (1999). Problem by Kim Morrison.","id":"open-five_transitive_card_classification","name":"Possible orders of 5-transitive finite permutation groups","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"description":"Unsolved. Group: formalization-evaluation. Problem by Oliver Nash.","id":"open-four_manifold_not_smooth","name":"Freedman's non-smoothability theorem","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"description":"Unsolved. Group: formalization-evaluation. Source: Kepler conjecture 1611 (J. Kepler, *Strena seu de Nive Sexangula*); proof T. Hales, 'A proof of the Kepler conjecture', Ann. of Math. (2) 162 (2005) 1065–1185;  Problem by Kim Morrison.","id":"open-kepler_conjecture","name":"Kepler conjecture (optimal sphere packing in ℝ³)","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"description":"Unsolved. Group: formalization-evaluation. Source: János Kollár, Max Lieblich, Martin Olsson, and Will Sawin. The Zariski topology, linear systems, and algebraic varieties. Problem by Junyan Xu.","id":"open-kollar_lieblich_olsson_sawin","name":"Topological reconstruction theorems for varieties","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"description":"Unsolved. Group: formalization-evaluation. Source: C. Manolescu, *Pin(2)-equivariant Seiberg–Witten Floer homology and the triangulation conjecture*, J. Amer. Math. Soc. 29 (2016), via the Galewski–Stern and Mat Problem by Kim Morrison.","id":"open-manolescu_triangulation_disproof","name":"Manolescu's disproof of the triangulation conjecture","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"description":"Unsolved. Group: formalization-evaluation. Source: B. Mazur, *Modular curves and the Eisenstein ideal*, Publ. IHÉS 47 (1977). Knill, *Some fundamental theorems in mathematics*, §139. Problem by Kim Morrison.","id":"open-mazur_torsion","name":"Mazur's torsion theorem","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"description":"Unsolved. Group: formalization-evaluation. Source: Nikolay Nikolov and Dan Segal, *On finitely generated profinite groups, I: strong completeness and uniform bounds*, Annals of Mathematics 165 (2007), no. 1, 171 Problem by Adam Topaz.","id":"open-nikolov_segal","name":"Nikolov–Segal strong completeness theorem","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"description":"Unsolved. Group: formalization-evaluation. Source: Martin W. Liebeck, E. A. O'Brien, Aner Shalev, and Pham Huu Tiep, *The Ore conjecture*, Journal of the European Mathematical Society 12 (2010), no. 4, 939–1008, Problem by Adam Topaz.","id":"open-ore_conjecture","name":"The Ore conjecture: every element of a finite nonabelian simple group is a commutator","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"description":"Unsolved. Group: formalization-evaluation. Source: G. Perelman, three arXiv preprints 2002-2003 (math/0211159, math/0303109, math/0307245). Recorded as a `proof_wanted` in Mathlib/Geometry/Manifold/PoincareConje Problem by Kim Morrison.","id":"open-poincare_3d_smooth","name":"3D smooth Poincaré conjecture (Perelman)","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"description":"Unsolved. Group: formalization-evaluation. Source: G. Perelman, The entropy formula for the Ricci flow and its geometric applications (arXiv:math/0211159, 2002); Ricci flow with surgery on three-manifolds (arXiv Problem by Kim Morrison.","id":"open-poincare_3d_topological","name":"3D topological Poincaré conjecture (Perelman)","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"description":"Unsolved. Group: formalization-evaluation. Source: M. H. Freedman, The topology of four-dimensional manifolds, J. Differential Geom. 17 (1982), 357-453. Specialization of `proof_wanted ContinuousMap.HomotopyEqui Problem by Kim Morrison.","id":"open-poincare_4d_topological","name":"4D topological Poincaré conjecture (Freedman)","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"description":"Unsolved. Group: formalization-evaluation. Source: S. Ramanujan, 'On certain arithmetical functions', Trans. Cambridge Philos. Soc. 22 (1916) 159–184. H. Petersson, 'Theorie der automorphen Formen beliebiger ree Problem by Seewoo Lee.","id":"open-ramanujan_petersson","name":"Ramanujan–Petersson conjecture for the τ-function (Deligne's theorem)","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"description":"Unsolved. Group: formalization-evaluation. Source: O. Schreier, Über die Erweiterung von Gruppen II, Abh. Math. Sem. Univ. Hamburg 4 (1926); CFSG, completed c. 2004. Problem by Kim Morrison.","id":"open-schreier_conjecture","name":"Schreier's conjecture: outer automorphism group of a finite simple group is solvable","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"description":"Unsolved. Group: formalization-evaluation. Source: A. Hatcher, A proof of the Smale conjecture, Diff(S3) = O(4), Ann. of Math. 117 (1983). Problem by Kim Morrison.","id":"open-smale_conjecture","name":"Smale conjecture (Hatcher) in relative parameterized form","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"description":"Unsolved. Group: formalization-evaluation. Source: M. Bhargava, J. Hanke, Universal quadratic forms and the 290-theorem, preprint (2011). See also https://en.wikipedia.org/wiki/15_and_290_theorems Problem by Bolton Bailey/Project Numina.","id":"open-two_ninety_theorem","name":"The 290 theorem","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"description":"Unsolved. Group: formalization-evaluation. Source: T. Watanabe, *Some exotic nontrivial elements of the rational homotopy groups of Diff(S⁴)*, arXiv:1812.02448 (2018). Pairs with the positive S³ statement of Hat Problem by Kim Morrison.","id":"open-watanabe_four_dim_smale_disproof","name":"Watanabe's disproof of the 4-dimensional Smale conjecture","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"description":"Unsolved. Group: formalization-evaluation. Source: H. A. Helfgott, The ternary Goldbach conjecture is true, arXiv:1312.7748 (2013), https://arxiv.org/abs/1312.7748; H. A. Helfgott and D. J. Platt, Numerical Veri Problem by Vasily Ilin.","id":"open-weak_goldbach","name":"Weak Goldbach theorem","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"description":"Unsolved. Group: formalization-evaluation. Source: P. Deligne, 'La conjecture de Weil. I', Publications Mathématiques de l'Institut des Hautes Études Scientifiques, Volume 43, pages 273–307 (1974), https://www.n Problem by Junyan Xu.","id":"open-weil_conjectures","name":"Weil conjectures in terms of point counts","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"description":"Unsolved. Group: formalization-evaluation. Source: C.H. Taubes, 'The Seiberg–Witten equations and the Weinstein conjecture', Geom. Topol. 11 (2007) 2117–2202; arXiv:math/0611007. Conjecture: A. Weinstein, 'On th Problem by Kim Morrison.","id":"open-weinstein_conjecture_dim3","name":"Weinstein conjecture in dimension three (Taubes 2007)","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"description":"Official leaderboard: each row is a model (or harness + model) ranked by the number of distinct benchmark problems it has solved; 304 problems in the catalog, 130 active.","id":"models","name":"Models — problems solved","metricName":"problems solved","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"Axiom Prover (Axiom Math)","description":"1 submitter: Vilin97 (263)","date":"2026-10-13","value":263,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":true,"agent":{"model":"Axiom Prover (Axiom Math)","evidence":"lean-lang.org/eval leaderboard.json: \"Axiom Prover (Axiom Math)\" solved 263","family":"axiom","confidence":"high","role":"subject"}},{"contributor":"Humanifa + GPT 5.6 sol","description":"4 submitters: ZhengyangZhang06 (215), hongzhoulin89 (34), antoinegg1 (9), juihuichung (6)","date":"2026-10-14","value":260,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"GPT 5.6 sol","tool":"Humanifa","evidence":"lean-lang.org/eval leaderboard.json: \"Humanifa + GPT 5.6 sol\" solved 260","family":"openai","confidence":"high","role":"subject"}},{"contributor":"Tau (caj.al)","description":"1 submitter: lukerj00 (200)","date":"2026-09-07","value":200,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"Tau (caj.al)","evidence":"lean-lang.org/eval leaderboard.json: \"Tau (caj.al)\" solved 200","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"Aristotle (Harmonic)","description":"7 submitters: LorenzoLuccioli (105), parabamoghv (42), JohnEdwardJennings (7), sqrt-of-2 (7), kim-em (6), adrianmartir (3), Parcly-Taxel (1)","date":"2026-08-20","value":162,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"Aristotle (Harmonic)","evidence":"lean-lang.org/eval leaderboard.json: \"Aristotle (Harmonic)\" solved 162","family":"harmonic","confidence":"high","role":"subject"}},{"contributor":"Seed Prover (ByteDance)","description":"2 submitters: GanjinZero (135), hanwenzhu (21)","date":"2026-08-13","value":156,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"Seed Prover (ByteDance)","evidence":"lean-lang.org/eval leaderboard.json: \"Seed Prover (ByteDance)\" solved 156","family":"bytedance","confidence":"high","role":"subject"}},{"contributor":"GPT-5.6","description":"2 submitters: Morgan-Griffiths (111), eohjelle (3)","date":"2026-07-21","value":112,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"GPT-5.6","evidence":"lean-lang.org/eval leaderboard.json: \"GPT-5.6\" solved 112","family":"openai","confidence":"high","role":"subject"}},{"contributor":"Stealth Model","description":"1 submitter: rishistyping (104)","date":"2026-08-20","value":104,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"Stealth Model","evidence":"lean-lang.org/eval leaderboard.json: \"Stealth Model\" solved 104","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"NEAR AI w/ DeepSeek V4","description":"1 submitter: skidanovalex (88)","date":"2026-10-14","value":88,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"DeepSeek V4","tool":"NEAR AI","evidence":"lean-lang.org/eval leaderboard.json: \"NEAR AI w/ DeepSeek V4\" solved 88","family":"deepseek","confidence":"high","role":"subject"}},{"contributor":"max gp (GPT-6)","description":"1 submitter: ma4ypic4y (83)","date":"2026-10-14","value":83,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"max gp (GPT-6)","evidence":"lean-lang.org/eval leaderboard.json: \"max gp (GPT-6)\" solved 83","family":"openai","confidence":"high","role":"subject"}},{"contributor":"Vasily-24-hour-gpt-5.6-speedrun","description":"1 submitter: Vilin97 (79)","date":"2026-08-17","value":79,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"Vasily-24-hour-gpt-5.6-speedrun","evidence":"lean-lang.org/eval leaderboard.json: \"Vasily-24-hour-gpt-5.6-speedrun\" solved 79","family":"openai","confidence":"high","role":"subject"}},{"contributor":"Aleph Prover(logicalintelligence.com)","description":"2 submitters: mayorov-m-a (52), antpavzhi (4)","date":"2026-06-05","value":56,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"Aleph Prover(logicalintelligence.com)","evidence":"lean-lang.org/eval leaderboard.json: \"Aleph Prover(logicalintelligence.com)\" solved 56","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"Midas","description":"1 submitter: sarjinius (39)","date":"2026-10-01","value":39,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"Midas","evidence":"lean-lang.org/eval leaderboard.json: \"Midas\" solved 39","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"github.com/mlgraham (various models)","description":"1 submitter: mlgraham (31)","date":"2026-08-29","value":31,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"github.com/mlgraham (various models)","evidence":"lean-lang.org/eval leaderboard.json: \"github.com/mlgraham (various models)\" solved 31","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"MerLean-Prover","description":"1 submitter: doxtor6 (30)","date":"2026-06-25","value":30,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"MerLean-Prover","evidence":"lean-lang.org/eval leaderboard.json: \"MerLean-Prover\" solved 30","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"Antigravity (Multi-Model Ensemble: Gemini 3.1 Pro, Gemini 3 Flash, Claude 4.6 Sonnet/Opus)","description":"1 submitter: daouid (21)","date":"2026-05-22","value":21,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"Antigravity (Multi-Model Ensemble: Gemini 3.1 Pro, Gemini 3 Flash, Claude 4.6 Sonnet/Opus)","evidence":"lean-lang.org/eval leaderboard.json: \"Antigravity (Multi-Model Ensemble: Gemini 3.1 Pro, Gemini 3 Flash, Claude 4.6 Sonnet/Opus)\" solved 21","family":"anthropic","confidence":"high","role":"subject"}},{"contributor":"savarin-6-hour-gpt-5.6-speedrun","description":"1 submitter: savarin (21)","date":"2026-08-17","value":21,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"savarin-6-hour-gpt-5.6-speedrun","evidence":"lean-lang.org/eval leaderboard.json: \"savarin-6-hour-gpt-5.6-speedrun\" solved 21","family":"openai","confidence":"high","role":"subject"}},{"contributor":"Claude Opus 4.7 (1M context)","description":"2 submitters: rkirov (15), jzuiddam (2)","date":"2026-05-24","value":17,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"Claude Opus 4.7 (1M context)","evidence":"lean-lang.org/eval leaderboard.json: \"Claude Opus 4.7 (1M context)\" solved 17","family":"anthropic","confidence":"high","role":"subject"}},{"contributor":"GPT-5.5","description":"3 submitters: Morgan-Griffiths (10), sqrt-of-2 (9), A-M-Berns (4)","date":"2026-06-08","value":16,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"GPT-5.5","evidence":"lean-lang.org/eval leaderboard.json: \"GPT-5.5\" solved 16","family":"openai","confidence":"high","role":"subject"}},{"contributor":"github.com/mlgraham (Claude Fable 5)","description":"1 submitter: mlgraham (16)","date":"2026-08-28","value":16,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"github.com/mlgraham (Claude Fable 5)","evidence":"lean-lang.org/eval leaderboard.json: \"github.com/mlgraham (Claude Fable 5)\" solved 16","family":"anthropic","confidence":"high","role":"subject"}},{"contributor":"EVO (deepthought.com.au)","description":"1 submitter: test1-deepthought (8)","date":"2026-06-23","value":8,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"EVO (deepthought.com.au)","evidence":"lean-lang.org/eval leaderboard.json: \"EVO (deepthought.com.au)\" solved 8","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"Public accepted source + Codex packaging","description":"1 submitter: rishistyping (7)","date":"2026-06-19","value":7,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"Codex packaging","tool":"Public accepted source","evidence":"lean-lang.org/eval leaderboard.json: \"Public accepted source + Codex packaging\" solved 7","family":"openai","confidence":"high","role":"subject"}},{"contributor":"Humanize2 flame chase: GPT-5.6-sol + Claude Opus 5","description":"1 submitter: ZhengyangZhang06 (7)","date":"2026-08-21","value":7,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"Claude Opus 5","tool":"Humanize2 flame chase: GPT-5.6-sol","evidence":"lean-lang.org/eval leaderboard.json: \"Humanize2 flame chase: GPT-5.6-sol + Claude Opus 5\" solved 7","family":"anthropic","confidence":"high","role":"subject"}},{"contributor":"GPT-5.6 Sol","description":"2 submitters: Vilin97 (5), n-yamaguchi-0729 (2)","date":"2026-09-08","value":7,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"GPT-5.6 Sol","evidence":"lean-lang.org/eval leaderboard.json: \"GPT-5.6 Sol\" solved 7","family":"openai","confidence":"high","role":"subject"}},{"contributor":"trureturing","description":"1 submitter: AlyciaBHZ (7)","date":"2026-10-05","value":7,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"trureturing","evidence":"lean-lang.org/eval leaderboard.json: \"trureturing\" solved 7","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"Opus-5","description":"1 submitter: Vilin97 (6)","date":"2026-08-13","value":6,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"Opus-5","evidence":"lean-lang.org/eval leaderboard.json: \"Opus-5\" solved 6","family":"anthropic","confidence":"high","role":"subject"}},{"contributor":"Claude Fable 5","description":"2 submitters: rkirov (4), will1491 (1)","date":"2026-07-28","value":5,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"Claude Fable 5","evidence":"lean-lang.org/eval leaderboard.json: \"Claude Fable 5\" solved 5","family":"anthropic","confidence":"high","role":"subject"}},{"contributor":"Forge","description":"1 submitter: xiaoyangliu-sjtu (5)","date":"2026-09-15","value":5,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"Forge","evidence":"lean-lang.org/eval leaderboard.json: \"Forge\" solved 5","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"Ivan Bogatyy","description":"1 submitter: bogatyy (4)","date":"2026-09-19","value":4,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"Ivan Bogatyy","evidence":"lean-lang.org/eval leaderboard.json: \"Ivan Bogatyy\" solved 4","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"DeepSeek V4 Flash","description":"1 submitter: rwst (3)","date":"2026-07-12","value":3,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"DeepSeek V4 Flash","evidence":"lean-lang.org/eval leaderboard.json: \"DeepSeek V4 Flash\" solved 3","family":"deepseek","confidence":"high","role":"subject"}},{"contributor":"Grok 4.5","description":"1 submitter: rwst (3)","date":"2026-07-23","value":3,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"Grok 4.5","evidence":"lean-lang.org/eval leaderboard.json: \"Grok 4.5\" solved 3","family":"xai","confidence":"high","role":"subject"}},{"contributor":"deepseek-v4 and gpt-5.x","description":"1 submitter: ivwumupy (3)","date":"2026-07-27","value":3,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"deepseek-v4 and gpt-5.x","evidence":"lean-lang.org/eval leaderboard.json: \"deepseek-v4 and gpt-5.x\" solved 3","family":"openai","confidence":"high","role":"subject"}},{"contributor":"mostly ChatGPT 5.6 Sol","description":"1 submitter: mccorvie (3)","date":"2026-08-03","value":3,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"mostly ChatGPT 5.6 Sol","evidence":"lean-lang.org/eval leaderboard.json: \"mostly ChatGPT 5.6 Sol\" solved 3","family":"openai","confidence":"high","role":"subject"}},{"contributor":"GPT-5.5 Codex","description":"1 submitter: A-M-Berns (2)","date":"2026-05-07","value":2,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"GPT-5.5 Codex","evidence":"lean-lang.org/eval leaderboard.json: \"GPT-5.5 Codex\" solved 2","family":"openai","confidence":"high","role":"subject"}},{"contributor":"Gemini 3.1 Pro","description":"1 submitter: sqrt-of-2 (2)","date":"2026-05-10","value":2,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"Gemini 3.1 Pro","evidence":"lean-lang.org/eval leaderboard.json: \"Gemini 3.1 Pro\" solved 2","family":"google","confidence":"high","role":"subject"}},{"contributor":"[submission] aegis-of-the-unit-circle-logos","description":"1 submitter: rishistyping (2)","date":"2026-05-12","value":2,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"[submission] aegis-of-the-unit-circle-logos","evidence":"lean-lang.org/eval leaderboard.json: \"[submission] aegis-of-the-unit-circle-logos\" solved 2","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"EVO","description":"2 submitters: machinelearning2014 (2), test1-deepthought (2)","date":"2026-05-30","value":2,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"EVO","evidence":"lean-lang.org/eval leaderboard.json: \"EVO\" solved 2","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"Claude Opus 4.8 (1M context)","description":"1 submitter: rkirov (2)","date":"2026-07-01","value":2,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"Claude Opus 4.8 (1M context)","evidence":"lean-lang.org/eval leaderboard.json: \"Claude Opus 4.8 (1M context)\" solved 2","family":"anthropic","confidence":"high","role":"subject"}},{"contributor":"Kimi K2.7","description":"1 submitter: rwst (2)","date":"2026-07-10","value":2,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"Kimi K2.7","evidence":"lean-lang.org/eval leaderboard.json: \"Kimi K2.7\" solved 2","family":"moonshot","confidence":"high","role":"subject"}},{"contributor":"GPT-5.6 and Fable 5 (human-in-the-loop)","description":"1 submitter: KitaKen1 (2)","date":"2026-08-06","value":2,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"GPT-5.6 and Fable 5 (human-in-the-loop)","evidence":"lean-lang.org/eval leaderboard.json: \"GPT-5.6 and Fable 5 (human-in-the-loop)\" solved 2","family":"anthropic","confidence":"high","role":"subject"}},{"contributor":"Humanize RLCR: GPT-5.6-sol (max) worker + reviewer","description":"1 submitter: ZhengyangZhang06 (2)","date":"2026-08-22","value":2,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"reviewer","tool":"Humanize RLCR: GPT-5.6-sol (max) worker","evidence":"lean-lang.org/eval leaderboard.json: \"Humanize RLCR: GPT-5.6-sol (max) worker + reviewer\" solved 2","family":"openai","confidence":"high","role":"subject"}},{"contributor":"Forall (Astrio)","description":"1 submitter: nolanlwin (2)","date":"2026-09-07","value":2,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"Forall (Astrio)","evidence":"lean-lang.org/eval leaderboard.json: \"Forall (Astrio)\" solved 2","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"trureturning","description":"1 submitter: AlyciaBHZ (2)","date":"2026-10-05","value":2,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"trureturning","evidence":"lean-lang.org/eval leaderboard.json: \"trureturning\" solved 2","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"Claude Opus 4.7","description":"1 submitter: rkirov (1)","date":"2026-04-30","value":1,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"Claude Opus 4.7","evidence":"lean-lang.org/eval leaderboard.json: \"Claude Opus 4.7\" solved 1","family":"anthropic","confidence":"high","role":"subject"}},{"contributor":"Claude Opus 4.7 + GPT-5.5 (human-in-the-loop)","description":"1 submitter: yawara (1)","date":"2026-05-29","value":1,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"GPT-5.5 (human-in-the-loop)","tool":"Claude Opus 4.7","evidence":"lean-lang.org/eval leaderboard.json: \"Claude Opus 4.7 + GPT-5.5 (human-in-the-loop)\" solved 1","family":"anthropic","confidence":"high","role":"subject"}},{"contributor":"GPT-5 Codex + Aristotle","description":"1 submitter: rishistyping (1)","date":"2026-06-01","value":1,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"Aristotle","tool":"GPT-5 Codex","evidence":"lean-lang.org/eval leaderboard.json: \"GPT-5 Codex + Aristotle\" solved 1","family":"openai","confidence":"high","role":"subject"}},{"contributor":"Autoform-Bot","description":"1 submitter: niketp03 (1)","date":"2026-06-02","value":1,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"Autoform-Bot","evidence":"lean-lang.org/eval leaderboard.json: \"Autoform-Bot\" solved 1","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"Leanstral 1.4","description":"1 submitter: albertqjiang (1)","date":"2026-06-06","value":1,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"Leanstral 1.4","evidence":"lean-lang.org/eval leaderboard.json: \"Leanstral 1.4\" solved 1","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"Claude Opus 4.7, 4.8 and Fable 5 + OSS contributions","description":"1 submitter: rkirov (1)","date":"2026-06-11","value":1,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"OSS contributions","tool":"Claude Opus 4.7, 4.8 and Fable 5","evidence":"lean-lang.org/eval leaderboard.json: \"Claude Opus 4.7, 4.8 and Fable 5 + OSS contributions\" solved 1","family":"anthropic","confidence":"high","role":"subject"}},{"contributor":"Community multi-model project","description":"1 submitter: mrdouglasny (1)","date":"2026-06-15","value":1,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"Community multi-model project","evidence":"lean-lang.org/eval leaderboard.json: \"Community multi-model project\" solved 1","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"Codex (with human in the loop)","description":"1 submitter: plby (1)","date":"2026-06-26","value":1,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"Codex (with human in the loop)","evidence":"lean-lang.org/eval leaderboard.json: \"Codex (with human in the loop)\" solved 1","family":"openai","confidence":"high","role":"subject"}},{"contributor":"Claude Opus 4.8 + Fable 5 (multi-agent)","description":"1 submitter: ldct (1)","date":"2026-07-05","value":1,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"Fable 5 (multi-agent)","tool":"Claude Opus 4.8","evidence":"lean-lang.org/eval leaderboard.json: \"Claude Opus 4.8 + Fable 5 (multi-agent)\" solved 1","family":"anthropic","confidence":"high","role":"subject"}},{"contributor":"Hy3","description":"1 submitter: rwst (1)","date":"2026-07-12","value":1,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"Hy3","evidence":"lean-lang.org/eval leaderboard.json: \"Hy3\" solved 1","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"UNICO/NOUS pipeline - Claude (Anthropic)","description":"1 submitter: Solarys431 (1)","date":"2026-07-13","value":1,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"UNICO/NOUS pipeline - Claude (Anthropic)","evidence":"lean-lang.org/eval leaderboard.json: \"UNICO/NOUS pipeline - Claude (Anthropic)\" solved 1","family":"anthropic","confidence":"high","role":"subject"}},{"contributor":"Claude Fable 5 (orchestrating Opus/Sonnet 4.8)","description":"1 submitter: ianklatzco (1)","date":"2026-07-14","value":1,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"Claude Fable 5 (orchestrating Opus/Sonnet 4.8)","evidence":"lean-lang.org/eval leaderboard.json: \"Claude Fable 5 (orchestrating Opus/Sonnet 4.8)\" solved 1","family":"anthropic","confidence":"high","role":"subject"}},{"contributor":"GPT-5.6 (human-in-the-loop)","description":"1 submitter: matthewjasper (1)","date":"2026-07-14","value":1,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"GPT-5.6 (human-in-the-loop)","evidence":"lean-lang.org/eval leaderboard.json: \"GPT-5.6 (human-in-the-loop)\" solved 1","family":"openai","confidence":"high","role":"subject"}},{"contributor":"Various models ( with human in the loop )","description":"1 submitter: yawara (1)","date":"2026-07-16","value":1,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"Various models ( with human in the loop )","evidence":"lean-lang.org/eval leaderboard.json: \"Various models ( with human in the loop )\" solved 1","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"Trellis","description":"1 submitter: wpegden (1)","date":"2026-07-20","value":1,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"Trellis","evidence":"lean-lang.org/eval leaderboard.json: \"Trellis\" solved 1","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"UNICO/NOUS: Claude (Fable 5 + Opus 4.8) + GPT-5.6-Sol","description":"1 submitter: Solarys431 (1)","date":"2026-07-21","value":1,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"Opus 4.8) + GPT-5.6-Sol","tool":"UNICO/NOUS: Claude (Fable 5","evidence":"lean-lang.org/eval leaderboard.json: \"UNICO/NOUS: Claude (Fable 5 + Opus 4.8) + GPT-5.6-Sol\" solved 1","family":"anthropic","confidence":"high","role":"subject"}},{"contributor":"various model (deepseek-v4, gpt-5.*)","description":"1 submitter: ivwumupy (1)","date":"2026-07-24","value":1,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"various model (deepseek-v4, gpt-5.*)","evidence":"lean-lang.org/eval leaderboard.json: \"various model (deepseek-v4, gpt-5.*)\" solved 1","family":"openai","confidence":"high","role":"subject"}},{"contributor":"Fable 5 and GPT-5.6","description":"1 submitter: adamtopaz (1)","date":"2026-07-25","value":1,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"Fable 5 and GPT-5.6","evidence":"lean-lang.org/eval leaderboard.json: \"Fable 5 and GPT-5.6\" solved 1","family":"anthropic","confidence":"high","role":"subject"}},{"contributor":"OpenAI Codex (GPT-5.6 Sol), Claude Code (Opus 5 and Opus 4.8)","description":"1 submitter: rigwild (1)","date":"2026-08-04","value":1,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"OpenAI Codex (GPT-5.6 Sol), Claude Code (Opus 5 and Opus 4.8)","evidence":"lean-lang.org/eval leaderboard.json: \"OpenAI Codex (GPT-5.6 Sol), Claude Code (Opus 5 and Opus 4.8)\" solved 1","family":"anthropic","confidence":"high","role":"subject"}},{"contributor":"GPT-5.6 Codex (public-proof reproduction)","description":"1 submitter: Vilin97 (1)","date":"2026-08-07","value":1,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"GPT-5.6 Codex (public-proof reproduction)","evidence":"lean-lang.org/eval leaderboard.json: \"GPT-5.6 Codex (public-proof reproduction)\" solved 1","family":"openai","confidence":"high","role":"subject"}},{"contributor":"GPT-5 Codex","description":"1 submitter: Vilin97 (1)","date":"2026-08-12","value":1,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"GPT-5 Codex","evidence":"lean-lang.org/eval leaderboard.json: \"GPT-5 Codex\" solved 1","family":"openai","confidence":"high","role":"subject"}},{"contributor":"OpenAI Codex (GPT-5)","description":"1 submitter: Vilin97 (1)","date":"2026-08-13","value":1,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"OpenAI Codex (GPT-5)","evidence":"lean-lang.org/eval leaderboard.json: \"OpenAI Codex (GPT-5)\" solved 1","family":"openai","confidence":"high","role":"subject"}},{"contributor":"Claude Fable 5 (Claude Code)","description":"1 submitter: homeowmorphism (1)","date":"2026-08-18","value":1,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"Claude Fable 5 (Claude Code)","evidence":"lean-lang.org/eval leaderboard.json: \"Claude Fable 5 (Claude Code)\" solved 1","family":"anthropic","confidence":"high","role":"subject"}},{"contributor":"github.com/mlgraham — public accepted source, packaged (Claude Fable 5)","description":"1 submitter: mlgraham (1)","date":"2026-08-20","value":1,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"github.com/mlgraham — public accepted source, packaged (Claude Fable 5)","evidence":"lean-lang.org/eval leaderboard.json: \"github.com/mlgraham — public accepted source, packaged (Claude Fable 5)\" solved 1","family":"anthropic","confidence":"high","role":"subject"}},{"contributor":"DeepScientist + GPT-5.6","description":"1 submitter: LikC1606 (1)","date":"2026-08-22","value":1,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"GPT-5.6","tool":"DeepScientist","evidence":"lean-lang.org/eval leaderboard.json: \"DeepScientist + GPT-5.6\" solved 1","family":"openai","confidence":"high","role":"subject"}},{"contributor":"OpenAI Codex (GPT-5.6 Sol)","description":"1 submitter: Cond-t (1)","date":"2026-08-31","value":1,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"OpenAI Codex (GPT-5.6 Sol)","evidence":"lean-lang.org/eval leaderboard.json: \"OpenAI Codex (GPT-5.6 Sol)\" solved 1","family":"openai","confidence":"high","role":"subject"}},{"contributor":"LeanEval production launch canary 3dc17f41ccd0c86fa443687fac784a42f3798183","description":"1 submitter: kim-em (1)","date":"2026-09-02","value":1,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"LeanEval production launch canary 3dc17f41ccd0c86fa443687fac784a42f3798183","evidence":"lean-lang.org/eval leaderboard.json: \"LeanEval production launch canary 3dc17f41ccd0c86fa443687fac784a42f3798183\" solved 1","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"BRILLiant + ChatGPT 5.6 Sol","description":"1 submitter: LexBrill (1)","date":"2026-09-02","value":1,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"ChatGPT 5.6 Sol","tool":"BRILLiant","evidence":"lean-lang.org/eval leaderboard.json: \"BRILLiant + ChatGPT 5.6 Sol\" solved 1","family":"openai","confidence":"high","role":"subject"}},{"contributor":"OpenAI Codex (GPT-6 + GPT-5.6 Sol)","description":"1 submitter: lixiang90 (1)","date":"2026-09-07","value":1,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"GPT-5.6 Sol)","tool":"OpenAI Codex (GPT-6","evidence":"lean-lang.org/eval leaderboard.json: \"OpenAI Codex (GPT-6 + GPT-5.6 Sol)\" solved 1","family":"openai","confidence":"high","role":"subject"}},{"contributor":"(various models)","description":"1 submitter: mlgraham (1)","date":"2026-09-13","value":1,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"(various models)","evidence":"lean-lang.org/eval leaderboard.json: \"(various models)\" solved 1","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"LeanEval P1 launch canary","description":"1 submitter: kim-em (1)","date":"2026-09-13","value":1,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"LeanEval P1 launch canary","evidence":"lean-lang.org/eval leaderboard.json: \"LeanEval P1 launch canary\" solved 1","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"Codex 5.6 Sol + HITL","description":"1 submitter: kimihiro64 (1)","date":"2026-09-13","value":1,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"HITL","tool":"Codex 5.6 Sol","evidence":"lean-lang.org/eval leaderboard.json: \"Codex 5.6 Sol + HITL\" solved 1","family":"openai","confidence":"high","role":"subject"}},{"contributor":"Fable 5.1 and GPT-6","description":"1 submitter: will1491 (1)","date":"2026-09-26","value":1,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"Fable 5.1 and GPT-6","evidence":"lean-lang.org/eval leaderboard.json: \"Fable 5.1 and GPT-6\" solved 1","family":"anthropic","confidence":"high","role":"subject"}},{"contributor":"DeepScientist Math + GPT-6.1 (gpt-6.1-sol)","description":"1 submitter: LikC1606 (1)","date":"2026-10-14","value":1,"contributorKind":"model","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"model":"GPT-6.1 (gpt-6.1-sol)","tool":"DeepScientist Math","evidence":"lean-lang.org/eval leaderboard.json: \"DeepScientist Math + GPT-6.1 (gpt-6.1-sol)\" solved 1","family":"openai","confidence":"high","role":"subject"}}],"contested":true},{"description":"People who ran a model and got a proof accepted, ranked by distinct problems solved. Attribution is the declared model on their accepted results.","id":"submitters","name":"Submitters — problems solved","metricName":"problems solved","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"Vilin97","description":"models: Axiom Prover (Axiom Math) ×264, Vasily-24-hour-gpt-5.6-speedrun ×79, Opus-5 ×6, gpt-5.6-Sol ×5, GPT-5 Codex ×1, GPT-5.6 Codex (public-proof reproduction) ×1, OpenAI Codex (GPT-5) ×1","date":"2026-10-15","value":264,"contributorKind":"person","contributorUrl":"https://github.com/Vilin97","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/Vilin97.json","https://github.com/Vilin97/lean-eval-speedrun","https://github.com/Vilin97/lean-eval-ado"],"isCurrentBest":true,"agent":{"model":"Axiom Prover (Axiom Math)","evidence":"results/Vilin97.json declared_model=\"Axiom Prover (Axiom Math)\" on 264 of 264 problems","family":"axiom","confidence":"high","role":"author"}},{"contributor":"ZhengyangZhang06","description":"models: Humanifa + GPT 5.6 sol ×215, Humanize2 flame chase: GPT-5.6-sol + Claude Opus 5 ×7, Humanize RLCR: GPT-5.6-sol (max) worker + reviewer ×2","date":"2026-09-25","value":215,"contributorKind":"person","contributorUrl":"https://github.com/ZhengyangZhang06","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/ZhengyangZhang06.json","https://github.com/humanfia/lean-eval"],"agent":{"model":"GPT 5.6 sol","tool":"Humanifa","evidence":"results/ZhengyangZhang06.json declared_model=\"Humanifa + GPT 5.6 sol\" on 215 of 215 problems","family":"openai","confidence":"high","role":"author"}},{"contributor":"lukerj00","description":"models: Tau (caj.al) ×200","date":"2026-09-07","value":200,"contributorKind":"person","contributorUrl":"https://github.com/lukerj00","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/lukerj00.json","https://github.com/lukerj00/tau-lean-eval-submission"],"agent":{"model":"Tau (caj.al)","evidence":"results/lukerj00.json declared_model=\"Tau (caj.al)\" on 200 of 200 problems","family":"other-ai","confidence":"high","role":"author"}},{"contributor":"GanjinZero","description":"models: Seed Prover (ByteDance) ×135","date":"2026-08-02","value":135,"contributorKind":"person","contributorUrl":"https://github.com/GanjinZero","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/GanjinZero.json","https://github.com/GanjinZero/lidskii_last","https://github.com/GanjinZero/frobenius_group_determinant"],"agent":{"model":"Seed Prover (ByteDance)","evidence":"results/GanjinZero.json declared_model=\"Seed Prover (ByteDance)\" on 135 of 135 problems","family":"bytedance","confidence":"high","role":"author"}},{"contributor":"Morgan-Griffiths","description":"models: GPT-5.6 ×111, GPT-5.5 ×10","date":"2026-07-16","value":111,"contributorKind":"person","contributorUrl":"https://github.com/Morgan-Griffiths","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/Morgan-Griffiths.json","https://github.com/Morgan-Griffiths/4bdee1299a2cb3e5ad3ee11129659885","https://github.com/Morgan-Griffiths/540d938ce971aabb2511aae2ea81e911"],"agent":{"model":"GPT-5.6","evidence":"results/Morgan-Griffiths.json declared_model=\"GPT-5.6\" on 111 of 111 problems","family":"openai","confidence":"high","role":"author"}},{"contributor":"LorenzoLuccioli","description":"models: Aristotle (Harmonic) ×105","date":"2026-07-18","value":105,"contributorKind":"person","contributorUrl":"https://github.com/LorenzoLuccioli","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/LorenzoLuccioli.json","https://github.com/LorenzoLuccioli/lean-eval-private","https://github.com/LorenzoLuccioli/lean-eval"],"agent":{"model":"Aristotle (Harmonic)","evidence":"results/LorenzoLuccioli.json declared_model=\"Aristotle (Harmonic)\" on 105 of 105 problems","family":"harmonic","confidence":"high","role":"author"}},{"contributor":"rishistyping","description":"models: Stealth Model ×104, Public accepted source + Codex packaging ×7, [submission] aegis-of-the-unit-circle-logos ×2, GPT-5 Codex + Aristotle ×1","date":"2026-08-20","value":104,"contributorKind":"person","contributorUrl":"https://github.com/rishistyping","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/rishistyping.json","https://github.com/rishistyping/autoformalization-machine-intelligence","https://github.com/rishistyping/lean-eval-symplectic-matrix-det-submission"],"agent":{"model":"Stealth Model","evidence":"results/rishistyping.json declared_model=\"Stealth Model\" on 104 of 104 problems","family":"other-ai","confidence":"high","role":"author"}},{"contributor":"ma4ypic4y","description":"models: max gp (GPT-6) ×90","date":"2026-10-18","value":90,"contributorKind":"person","contributorUrl":"https://github.com/ma4ypic4y","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/ma4ypic4y.json","https://github.com/ma4ypic4y/lean-eval-top1"],"agent":{"model":"max gp (GPT-6)","evidence":"results/ma4ypic4y.json declared_model=\"max gp (GPT-6)\" on 90 of 90 problems","family":"openai","confidence":"high","role":"author"}},{"contributor":"skidanovalex","description":"models: NEAR AI w/ DeepSeek V4 ×88","date":"2026-10-14","value":88,"contributorKind":"person","contributorUrl":"https://github.com/skidanovalex","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/skidanovalex.json","https://github.com/SkidanovAlex/leaneval-deepseek"],"agent":{"model":"DeepSeek V4","tool":"NEAR AI","evidence":"results/skidanovalex.json declared_model=\"NEAR AI w/ DeepSeek V4\" on 88 of 88 problems","family":"deepseek","confidence":"high","role":"author"}},{"contributor":"mayorov-m-a","description":"models: Aleph Prover(logicalintelligence.com) ×52","date":"2026-06-05","value":52,"contributorKind":"person","contributorUrl":"https://github.com/mayorov-m-a","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/mayorov-m-a.json","https://github.com/mayorov-m-a/aleph-lean-eval-subm-2026-may-08"],"agent":{"model":"Aleph Prover(logicalintelligence.com)","evidence":"results/mayorov-m-a.json declared_model=\"Aleph Prover(logicalintelligence.com)\" on 52 of 52 problems","family":"other-ai","confidence":"high","role":"author"}},{"contributor":"parabamoghv","description":"models: Aristotle (Harmonic) ×42","date":"2026-08-20","value":42,"contributorKind":"person","contributorUrl":"https://github.com/parabamoghv","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/parabamoghv.json","https://github.com/parabamoghv/lean-eval","https://github.com/parabamoghv/lean-eval-private"],"agent":{"model":"Aristotle (Harmonic)","evidence":"results/parabamoghv.json declared_model=\"Aristotle (Harmonic)\" on 42 of 42 problems","family":"harmonic","confidence":"high","role":"author"}},{"contributor":"sarjinius","description":"models: Midas ×39","date":"2026-10-01","value":39,"contributorKind":"person","contributorUrl":"https://github.com/sarjinius","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/sarjinius.json","https://github.com/midas-intelligence/midas-leaneval-submission"],"agent":{"model":"Midas","evidence":"results/sarjinius.json declared_model=\"Midas\" on 39 of 39 problems","family":"other-ai","confidence":"high","role":"author"}},{"contributor":"hongzhoulin89","description":"models: Humanifa + GPT 5.6 sol ×34, Humanifa + GPT-5.6 sol ×2","date":"2026-10-04","value":34,"contributorKind":"person","contributorUrl":"https://github.com/hongzhoulin89","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/hongzhoulin89.json","https://github.com/hongzhoulin89/lean-eval-bourgain-polynomial-ergodic","https://github.com/hongzhoulin89/lean-eval-pardon-torus-knot-distortion"],"agent":{"model":"GPT 5.6 sol","tool":"Humanifa","evidence":"results/hongzhoulin89.json declared_model=\"Humanifa + GPT 5.6 sol\" on 34 of 34 problems","family":"openai","confidence":"high","role":"author"}},{"contributor":"mlgraham","description":"models: github.com/mlgraham (various models) ×31, github.com/mlgraham (Claude Fable 5) ×16, github.com/mlgraham — public accepted source, packaged (Claude Fable 5) ×1, (various models) ×1","date":"2026-09-13","value":32,"contributorKind":"person","contributorUrl":"https://github.com/mlgraham","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/mlgraham.json","https://github.com/mlgraham/lean-eval-submission","https://github.com/mlgraham/lean-eval-coc"],"agent":{"model":"github.com/mlgraham (various models)","evidence":"results/mlgraham.json declared_model=\"github.com/mlgraham (various models)\" on 31 of 32 problems","family":"other-ai","confidence":"high","role":"author"}},{"contributor":"doxtor6","description":"models: MerLean-Prover ×30","date":"2026-06-25","value":30,"contributorKind":"person","contributorUrl":"https://github.com/doxtor6","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/doxtor6.json","https://github.com/doxtor6/lean-eval"],"agent":{"model":"MerLean-Prover","evidence":"results/doxtor6.json declared_model=\"MerLean-Prover\" on 30 of 30 problems","family":"other-ai","confidence":"high","role":"author"}},{"contributor":"daouid","description":"models: Antigravity (Multi-Model Ensemble: Gemini 3.1 Pro, Gemini 3 Flash, Claude 4.6 Sonnet/Opus) ×21","date":"2026-05-22","value":21,"contributorKind":"person","contributorUrl":"https://github.com/daouid","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/daouid.json","https://github.com/daouid/lean-eval","https://github.com/daouid/73182095262a4e329bcec8579bdd573a"],"agent":{"model":"Antigravity (Multi-Model Ensemble: Gemini 3.1 Pro, Gemini 3 Flash, Claude 4.6 Sonnet/Opus)","evidence":"results/daouid.json declared_model=\"Antigravity (Multi-Model Ensemble: Gemini 3.1 Pro, Gemini 3 Flash, Claude 4.6 Sonnet/Opus)\" on 21 of 21 problems","family":"anthropic","confidence":"high","role":"author"}},{"contributor":"hanwenzhu","description":"models: Seed Prover (ByteDance) ×21","date":"2026-08-13","value":21,"contributorKind":"person","contributorUrl":"https://github.com/hanwenzhu","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/hanwenzhu.json","https://github.com/hanwenzhu/lean-eval-seedprover"],"agent":{"model":"Seed Prover (ByteDance)","evidence":"results/hanwenzhu.json declared_model=\"Seed Prover (ByteDance)\" on 21 of 21 problems","family":"bytedance","confidence":"high","role":"author"}},{"contributor":"savarin","description":"models: savarin-6-hour-gpt-5.6-speedrun ×21","date":"2026-08-17","value":21,"contributorKind":"person","contributorUrl":"https://github.com/savarin","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/savarin.json","https://github.com/savarin/lean-eval-speedrun"],"agent":{"model":"savarin-6-hour-gpt-5.6-speedrun","evidence":"results/savarin.json declared_model=\"savarin-6-hour-gpt-5.6-speedrun\" on 21 of 21 problems","family":"openai","confidence":"high","role":"author"}},{"contributor":"rkirov","description":"models: Claude Opus 4.7 (1M context) ×15, Claude Fable 5 ×4, Claude Opus 4.8 (1M context) ×2, Claude Opus 4.7 ×1, Claude Opus 4.7, 4.8 and Fable 5 + OSS contributions ×1","date":"2026-07-10","value":19,"contributorKind":"person","contributorUrl":"https://github.com/rkirov","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/rkirov.json","https://github.com/rkirov/lean-eval","https://github.com/rkirov/jordan_pick"],"agent":{"model":"Claude Opus 4.7 (1M context)","evidence":"results/rkirov.json declared_model=\"Claude Opus 4.7 (1M context)\" on 15 of 19 problems","family":"anthropic","confidence":"high","role":"author"}},{"contributor":"sqrt-of-2","description":"models: Aristotle (Harmonic) ×7, GPT-5.5 ×7, GPT 5.5 ×2, Gemini 3.1 Pro ×2","date":"2026-05-12","value":11,"contributorKind":"person","contributorUrl":"https://github.com/sqrt-of-2","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/sqrt-of-2.json","https://github.com/sqrt-of-2/931556b393de1dc23de63d1c5aefc109","https://github.com/sqrt-of-2/5d253653afb78ab2e57ae1446599c9eb"],"agent":{"model":"Aristotle (Harmonic)","evidence":"results/sqrt-of-2.json declared_model=\"Aristotle (Harmonic)\" on 7 of 11 problems","family":"harmonic","confidence":"high","role":"author"}},{"contributor":"antoinegg1","description":"models: Humanifa + GPT 5.6 sol ×10","date":"2026-10-15","value":10,"contributorKind":"person","contributorUrl":"https://github.com/antoinegg1","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/antoinegg1.json","https://github.com/antoinegg1/lean-eval-lot-solutions"],"agent":{"model":"GPT 5.6 sol","tool":"Humanifa","evidence":"results/antoinegg1.json declared_model=\"Humanifa + GPT 5.6 sol\" on 10 of 10 problems","family":"openai","confidence":"high","role":"author"}},{"contributor":"LikC1606","description":"models: DeepScientist Math + GPT-6.1 (gpt-6.1-sol) ×9, DeepScientist + GPT-5.6 ×1","date":"2026-10-16","value":10,"contributorKind":"person","contributorUrl":"https://github.com/LikC1606","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/LikC1606.json","https://github.com/LikC1606/7570e284ff8b474a4139db2092de6eec","https://github.com/LikC1606/deepscientist-math-proofs"],"agent":{"model":"GPT-6.1 (gpt-6.1-sol)","tool":"DeepScientist Math","evidence":"results/LikC1606.json declared_model=\"DeepScientist Math + GPT-6.1 (gpt-6.1-sol)\" on 9 of 10 problems","family":"openai","confidence":"high","role":"author"}},{"contributor":"test1-deepthought","description":"models: EVO (deepthought.com.au) ×8, EVO ×2","date":"2026-06-23","value":8,"contributorKind":"person","contributorUrl":"https://github.com/test1-deepthought","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/test1-deepthought.json","https://github.com/test1-deepthought/lean-eval-solutions","https://github.com/machinelearning2014/deepthought_lean_eval"],"agent":{"model":"EVO (deepthought.com.au)","evidence":"results/test1-deepthought.json declared_model=\"EVO (deepthought.com.au)\" on 8 of 8 problems","family":"other-ai","confidence":"high","role":"author"}},{"contributor":"AlyciaBHZ","description":"models: trureturing ×7, trureturning ×2","date":"2026-10-05","value":7,"contributorKind":"person","contributorUrl":"https://github.com/AlyciaBHZ","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/AlyciaBHZ.json","https://github.com/the-omega-institute/trureturning-lean-eval","https://github.com/the-omega-institute/trureturing-lean-eval"],"agent":{"model":"trureturing","evidence":"results/AlyciaBHZ.json declared_model=\"trureturing\" on 7 of 7 problems","family":"other-ai","confidence":"high","role":"author"}},{"contributor":"JohnEdwardJennings","description":"models: Aristotle (Harmonic) ×7","date":"2026-06-14","value":7,"contributorKind":"person","contributorUrl":"https://github.com/JohnEdwardJennings","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/JohnEdwardJennings.json","https://github.com/JohnEdwardJennings/lean-eval-personal"],"agent":{"model":"Aristotle (Harmonic)","evidence":"results/JohnEdwardJennings.json declared_model=\"Aristotle (Harmonic)\" on 7 of 7 problems","family":"harmonic","confidence":"high","role":"author"}},{"contributor":"kim-em","description":"models: Aristotle (Harmonic) ×6, LeanEval production launch canary 3dc17f41ccd0c86fa443687fac784a42f3798183 ×1, LeanEval P1 launch canary ×1","date":"2026-09-13","value":7,"contributorKind":"person","contributorUrl":"https://github.com/kim-em","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/kim-em.json","https://github.com/kim-em/0e2146482105e90a2bb4b63f137975fb","https://github.com/kim-em/ba837b9520ea4ec64c7945bb1f0cf10f"],"agent":{"model":"Aristotle (Harmonic)","evidence":"results/kim-em.json declared_model=\"Aristotle (Harmonic)\" on 6 of 7 problems","family":"harmonic","confidence":"high","role":"author"}},{"contributor":"juihuichung","description":"models: Humanifa + GPT 5.6 sol ×6","date":"2026-08-31","value":6,"contributorKind":"person","contributorUrl":"https://github.com/juihuichung","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/juihuichung.json","https://github.com/juihuichung/lean-eval-submissions-private"],"agent":{"model":"GPT 5.6 sol","tool":"Humanifa","evidence":"results/juihuichung.json declared_model=\"Humanifa + GPT 5.6 sol\" on 6 of 6 problems","family":"openai","confidence":"high","role":"author"}},{"contributor":"xiaoyangliu-sjtu","description":"models: Forge ×5","date":"2026-09-15","value":5,"contributorKind":"person","contributorUrl":"https://github.com/xiaoyangliu-sjtu","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/xiaoyangliu-sjtu.json","https://github.com/XiaoyangLiu-sjtu/forge-leaneval-submissions"],"agent":{"model":"Forge","evidence":"results/xiaoyangliu-sjtu.json declared_model=\"Forge\" on 5 of 5 problems","family":"other-ai","confidence":"high","role":"author"}},{"contributor":"A-M-Berns","description":"models: GPT-5.5 ×4, GPT-5.5 Codex ×2","date":"2026-05-16","value":4,"contributorKind":"person","contributorUrl":"https://github.com/A-M-Berns","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/A-M-Berns.json","https://github.com/A-M-Berns/lean-eval-submissions"],"agent":{"model":"GPT-5.5","evidence":"results/A-M-Berns.json declared_model=\"GPT-5.5\" on 4 of 4 problems","family":"openai","confidence":"high","role":"author"}},{"contributor":"antpavzhi","description":"models: Aleph Prover(logicalintelligence.com) ×4","date":"2026-05-11","value":4,"contributorKind":"person","contributorUrl":"https://github.com/antpavzhi","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/antpavzhi.json","https://github.com/antpavzhi/lean-eval"],"agent":{"model":"Aleph Prover(logicalintelligence.com)","evidence":"results/antpavzhi.json declared_model=\"Aleph Prover(logicalintelligence.com)\" on 4 of 4 problems","family":"other-ai","confidence":"high","role":"author"}},{"contributor":"bogatyy","description":"models: Ivan Bogatyy ×4","date":"2026-09-19","value":4,"contributorKind":"person","contributorUrl":"https://github.com/bogatyy","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/bogatyy.json","https://github.com/bogatyy/lean-eval-proofs-intake"],"agent":{"model":"Ivan Bogatyy","evidence":"results/bogatyy.json declared_model=\"Ivan Bogatyy\" on 4 of 4 problems","family":"other-ai","confidence":"high","role":"author"}},{"contributor":"adrianmartir","description":"models: Aristotle (Harmonic) ×3","date":"2026-06-29","value":3,"contributorKind":"person","contributorUrl":"https://github.com/adrianmartir","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/adrianmartir.json","https://github.com/adrianmartir/lean-eval-private","https://github.com/LorenzoLuccioli/lean-eval-private"],"agent":{"model":"Aristotle (Harmonic)","evidence":"results/adrianmartir.json declared_model=\"Aristotle (Harmonic)\" on 3 of 3 problems","family":"harmonic","confidence":"high","role":"author"}},{"contributor":"eohjelle","description":"models: GPT-5.6 ×3","date":"2026-07-21","value":3,"contributorKind":"person","contributorUrl":"https://github.com/eohjelle","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/eohjelle.json","https://github.com/eohjelle/feit-thompson-odd-order-theorem"],"agent":{"model":"GPT-5.6","evidence":"results/eohjelle.json declared_model=\"GPT-5.6\" on 3 of 3 problems","family":"openai","confidence":"high","role":"author"}},{"contributor":"ivwumupy","description":"models: deepseek-v4 and gpt-5.x ×3, various model (deepseek-v4, gpt-5.*) ×1","date":"2026-07-27","value":3,"contributorKind":"person","contributorUrl":"https://github.com/ivwumupy","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/ivwumupy.json","https://github.com/Qiuzhen-CFSG/CFSG"],"agent":{"model":"deepseek-v4 and gpt-5.x","evidence":"results/ivwumupy.json declared_model=\"deepseek-v4 and gpt-5.x\" on 3 of 3 problems","family":"openai","confidence":"high","role":"author"}},{"contributor":"mccorvie","description":"models: mostly ChatGPT 5.6 Sol ×3","date":"2026-08-03","value":3,"contributorKind":"person","contributorUrl":"https://github.com/mccorvie","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/mccorvie.json","https://github.com/mccorvie/classification-of-surfaces","https://github.com/mccorvie/tcos-submission"],"agent":{"model":"mostly ChatGPT 5.6 Sol","evidence":"results/mccorvie.json declared_model=\"mostly ChatGPT 5.6 Sol\" on 3 of 3 problems","family":"openai","confidence":"high","role":"author"}},{"contributor":"rwst","description":"models: Grok 4.5 ×3, DeepSeek V4 Flash ×3, Kimi K2.7 ×2, Hy3 ×1","date":"2026-07-23","value":3,"contributorKind":"person","contributorUrl":"https://github.com/rwst","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/rwst.json","https://github.com/rwst/lean-eval"],"agent":{"model":"Grok 4.5","evidence":"results/rwst.json declared_model=\"Grok 4.5\" on 3 of 3 problems","family":"xai","confidence":"high","role":"author"}},{"contributor":"jzuiddam","description":"models: Claude Opus 4.7 (1M context) ×2","date":"2026-05-07","value":2,"contributorKind":"person","contributorUrl":"https://github.com/jzuiddam","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/jzuiddam.json","https://github.com/jzuiddam/symAction-eval-submission","https://github.com/jzuiddam/glAction-eval-submission"],"agent":{"model":"Claude Opus 4.7 (1M context)","evidence":"results/jzuiddam.json declared_model=\"Claude Opus 4.7 (1M context)\" on 2 of 2 problems","family":"anthropic","confidence":"high","role":"author"}},{"contributor":"KitaKen1","description":"models: GPT-5.6 and Fable 5 (human-in-the-loop) ×2","date":"2026-08-06","value":2,"contributorKind":"person","contributorUrl":"https://github.com/KitaKen1","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/KitaKen1.json","https://github.com/KitaKen1/lean-eval-pi-succ-sphere-n-mulequiv-zmod-two","https://github.com/KitaKen1/lean-eval-conway-schneeberger-fifteen"],"agent":{"model":"GPT-5.6 and Fable 5 (human-in-the-loop)","evidence":"results/KitaKen1.json declared_model=\"GPT-5.6 and Fable 5 (human-in-the-loop)\" on 2 of 2 problems","family":"anthropic","confidence":"high","role":"author"}},{"contributor":"machinelearning2014","description":"models: EVO ×2","date":"2026-05-30","value":2,"contributorKind":"person","contributorUrl":"https://github.com/machinelearning2014","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/machinelearning2014.json","https://github.com/machinelearning2014/deepthought_lean_eval"],"agent":{"model":"EVO","evidence":"results/machinelearning2014.json declared_model=\"EVO\" on 2 of 2 problems","family":"other-ai","confidence":"high","role":"author"}},{"contributor":"n-yamaguchi-0729","description":"models: GPT-5.6 Sol ×2","date":"2026-09-08","value":2,"contributorKind":"person","contributorUrl":"https://github.com/n-yamaguchi-0729","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/n-yamaguchi-0729.json","https://github.com/n-yamaguchi-0729/LeanEvalSubmissions"],"agent":{"model":"GPT-5.6 Sol","evidence":"results/n-yamaguchi-0729.json declared_model=\"GPT-5.6 Sol\" on 2 of 2 problems","family":"openai","confidence":"high","role":"author"}},{"contributor":"nolanlwin","description":"models: Forall (Astrio) ×2","date":"2026-09-07","value":2,"contributorKind":"person","contributorUrl":"https://github.com/nolanlwin","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/nolanlwin.json","https://github.com/nolanlwin/lean-eval"],"agent":{"model":"Forall (Astrio)","evidence":"results/nolanlwin.json declared_model=\"Forall (Astrio)\" on 2 of 2 problems","family":"other-ai","confidence":"high","role":"author"}},{"contributor":"Solarys431","description":"models: UNICO/NOUS: Claude (Fable 5 + Opus 4.8) + GPT-5.6-Sol ×1, UNICO/NOUS pipeline - Claude (Anthropic) ×1","date":"2026-07-21","value":2,"contributorKind":"person","contributorUrl":"https://github.com/Solarys431","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/Solarys431.json","https://github.com/Solarys431/lean-eval-platonic-classification","https://github.com/Solarys431/unico-lean-proofs"],"agent":{"model":"Opus 4.8) + GPT-5.6-Sol","tool":"UNICO/NOUS: Claude (Fable 5","evidence":"results/Solarys431.json declared_model=\"UNICO/NOUS: Claude (Fable 5 + Opus 4.8) + GPT-5.6-Sol\" on 1 of 2 problems","family":"anthropic","confidence":"high","role":"author"}},{"contributor":"will1491","description":"models: Claude Fable 5 ×1, Fable 5.1 and GPT-6 ×1","date":"2026-09-26","value":2,"contributorKind":"person","contributorUrl":"https://github.com/will1491","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/will1491.json","https://github.com/will1491/uniformization","https://github.com/will1491/Poincare-High-Dim"],"agent":{"model":"Claude Fable 5","evidence":"results/will1491.json declared_model=\"Claude Fable 5\" on 1 of 2 problems","family":"anthropic","confidence":"high","role":"author"}},{"contributor":"yawara","description":"models: Various models ( with human in the loop ) ×1, Claude Opus 4.7 + GPT-5.5 (human-in-the-loop) ×1","date":"2026-07-16","value":2,"contributorKind":"person","contributorUrl":"https://github.com/yawara","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/yawara.json","https://github.com/yawara/odd-order-submission","https://github.com/yawara/lean-eval-submissions"],"agent":{"model":"Various models ( with human in the loop )","evidence":"results/yawara.json declared_model=\"Various models ( with human in the loop )\" on 1 of 2 problems","family":"other-ai","confidence":"high","role":"author"}},{"contributor":"adamtopaz","description":"models: Fable 5 and GPT-5.6 ×1","date":"2026-07-25","value":1,"contributorKind":"person","contributorUrl":"https://github.com/adamtopaz","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/adamtopaz.json","https://github.com/adamtopaz/NeukirchUchida"],"agent":{"model":"Fable 5 and GPT-5.6","evidence":"results/adamtopaz.json declared_model=\"Fable 5 and GPT-5.6\" on 1 of 1 problems","family":"anthropic","confidence":"high","role":"author"}},{"contributor":"albertqjiang","description":"models: Leanstral 1.4 ×1","date":"2026-06-06","value":1,"contributorKind":"person","contributorUrl":"https://github.com/albertqjiang","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/albertqjiang.json","https://github.com/albertqjiang/lean-eval-submissions"],"agent":{"model":"Leanstral 1.4","evidence":"results/albertqjiang.json declared_model=\"Leanstral 1.4\" on 1 of 1 problems","family":"other-ai","confidence":"high","role":"author"}},{"contributor":"Cond-t","description":"models: OpenAI Codex (GPT-5.6 Sol) ×1","date":"2026-08-31","value":1,"contributorKind":"person","contributorUrl":"https://github.com/Cond-t","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/Cond-t.json","https://github.com/Cond-t/thema-chen-theorem-lean-eval"],"agent":{"model":"OpenAI Codex (GPT-5.6 Sol)","evidence":"results/Cond-t.json declared_model=\"OpenAI Codex (GPT-5.6 Sol)\" on 1 of 1 problems","family":"openai","confidence":"high","role":"author"}},{"contributor":"homeowmorphism","description":"models: Claude Fable 5 (Claude Code) ×1","date":"2026-08-18","value":1,"contributorKind":"person","contributorUrl":"https://github.com/homeowmorphism","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/homeowmorphism.json","https://github.com/homeowmorphism/lean-eval-annals-marton"],"agent":{"model":"Claude Fable 5 (Claude Code)","evidence":"results/homeowmorphism.json declared_model=\"Claude Fable 5 (Claude Code)\" on 1 of 1 problems","family":"anthropic","confidence":"high","role":"author"}},{"contributor":"ianklatzco","description":"models: Claude Fable 5 (orchestrating Opus/Sonnet 4.8) ×1","date":"2026-07-14","value":1,"contributorKind":"person","contributorUrl":"https://github.com/ianklatzco","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/ianklatzco.json","https://github.com/ianklatzco/odd-order-lean"],"agent":{"model":"Claude Fable 5 (orchestrating Opus/Sonnet 4.8)","evidence":"results/ianklatzco.json declared_model=\"Claude Fable 5 (orchestrating Opus/Sonnet 4.8)\" on 1 of 1 problems","family":"anthropic","confidence":"high","role":"author"}},{"contributor":"kimihiro64","description":"models: Codex 5.6 Sol + HITL ×1","date":"2026-09-13","value":1,"contributorKind":"person","contributorUrl":"https://github.com/kimihiro64","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/kimihiro64.json","https://github.com/kimihiro64/Robin1984"],"agent":{"model":"HITL","tool":"Codex 5.6 Sol","evidence":"results/kimihiro64.json declared_model=\"Codex 5.6 Sol + HITL\" on 1 of 1 problems","family":"openai","confidence":"high","role":"author"}},{"contributor":"ldct","description":"models: Claude Opus 4.8 + Fable 5 (multi-agent) ×1","date":"2026-07-05","value":1,"contributorKind":"person","contributorUrl":"https://github.com/ldct","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/ldct.json","https://github.com/ldct/lean-eval-chudnovsky"],"agent":{"model":"Fable 5 (multi-agent)","tool":"Claude Opus 4.8","evidence":"results/ldct.json declared_model=\"Claude Opus 4.8 + Fable 5 (multi-agent)\" on 1 of 1 problems","family":"anthropic","confidence":"high","role":"author"}},{"contributor":"LexBrill","description":"models: BRILLiant + ChatGPT 5.6 Sol ×1","date":"2026-09-02","value":1,"contributorKind":"person","contributorUrl":"https://github.com/LexBrill","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/LexBrill.json","https://github.com/LexBrill/lean-eval-annals-unit-submission"],"agent":{"model":"ChatGPT 5.6 Sol","tool":"BRILLiant","evidence":"results/LexBrill.json declared_model=\"BRILLiant + ChatGPT 5.6 Sol\" on 1 of 1 problems","family":"openai","confidence":"high","role":"author"}},{"contributor":"lixiang90","description":"models: OpenAI Codex (GPT-6 + GPT-5.6 Sol) ×1","date":"2026-09-07","value":1,"contributorKind":"person","contributorUrl":"https://github.com/lixiang90","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/lixiang90.json","https://github.com/lixiang90/chen_theorem"],"agent":{"model":"GPT-5.6 Sol)","tool":"OpenAI Codex (GPT-6","evidence":"results/lixiang90.json declared_model=\"OpenAI Codex (GPT-6 + GPT-5.6 Sol)\" on 1 of 1 problems","family":"openai","confidence":"high","role":"author"}},{"contributor":"matthewjasper","description":"models: GPT-5.6 (human-in-the-loop) ×1","date":"2026-07-14","value":1,"contributorKind":"person","contributorUrl":"https://github.com/matthewjasper","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/matthewjasper.json","https://github.com/matthewjasper/m23_irrep_tensor_square_decomp"],"agent":{"model":"GPT-5.6 (human-in-the-loop)","evidence":"results/matthewjasper.json declared_model=\"GPT-5.6 (human-in-the-loop)\" on 1 of 1 problems","family":"openai","confidence":"high","role":"author"}},{"contributor":"mrdouglasny","description":"models: Community multi-model project ×1","date":"2026-06-15","value":1,"contributorKind":"person","contributorUrl":"https://github.com/mrdouglasny","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/mrdouglasny.json","https://github.com/mrdouglasny/jacobian-challenge"],"agent":{"model":"Community multi-model project","evidence":"results/mrdouglasny.json declared_model=\"Community multi-model project\" on 1 of 1 problems","family":"other-ai","confidence":"high","role":"author"}},{"contributor":"niketp03","description":"models: Autoform-Bot ×1","date":"2026-06-02","value":1,"contributorKind":"person","contributorUrl":"https://github.com/niketp03","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/niketp03.json","https://github.com/niketp03/fourier-dirichlet-fejer-submission"],"agent":{"model":"Autoform-Bot","evidence":"results/niketp03.json declared_model=\"Autoform-Bot\" on 1 of 1 problems","family":"other-ai","confidence":"high","role":"author"}},{"contributor":"Parcly-Taxel","description":"models: Aristotle (Harmonic) ×1","date":"2026-05-23","value":1,"contributorKind":"person","contributorUrl":"https://github.com/Parcly-Taxel","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/Parcly-Taxel.json","https://github.com/Parcly-Taxel/lean-eval-bbp"],"agent":{"model":"Aristotle (Harmonic)","evidence":"results/Parcly-Taxel.json declared_model=\"Aristotle (Harmonic)\" on 1 of 1 problems","family":"harmonic","confidence":"high","role":"author"}},{"contributor":"plby","description":"models: Codex (with human in the loop) ×1","date":"2026-06-26","value":1,"contributorKind":"person","contributorUrl":"https://github.com/plby","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/plby.json","https://github.com/plby/Erdos90"],"agent":{"model":"Codex (with human in the loop)","evidence":"results/plby.json declared_model=\"Codex (with human in the loop)\" on 1 of 1 problems","family":"openai","confidence":"high","role":"author"}},{"contributor":"rigwild","description":"models: OpenAI Codex (GPT-5.6 Sol), Claude Code (Opus 5 and Opus 4.8) ×1","date":"2026-08-04","value":1,"contributorKind":"person","contributorUrl":"https://github.com/rigwild","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/rigwild.json","https://github.com/rigwild/lean-math-problems"],"agent":{"model":"OpenAI Codex (GPT-5.6 Sol), Claude Code (Opus 5 and Opus 4.8)","evidence":"results/rigwild.json declared_model=\"OpenAI Codex (GPT-5.6 Sol), Claude Code (Opus 5 and Opus 4.8)\" on 1 of 1 problems","family":"anthropic","confidence":"high","role":"author"}},{"contributor":"weiyangzen","description":"models: Deepseek-v4.1 Flash-1017 ×1","date":"2026-10-17","value":1,"contributorKind":"person","contributorUrl":"https://github.com/weiyangzen","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/weiyangzen.json","https://github.com/weiyangzen/lean-eval-v1-zenpi"],"agent":{"model":"Deepseek-v4.1 Flash-1017","evidence":"results/weiyangzen.json declared_model=\"Deepseek-v4.1 Flash-1017\" on 1 of 1 problems","family":"deepseek","confidence":"high","role":"author"}},{"contributor":"wpegden","description":"models: Trellis ×1","date":"2026-07-20","value":1,"contributorKind":"person","contributorUrl":"https://github.com/wpegden","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/wpegden.json","https://github.com/wpegden/unitdistance_challenge"],"agent":{"model":"Trellis","evidence":"results/wpegden.json declared_model=\"Trellis\" on 1 of 1 problems","family":"other-ai","confidence":"high","role":"author"}}],"contested":true}],"agentStats":[{"family":"axiom","records":1,"currentBests":1,"models":["Axiom Prover (Axiom Math)"],"contributors":["Vilin97"]},{"family":"openai","records":19,"currentBests":0,"models":["GPT 5.6 sol","GPT-5.6","max gp (GPT-6)","savarin-6-hour-gpt-5.6-speedrun","GPT-6.1 (gpt-6.1-sol)","GPT-5.5","deepseek-v4 and gpt-5.x","mostly ChatGPT 5.6 Sol","GPT-5.6 Sol","OpenAI Codex (GPT-5.6 Sol)","HITL","ChatGPT 5.6 Sol","GPT-5.6 Sol)","GPT-5.6 (human-in-the-loop)","Codex (with human in the loop)"],"contributors":["ZhengyangZhang06","Morgan-Griffiths","ma4ypic4y","hongzhoulin89","savarin","antoinegg1","LikC1606","juihuichung","A-M-Berns","eohjelle","ivwumupy","mccorvie","n-yamaguchi-0729","Cond-t","kimihiro64","LexBrill","lixiang90","matthewjasper","plby"]},{"family":"other-ai","records":18,"currentBests":0,"models":["Tau (caj.al)","Stealth Model","Aleph Prover(logicalintelligence.com)","Midas","github.com/mlgraham (various models)","MerLean-Prover","EVO (deepthought.com.au)","trureturing","Forge","Ivan Bogatyy","EVO","Forall (Astrio)","Various models ( with human in the loop )","Leanstral 1.4","Community multi-model project","Autoform-Bot","Trellis"],"contributors":["lukerj00","rishistyping","mayorov-m-a","sarjinius","mlgraham","doxtor6","test1-deepthought","AlyciaBHZ","xiaoyangliu-sjtu","antpavzhi","bogatyy","machinelearning2014","nolanlwin","yawara","albertqjiang","mrdouglasny","niketp03","wpegden"]},{"family":"anthropic","records":11,"currentBests":0,"models":["Antigravity (Multi-Model Ensemble: Gemini 3.1 Pro, Gemini 3 Flash, Claude 4.6 Sonnet/Opus)","Claude Opus 4.7 (1M context)","GPT-5.6 and Fable 5 (human-in-the-loop)","Opus 4.8) + GPT-5.6-Sol","Claude Fable 5","Fable 5 and GPT-5.6","Claude Fable 5 (Claude Code)","Claude Fable 5 (orchestrating Opus/Sonnet 4.8)","Fable 5 (multi-agent)","OpenAI Codex (GPT-5.6 Sol), Claude Code (Opus 5 and Opus 4.8)"],"contributors":["daouid","rkirov","jzuiddam","KitaKen1","Solarys431","will1491","adamtopaz","homeowmorphism","ianklatzco","ldct","rigwild"]},{"family":"harmonic","records":7,"currentBests":0,"models":["Aristotle (Harmonic)"],"contributors":["LorenzoLuccioli","parabamoghv","sqrt-of-2","JohnEdwardJennings","kim-em","adrianmartir","Parcly-Taxel"]},{"family":"bytedance","records":2,"currentBests":0,"models":["Seed Prover (ByteDance)"],"contributors":["GanjinZero","hanwenzhu"]},{"family":"deepseek","records":2,"currentBests":0,"models":["DeepSeek V4","Deepseek-v4.1 Flash-1017"],"contributors":["skidanovalex","weiyangzen"]},{"family":"xai","records":1,"currentBests":0,"models":["Grok 4.5"],"contributors":["rwst"]}],"repo":{"owner":"leanprover","name":"lean-eval-submissions","branch":"main"},"participants":[{"name":"Vilin97","url":"https://github.com/Vilin97","submissions":1,"problems":["submitters"],"bestRank":1,"lastActive":"2026-10-15"},{"name":"ZhengyangZhang06","url":"https://github.com/ZhengyangZhang06","submissions":1,"problems":["submitters"],"lastActive":"2026-09-25"},{"name":"lukerj00","url":"https://github.com/lukerj00","submissions":1,"problems":["submitters"],"lastActive":"2026-09-07"},{"name":"GanjinZero","url":"https://github.com/GanjinZero","submissions":1,"problems":["submitters"],"lastActive":"2026-08-02"},{"name":"Morgan-Griffiths","url":"https://github.com/Morgan-Griffiths","submissions":1,"problems":["submitters"],"lastActive":"2026-07-16"},{"name":"LorenzoLuccioli","url":"https://github.com/LorenzoLuccioli","submissions":1,"problems":["submitters"],"lastActive":"2026-07-18"},{"name":"rishistyping","url":"https://github.com/rishistyping","submissions":1,"problems":["submitters"],"lastActive":"2026-08-20"},{"name":"ma4ypic4y","url":"https://github.com/ma4ypic4y","submissions":1,"problems":["submitters"],"lastActive":"2026-10-18"},{"name":"skidanovalex","url":"https://github.com/skidanovalex","submissions":1,"problems":["submitters"],"lastActive":"2026-10-14"},{"name":"mayorov-m-a","url":"https://github.com/mayorov-m-a","submissions":1,"problems":["submitters"],"lastActive":"2026-06-05"},{"name":"parabamoghv","url":"https://github.com/parabamoghv","submissions":1,"problems":["submitters"],"lastActive":"2026-08-20"},{"name":"sarjinius","url":"https://github.com/sarjinius","submissions":1,"problems":["submitters"],"lastActive":"2026-10-01"},{"name":"hongzhoulin89","url":"https://github.com/hongzhoulin89","submissions":1,"problems":["submitters"],"lastActive":"2026-10-04"},{"name":"mlgraham","url":"https://github.com/mlgraham","submissions":1,"problems":["submitters"],"lastActive":"2026-09-13"},{"name":"doxtor6","url":"https://github.com/doxtor6","submissions":1,"problems":["submitters"],"lastActive":"2026-06-25"},{"name":"daouid","url":"https://github.com/daouid","submissions":1,"problems":["submitters"],"lastActive":"2026-05-22"},{"name":"hanwenzhu","url":"https://github.com/hanwenzhu","submissions":1,"problems":["submitters"],"lastActive":"2026-08-13"},{"name":"savarin","url":"https://github.com/savarin","submissions":1,"problems":["submitters"],"lastActive":"2026-08-17"},{"name":"rkirov","url":"https://github.com/rkirov","submissions":1,"problems":["submitters"],"lastActive":"2026-07-10"},{"name":"sqrt-of-2","url":"https://github.com/sqrt-of-2","submissions":1,"problems":["submitters"],"lastActive":"2026-05-12"},{"name":"antoinegg1","url":"https://github.com/antoinegg1","submissions":1,"problems":["submitters"],"lastActive":"2026-10-15"},{"name":"LikC1606","url":"https://github.com/LikC1606","submissions":1,"problems":["submitters"],"lastActive":"2026-10-16"},{"name":"test1-deepthought","url":"https://github.com/test1-deepthought","submissions":1,"problems":["submitters"],"lastActive":"2026-06-23"},{"name":"AlyciaBHZ","url":"https://github.com/AlyciaBHZ","submissions":1,"problems":["submitters"],"lastActive":"2026-10-05"},{"name":"JohnEdwardJennings","url":"https://github.com/JohnEdwardJennings","submissions":1,"problems":["submitters"],"lastActive":"2026-06-14"},{"name":"kim-em","url":"https://github.com/kim-em","submissions":1,"problems":["submitters"],"lastActive":"2026-09-13"},{"name":"juihuichung","url":"https://github.com/juihuichung","submissions":1,"problems":["submitters"],"lastActive":"2026-08-31"},{"name":"xiaoyangliu-sjtu","url":"https://github.com/xiaoyangliu-sjtu","submissions":1,"problems":["submitters"],"lastActive":"2026-09-15"},{"name":"A-M-Berns","url":"https://github.com/A-M-Berns","submissions":1,"problems":["submitters"],"lastActive":"2026-05-16"},{"name":"antpavzhi","url":"https://github.com/antpavzhi","submissions":1,"problems":["submitters"],"lastActive":"2026-05-11"},{"name":"bogatyy","url":"https://github.com/bogatyy","submissions":1,"problems":["submitters"],"lastActive":"2026-09-19"},{"name":"adrianmartir","url":"https://github.com/adrianmartir","submissions":1,"problems":["submitters"],"lastActive":"2026-06-29"},{"name":"eohjelle","url":"https://github.com/eohjelle","submissions":1,"problems":["submitters"],"lastActive":"2026-07-21"},{"name":"ivwumupy","url":"https://github.com/ivwumupy","submissions":1,"problems":["submitters"],"lastActive":"2026-07-27"},{"name":"mccorvie","url":"https://github.com/mccorvie","submissions":1,"problems":["submitters"],"lastActive":"2026-08-03"},{"name":"rwst","url":"https://github.com/rwst","submissions":1,"problems":["submitters"],"lastActive":"2026-07-23"},{"name":"jzuiddam","url":"https://github.com/jzuiddam","submissions":1,"problems":["submitters"],"lastActive":"2026-05-07"},{"name":"KitaKen1","url":"https://github.com/KitaKen1","submissions":1,"problems":["submitters"],"lastActive":"2026-08-06"},{"name":"machinelearning2014","url":"https://github.com/machinelearning2014","submissions":1,"problems":["submitters"],"lastActive":"2026-05-30"},{"name":"n-yamaguchi-0729","url":"https://github.com/n-yamaguchi-0729","submissions":1,"problems":["submitters"],"lastActive":"2026-09-08"},{"name":"nolanlwin","url":"https://github.com/nolanlwin","submissions":1,"problems":["submitters"],"lastActive":"2026-09-07"},{"name":"Solarys431","url":"https://github.com/Solarys431","submissions":1,"problems":["submitters"],"lastActive":"2026-07-21"},{"name":"will1491","url":"https://github.com/will1491","submissions":1,"problems":["submitters"],"lastActive":"2026-09-26"},{"name":"yawara","url":"https://github.com/yawara","submissions":1,"problems":["submitters"],"lastActive":"2026-07-16"},{"name":"adamtopaz","url":"https://github.com/adamtopaz","submissions":1,"problems":["submitters"],"lastActive":"2026-07-25"},{"name":"albertqjiang","url":"https://github.com/albertqjiang","submissions":1,"problems":["submitters"],"lastActive":"2026-06-06"},{"name":"Cond-t","url":"https://github.com/Cond-t","submissions":1,"problems":["submitters"],"lastActive":"2026-08-31"},{"name":"homeowmorphism","url":"https://github.com/homeowmorphism","submissions":1,"problems":["submitters"],"lastActive":"2026-08-18"},{"name":"ianklatzco","url":"https://github.com/ianklatzco","submissions":1,"problems":["submitters"],"lastActive":"2026-07-14"},{"name":"kimihiro64","url":"https://github.com/kimihiro64","submissions":1,"problems":["submitters"],"lastActive":"2026-09-13"},{"name":"ldct","url":"https://github.com/ldct","submissions":1,"problems":["submitters"],"lastActive":"2026-07-05"},{"name":"LexBrill","url":"https://github.com/LexBrill","submissions":1,"problems":["submitters"],"lastActive":"2026-09-02"},{"name":"lixiang90","url":"https://github.com/lixiang90","submissions":1,"problems":["submitters"],"lastActive":"2026-09-07"},{"name":"matthewjasper","url":"https://github.com/matthewjasper","submissions":1,"problems":["submitters"],"lastActive":"2026-07-14"},{"name":"mrdouglasny","url":"https://github.com/mrdouglasny","submissions":1,"problems":["submitters"],"lastActive":"2026-06-15"},{"name":"niketp03","url":"https://github.com/niketp03","submissions":1,"problems":["submitters"],"lastActive":"2026-06-02"},{"name":"Parcly-Taxel","url":"https://github.com/Parcly-Taxel","submissions":1,"problems":["submitters"],"lastActive":"2026-05-23"},{"name":"plby","url":"https://github.com/plby","submissions":1,"problems":["submitters"],"lastActive":"2026-06-26"},{"name":"rigwild","url":"https://github.com/rigwild","submissions":1,"problems":["submitters"],"lastActive":"2026-08-04"},{"name":"weiyangzen","url":"https://github.com/weiyangzen","submissions":1,"problems":["submitters"],"lastActive":"2026-10-17"},{"name":"wpegden","url":"https://github.com/wpegden","submissions":1,"problems":["submitters"],"lastActive":"2026-07-20"}],"stats":{"totalRecords":137,"uniqueParticipants":61,"firstSubmission":"2026-04-30","lastSubmission":"2026-10-18","recordsLast30d":20,"recordsLast90d":80},"lastUpdated":"2026-10-19T09:11:13.026Z"},{"id":"leansolving-squeezebench-04bec82398b15ff6","name":"SqueezeBench","tagline":"Prove Lean-verified lower and upper bounds that squeeze intractable extremal-combinatorics functions f(n).","description":"SqueezeBench keeps, for each extremal-combinatorics problem, the best Lean-proved lower bound and the best Lean-proved upper bound on a function f : ℕ → ℕ (Ramsey, Schur, cap sets, Golomb rulers, ex(n; C₄), etc.). A submission is a new record version: a Lean function plus machine-checked proofs of valid, improves and strict, judged by Comparator against a frozen statement and checked by two independent kernels (Lean and nanoda). It is run by the SqueezeBench team (Pablo Manrique, Hai Xia, Stefan Szeider) and doubles as a saturation-resilient evaluation of AI mathematical capability.","url":"https://github.com/leansolving/SqueezeBench","host":"github","status":"active","domain":"formal-methods","tags":["lean","formal-methods","combinatorics","theorem-proving","bounds","ai-eval"],"images":["https://raw.githubusercontent.com/leansolving/SqueezeBench/main/docs/img/squeeze-light.svg","https://raw.githubusercontent.com/leansolving/SqueezeBench/main/docs/img/squeeze-dark.svg"],"participation":{"howToSubmit":"Run `lake exe squeeze new <Problem> <lower|upper>` to lay out the next version, write Def.lean and Proof.lean, fill in REPORT.md and metadata.toml, run `lake exe squeeze verify <record-dir> --base upstream/main`, then open a PR that adds your record directory and nothing else.","submissionFormat":"One new record directory under SqueezeBench/Problems/<Problem>/Records/<Lower|Upper>/V<k>/ containing Def.lean, Proof.lean, REPORT.md and metadata.toml.","computeDetails":"Proof checking is CPU-only; `squeeze verify`'s fuel check alone peaks near 7 GB, and the Comparator runs in its own 16 GB unit.","deadline":"rolling","prizes":"recognition only","cost":"free","verification":"Comparator judges against a statement frozen for that version; every proof is checked by Lean's kernel and by nanoda (Rust re-implementation). No sorry, no new axioms, no native_decide.","requirements":["Lean 4 (v4.31.0)","Mathlib","elan/lake","Rust/cargo (for bootstrap)","C compiler","Linux with Landlock, non-root"],"compute":"cpu"},"links":[{"label":"Contributing guide","url":"https://github.com/leansolving/SqueezeBench/blob/main/docs/CONTRIBUTING.md"},{"label":"Adding a problem","url":"https://github.com/leansolving/SqueezeBench/blob/main/docs/ADDING_A_PROBLEM.md"},{"label":"Paper (arXiv:2610.13461)","url":"https://arxiv.org/abs/2610.13461"},{"label":"Comparator","url":"https://github.com/leanprover/comparator"},{"label":"nanoda","url":"https://github.com/robsimmons/nanoda_lib"},{"label":"Baseline evaluation","url":"https://github.com/leansolving/SqueezeBench/blob/main/evaluation/README.md"}],"organizer":{"name":"SqueezeBench team (Pablo Manrique, Hai Xia, Stefan Szeider)","url":"https://github.com/leansolving/SqueezeBench","type":"academic"},"venue":{"type":"online"},"quickstart":"git clone https://github.com/leansolving/SqueezeBench.git\ncd SqueezeBench\nlake exe cache get\nlake build\nlake exe squeeze bootstrap\nlake exe squeeze new <Problem> <lower|upper>\nlake exe squeeze verify <record-dir> --base upstream/main","problems":[{"group":"capset","description":"Best proved lower bound on the largest cap in F₃ⁿ.","id":"capset-lower","name":"Cap sets — lower bound","metricName":"lower bound","metricDirection":"maximize","isOpen":true,"records":[{"contributor":"Baseline (by SB team)","description":"Record V005","value":0,"contributorKind":"team","submissionUrls":["https://github.com/leansolving/SqueezeBench/blob/main/SqueezeBench/Problems/CapSet/Records/Lower/V005"],"isBaseline":true,"isCurrentBest":true,"agent":{"model":"claude-fable","evidence":"AI system column: claude-fable (autonomous)","family":"anthropic","confidence":"high","role":"author"}}],"contested":false},{"group":"capset","description":"Best proved upper bound on the largest cap in F₃ⁿ.","id":"capset-upper","name":"Cap sets — upper bound","metricName":"upper bound","metricDirection":"minimize","isOpen":true,"records":[{"contributor":"Baseline (by SB team)","description":"Record V008","value":0,"contributorKind":"team","submissionUrls":["https://github.com/leansolving/SqueezeBench/blob/main/SqueezeBench/Problems/CapSet/Records/Upper/V008"],"isBaseline":true,"isCurrentBest":true,"agent":{"model":"claude-fable","evidence":"AI system column: claude-fable (autonomous)","family":"anthropic","confidence":"high","role":"author"}}],"contested":false},{"group":"exc4","description":"Best proved lower bound on the most edges in an n-vertex graph with no 4-cycle.","id":"exc4-lower","name":"ex(n; C₄) — lower bound","metricName":"lower bound","metricDirection":"maximize","isOpen":true,"records":[{"contributor":"Baseline (by SB team)","description":"Record V008","value":0,"contributorKind":"team","submissionUrls":["https://github.com/leansolving/SqueezeBench/blob/main/SqueezeBench/Problems/ExC4/Records/Lower/V008"],"isBaseline":true,"isCurrentBest":true,"agent":{"model":"claude-fable","evidence":"AI system column: claude-fable (autonomous)","family":"anthropic","confidence":"high","role":"author"}}],"contested":false},{"group":"exc4","description":"Best proved upper bound on the most edges in an n-vertex graph with no 4-cycle.","id":"exc4-upper","name":"ex(n; C₄) — upper bound","metricName":"upper bound","metricDirection":"minimize","isOpen":true,"records":[{"contributor":"Baseline (by SB team)","description":"Record V005","value":0,"contributorKind":"team","submissionUrls":["https://github.com/leansolving/SqueezeBench/blob/main/SqueezeBench/Problems/ExC4/Records/Upper/V005"],"isBaseline":true,"isCurrentBest":true,"agent":{"model":"claude-fable","evidence":"AI system column: claude-fable (autonomous)","family":"anthropic","confidence":"high","role":"author"}}],"contested":false},{"group":"exgirthfive","description":"Best proved lower bound on the most edges in an n-vertex graph of girth at least 5.","id":"exgirthfive-lower","name":"ex(n; {C₃,C₄}) — lower bound","metricName":"lower bound","metricDirection":"maximize","isOpen":true,"records":[{"contributor":"Baseline (by SB team)","description":"Record V011","value":0,"contributorKind":"team","submissionUrls":["https://github.com/leansolving/SqueezeBench/blob/main/SqueezeBench/Problems/ExGirthFive/Records/Lower/V011"],"isBaseline":true,"isCurrentBest":true,"agent":{"model":"claude-fable","evidence":"AI system column: claude-fable (autonomous)","family":"anthropic","confidence":"high","role":"author"}}],"contested":false},{"group":"exgirthfive","description":"Best proved upper bound on the most edges in an n-vertex graph of girth at least 5.","id":"exgirthfive-upper","name":"ex(n; {C₃,C₄}) — upper bound","metricName":"upper bound","metricDirection":"minimize","isOpen":true,"records":[{"contributor":"Baseline (by SB team)","description":"Record V006","value":0,"contributorKind":"team","submissionUrls":["https://github.com/leansolving/SqueezeBench/blob/main/SqueezeBench/Problems/ExGirthFive/Records/Upper/V006"],"isBaseline":true,"isCurrentBest":true,"agent":{"model":"claude-fable","evidence":"AI system column: claude-fable (autonomous)","family":"anthropic","confidence":"high","role":"author"}}],"contested":false},{"group":"golomb","description":"Best proved lower bound on the least length of a Golomb ruler with m marks.","id":"golomb-lower","name":"Golomb rulers — lower bound","metricName":"lower bound","metricDirection":"maximize","isOpen":true,"records":[{"contributor":"Baseline (by SB team)","description":"Record V008","value":0,"contributorKind":"team","submissionUrls":["https://github.com/leansolving/SqueezeBench/blob/main/SqueezeBench/Problems/Golomb/Records/Lower/V008"],"isBaseline":true,"isCurrentBest":true,"agent":{"model":"claude-opus-5.5","evidence":"AI system column: claude-opus-5.5 (autonomous)","family":"anthropic","confidence":"high","role":"author"}}],"contested":false},{"group":"golomb","description":"Best proved upper bound on the least length of a Golomb ruler with m marks.","id":"golomb-upper","name":"Golomb rulers — upper bound","metricName":"upper bound","metricDirection":"minimize","isOpen":true,"records":[{"contributor":"Baseline (by SB team)","description":"Record V013","value":0,"contributorKind":"team","submissionUrls":["https://github.com/leansolving/SqueezeBench/blob/main/SqueezeBench/Problems/Golomb/Records/Upper/V013"],"isBaseline":true,"isCurrentBest":true,"agent":{"model":"claude-fable","evidence":"AI system column: claude-fable (autonomous)","family":"anthropic","confidence":"high","role":"author"}}],"contested":false},{"group":"hypercubec4","description":"Best proved lower bound on the most edges in a C₄-free subgraph of the n-cube.","id":"hypercubec4-lower","name":"ex(Qₙ, C₄) — lower bound","metricName":"lower bound","metricDirection":"maximize","isOpen":true,"records":[{"contributor":"Baseline (by SB team)","description":"Record V007","value":0,"contributorKind":"team","submissionUrls":["https://github.com/leansolving/SqueezeBench/blob/main/SqueezeBench/Problems/HypercubeC4/Records/Lower/V007"],"isBaseline":true,"isCurrentBest":true,"agent":{"model":"claude-fable","evidence":"AI system column: claude-fable (autonomous)","family":"anthropic","confidence":"high","role":"author"}}],"contested":false},{"group":"hypercubec4","description":"Best proved upper bound on the most edges in a C₄-free subgraph of the n-cube.","id":"hypercubec4-upper","name":"ex(Qₙ, C₄) — upper bound","metricName":"upper bound","metricDirection":"minimize","isOpen":true,"records":[{"contributor":"Baseline (by SB team)","description":"Record V007","value":0,"contributorKind":"team","submissionUrls":["https://github.com/leansolving/SqueezeBench/blob/main/SqueezeBench/Problems/HypercubeC4/Records/Upper/V007"],"isBaseline":true,"isCurrentBest":true,"agent":{"model":"claude-fable","evidence":"AI system column: claude-fable (autonomous)","family":"anthropic","confidence":"high","role":"author"}}],"contested":false},{"group":"ramsey","description":"Best proved lower bound on the least N such that every 2-colouring of K_N has a monochromatic K_n.","id":"ramsey-lower","name":"Ramsey — lower bound","metricName":"lower bound","metricDirection":"maximize","isOpen":true,"records":[{"contributor":"Baseline (by SB team)","description":"Record V013","value":0,"contributorKind":"team","submissionUrls":["https://github.com/leansolving/SqueezeBench/blob/main/SqueezeBench/Problems/Ramsey/Records/Lower/V013"],"isBaseline":true,"isCurrentBest":true,"agent":{"model":"claude-fable","evidence":"AI system column: claude-fable (autonomous)","family":"anthropic","confidence":"high","role":"author"}}],"contested":false},{"group":"ramsey","description":"Best proved upper bound on the least N such that every 2-colouring of K_N has a monochromatic K_n.","id":"ramsey-upper","name":"Ramsey — upper bound","metricName":"upper bound","metricDirection":"minimize","isOpen":true,"records":[{"contributor":"Baseline (by SB team)","description":"Record V002","value":0,"contributorKind":"team","submissionUrls":["https://github.com/leansolving/SqueezeBench/blob/main/SqueezeBench/Problems/Ramsey/Records/Upper/V002"],"isBaseline":true,"isCurrentBest":true,"agent":{"model":"claude-fable","evidence":"AI system column: claude-fable (autonomous)","family":"anthropic","confidence":"high","role":"author"}}],"contested":false},{"group":"ramsey3","description":"Best proved lower bound on the least N forcing a red K₃ or a blue K_k.","id":"ramsey3-lower","name":"R(3,k) — lower bound","metricName":"lower bound","metricDirection":"maximize","isOpen":true,"records":[{"contributor":"Baseline (by SB team)","description":"Record V007","value":0,"contributorKind":"team","submissionUrls":["https://github.com/leansolving/SqueezeBench/blob/main/SqueezeBench/Problems/Ramsey3/Records/Lower/V007"],"isBaseline":true,"isCurrentBest":true,"agent":{"model":"claude-fable","evidence":"AI system column: claude-fable (autonomous)","family":"anthropic","confidence":"high","role":"author"}}],"contested":false},{"group":"ramsey3","description":"Best proved upper bound on the least N forcing a red K₃ or a blue K_k.","id":"ramsey3-upper","name":"R(3,k) — upper bound","metricName":"upper bound","metricDirection":"minimize","isOpen":true,"records":[{"contributor":"Baseline (by SB team)","description":"Record V004","value":0,"contributorKind":"team","submissionUrls":["https://github.com/leansolving/SqueezeBench/blob/main/SqueezeBench/Problems/Ramsey3/Records/Upper/V004"],"isBaseline":true,"isCurrentBest":true,"agent":{"model":"claude-fable","evidence":"AI system column: claude-fable (autonomous)","family":"anthropic","confidence":"high","role":"author"}}],"contested":false},{"group":"roth","description":"Best proved lower bound on the largest subset of {0,…,n−1} with no 3-term arithmetic progression.","id":"roth-lower","name":"Roth — lower bound","metricName":"lower bound","metricDirection":"maximize","isOpen":true,"records":[{"contributor":"Baseline (by SB team)","description":"Record V009","value":0,"contributorKind":"team","submissionUrls":["https://github.com/leansolving/SqueezeBench/blob/main/SqueezeBench/Problems/Roth/Records/Lower/V009"],"isBaseline":true,"isCurrentBest":true,"agent":{"model":"claude-fable","evidence":"AI system column: claude-fable (autonomous)","family":"anthropic","confidence":"high","role":"author"}}],"contested":false},{"group":"roth","description":"Best proved upper bound on the largest subset of {0,…,n−1} with no 3-term arithmetic progression.","id":"roth-upper","name":"Roth — upper bound","metricName":"upper bound","metricDirection":"minimize","isOpen":true,"records":[{"contributor":"Baseline (by SB team)","description":"Record V007","value":0,"contributorKind":"team","submissionUrls":["https://github.com/leansolving/SqueezeBench/blob/main/SqueezeBench/Problems/Roth/Records/Upper/V007"],"isBaseline":true,"isCurrentBest":true,"agent":{"model":"claude-fable","evidence":"AI system column: claude-fable (autonomous)","family":"anthropic","confidence":"high","role":"author"}}],"contested":false},{"group":"ruzsa-szemeredi","description":"Best proved upper bound on the most triangles in a locally linear graph on n vertices.","id":"ruzsa-szemeredi-upper","name":"Ruzsa–Szemerédi — upper bound","metricName":"upper bound","metricDirection":"minimize","isOpen":true,"records":[{"contributor":"Baseline (by SB team)","description":"Record V008","value":0,"contributorKind":"team","submissionUrls":["https://github.com/leansolving/SqueezeBench/blob/main/SqueezeBench/Problems/RuzsaSzemeredi/Records/Upper/V008"],"isBaseline":true,"isCurrentBest":true,"agent":{"model":"claude-fable","evidence":"AI system column: claude-fable (autonomous)","family":"anthropic","confidence":"high","role":"author"}}],"contested":false},{"group":"ruzsa-szemeredi","description":"Best proved lower bound on the most triangles in a locally linear graph on n vertices.","id":"ruzsa-szemeredi-lower","name":"Ruzsa–Szemerédi — lower bound","metricName":"lower bound","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"Pablo Manrique","description":"Record V009","value":0,"contributorKind":"person","submissionUrls":["https://github.com/leansolving/SqueezeBench/blob/main/SqueezeBench/Problems/RuzsaSzemeredi/Records/Lower/V009"],"isCurrentBest":true,"agent":{"model":"claude-opus-5.5","evidence":"AI system column: claude-opus-5.5 (autonomous)","family":"anthropic","confidence":"high","role":"author"}}],"contested":false},{"group":"schur","description":"Best proved lower bound on the largest N such that {1,…,N} splits into k sum-free parts.","id":"schur-lower","name":"Schur — lower bound","metricName":"lower bound","metricDirection":"maximize","isOpen":true,"records":[{"contributor":"Baseline (by SB team)","description":"Record V003","value":0,"contributorKind":"team","submissionUrls":["https://github.com/leansolving/SqueezeBench/blob/main/SqueezeBench/Problems/Schur/Records/Lower/V003"],"isBaseline":true,"isCurrentBest":true,"agent":{"model":"claude-fable","evidence":"AI system column: claude-fable (autonomous)","family":"anthropic","confidence":"high","role":"author"}}],"contested":false},{"group":"schur","description":"Best proved upper bound on the largest N such that {1,…,N} splits into k sum-free parts.","id":"schur-upper","name":"Schur — upper bound","metricName":"upper bound","metricDirection":"minimize","isOpen":true,"records":[{"contributor":"Baseline (by SB team)","description":"Record V003","value":0,"contributorKind":"team","submissionUrls":["https://github.com/leansolving/SqueezeBench/blob/main/SqueezeBench/Problems/Schur/Records/Upper/V003"],"isBaseline":true,"isCurrentBest":true,"agent":{"model":"claude-opus-5","evidence":"AI system column: claude-opus-5 (autonomous)","family":"anthropic","confidence":"high","role":"author"}}],"contested":false}],"agentStats":[{"family":"anthropic","records":1,"currentBests":0,"models":["claude-opus-5.5"],"contributors":["Pablo Manrique"],"uncontestedBests":1}],"repo":{"owner":"leansolving","name":"SqueezeBench","branch":"main"},"participants":[{"name":"Baseline (by SB team)","submissions":19,"problems":["capset-lower","capset-upper","exc4-lower","exc4-upper","exgirthfive-lower","exgirthfive-upper","golomb-lower","golomb-upper","hypercubec4-lower","hypercubec4-upper","ramsey-lower","ramsey-upper","ramsey3-lower","ramsey3-upper","roth-lower","roth-upper","ruzsa-szemeredi-upper","schur-lower","schur-upper"]},{"name":"Pablo Manrique","submissions":1,"problems":["ruzsa-szemeredi-lower"]}],"stats":{"totalRecords":20,"uniqueParticipants":2,"recordsLast30d":0,"recordsLast90d":0},"lastUpdated":"2026-10-19T09:02:20.239Z"},{"id":"s2n-bignum-bench","name":"s2n-bignum-bench","tagline":"Benchmark LLM low-level code reasoning by synthesizing HOL Light tactics for AWS s2n-bignum cryptographic proofs.","description":"s2n-bignum-bench evaluates large language models on their ability to generate HOL Light tactic proofs for cryptographic verification goals from the AWS s2n-bignum library. Participants are given a HOL Light goal and must produce a tactic chain that proves it; correctness is checked by running the proof in HOL Light, with Pass@K scoring supported. Maintained by researchers including Balaji Rao, John Harrison, Soonho Kong, Juneyoung Lee, and Carlo Lipizzi.","url":"https://github.com/kings-crown/s2n-bignum-bench","host":"github","status":"unknown","domain":"formal-methods","tags":["formal-methods","hol-light","tactic-synthesis","cryptography","llm-eval","proof-verification"],"images":[],"participation":{"howToSubmit":"Zip your answers CSV and submit via GitHub issue template at https://github.com/kings-crown/s2n-bignum-leaderboard/issues/new?template=leaderboard-submission.yml","submissionFormat":"CSV with columns problem_id, category, query, answer; answers must be HOL Light tactic proofs in answer.txt files per problem.","computeDetails":"HOL Light proof checking is CPU-bound; each proof can use ~5 GB RAM. GPU+CUDA only if hosting a local LLM to generate answers.","deadline":"rolling","cost":"free","verification":"Proofs are compiled and run in HOL Light; verdicts recorded as OK, FAIL, CHEATING, or ERROR.","requirements":["Linux","bash","git","make","gcc","opam","Python 3.9+","HOL Light","s2n-bignum","~20 GB disk space"],"compute":"cpu"},"links":[{"label":"Paper","url":"https://arxiv.org/pdf/2603.14628"},{"label":"Leaderboard","url":"https://kings-crown.github.io/s2n-bignum-leaderboard/"},{"label":"Dataset on Hugging Face","url":"https://huggingface.co/datasets/kings-crown/s2n-bignum-bench"},{"label":"GUIDE.md","url":"https://github.com/kings-crown/s2n-bignum-bench/blob/main/GUIDE.md"}],"organizer":{"name":"s2n-bignum-bench maintainers","url":"https://github.com/kings-crown","type":"community"},"problems":[{"description":"Given a HOL Light goal from s2n-bignum cryptographic proofs, generate a tactic proof that proves the goal. Scored by pass rate (correct proofs / total problems).","metricUnit":"%","id":"tactic-synthesis","name":"HOL Light Tactic Synthesis","metricName":"pass rate","metricDirection":"maximize","isOpen":true,"records":[],"contested":false}],"agentStats":[],"repo":{"owner":"kings-crown","name":"s2n-bignum-bench","branch":"main"},"participants":[],"stats":{"totalRecords":0,"uniqueParticipants":0,"recordsLast30d":0,"recordsLast90d":0},"lastUpdated":"2026-10-19T09:11:20.116Z"},{"id":"agent-anvil-leaderboard","name":"Agent Anvil Public Leaderboard","tagline":"Score agent tool-use safety with trace-aware assertions instead of final-answer-only checks.","description":"Agent Anvil is a compact offline benchmark that compares final-answer-only grading with trace-aware assertions across tool-use safety scenarios. Authors run the benchmark in their own repo or CI, export a leaderboard_submission.json, and open a PR to this repository, where CI validates provenance and rebuilds the aggregate index. Rows are labeled self_reported, github_actions (with verified run URL and artifact attestation), or maintainer_rerun. The headline metric is trace-aware pass rate, which exposes policy violations that answer-only checks miss.","url":"https://github.com/agent-axiom/agent-anvil-leaderboard","host":"github","status":"active","domain":"llm-eval","tags":["agent-eval","tool-use","safety","trace-eval","leaderboard","provenance"],"images":[],"participation":{"howToSubmit":"Run the benchmark in your repo, export leaderboard_submission.json with `anvil leaderboard export`, validate it, copy it to submissions/<agent-name>.json, and open a PR to main.","submissionFormat":"One JSON file per agent under submissions/ following schema agent-anvil.leaderboard.v1 (submitter, benchmark, metrics, artifacts, verification).","computeDetails":"Compact offline benchmark of 5 scenarios / 100 trials; no GPU required.","deadline":"rolling","prizes":"recognition only","eligibility":"Open to any agent author; submissions via public PR.","cost":"free","verification":"CI validates the JSON, verifies GitHub run URLs via the GitHub API, checks artifact attestations for github_actions rows, and runs a leaderboard audit (accept/review/reject). Maintainer reruns are stored separately.","requirements":["Python 3.10+","uv / uvx","Agent Anvil CLI (v0.2.53)","GitHub Actions (for github_actions trust level)"],"compute":"cpu"},"links":[{"label":"Hugging Face Space","url":"https://huggingface.co/spaces/ifif/agent-anvil-leaderboard"},{"label":"Hugging Face Dataset","url":"https://huggingface.co/datasets/ifif/agent-anvil-leaderboard-data"},{"label":"Demo agent repo","url":"https://github.com/agent-axiom/agent-anvil-demo-agent"},{"label":"Agent Anvil CLI","url":"https://github.com/agent-axiom/agent-anvil"},{"label":"Contributing guide","url":"https://github.com/agent-axiom/agent-anvil-leaderboard/blob/main/CONTRIBUTING.md"},{"label":"Submissions contract","url":"https://github.com/agent-axiom/agent-anvil-leaderboard/blob/main/submissions/README.md"}],"organizer":{"name":"agent-axiom","url":"https://github.com/agent-axiom","type":"community"},"venue":{"type":"online"},"quickstart":"uv run anvil paper reproduce\nuv run anvil leaderboard export docs/paper/results.json \\\n  --manifest experiments/paper.yaml \\\n  --out leaderboard_submission.json \\\n  --agent-name \"My Agent\" \\\n  --agent-version \"2026-05-22\" \\\n  --repo-url \"https://github.com/acme/my-agent\" \\\n  --commit-sha \"$(git rev-parse HEAD)\"\nuv run anvil leaderboard validate leaderboard_submission.json","problems":[{"description":"Trace-aware pass rate across 5 tool-use safety scenarios (100 trials), comparing full trace-aware assertions against final-answer-only grading.","metricUnit":"%","id":"agent_anvil_trace_eval_benchmark","name":"Agent Anvil Trace Eval Benchmark","metricName":"trace_aware_pass_rate","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"Agent Anvil Demo Agent","description":"Final-answer pass rate 100.0%, trace-aware pass rate 30.0%, 70 answer-only missed failures (policy violations) over 100 trials; trust level github_actions, attested.","date":"2026-05-29","value":30,"contributorKind":"model","contributorUrl":"https://github.com/agent-axiom/agent-anvil-demo-agent","submissionUrls":["https://github.com/agent-axiom/agent-anvil-leaderboard/blob/main/submissions/agent-anvil-demo-agent.json"],"isCurrentBest":true,"agent":{"model":"Agent Anvil Demo Agent","evidence":"Refresh Agent Anvil Demo Agent leaderboard submission ... Trace-aware pass rate: 30.0%","family":"other-ai","confidence":"high","role":"subject"}}],"contested":false}],"agentStats":[],"repo":{"owner":"agent-axiom","name":"agent-anvil-leaderboard","branch":"main"},"participants":[],"stats":{"totalRecords":1,"uniqueParticipants":0,"firstSubmission":"2026-05-29","lastSubmission":"2026-05-29","recordsLast30d":0,"recordsLast90d":0},"lastUpdated":"2026-10-19T09:02:20.236Z"},{"id":"programbench","name":"ProgramBench","tagline":"Score AI coding agents on real-world program-repair tasks; maximize mean per-instance score.","description":"ProgramBench evaluates AI models and agent harnesses on a suite of software-engineering instances, scoring each instance by test pass rate. Submissions are registered as PRs to this registry repo, each contributing a folder with a pointer.yaml, submission.yaml manifest, and per-instance _stats JSON files. The public leaderboard at programbench.com is compiled directly from merged entries, and submissions are verified in two tiers (Tier-0 recomputes the headline; Tier-1 re-runs the full evaluation).","url":"https://github.com/programbench/submissions","host":"github","status":"active","domain":"coding-agents","tags":["coding-agents","llm-eval","software-engineering","swe-bench","leaderboard"],"images":[],"participation":{"howToSubmit":"Open a PR to main adding a folder submissions/<submission_id>/ containing pointer.yaml, submission.yaml, and _stats/ (score.json required; cost.json and calls.json optional). Use `programbench submit register <your-run-dir>` to open the PR automatically.","submissionFormat":"submissions/<submission_id>/ with pointer.yaml (fork URL + pinned commit SHA), submission.yaml (model, provider, score headline), and _stats/score.json (flat {instance_id: value} map).","computeDetails":"Tier-0 verification requires no Docker; Tier-1 re-runs `programbench eval` on each submission.tar.gz.","deadline":"rolling","cost":"free","verification":"Tier-0: `uvx programbench submit verify .` recomputes headline from eval.json and checks against submission.yaml. Tier-1: `uvx programbench submit verify . --tier1` re-runs evaluation and confirms fresh scores match.","requirements":["Python","uvx / programbench CLI","Docker (for Tier-1 verification)","A public submission fork"],"compute":"unknown"},"links":[{"label":"Leaderboard","url":"https://programbench.com/"},{"label":"Submission guide","url":"https://programbench.com/blog/submission-guide"},{"label":"ProgramBench repo","url":"https://github.com/facebookresearch/ProgramBench"}],"organizer":{"name":"ProgramBench","url":"https://programbench.com/","type":"community"},"venue":{"type":"online"},"quickstart":"git clone <a-public-submission-fork> && cd <fork>\nuvx programbench submit verify .            # Tier-0: recompute headline vs manifest. No Docker.\nuvx programbench submit verify . --tier1    # Tier-1: re-run eval and confirm the artifacts reproduce it.","problems":[{"description":"Mean per-instance score across the ProgramBench Mini instance set, evaluated with the mini-SWE-agent harness.","metricUnit":"%","id":"programbench-mini","name":"ProgramBench Mini","metricName":"mean score","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"Gemini 3.7 Flash","description":"Google Gemini 3.7 Flash + mini-SWE-agent; 199 instances attempted; Tier-0 verified.","date":"2026-08-16","value":0,"contributorKind":"model","submissionUrls":["https://github.com/programbench/submissions/blob/main/submissions/20260815_mini-v2.4.2_gemini-3-7-flash/"],"agent":{"model":"Gemini 3.7 Flash","tool":"mini-SWE-agent","evidence":"Registers Gemini 3.7 Flash (Google) + mini-SWE-agent.","family":"google","confidence":"high","role":"subject"}},{"contributor":"GPT-5.6 Sol (xhigh)","description":"OpenAI GPT-5.6 Sol (xhigh) + mini-SWE-agent; 200 instances attempted; Tier-0 verified.","date":"2026-08-03","value":0,"contributorKind":"model","submissionUrls":["https://github.com/programbench/submissions/blob/main/submissions/20260802_mini-v2.4.2_gpt-5-6-sol-xhigh/"],"agent":{"model":"GPT-5.6 Sol","tool":"mini-SWE-agent","evidence":"Registers GPT-5.6 Sol (xhigh) (OpenAI) + mini-SWE-agent.","family":"openai","confidence":"high","role":"subject"}},{"contributor":"GPT-5.6 Sol","description":"OpenAI GPT-5.6 Sol + mini-SWE-agent; 197 instances attempted; Tier-0 verified.","date":"2026-08-03","value":0,"contributorKind":"model","submissionUrls":["https://github.com/programbench/submissions/blob/main/submissions/20260730_mini-v2.2.8_gpt-5-6-sol/"],"agent":{"model":"GPT-5.6 Sol","tool":"mini-SWE-agent","evidence":"Registers GPT-5.6 Sol (OpenAI) + mini-SWE-agent.","family":"openai","confidence":"high","role":"subject"}},{"contributor":"Claude Opus 5 (xhigh)","description":"Anthropic Claude Opus 5 (xhigh) + mini-SWE-agent; 198 instances attempted; Tier-0 verified.","date":"2026-08-03","value":0,"contributorKind":"model","submissionUrls":["https://github.com/programbench/submissions/blob/main/submissions/20260731_mini-v2.4.2_opus-5-xhigh/"],"agent":{"model":"Claude Opus 5","tool":"mini-SWE-agent","evidence":"Registers Claude Opus 5 (xhigh) (Anthropic) + mini-SWE-agent.","family":"anthropic","confidence":"high","role":"subject"}},{"contributor":"Gemini 3.6 Flash","description":"Google Gemini 3.6 Flash + mini-SWE-agent; 198 instances attempted; Tier-0 verified.","date":"2026-08-03","value":0,"contributorKind":"model","submissionUrls":["https://github.com/programbench/submissions/blob/main/submissions/20260731_mini-v2.4.2_gemini-3-6-flash/"],"agent":{"model":"Gemini 3.6 Flash","tool":"mini-SWE-agent","evidence":"Registers Gemini 3.6 Flash (Google) + mini-SWE-agent.","family":"google","confidence":"high","role":"subject"}},{"contributor":"Gemini 3.5 Flash","description":"Google Gemini 3.5 Flash + mini-SWE-agent; 198 instances attempted; Tier-0 verified.","date":"2026-08-03","value":0,"contributorKind":"model","submissionUrls":["https://github.com/programbench/submissions/blob/main/submissions/20260729_mini-v2.2.8_gemini-3-5-flash/"],"agent":{"model":"Gemini 3.5 Flash","tool":"mini-SWE-agent","evidence":"Registers Gemini 3.5 Flash (Google) + mini-SWE-agent.","family":"google","confidence":"high","role":"subject"}},{"contributor":"Claude Opus 4.8 (xhigh)","description":"Anthropic Claude Opus 4.8 (xhigh reasoning) + mini-SWE-agent; 200 instances attempted; Tier-0 verified.","date":"2026-07-21","value":0,"contributorKind":"model","submissionUrls":["https://github.com/programbench/submissions/blob/main/submissions/20260717_mini-v2.4.5_opus-4.8-xhigh/"],"agent":{"model":"Claude Opus 4.8","tool":"mini-SWE-agent","evidence":"Registers Claude Opus 4.8 (xhigh reasoning) (Anthropic) + mini-SWE-agent.","family":"anthropic","confidence":"high","role":"subject"}},{"contributor":"GLM-5.2","description":"Z.ai (Zhipu AI) GLM-5.2 + mini-SWE-agent; 199 instances attempted; Tier-0 verified.","date":"2026-07-21","value":0,"contributorKind":"model","submissionUrls":["https://github.com/programbench/submissions/blob/main/submissions/20260717_mini-v2.4.5_glm-5.2/"],"agent":{"model":"GLM-5.2","tool":"mini-SWE-agent","evidence":"Registers GLM-5.2 (Z.ai (Zhipu AI)) + mini-SWE-agent.","family":"zhipu","confidence":"high","role":"subject"}},{"contributor":"GPT-5.5 (xhigh)","description":"OpenAI GPT-5.5 (xhigh reasoning) + mini-SWE-agent; resolved 0.5%, near-resolved 13.5%; 198/200 instances; Tier-0 verified.","date":"2026-06-22","value":69.8,"contributorKind":"model","submissionUrls":["https://github.com/programbench/submissions/blob/main/submissions/20260508_mini-v2.2.6_gpt-5-5-xhigh/"],"agent":{"model":"GPT-5.5","tool":"mini-SWE-agent","evidence":"Registers GPT-5.5 (xhigh reasoning) (OpenAI) + mini-SWE-agent. - mean score: 69.8%","family":"openai","confidence":"high","role":"subject"},"isCurrentBest":true},{"contributor":"GPT-5.5 (high)","description":"OpenAI GPT-5.5 (high reasoning) + mini-SWE-agent; resolved 0.5%, near-resolved 5.0%; 196/200 instances; Tier-0 verified.","date":"2026-06-22","value":67,"contributorKind":"model","submissionUrls":["https://github.com/programbench/submissions/blob/main/submissions/20260507_mini-v2.2.6_gpt-5-5-high/"],"agent":{"model":"GPT-5.5","tool":"mini-SWE-agent","evidence":"Registers GPT-5.5 (high reasoning) (OpenAI) + mini-SWE-agent. - mean score: 67.0%","family":"openai","confidence":"high","role":"subject"}},{"contributor":"GPT-5.5","description":"OpenAI GPT-5.5 + mini-SWE-agent; resolved 0.0%, near-resolved 1.5%; 199/200 instances; Tier-0 verified.","date":"2026-06-22","value":56.8,"contributorKind":"model","submissionUrls":["https://github.com/programbench/submissions/blob/main/submissions/20260507_mini-v2.2.6_gpt-5-5/"],"agent":{"model":"GPT-5.5","tool":"mini-SWE-agent","evidence":"Registers GPT-5.5 (OpenAI) + mini-SWE-agent. - mean score: 56.8%","family":"openai","confidence":"high","role":"subject"}},{"contributor":"Claude Opus 4.6","description":"Anthropic Claude Opus 4.6 + mini-SWE-agent; resolved 0.0%, near-resolved 2.5%; 200/200 instances; Tier-0 verified.","date":"2026-06-22","value":52.1,"contributorKind":"model","submissionUrls":["https://github.com/programbench/submissions/blob/main/submissions/20260430_mini-v2.2.6_opus-4-6/"],"agent":{"model":"Claude Opus 4.6","tool":"mini-SWE-agent","evidence":"Registers Claude Opus 4.6 (Anthropic) + mini-SWE-agent. - mean score: 52.1%","family":"anthropic","confidence":"high","role":"subject"}},{"contributor":"Claude Sonnet 4.6","description":"Anthropic Claude Sonnet 4.6 + mini-SWE-agent; resolved 0.0%, near-resolved 1.5%; 192/200 instances; Tier-0 verified.","date":"2026-06-22","value":48.6,"contributorKind":"model","submissionUrls":["https://github.com/programbench/submissions/blob/main/submissions/20260429_mini-v2.2.6_sonnet-4-6/"],"agent":{"model":"Claude Sonnet 4.6","tool":"mini-SWE-agent","evidence":"Registers Claude Sonnet 4.6 (Anthropic) + mini-SWE-agent. - mean score: 48.6%","family":"anthropic","confidence":"high","role":"subject"}},{"contributor":"GPT-5.4","description":"OpenAI GPT-5.4 + mini-SWE-agent; resolved 0.0%, near-resolved 0.0%; 200/200 instances; Tier-0 verified.","date":"2026-06-22","value":38.7,"contributorKind":"model","submissionUrls":["https://github.com/programbench/submissions/blob/main/submissions/20260429_mini-v2.2.6_gpt-5-4/"],"agent":{"model":"GPT-5.4","tool":"mini-SWE-agent","evidence":"Registers GPT-5.4 (OpenAI) + mini-SWE-agent. - mean score: 38.7%","family":"openai","confidence":"high","role":"subject"}},{"contributor":"Gemini 3 Flash","description":"Google Gemini 3 Flash + mini-SWE-agent; resolved 0.0%, near-resolved 0.0%; 199/200 instances; Tier-0 verified.","date":"2026-06-22","value":32.6,"contributorKind":"model","submissionUrls":["https://github.com/programbench/submissions/blob/main/submissions/20260429_mini-v2.2.6_gemini-3-flash/"],"agent":{"model":"Gemini 3 Flash","tool":"mini-SWE-agent","evidence":"Registers Gemini 3 Flash (Google) + mini-SWE-agent. - mean score: 32.6%","family":"google","confidence":"high","role":"subject"}},{"contributor":"Claude Haiku 4.5","description":"Anthropic Claude Haiku 4.5 + mini-SWE-agent; resolved 0.0%, near-resolved 0.0%; 200/200 instances; Tier-0 verified.","date":"2026-06-22","value":30.4,"contributorKind":"model","submissionUrls":["https://github.com/programbench/submissions/blob/main/submissions/20260429_mini-v2.2.6_haiku-4-5/"],"agent":{"model":"Claude Haiku 4.5","tool":"mini-SWE-agent","evidence":"Registers Claude Haiku 4.5 (Anthropic) + mini-SWE-agent. - mean score: 30.4%","family":"anthropic","confidence":"high","role":"subject"}},{"contributor":"GPT-5.4 mini","description":"OpenAI GPT-5.4 mini + mini-SWE-agent; resolved 0.0%, near-resolved 0.0%; 199/200 instances; Tier-0 verified.","date":"2026-06-22","value":17.1,"contributorKind":"model","submissionUrls":["https://github.com/programbench/submissions/blob/main/submissions/20260429_mini-v2.2.6_gpt-5-4-mini/"],"agent":{"model":"GPT-5.4 mini","tool":"mini-SWE-agent","evidence":"Registers GPT-5.4 mini (OpenAI) + mini-SWE-agent. - mean score: 17.1%","family":"openai","confidence":"high","role":"subject"}},{"contributor":"GPT-5 mini","description":"OpenAI GPT-5 mini + mini-SWE-agent; resolved 0.0%, near-resolved 0.0%; 200/200 instances; Tier-0 verified.","date":"2026-06-22","value":16.2,"contributorKind":"model","submissionUrls":["https://github.com/programbench/submissions/blob/main/submissions/20260429_mini-v2.2.6_gpt-5-mini/"],"agent":{"model":"GPT-5 mini","tool":"mini-SWE-agent","evidence":"Registers GPT-5 mini (OpenAI) + mini-SWE-agent. - mean score: 16.2%","family":"openai","confidence":"high","role":"subject"}},{"contributor":"Claude Opus 4.7","description":"Anthropic Claude Opus 4.7 + mini-SWE-agent; Tier-0 verified.","date":"2026-06-22","value":0,"contributorKind":"model","submissionUrls":["https://github.com/programbench/submissions/blob/main/submissions/20260429_mini-v2.2.6_opus-4-7/"],"agent":{"model":"Claude Opus 4.7","tool":"mini-SWE-agent","evidence":"Registers Claude Opus 4.7 (Anthropic) + mini-SWE-agent.","family":"anthropic","confidence":"high","role":"subject"}},{"contributor":"Gemini 3.1 Pro","description":"Google Gemini 3.1 Pro + mini-SWE-agent; Tier-0 verified.","date":"2026-06-22","value":0,"contributorKind":"model","submissionUrls":["https://github.com/programbench/submissions/blob/main/submissions/20260429_mini-v2.2.6_gemini-3-1-pro/"],"agent":{"model":"Gemini 3.1 Pro","tool":"mini-SWE-agent","evidence":"Registers Gemini 3.1 Pro (Google) + mini-SWE-agent.","family":"google","confidence":"high","role":"subject"}}],"contested":true}],"agentStats":[],"repo":{"owner":"programbench","name":"submissions","branch":"main"},"participants":[],"stats":{"totalRecords":20,"uniqueParticipants":0,"firstSubmission":"2026-06-22","lastSubmission":"2026-08-16","recordsLast30d":0,"recordsLast90d":6},"lastUpdated":"2026-10-19T09:13:00.604Z"},{"id":"chi-bench","name":"CHI-Bench Leaderboard","tagline":"Maximize pass@1 on healthcare administration agent tasks across prior authorization, utilization management, and care management.","description":"CHI-Bench is a benchmark of 75 healthcare-administration tasks split into three domains: provider prior authorization (25), payer utilization management (25), and care management (25). Agents run each task once and are scored by pass@1, with per-domain and overall pass rates reported. Submissions are produced with the chi-bench producer tooling and merged into this repo as audit packets via pull request; the rendered leaderboard is hosted at actava.ai.","url":"https://github.com/actava-ai/leaderboard","host":"github","status":"active","domain":"llm-eval","tags":["healthcare","agents","pass@1","llm-eval","prior-authorization","care-management"],"images":[],"participation":{"howToSubmit":"Fork actava-ai/leaderboard, add your packet under benchmarks/chi-bench/submissions/<YYYY-MM-DD>-<slug>/ (one new directory per PR), then open a PR to main. Helper: python scripts/submit.py /path/to/packet/","submissionFormat":"A packet directory containing submission.json, results.csv, sub.yaml, provenance.json, README.md, and >=1 trial result (zstd-compressed JSONL trajectories)","computeDetails":"Agent runs are executed by the submitter; full-run costs reported range from ~$35 to ~$359 depending on model","deadline":"rolling","prizes":"recognition only","cost":"free","verification":"CI schema/integrity validation (.github/workflows/validate.yml) plus maintainer review","requirements":["Python 3.10+","uv","gh CLI","git","zstd"],"compute":"unknown"},"links":[{"label":"Live leaderboard","url":"https://actava.ai/benchmarks/leaderboards"},{"label":"Submit guide","url":"https://actava.ai/benchmarks/submit"},{"label":"CHI-Bench overview","url":"https://actava.ai/benchmarks/chi-bench"},{"label":"Producer repo (chi-bench)","url":"https://github.com/actava-ai/chi-bench"},{"label":"Submission packet contract","url":"https://github.com/actava-ai/chi-bench/blob/main/docs/submission-packet.md"},{"label":"Task explorer","url":"https://actava.ai/benchmarks/tasks"}],"organizer":{"name":"Actava AI","url":"https://actava.ai/benchmarks/chi-bench","type":"company"},"venue":{"type":"online"},"quickstart":"gh auth login\ngh repo fork actava-ai/leaderboard --clone=true\ncd leaderboard\nuv run cb submission prepare -f configs/submissions/<id>.yaml\npython scripts/submit.py /path/to/packet/2026-05-12-<slug>/","problems":[{"group":"chi-bench","description":"Overall pass@1 across all 75 CHI-Bench tasks (provider PA + payer UM + care management).","metricUnit":"%","id":"chi-bench-overall","name":"Overall","metricName":"pass@1","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"HG-Market-System","description":"Repaired v1.1.0 packet, 20/75","date":"2026-10-06","value":26.67,"contributorKind":"method","isCurrentBest":false,"agent":{"model":"Claude Opus 4.7","evidence":"one `claude-opus-4-7` vote, valid 0/5 rubrics","family":"anthropic","confidence":"medium","role":"author"}},{"contributor":"U2MedFellow v6","description":"U2Med team, openai/deepseek-v4-pro","date":"2026-09-23","value":37.3,"contributorKind":"model","isCurrentBest":false,"agent":{"model":"openai/deepseek-v4-pro","tool":"U2MedFellow v6","family":"deepseek","confidence":"high","role":"subject"}},{"contributor":"NVIDIA-Nemotron-3-Ultra-550B-A55B-BF16:peft:262144","description":"Actava, openai-agents harness, 0/75","date":"2026-08-12","value":0,"contributorKind":"model","isCurrentBest":false,"agent":{"model":"nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-BF16:peft:262144","tool":"openai-agents","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"claude-opus-5","description":"Actava, claude-code harness, 28/75","date":"2026-08-12","value":37.3,"contributorKind":"model","isCurrentBest":false,"agent":{"model":"anthropic/claude-opus-5","tool":"claude-code","family":"anthropic","confidence":"high","role":"subject"}},{"contributor":"kimi-k3","description":"Actava, openai-agents harness, 19/75","date":"2026-08-12","value":25.3,"contributorKind":"model","isCurrentBest":false,"agent":{"model":"moonshotai/kimi-k3","tool":"openai-agents","family":"moonshot","confidence":"high","role":"subject"}},{"contributor":"erius","description":"Michael Johnson (MJ), GAMPO on anthropic/claude-opus-5","date":"2026-07-27","value":54.7,"contributorKind":"method","isCurrentBest":true,"agent":{"model":"anthropic/claude-opus-5","tool":"erius","family":"anthropic","confidence":"high","role":"author"}},{"contributor":"Inkling:peft:262144","description":"Actava, openai-agents harness, 6/75","date":"2026-07-24","value":8,"contributorKind":"model","isCurrentBest":false,"agent":{"model":"thinkingmachines/Inkling:peft:262144","tool":"openai-agents","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"gpt-5.6-luna","description":"Actava, codex harness, 10/75","date":"2026-07-24","value":13.3,"contributorKind":"model","isCurrentBest":false,"agent":{"model":"openai/gpt-5.6-luna","tool":"codex","family":"openai","confidence":"high","role":"subject"}},{"contributor":"gpt-5.6-terra","description":"Actava, codex harness, 10/75","date":"2026-07-24","value":13.3,"contributorKind":"model","isCurrentBest":false,"agent":{"model":"openai/gpt-5.6-terra","tool":"codex","family":"openai","confidence":"high","role":"subject"}},{"contributor":"gpt-5.6-sol","description":"Actava, codex harness, 19/75","date":"2026-07-24","value":25.3,"contributorKind":"model","isCurrentBest":false,"agent":{"model":"openai/gpt-5.6-sol","tool":"codex","family":"openai","confidence":"high","role":"subject"}},{"contributor":"claude-fable-5","description":"Actava, claude-code harness, 18/75","date":"2026-07-22","value":24,"contributorKind":"model","isCurrentBest":false,"agent":{"model":"anthropic/claude-fable-5","tool":"claude-code","family":"anthropic","confidence":"high","role":"subject"}},{"contributor":"claude-sonnet-5","description":"Actava, claude-code harness, 15/75","date":"2026-07-06","value":20,"contributorKind":"model","isCurrentBest":false,"agent":{"model":"anthropic/claude-sonnet-5","tool":"claude-code","family":"anthropic","confidence":"high","role":"subject"}},{"contributor":"glm-5.2","description":"Actava, openai-agents harness, 14/75","date":"2026-07-06","value":18.7,"contributorKind":"model","isCurrentBest":false,"agent":{"model":"z-ai/glm-5.2","tool":"openai-agents","family":"zhipu","confidence":"high","role":"subject"}},{"contributor":"MedGuard","description":"cuilinke, hermes harness, 17/75","date":"2026-07-06","value":22.7,"contributorKind":"model","isCurrentBest":false,"agent":{"model":"MedGuard","tool":"hermes","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"erius","description":"Michael Johnson (MJ), GAMPO on anthropic/claude-opus-4-8, corrected CM","date":"2026-06-05","value":37.3,"contributorKind":"method","isCurrentBest":false,"agent":{"model":"anthropic/claude-opus-4-8","tool":"erius","family":"anthropic","confidence":"high","role":"author"}},{"contributor":"erius","description":"Michael Johnson (MJ), GAMPO on anthropic/claude-opus-4-8, superseded","date":"2026-06-02","value":18.7,"contributorKind":"method","isCurrentBest":false,"agent":{"model":"anthropic/claude-opus-4-8","tool":"erius","family":"anthropic","confidence":"high","role":"author"}},{"contributor":"claude-opus-4-8","description":"Actava, claude-code harness, 25/75","date":"2026-05-28","value":33.3,"contributorKind":"model","isCurrentBest":false,"agent":{"model":"anthropic/claude-opus-4-8","tool":"claude-code","family":"anthropic","confidence":"high","role":"subject"}},{"contributor":"claude-opus-4-6","description":"My Team, claude-code harness, 21/75","date":"2026-05-12","value":28,"contributorKind":"model","isCurrentBest":false,"agent":{"model":"anthropic/claude-opus-4-6","tool":"claude-code","family":"anthropic","confidence":"high","role":"subject"}}],"contested":true},{"group":"chi-bench","description":"pass@1 on the 25 provider prior-authorization tasks.","metricUnit":"%","id":"chi-bench-pa-provider","name":"Provider Prior Authorization","metricName":"pass@1","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"HG-Market-System","description":"11/25","date":"2026-10-06","value":44,"contributorKind":"method","isCurrentBest":false,"agent":{"model":"Claude Opus 4.7","evidence":"one `claude-opus-4-7` vote, valid 0/5 rubrics","family":"anthropic","confidence":"medium","role":"author"}},{"contributor":"U2MedFellow v6","description":"U2Med, openai/deepseek-v4-pro","date":"2026-09-23","value":24,"contributorKind":"model","isCurrentBest":false,"agent":{"model":"openai/deepseek-v4-pro","tool":"U2MedFellow v6","family":"deepseek","confidence":"high","role":"subject"}},{"contributor":"NVIDIA-Nemotron-3-Ultra-550B-A55B-BF16:peft:262144","description":"Actava, openai-agents, 0/25","date":"2026-08-12","value":0,"contributorKind":"model","isCurrentBest":false,"agent":{"model":"nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-BF16:peft:262144","tool":"openai-agents","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"claude-opus-5","description":"Actava, claude-code, 5/25","date":"2026-08-12","value":20,"contributorKind":"model","isCurrentBest":false,"agent":{"model":"anthropic/claude-opus-5","tool":"claude-code","family":"anthropic","confidence":"high","role":"subject"}},{"contributor":"kimi-k3","description":"Actava, openai-agents, 7/25","date":"2026-08-12","value":28,"contributorKind":"model","isCurrentBest":false,"agent":{"model":"moonshotai/kimi-k3","tool":"openai-agents","family":"moonshot","confidence":"high","role":"subject"}},{"contributor":"erius","description":"Michael Johnson (MJ), GAMPO on claude-opus-5","date":"2026-07-27","value":72,"contributorKind":"method","isCurrentBest":true,"agent":{"model":"anthropic/claude-opus-5","tool":"erius","family":"anthropic","confidence":"high","role":"author"}},{"contributor":"Inkling:peft:262144","description":"Actava, openai-agents, 1/25","date":"2026-07-24","value":4,"contributorKind":"model","isCurrentBest":false,"agent":{"model":"thinkingmachines/Inkling:peft:262144","tool":"openai-agents","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"gpt-5.6-luna","description":"Actava, codex, 5/25","date":"2026-07-24","value":20,"contributorKind":"model","isCurrentBest":false,"agent":{"model":"openai/gpt-5.6-luna","tool":"codex","family":"openai","confidence":"high","role":"subject"}},{"contributor":"gpt-5.6-terra","description":"Actava, codex, 3/25","date":"2026-07-24","value":12,"contributorKind":"model","isCurrentBest":false,"agent":{"model":"openai/gpt-5.6-terra","tool":"codex","family":"openai","confidence":"high","role":"subject"}},{"contributor":"gpt-5.6-sol","description":"Actava, codex, 9/25","date":"2026-07-24","value":36,"contributorKind":"model","isCurrentBest":false,"agent":{"model":"openai/gpt-5.6-sol","tool":"codex","family":"openai","confidence":"high","role":"subject"}},{"contributor":"claude-fable-5","description":"Actava, claude-code, 6/25","date":"2026-07-22","value":24,"contributorKind":"model","isCurrentBest":false,"agent":{"model":"anthropic/claude-fable-5","tool":"claude-code","family":"anthropic","confidence":"high","role":"subject"}},{"contributor":"claude-sonnet-5","description":"Actava, claude-code, 6/25","date":"2026-07-06","value":24,"contributorKind":"model","isCurrentBest":false,"agent":{"model":"anthropic/claude-sonnet-5","tool":"claude-code","family":"anthropic","confidence":"high","role":"subject"}},{"contributor":"glm-5.2","description":"Actava, openai-agents, 5/25","date":"2026-07-06","value":20,"contributorKind":"model","isCurrentBest":false,"agent":{"model":"z-ai/glm-5.2","tool":"openai-agents","family":"zhipu","confidence":"high","role":"subject"}},{"contributor":"MedGuard","description":"cuilinke, hermes, 1/25","date":"2026-07-06","value":4,"contributorKind":"model","isCurrentBest":false,"agent":{"model":"MedGuard","tool":"hermes","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"MedArise","description":"PA Codex agent on openai/gpt-5.5, 17/25","date":"2026-06-05","value":68,"contributorKind":"team","isCurrentBest":false,"agent":{"model":"openai/gpt-5.5","tool":"codex","family":"openai","confidence":"high","role":"author"}},{"contributor":"erius","description":"Michael Johnson (MJ), GAMPO on claude-opus-4-8, corrected CM","date":"2026-06-05","value":40,"contributorKind":"method","isCurrentBest":false,"agent":{"model":"anthropic/claude-opus-4-8","tool":"erius","family":"anthropic","confidence":"high","role":"author"}},{"contributor":"erius","description":"Michael Johnson (MJ), GAMPO on claude-opus-4-8, superseded","date":"2026-06-02","value":40,"contributorKind":"method","isCurrentBest":false,"agent":{"model":"anthropic/claude-opus-4-8","tool":"erius","family":"anthropic","confidence":"high","role":"author"}},{"contributor":"claude-opus-4-8","description":"Actava, claude-code, 8/25","date":"2026-05-28","value":32,"contributorKind":"model","isCurrentBest":false,"agent":{"model":"anthropic/claude-opus-4-8","tool":"claude-code","family":"anthropic","confidence":"high","role":"subject"}},{"contributor":"claude-opus-4-6","description":"My Team, claude-code, 5/25","date":"2026-05-12","value":20,"contributorKind":"model","isCurrentBest":false,"agent":{"model":"anthropic/claude-opus-4-6","tool":"claude-code","family":"anthropic","confidence":"high","role":"subject"}}],"contested":true},{"group":"chi-bench","description":"pass@1 on the 25 payer utilization-management tasks.","metricUnit":"%","id":"chi-bench-pa-um","name":"Payer Utilization Management","metricName":"pass@1","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"HG-Market-System","description":"7/25","date":"2026-10-06","value":28,"contributorKind":"method","isCurrentBest":false,"agent":{"model":"Claude Opus 4.7","evidence":"one `claude-opus-4-7` vote, valid 0/5 rubrics","family":"anthropic","confidence":"medium","role":"author"}},{"contributor":"U2MedFellow v6","description":"U2Med, openai/deepseek-v4-pro","date":"2026-09-23","value":24,"contributorKind":"model","isCurrentBest":false,"agent":{"model":"openai/deepseek-v4-pro","tool":"U2MedFellow v6","family":"deepseek","confidence":"high","role":"subject"}},{"contributor":"NVIDIA-Nemotron-3-Ultra-550B-A55B-BF16:peft:262144","description":"Actava, openai-agents, 0/25","date":"2026-08-12","value":0,"contributorKind":"model","isCurrentBest":false,"agent":{"model":"nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-BF16:peft:262144","tool":"openai-agents","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"claude-opus-5","description":"Actava, claude-code, 8/25","date":"2026-08-12","value":32,"contributorKind":"model","isCurrentBest":false,"agent":{"model":"anthropic/claude-opus-5","tool":"claude-code","family":"anthropic","confidence":"high","role":"subject"}},{"contributor":"kimi-k3","description":"Actava, openai-agents, 8/25","date":"2026-08-12","value":32,"contributorKind":"model","isCurrentBest":false,"agent":{"model":"moonshotai/kimi-k3","tool":"openai-agents","family":"moonshot","confidence":"high","role":"subject"}},{"contributor":"erius","description":"Michael Johnson (MJ), GAMPO on claude-opus-5","date":"2026-07-27","value":36,"contributorKind":"method","isCurrentBest":true,"agent":{"model":"anthropic/claude-opus-5","tool":"erius","family":"anthropic","confidence":"high","role":"author"}},{"contributor":"Inkling:peft:262144","description":"Actava, openai-agents, 4/25","date":"2026-07-24","value":16,"contributorKind":"model","isCurrentBest":false,"agent":{"model":"thinkingmachines/Inkling:peft:262144","tool":"openai-agents","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"gpt-5.6-luna","description":"Actava, codex, 4/25","date":"2026-07-24","value":16,"contributorKind":"model","isCurrentBest":false,"agent":{"model":"openai/gpt-5.6-luna","tool":"codex","family":"openai","confidence":"high","role":"subject"}},{"contributor":"gpt-5.6-terra","description":"Actava, codex, 5/25","date":"2026-07-24","value":20,"contributorKind":"model","isCurrentBest":false,"agent":{"model":"openai/gpt-5.6-terra","tool":"codex","family":"openai","confidence":"high","role":"subject"}},{"contributor":"gpt-5.6-sol","description":"Actava, codex, 7/25","date":"2026-07-24","value":28,"contributorKind":"model","isCurrentBest":false,"agent":{"model":"openai/gpt-5.6-sol","tool":"codex","family":"openai","confidence":"high","role":"subject"}},{"contributor":"claude-fable-5","description":"Actava, claude-code, 6/25","date":"2026-07-22","value":24,"contributorKind":"model","isCurrentBest":false,"agent":{"model":"anthropic/claude-fable-5","tool":"claude-code","family":"anthropic","confidence":"high","role":"subject"}},{"contributor":"claude-sonnet-5","description":"Actava, claude-code, 6/25","date":"2026-07-06","value":24,"contributorKind":"model","isCurrentBest":false,"agent":{"model":"anthropic/claude-sonnet-5","tool":"claude-code","family":"anthropic","confidence":"high","role":"subject"}},{"contributor":"glm-5.2","description":"Actava, openai-agents, 8/25","date":"2026-07-06","value":32,"contributorKind":"model","isCurrentBest":false,"agent":{"model":"z-ai/glm-5.2","tool":"openai-agents","family":"zhipu","confidence":"high","role":"subject"}},{"contributor":"MedGuard","description":"cuilinke, hermes, 1/25","date":"2026-07-06","value":4,"contributorKind":"model","isCurrentBest":false,"agent":{"model":"MedGuard","tool":"hermes","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"erius","description":"Michael Johnson (MJ), GAMPO on claude-opus-4-8, corrected CM","date":"2026-06-05","value":16,"contributorKind":"method","isCurrentBest":false,"agent":{"model":"anthropic/claude-opus-4-8","tool":"erius","family":"anthropic","confidence":"high","role":"author"}},{"contributor":"erius","description":"Michael Johnson (MJ), GAMPO on claude-opus-4-8, superseded","date":"2026-06-02","value":16,"contributorKind":"method","isCurrentBest":false,"agent":{"model":"anthropic/claude-opus-4-8","tool":"erius","family":"anthropic","confidence":"high","role":"author"}},{"contributor":"claude-opus-4-8","description":"Actava, claude-code, 7/25","date":"2026-05-28","value":28,"contributorKind":"model","isCurrentBest":false,"agent":{"model":"anthropic/claude-opus-4-8","tool":"claude-code","family":"anthropic","confidence":"high","role":"subject"}},{"contributor":"claude-opus-4-6","description":"My Team, claude-code, 9/25","date":"2026-05-12","value":36,"contributorKind":"model","isCurrentBest":false,"agent":{"model":"anthropic/claude-opus-4-6","tool":"claude-code","family":"anthropic","confidence":"high","role":"subject"}}],"contested":true},{"group":"chi-bench","description":"pass@1 on the 25 care-management tasks.","metricUnit":"%","id":"chi-bench-cm","name":"Care Management","metricName":"pass@1","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"HG-Market-System","description":"2/25","date":"2026-10-06","value":8,"contributorKind":"method","isCurrentBest":false,"agent":{"model":"Claude Opus 4.7","evidence":"one `claude-opus-4-7` vote, valid 0/5 rubrics","family":"anthropic","confidence":"medium","role":"author"}},{"contributor":"U2MedFellow v6","description":"U2Med, openai/deepseek-v4-pro","date":"2026-09-23","value":24,"contributorKind":"model","isCurrentBest":false,"agent":{"model":"openai/deepseek-v4-pro","tool":"U2MedFellow v6","family":"deepseek","confidence":"high","role":"subject"}},{"contributor":"NVIDIA-Nemotron-3-Ultra-550B-A55B-BF16:peft:262144","description":"Actava, openai-agents, 0/25","date":"2026-08-12","value":0,"contributorKind":"model","isCurrentBest":false,"agent":{"model":"nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-BF16:peft:262144","tool":"openai-agents","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"claude-opus-5","description":"Actava, claude-code, 15/25","date":"2026-08-12","value":60,"contributorKind":"model","isCurrentBest":false,"agent":{"model":"anthropic/claude-opus-5","tool":"claude-code","family":"anthropic","confidence":"high","role":"subject"}},{"contributor":"kimi-k3","description":"Actava, openai-agents, 4/25","date":"2026-08-12","value":16,"contributorKind":"model","isCurrentBest":false,"agent":{"model":"moonshotai/kimi-k3","tool":"openai-agents","family":"moonshot","confidence":"high","role":"subject"}},{"contributor":"erius","description":"Michael Johnson (MJ), GAMPO on claude-opus-5","date":"2026-07-27","value":56,"contributorKind":"method","isCurrentBest":false,"agent":{"model":"anthropic/claude-opus-5","tool":"erius","family":"anthropic","confidence":"high","role":"author"}},{"contributor":"Inkling:peft:262144","description":"Actava, openai-agents, 1/25","date":"2026-07-24","value":4,"contributorKind":"model","isCurrentBest":false,"agent":{"model":"thinkingmachines/Inkling:peft:262144","tool":"openai-agents","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"gpt-5.6-luna","description":"Actava, codex, 1/25","date":"2026-07-24","value":4,"contributorKind":"model","isCurrentBest":false,"agent":{"model":"openai/gpt-5.6-luna","tool":"codex","family":"openai","confidence":"high","role":"subject"}},{"contributor":"gpt-5.6-terra","description":"Actava, codex, 2/25","date":"2026-07-24","value":8,"contributorKind":"model","isCurrentBest":false,"agent":{"model":"openai/gpt-5.6-terra","tool":"codex","family":"openai","confidence":"high","role":"subject"}},{"contributor":"gpt-5.6-sol","description":"Actava, codex, 3/25","date":"2026-07-24","value":12,"contributorKind":"model","isCurrentBest":false,"agent":{"model":"openai/gpt-5.6-sol","tool":"codex","family":"openai","confidence":"high","role":"subject"}},{"contributor":"claude-fable-5","description":"Actava, claude-code, 6/25","date":"2026-07-22","value":24,"contributorKind":"model","isCurrentBest":false,"agent":{"model":"anthropic/claude-fable-5","tool":"claude-code","family":"anthropic","confidence":"high","role":"subject"}},{"contributor":"claude-sonnet-5","description":"Actava, claude-code, 3/25","date":"2026-07-06","value":12,"contributorKind":"model","isCurrentBest":false,"agent":{"model":"anthropic/claude-sonnet-5","tool":"claude-code","family":"anthropic","confidence":"high","role":"subject"}},{"contributor":"glm-5.2","description":"Actava, openai-agents, 1/25","date":"2026-07-06","value":4,"contributorKind":"model","isCurrentBest":false,"agent":{"model":"z-ai/glm-5.2","tool":"openai-agents","family":"zhipu","confidence":"high","role":"subject"}},{"contributor":"MedGuard","description":"cuilinke, hermes, 15/25","date":"2026-07-06","value":60,"contributorKind":"model","isCurrentBest":true,"agent":{"model":"MedGuard","tool":"hermes","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"erius","description":"Michael Johnson (MJ), GAMPO on claude-opus-4-8, corrected CM","date":"2026-06-05","value":56,"contributorKind":"method","isCurrentBest":false,"agent":{"model":"anthropic/claude-opus-4-8","tool":"erius","family":"anthropic","confidence":"high","role":"author"}},{"contributor":"erius","description":"Michael Johnson (MJ), GAMPO on claude-opus-4-8, superseded (simulator artifact)","date":"2026-06-02","value":0,"contributorKind":"method","isCurrentBest":false,"agent":{"model":"anthropic/claude-opus-4-8","tool":"erius","family":"anthropic","confidence":"high","role":"author"}},{"contributor":"claude-opus-4-8","description":"Actava, claude-code, 10/25","date":"2026-05-28","value":40,"contributorKind":"model","isCurrentBest":false,"agent":{"model":"anthropic/claude-opus-4-8","tool":"claude-code","family":"anthropic","confidence":"high","role":"subject"}},{"contributor":"claude-opus-4-6","description":"My Team, claude-code, 7/25","date":"2026-05-12","value":28,"contributorKind":"model","isCurrentBest":false,"agent":{"model":"anthropic/claude-opus-4-6","tool":"claude-code","family":"anthropic","confidence":"high","role":"subject"}}],"contested":true}],"agentStats":[{"family":"anthropic","records":16,"currentBests":3,"models":["Claude Opus 4.7","anthropic/claude-opus-5","anthropic/claude-opus-4-8"],"contributors":["HG-Market-System","erius"]},{"family":"openai","records":1,"currentBests":0,"models":["openai/gpt-5.5"],"contributors":["MedArise"]}],"repo":{"owner":"actava-ai","name":"leaderboard","branch":"main"},"participants":[{"name":"MedArise","submissions":1,"problems":["chi-bench-pa-provider"],"lastActive":"2026-06-05"}],"stats":{"totalRecords":73,"uniqueParticipants":1,"firstSubmission":"2026-05-12","lastSubmission":"2026-10-06","recordsLast30d":8,"recordsLast90d":44},"lastUpdated":"2026-10-19T09:13:40.944Z"},{"id":"geo-bench-2-leaderboard","name":"GEO-Bench 2 Leaderboard","tagline":"Benchmark geospatial foundation models across GEO-Bench datasets and metrics.","description":"The GEO-Bench 2 Leaderboard tracks performance of geospatial foundation models on various benchmark datasets using the GEO-Bench benchmarking framework. Submissions consist of a results CSV and an additional_info.json detailing backbone, hyperparameters, and metrics. Results are aggregated and displayed on a Hugging Face Space leaderboard. It matters because it provides a standardized comparison of geospatial foundation models for Earth observation tasks.","url":"https://github.com/The-AI-Alliance/GEO-Bench-2-Leaderboard","host":"github","status":"active","domain":"ml-research","tags":["geospatial","remote-sensing","foundation-models","benchmark","earth-observation"],"images":[],"participation":{"howToSubmit":"Fork the repository, add results to new_submission/results_and_parameters.csv and new_submission/additional_info.json, then open a pull request to main.","submissionFormat":"A CSV file (results_and_parameters.csv) with columns: backbone, dataset, Metric, experiment_name, batch_size_selection, early_stop_patience, n_trials, Seed, batch_size, weight_decay, lr, test metric; and a JSON file (additional_info.json) with submission details.","computeDetails":"Hyperparameter optimization and repeated experiments; compute details depend on backbone and dataset.","deadline":"rolling","eligibility":"Open to all","cost":"free","verification":"Human review of pull requests","requirements":["Python 3.10+","TerraTorch","TerraTorch-Iterate"],"compute":"unknown"},"links":[{"label":"Leaderboard Space","url":"https://huggingface.co/spaces/aialliance/GEO-Bench-2-Leaderboard"},{"label":"TerraTorch-Iterate","url":"https://github.com/IBM/terratorch-iterate"},{"label":"TerraTorch","url":"https://github.com/IBM/terratorch"}],"organizer":{"name":"The AI Alliance","url":"https://thealliance.ai/","type":"community"},"venue":{"type":"online"},"quickstart":"terratorch iterate --hpo --repeat --config <config-file>\nterratorch iterate --summarize --config <summarize-config-file>","problems":[{"group":"biomassters-rmse","description":"Models are evaluated on RMSE for the Biomassters dataset, with de-normalized scores as per the scale_rmse function.","id":"biomassters-rmse","name":"Biomassters RMSE Track","metricName":"RMSE","metricDirection":"minimize","isOpen":true,"records":[],"contested":false},{"group":"multiclass-jaccard","description":"Models are evaluated on multiclass Jaccard index across GEO-Bench datasets using this metric.","id":"multiclass-jaccard","name":"Multiclass Jaccard Index Track","metricName":"Multiclass Jaccard Index","metricDirection":"maximize","isOpen":true,"records":[],"contested":false},{"group":"multilabel-f1","description":"Models are evaluated on multilabel F1 score across GEO-Bench datasets using this metric.","id":"multilabel-f1","name":"Multilabel F1 Score Track","metricName":"Multilabel F1 Score","metricDirection":"maximize","isOpen":true,"records":[],"contested":false},{"group":"overall-accuracy","description":"Models are evaluated on overall accuracy across GEO-Bench datasets using this metric.","metricUnit":"%","id":"overall-accuracy","name":"Overall Accuracy Track","metricName":"Overall Accuracy","metricDirection":"maximize","isOpen":true,"records":[],"contested":false}],"agentStats":[],"repo":{"owner":"The-AI-Alliance","name":"GEO-Bench-2-Leaderboard","branch":"main"},"participants":[],"stats":{"totalRecords":0,"uniqueParticipants":0,"recordsLast30d":0,"recordsLast90d":0},"lastUpdated":"2026-10-19T09:13:49.590Z"},{"id":"modded-nanogpt-npu","name":"Modded-NanoGPT · Ascend NPU Leaderboard","tagline":"Port and optimize NanoGPT speedrun records to run fastest on 8× Ascend 910C NPUs.","description":"A fork of KellerJordan/modded-nanogpt that reproduces and optimizes upstream GPU speedrun records on eight Ascend 910C NPUs (16 dies). Each accepted record must reach ≤3.28 mean FineWeb validation loss and beat the prior NPU time, with auditable source, launch commands, logs and metrics under records/track_1_short_npu/. The repo also preserves the upstream GPU leaderboards for reference.","url":"https://github.com/zhaoyiyidan/modded-nanogpt-npu","host":"github","status":"active","domain":"systems-perf","tags":["nanogpt","speedrun","ascend-npu","training-efficiency","llm-training","leaderboard"],"images":[],"participation":{"howToSubmit":"Open a PR adding a record folder under records/track_1_short_npu/<record>/ with source/ and logs/ (successful .train.log files), following the upstream submission rules.","submissionFormat":"Record folder containing source code, launch command, environment metadata, and successful training logs; multiple seeds averaged.","computeDetails":"Eight Ascend 910C NPUs (16 dies, 16 processes) via torchrun --nproc_per_node=16; reference image docker.cnb.cool/nilpotenter/docker/codeserver-mindspeed:v1.0.5.","deadline":"rolling","verification":"Human review of PR plus successful run logs; records must beat prior NPU time and attain ≤3.28 mean val loss with p<0.01.","requirements":["Python 3.10","PyTorch 2.7.1","torch_npu 2.7.1.post2","Docker","Ascend 910C NPU access"],"compute":"cluster"},"links":[{"label":"Upstream modded-nanogpt","url":"https://github.com/KellerJordan/modded-nanogpt"},{"label":"NPU_optimal_reference_skills","url":"https://github.com/YChen-zzz/NPU_optimal_reference_skills"}],"organizer":{"name":"zhaoyiyidan","url":"https://github.com/zhaoyiyidan","type":"individual"},"venue":{"type":"online"},"quickstart":"git clone https://github.com/zhaoyiyidan/modded-nanogpt-npu.git && cd modded-nanogpt-npu\n# use the reference Ascend image\ndocker build -t modded-nanogpt-npu .\ndocker run -it --rm -v $(pwd):/modded-nanogpt-npu modded-nanogpt-npu bash\n# inside container\ntorchrun --standalone --nproc_per_node=16 train_gpt.py","problems":[{"group":"track-1-short-npu","description":"Fastest training algorithm to reach ≤3.28 FineWeb validation loss on eight Ascend 910C NPUs (16 dies).","metricUnit":"minutes","id":"track-1-short-npu","name":"Track 1 Short (16x Ascend 910C NPU)","metricName":"time","metricDirection":"minimize","baseline":104.72,"isOpen":false,"records":[{"contributor":"llm.c baseline","description":"baseline/reference port, no specific speed-up trick.","value":104.72,"contributorKind":"method","submissionUrls":["https://github.com/zhaoyiyidan/modded-nanogpt-npu/blob/master/records/track_1_short_npu/record_001_llmc_baseline/logs/seed_007.train.log"],"isBaseline":true},{"contributor":"Tuned learning rate and rotary embeddings","value":61.6,"contributorKind":"method","submissionUrls":["https://github.com/zhaoyiyidan/modded-nanogpt-npu/blob/master/records/track_1_short_npu/record_002_adamw_rotary_lr/logs/seed_000.train.log"]},{"contributor":"Pad embeddings, ReLU2, zero-init projections, QK-norm","value":42.18,"contributorKind":"method","submissionUrls":["https://github.com/zhaoyiyidan/modded-nanogpt-npu/blob/master/records/track_1_short_npu/record_005_modern_arch/logs/seed_000.train.log"]},{"contributor":"Muon optimizer improvements","value":40.43,"contributorKind":"method","submissionUrls":["https://github.com/zhaoyiyidan/modded-nanogpt-npu/blob/master/records/track_1_short_npu/record_004_muon_improvements/logs/seed_000.train.log"]},{"contributor":"Distributed Muon overhead","value":40.33,"contributorKind":"method","submissionUrls":["https://github.com/zhaoyiyidan/modded-nanogpt-npu/blob/master/records/track_1_short_npu/record_006_distributed_muon/logs/seed_000.train.log"]},{"contributor":"PyTorch 2.5 runtime upgrade","value":40.12,"contributorKind":"method","submissionUrls":["https://github.com/zhaoyiyidan/modded-nanogpt-npu/blob/master/records/track_1_short_npu/record_007_pytorch25/logs/seed_018.train.log"]},{"contributor":"Tuned LR & rotary embeddings [optimized]","value":38.95,"contributorKind":"method","submissionUrls":["https://github.com/zhaoyiyidan/modded-nanogpt-npu/blob/master/records/track_1_short_npu/record_002_adamw_rotary_lr_optimized/logs/migration_20260927_214817.train.log"]},{"contributor":"Untied token embedding and LM head","value":36.6,"contributorKind":"method","submissionUrls":["https://github.com/zhaoyiyidan/modded-nanogpt-npu/blob/master/records/track_1_short_npu/record_008_untie_embed/logs/seed_000.train.log"]},{"contributor":"Cast activations to bfloat16","value":33.96,"contributorKind":"method","submissionUrls":["https://github.com/zhaoyiyidan/modded-nanogpt-npu/blob/master/records/track_1_short_npu/record_010_cast_bf16/logs/seed_000.train.log"]},{"contributor":"Value/embedding skip connections, momentum warmup, logit softcap","value":31.83,"contributorKind":"method","submissionUrls":["https://github.com/zhaoyiyidan/modded-nanogpt-npu/blob/master/records/track_1_short_npu/record_009_shortcuts_tweaks/logs/seed_000.train.log"]},{"contributor":"Muon improvements [optimized]","value":31.66,"contributorKind":"method","submissionUrls":["https://github.com/zhaoyiyidan/modded-nanogpt-npu/blob/master/records/track_1_short_npu/record_004_muon_improvements_optimized/logs/verify_migrate_20260929_001.log"]},{"contributor":"Reduced batch size","value":29.56,"contributorKind":"method","submissionUrls":["https://github.com/zhaoyiyidan/modded-nanogpt-npu/blob/master/records/track_1_short_npu/record_021_batch_size/logs/seed_006.train.log"]},{"contributor":"Modern arch [optimized]","value":22.61,"contributorKind":"method","submissionUrls":["https://github.com/zhaoyiyidan/modded-nanogpt-npu/blob/master/records/track_1_short_npu/record_005_modern_arch_optimized/logs/full_verify_20260928_record005_001.log"]},{"contributor":"PyTorch 2.5 [optimized]","value":21.87,"contributorKind":"method","submissionUrls":["https://github.com/zhaoyiyidan/modded-nanogpt-npu/blob/master/records/track_1_short_npu/record_007_pytorch25_optimized/logs/verify_20260928_100403_772149.log"]},{"contributor":"Distributed Muon [optimized]","value":21.58,"contributorKind":"method","submissionUrls":["https://github.com/zhaoyiyidan/modded-nanogpt-npu/blob/master/records/track_1_short_npu/record_006_distributed_muon_optimized/logs/verification_20260928.log"]},{"contributor":"Untied embed [optimized]","value":20.5,"contributorKind":"method","submissionUrls":["https://github.com/zhaoyiyidan/modded-nanogpt-npu/blob/master/records/track_1_short_npu/record_008_untie_embed_optimized/logs/verify_20260927_2322.console.log"]},{"contributor":"Sparsified value embeddings, improved rotary embeddings, dropped one attention layer","value":19.72,"contributorKind":"method","submissionUrls":["https://github.com/zhaoyiyidan/modded-nanogpt-npu/blob/master/records/track_1_short_npu/record_017_sparsify_embeds/logs/seed_003.train.log"]},{"contributor":"Shortcuts & tweaks [optimized]","value":17.52,"contributorKind":"method","submissionUrls":["https://github.com/zhaoyiyidan/modded-nanogpt-npu/blob/master/records/track_1_short_npu/record_009_shortcuts_tweaks_optimized/logs/migrate_record009_20260928_verify5.txt"]},{"contributor":"Bfloat16 cast [optimized]","value":16.03,"contributorKind":"method","submissionUrls":["https://github.com/zhaoyiyidan/modded-nanogpt-npu/blob/master/records/track_1_short_npu/record_010_cast_bf16_optimized/logs/record010_migration_verify_20260927_01.log"]},{"contributor":"Transposed one MLP matrix and added a symmetric matmul kernel","value":12.58,"contributorKind":"method","submissionUrls":["https://github.com/zhaoyiyidan/modded-nanogpt-npu/blob/master/records/track_1_short_npu/record_027_triton_muon/logs/seed_001.train.log"]},{"contributor":"Dropped the first MLP layer","value":11.65,"contributorKind":"method","submissionUrls":["https://github.com/zhaoyiyidan/modded-nanogpt-npu/blob/master/records/track_1_short_npu/record_030_skip_mlp_blocks/logs/seed_003.train.log"]},{"contributor":"Optimized distributed training, improved skip-connection gating, enhanced bfloat16 usage","value":11.06,"contributorKind":"method","submissionUrls":["https://github.com/zhaoyiyidan/modded-nanogpt-npu/blob/master/records/track_1_short_npu/record_032_vect_sigmoid_bfloat16/logs/seed_001.train.log"]},{"contributor":"Smeared token embeddings one position forward","value":11.05,"contributorKind":"method","submissionUrls":["https://github.com/zhaoyiyidan/modded-nanogpt-npu/blob/master/records/track_1_short_npu/record_034_smear/logs/seed_006.train.log"]},{"contributor":"Asynchronous data fetch/indexing and final-layer validation attention window extension","value":10.92,"contributorKind":"method","submissionUrls":["https://github.com/zhaoyiyidan/modded-nanogpt-npu/blob/master/records/track_1_short_npu/record_033_async_data_attn_final_window/logs/seed_100.train.log"]},{"contributor":"Dropped first attention layer, extended long validation windows, updated schedule","value":10.77,"contributorKind":"method","submissionUrls":["https://github.com/zhaoyiyidan/modded-nanogpt-npu/blob/master/records/track_1_short_npu/record_035_drop_attn/logs/seed_000.train.log"]},{"contributor":"Polar Express replacement for Newton-Schulz","value":10.76,"contributorKind":"method","submissionUrls":["https://github.com/zhaoyiyidan/modded-nanogpt-npu/blob/master/records/track_1_short_npu/record_038_polar_express/logs/seed_000.train.log"]},{"contributor":"Updated Adam params every other step and reduced batch size","value":10.53,"contributorKind":"method","submissionUrls":["https://github.com/zhaoyiyidan/modded-nanogpt-npu/blob/master/records/track_1_short_npu/record_039_custom_batching/logs/seed_000.train.log"]},{"contributor":"Backout changes, hyperparameter tuning, lambda padding optimization","value":10.16,"contributorKind":"method","submissionUrls":["https://github.com/zhaoyiyidan/modded-nanogpt-npu/blob/master/records/track_1_short_npu/record_040_backout/logs/seed_006.train.log"]},{"contributor":"Updated NorMuon learning rate and step logic","value":9.98,"contributorKind":"method","submissionUrls":["https://github.com/zhaoyiyidan/modded-nanogpt-npu/blob/master/records/track_1_short_npu/record_042_fix_muon_lr/logs/seed_001.train.log"]},{"contributor":"NorMuon optimizer","value":9.69,"contributorKind":"method","submissionUrls":["https://github.com/zhaoyiyidan/modded-nanogpt-npu/blob/master/records/track_1_short_npu/record_041_normuon/logs/seed_006v5.train.log"],"agent":{"evidence":"New WR (-15 steps/-0.7s): Implement NorMuon on latest version (#144) Co-authored-by: Zichong Li","family":"human","confidence":"medium","role":"author"}},{"contributor":"Refined skip architecture, updated exponential decay init","value":9.65,"contributorKind":"method","submissionUrls":["https://github.com/zhaoyiyidan/modded-nanogpt-npu/blob/master/records/track_1_short_npu/record_045_refine_skip/logs/seed_001.train.log"]},{"contributor":"Batch size schedule","value":9.57,"contributorKind":"method","submissionUrls":["https://github.com/zhaoyiyidan/modded-nanogpt-npu/blob/master/records/track_1_short_npu/record_046_batch_size_schedule/logs/seed_001.train.log"]},{"contributor":"Sped up Muon, pre-multiplied lambda, reshaped matrices, updated lr and NorMuon axis","value":9.56,"contributorKind":"method","submissionUrls":["https://github.com/zhaoyiyidan/modded-nanogpt-npu/blob/master/records/track_1_short_npu/record_048_normuon_optims_and_fixes/logs/seed_002.train.log"]},{"contributor":"Partial Key Offset","value":9.48,"contributorKind":"method","submissionUrls":["https://github.com/zhaoyiyidan/modded-nanogpt-npu/blob/master/records/track_1_short_npu/record_049_partial_key_offset/logs/seed_001.train.log"]},{"contributor":"Extended Cautious Weight Decay to Adam parameters","value":9.38,"contributorKind":"method","submissionUrls":["https://github.com/zhaoyiyidan/modded-nanogpt-npu/blob/master/records/track_1_short_npu/record_050_cautious_wd_adam/logs/seed_006.train.log"]},{"contributor":"Computed cross entropy in BF16 during training","value":9.34,"contributorKind":"method","submissionUrls":["https://github.com/zhaoyiyidan/modded-nanogpt-npu/blob/master/records/track_1_short_npu/record_037_bf16_ce/logs/seed_000.train.log"]},{"contributor":"Multiplied attention lambda with weight instead of data, fixed warmup","value":9.04,"contributorKind":"method","submissionUrls":["https://github.com/zhaoyiyidan/modded-nanogpt-npu/blob/master/records/track_1_short_npu/record_047_sa_lambda_on_weights/logs/seed_005.train.log"]},{"contributor":"Sparsified embeds [optimized]","value":7.63,"contributorKind":"method","submissionUrls":["https://github.com/zhaoyiyidan/modded-nanogpt-npu/blob/master/records/track_1_short_npu/record_017_sparsify_embeds_optimized/logs/migration_verify_20260928.console.log"]},{"contributor":"Triton Muon (NPU kernel) [optimized]","value":6.12,"contributorKind":"method","submissionUrls":["https://github.com/zhaoyiyidan/modded-nanogpt-npu/blob/master/records/track_1_short_npu/record_027_triton_muon_optimized/logs/full_train_20260928_record027_full_01.log"]},{"contributor":"Vect sigmoid bfloat16 [optimized]","value":5.69,"contributorKind":"method","submissionUrls":["https://github.com/zhaoyiyidan/modded-nanogpt-npu/blob/master/records/track_1_short_npu/record_032_vect_sigmoid_bfloat16_optimized/logs/migration_verify_20260927_194406.log"]},{"contributor":"Custom batching [optimized]","value":5.41,"contributorKind":"method","submissionUrls":["https://github.com/zhaoyiyidan/modded-nanogpt-npu/blob/master/records/track_1_short_npu/record_039_custom_batching_optimized/logs/migrated_20260928T034439Z_874982.log"]},{"contributor":"NorMuon [optimized]","value":5.22,"contributorKind":"method","submissionUrls":["https://github.com/zhaoyiyidan/modded-nanogpt-npu/blob/master/records/track_1_short_npu/record_041_normuon_optimized/logs/migration_full_20260927.log"]},{"contributor":"BF16 cross entropy [optimized]","value":5.21,"contributorKind":"method","submissionUrls":["https://github.com/zhaoyiyidan/modded-nanogpt-npu/blob/master/records/track_1_short_npu/record_037_bf16_ce_optimized/logs/migration_20260928_120304.log"]},{"contributor":"Polar Express [optimized]","value":5.2,"contributorKind":"method","submissionUrls":["https://github.com/zhaoyiyidan/modded-nanogpt-npu/blob/master/records/track_1_short_npu/record_038_polar_express_optimized/logs/migration_20260927_r68_full.driver.log"]},{"contributor":"Fix Muon LR [optimized]","value":4.97,"contributorKind":"method","submissionUrls":["https://github.com/zhaoyiyidan/modded-nanogpt-npu/blob/master/records/track_1_short_npu/record_042_fix_muon_lr_optimized/logs/verify_20260927_042.log"]},{"contributor":"Backout [optimized]","value":4.94,"contributorKind":"method","submissionUrls":["https://github.com/zhaoyiyidan/modded-nanogpt-npu/blob/master/records/track_1_short_npu/record_040_backout_optimized/logs/launcher.log"]},{"contributor":"SA lambda on weights [optimized]","value":4.63,"contributorKind":"method","submissionUrls":["https://github.com/zhaoyiyidan/modded-nanogpt-npu/blob/master/records/track_1_short_npu/record_047_sa_lambda_on_weights_optimized/logs/launcher.log"]},{"contributor":"Partial key offset [optimized]","value":4.63,"contributorKind":"method","submissionUrls":["https://github.com/zhaoyiyidan/modded-nanogpt-npu/blob/master/records/track_1_short_npu/record_049_partial_key_offset_optimized/logs/original_full_27252.log"]},{"contributor":"Cautious WD Adam: fused RMSNorm/rotary, custom Ascend C fused softcap CE, buffer reuse, bfloat16 weights, 2090-step schedule [optimized]","value":4.28,"contributorKind":"method","submissionUrls":["https://github.com/zhaoyiyidan/modded-nanogpt-npu/blob/master/records/track_1_short_npu/record_050_cautious_wd_adam_optimized/logs/805dac24-e75d-46d7-bf96-050f7077946c.txt"],"isCurrentBest":true}],"contested":true},{"group":"track-1-short-gpu","description":"Upstream GPU speedrun: fastest algorithm to reach ≤3.28 FineWeb validation loss on 8× NVIDIA H100 GPUs.","metricUnit":"minutes","id":"track-1-short-gpu","name":"Track 1 Short (8x NVIDIA H100 GPU)","metricName":"time","metricDirection":"minimize","baseline":45,"isOpen":false,"records":[{"contributor":"karpathy, llm.c contributors","description":"llm.c baseline","date":"2024-05-28","value":45,"contributorKind":"team","submissionUrls":["https://github.com/zhaoyiyidan/modded-nanogpt-npu/blob/master/records/track_1_short/2024-10-13_llmc/main.log"],"isBaseline":true},{"contributor":"kellerjordan0","description":"Tuned learning rate & rotary embeddings","date":"2024-06-06","value":31.4,"contributorKind":"person","contributorUrl":"https://github.com/kellerjordan0","submissionUrls":["https://github.com/zhaoyiyidan/modded-nanogpt-npu/blob/master/records/track_1_short/2024-06-06_AdamW/f66d43d7-e449-4029-8adf-e8537bab49ea.log"]},{"contributor":"kellerjordan0, jxbz","description":"Introduced the Muon optimizer","date":"2024-10-04","value":24.9,"contributorKind":"team"},{"contributor":"kellerjordan0, bozavlado","description":"Muon improvements","date":"2024-10-11","value":22.3,"contributorKind":"team","submissionUrls":["https://github.com/zhaoyiyidan/modded-nanogpt-npu/blob/master/records/track_1_short/2024-10-10_Muon/eb5659d0-fb6a-49e5-a311-f1f89412f726.txt"]},{"contributor":"Grad62304977, kellerjordan0","description":"Pad embeddings, ReLU², zero-init projections, QK-norm","date":"2024-10-14","value":15.2,"contributorKind":"team","submissionUrls":["https://github.com/zhaoyiyidan/modded-nanogpt-npu/blob/master/records/track_1_short/2024-10-14_ModernArch/dabaaddd-237c-4ec9-939d-6608a9ed5e27.txt"]},{"contributor":"kellerjordan0","description":"Distributed the overhead of Muon","date":"2024-10-18","value":13.1,"contributorKind":"person","contributorUrl":"https://github.com/kellerjordan0","submissionUrls":["https://github.com/zhaoyiyidan/modded-nanogpt-npu/blob/master/records/track_1_short/2024-10-17_DistributedMuon/22d24867-eb5a-4fcc-ae2c-263d0277dfd1.txt"]},{"contributor":"kellerjordan0","description":"Upgraded PyTorch 2.5.0","date":"2024-10-18","value":12,"contributorKind":"person","contributorUrl":"https://github.com/kellerjordan0","submissionUrls":["https://github.com/zhaoyiyidan/modded-nanogpt-npu/blob/master/records/track_1_short/2024-10-18_PyTorch25/d4bfb25f-688d-4da5-8743-33926fad4842.txt"]},{"contributor":"Grad62304977, kellerjordan0","description":"Untied embedding and head","date":"2024-11-03","value":10.8,"contributorKind":"team","submissionUrls":["https://github.com/zhaoyiyidan/modded-nanogpt-npu/blob/master/records/track_1_short/2024-11-03_UntieEmbed/d6b50d71-f419-4d26-bb39-a60d55ae7a04.txt"]},{"contributor":"Grad62304977, kellerjordan0","description":"Value and embedding skip connections, momentum warmup, logit softcap","date":"2024-11-06","value":8.2,"contributorKind":"team","submissionUrls":["https://github.com/zhaoyiyidan/modded-nanogpt-npu/blob/master/records/track_1_short/2024-11-06_ShortcutsTweaks/dd7304a6-cc43-4d5e-adb8-c070111464a1.txt"]},{"contributor":"kellerjordan0","description":"Bfloat16 activations","date":"2024-11-08","value":7.8,"contributorKind":"person","contributorUrl":"https://github.com/kellerjordan0","submissionUrls":["https://github.com/zhaoyiyidan/modded-nanogpt-npu/blob/master/records/track_1_short/2024-11-08_CastBf16/a833bed8-2fa8-4cfe-af05-58c1cc48bc30.txt"]},{"contributor":"brendanh0gan","description":"U-net pattern skip connections & double lr","date":"2024-11-10","value":7.2,"contributorKind":"person","contributorUrl":"https://github.com/brendanh0gan","submissionUrls":["https://github.com/zhaoyiyidan/modded-nanogpt-npu/blob/master/records/track_1_short/2024-11-10_UNetDoubleLr/c87bb826-797b-4f37-98c7-d3a5dad2de74.txt"]},{"contributor":"KoszarskyB","description":"1024-ctx dense causal attention → 64K-ctx FlexAttention","date":"2024-11-19","value":5.03,"contributorKind":"person","contributorUrl":"https://github.com/KoszarskyB","submissionUrls":["https://github.com/zhaoyiyidan/modded-nanogpt-npu/blob/master/records/track_1_short/2024-11-19_FlexAttention/8384493d-dba9-4991-b16b-8696953f5e6d.txt"]},{"contributor":"fernbear.bsky.social","description":"Attention window warmup","date":"2024-11-24","value":4.66,"contributorKind":"person","submissionUrls":["https://github.com/zhaoyiyidan/modded-nanogpt-npu/blob/master/records/track_1_short/2024-11-24_WindowWarmup/cf9e4571-c5fc-4323-abf3-a98d862ec6c8.txt"]},{"contributor":"KoszarskyB","description":"Value Embeddings","date":"2024-12-04","value":4.41,"contributorKind":"person","contributorUrl":"https://github.com/KoszarskyB","submissionUrls":["https://github.com/zhaoyiyidan/modded-nanogpt-npu/blob/master/records/track_1_short/2024-12-04_ValueEmbed"]},{"contributor":"leloykun, YouJiacheng","description":"U-net pattern value embeddings, assorted code optimizations","date":"2024-12-08","value":3.95,"contributorKind":"team","submissionUrls":["https://github.com/zhaoyiyidan/modded-nanogpt-npu/blob/master/records/track_1_short/2024-12-08_UNetValueEmbedsTweaks"]},{"contributor":"YouJiacheng","description":"Split value embeddings, block sliding window, separate block mask","date":"2024-12-10","value":3.8,"contributorKind":"person","contributorUrl":"https://github.com/YouJiacheng","submissionUrls":["https://github.com/zhaoyiyidan/modded-nanogpt-npu/blob/master/records/track_1_short/2024-12-10_MFUTweaks"]},{"contributor":"YouJiacheng","description":"Sparsify value embeddings, improve rotary embeddings, drop an attn layer","date":"2024-12-17","value":3.57,"contributorKind":"person","contributorUrl":"https://github.com/YouJiacheng","submissionUrls":["https://github.com/zhaoyiyidan/modded-nanogpt-npu/blob/master/records/track_1_short/2024-12-17_SparsifyEmbeds"]},{"contributor":"KoszarskyB","description":"Lower logit softcap from 30 to 15","date":"2025-01-04","value":3.4,"contributorKind":"person","contributorUrl":"https://github.com/KoszarskyB","submissionUrls":["https://github.com/zhaoyiyidan/modded-nanogpt-npu/blob/master/records/track_1_short/2025-01-04_SoftCap/31d6c427-f1f7-4d8a-91be-a67b5dcd13fd.txt"]},{"contributor":"YouJiacheng","description":"FP8 head, offset logits, lr decay to 0.1 instead of 0.0","date":"2025-01-13","value":3.142,"contributorKind":"person","contributorUrl":"https://github.com/YouJiacheng","submissionUrls":["https://github.com/zhaoyiyidan/modded-nanogpt-npu/blob/master/records/track_1_short/2025-01-13_Fp8LmHead/c51969c2-d04c-40a7-bcea-c092c3c2d11a.txt"]},{"contributor":"leloykun, fernbear.bsky.social, YouJiacheng, brendanh0gan, scottjmaddox, Grad62304977","description":"Merged QKV weights, long-short attention, attention scale, lower Adam epsilon, batched Muon","date":"2025-01-16","value":2.992,"contributorKind":"team","submissionUrls":["https://github.com/zhaoyiyidan/modded-nanogpt-npu/blob/master/records/track_1_short/2025-01-16_Sub3Min/1d3bd93b-a69e-4118-aeb8-8184239d7566.txt"]},{"contributor":"leloykun","description":"Reduced batch size","date":"2025-01-26","value":2.933,"contributorKind":"person","contributorUrl":"https://github.com/leloykun","submissionUrls":["https://github.com/zhaoyiyidan/modded-nanogpt-npu/blob/master/records/track_1_short/2025-01-26_BatchSize/c44090cc-1b99-4c95-8624-38fb4b5834f9.txt"]},{"contributor":"not a new record, just re-timing #21 with the updated rules","description":"21st record with new timing","date":"2025-02-01","value":2.997,"contributorKind":"method","submissionUrls":["https://github.com/zhaoyiyidan/modded-nanogpt-npu/blob/master/records/track_1_short/2025-02-01_RuleTweak/eff63a8c-2f7e-4fc5-97ce-7f600dae0bc7.txt"]},{"contributor":"not a new record, just re-timing #21 with latest torch","description":"21st record with latest torch","date":"2025-05-24","value":3.014,"contributorKind":"method","submissionUrls":["https://github.com/zhaoyiyidan/modded-nanogpt-npu/blob/master/records/track_1_short/2025-05-24_StableTorch/89d9f224-3b01-4581-966e-358d692335e0.txt"]},{"contributor":"KonstantinWilleke, alexrgilbert, adricarda, tuttyfrutyee, vdlad; The Enigma project","description":"Faster gradient all-reduce","date":"2025-05-24","value":2.99,"contributorKind":"team","submissionUrls":["https://github.com/zhaoyiyidan/modded-nanogpt-npu/blob/master/records/track_1_short/2025-05-24_FasterReduce/23f40b75-06fb-4c3f-87a8-743524769a35.txt"]},{"contributor":"ryanyang0","description":"Overlap computation and gradient communication","date":"2025-05-25","value":2.979,"contributorKind":"person","contributorUrl":"https://github.com/ryanyang0","submissionUrls":["https://github.com/zhaoyiyidan/modded-nanogpt-npu/blob/master/records/track_1_short/2025-05-25_EvenFasterReduce/6ae86d05-5cb2-4e40-a512-63246fd08e45.txt"]},{"contributor":"vagrawal","description":"Replace gradient all_reduce with reduce_scatter","date":"2025-05-30","value":2.966,"contributorKind":"person","contributorUrl":"https://github.com/vagrawal","submissionUrls":["https://github.com/zhaoyiyidan/modded-nanogpt-npu/blob/master/records/track_1_short/2025-05-30_noallreduce/8054c239-3a18-499e-b0c8-dbd27cb4b3ab.txt"]},{"contributor":"kellerjordan0","description":"Upgrade PyTorch to 2.9.0.dev20250713+cu126","date":"2025-07-13","value":2.896,"contributorKind":"person","contributorUrl":"https://github.com/kellerjordan0","submissionUrls":["https://github.com/zhaoyiyidan/modded-nanogpt-npu/blob/master/records/track_1_short/2025-07-13_UpgradeTorch190/692f80e0-5e64-4819-97d4-0dc83b7106b9.txt"]},{"contributor":"classiclarryd","description":"Align training batch starts with EoS, increase cooldown frac to .45","date":"2025-07-13","value":2.863,"contributorKind":"person","contributorUrl":"https://github.com/classiclarryd","submissionUrls":["https://github.com/zhaoyiyidan/modded-nanogpt-npu/blob/master/records/track_1_short/2025-07-12_BosAlign/c1fd8a38-bb9f-45c4-8af0-d37f70c993f3.txt"]},{"contributor":"byronxu99","description":"Transpose one of the MLP matrices + add Triton kernel for symmetric matmul","date":"2025-07-18","value":2.817,"contributorKind":"person","contributorUrl":"https://github.com/byronxu99","submissionUrls":["https://github.com/zhaoyiyidan/modded-nanogpt-npu/blob/master/records/track_1_short/2025-07-18_TritonMuon/record.txt"]},{"contributor":"classiclarryd","description":"Sparse attention gate","date":"2025-08-23","value":2.812,"contributorKind":"person","contributorUrl":"https://github.com/classiclarryd","submissionUrls":["https://github.com/zhaoyiyidan/modded-nanogpt-npu/blob/master/records/track_1_short/2025-08-23_SparseAttnGate/020630eb-2191-4ba2-9ee4-4cdc94316943.txt"]},{"contributor":"varunneal","description":"Flash Attention 3, 2048 max_doc_len, update ws schedule","date":"2025-09-03","value":2.731,"contributorKind":"person","contributorUrl":"https://github.com/varunneal","submissionUrls":["https://github.com/zhaoyiyidan/modded-nanogpt-npu/blob/master/records/track_1_short/2025-09-03_FA3/44fc1276-0510-4961-92c0-730c65e5feba.txt"]},{"contributor":"EmelyanenkoK","description":"Drop first MLP layer","date":"2025-09-05","value":2.717,"contributorKind":"person","contributorUrl":"https://github.com/EmelyanenkoK","submissionUrls":["https://github.com/zhaoyiyidan/modded-nanogpt-npu/blob/master/records/track_1_short/2025-09-05_SkipMLPBlocks/07e7ae76-b7d0-4481-b149-01e7d81b5ad4.txt"]},{"contributor":"classiclarryd","description":"Dynamically incorporate YaRN during training and validation","date":"2025-09-10","value":2.656,"contributorKind":"person","contributorUrl":"https://github.com/classiclarryd","submissionUrls":["https://github.com/zhaoyiyidan/modded-nanogpt-npu/blob/master/records/track_1_short/2025-09-10_Yarn/0ecdb695-510b-4c3b-b030-09861a162ce8.txt"]},{"contributor":"bernard24 & hiverge.ai","description":"Optimize distributed training, improve skip connection gating, and enhance bfloat16 usage","date":"2025-09-11","value":2.625,"contributorKind":"team","submissionUrls":["https://github.com/zhaoyiyidan/modded-nanogpt-npu/blob/master/records/track_1_short/2025-09-11_VectSigmoidBFloat16/0d0d9882-c34f-4d82-b961-a17d5659c988.txt"]},{"contributor":"classiclarryd","description":"Asynchronously fetch and index data batches, extend final layer attention window for validation","date":"2025-09-15","value":2.565,"contributorKind":"person","contributorUrl":"https://github.com/classiclarryd","submissionUrls":["https://github.com/zhaoyiyidan/modded-nanogpt-npu/blob/master/records/track_1_short/2025-09-15_AsyncDataLoadAttnFinalWindow/25db37c7-2bab-4ef4-ae63-d593590ef823.txt"]},{"contributor":"classiclarryd","description":"Smear token embeddings 1 position forward","date":"2025-09-18","value":2.547,"contributorKind":"person","contributorUrl":"https://github.com/classiclarryd","submissionUrls":["https://github.com/zhaoyiyidan/modded-nanogpt-npu/blob/master/records/track_1_short/2025-09-18_Smear/18a1e5c7-947e-479d-bc3a-a57a61a98fc9.txt"]},{"contributor":"classiclarryd","description":"Drop first attn layer, extend all long windows for validation, update schedule","date":"2025-09-21","value":2.527,"contributorKind":"person","contributorUrl":"https://github.com/classiclarryd","submissionUrls":["https://github.com/zhaoyiyidan/modded-nanogpt-npu/blob/master/records/track_1_short/2025-09-21_DropAttn/01fc4a96-f2a0-47a1-8a6a-c7d10bac99fe.txt"]},{"contributor":"classiclarryd","description":"MuonCustomSizing, perform mlp and attn reduce scatter in shared call","date":"2025-09-23","value":2.495,"contributorKind":"person","contributorUrl":"https://github.com/classiclarryd","submissionUrls":["https://github.com/zhaoyiyidan/modded-nanogpt-npu/blob/master/records/track_1_short/2025-09-23_MuonCustomSizing/b067b4ac-72a6-4436-a6f8-ea51c1efeef3.txt"]},{"contributor":"Gusarich","description":"Compute cross entropy in BF16 during training","date":"2025-09-27","value":2.483,"contributorKind":"person","contributorUrl":"https://github.com/Gusarich","submissionUrls":["https://github.com/zhaoyiyidan/modded-nanogpt-npu/blob/master/records/track_1_short/2025-09-27_BF16CE/08c0770f-17fc-44cd-971d-734a7a28a3e3.txt"]},{"contributor":"varunneal","description":"Polar Express, replacement for Newton-Schulz","date":"2025-09-29","value":2.476,"contributorKind":"person","contributorUrl":"https://github.com/varunneal","submissionUrls":["https://github.com/zhaoyiyidan/modded-nanogpt-npu/blob/master/records/track_1_short/2025-09-29_PolarExpress/0e3f0af5-ad08-47a6-813d-0c709b50d422.txt"]},{"contributor":"classiclarryd","description":"Only update Adam params every other step, reduce batch size","date":"2025-09-30","value":2.447,"contributorKind":"person","contributorUrl":"https://github.com/classiclarryd","submissionUrls":["https://github.com/zhaoyiyidan/modded-nanogpt-npu/blob/master/records/track_1_short/2025-09-30_CustomBatching/40b101b1-77ea-45ea-a089-1d3a647daa22.txt"]},{"contributor":"classiclarryd","description":"Backout, misc hyperparameter tuning, optimize lambda padding","date":"2025-10-04","value":2.358,"contributorKind":"person","contributorUrl":"https://github.com/classiclarryd","submissionUrls":["https://github.com/zhaoyiyidan/modded-nanogpt-npu/blob/master/records/track_1_short/2025-10-04_Backout/514e7581-fbd4-4338-a3e4-e556f9c958ce.txt"]},{"contributor":"li_zichong","description":"NorMuon","date":"2025-10-24","value":2.345,"contributorKind":"person","contributorUrl":"https://github.com/li_zichong","submissionUrls":["https://github.com/zhaoyiyidan/modded-nanogpt-npu/blob/master/records/track_1_short/2025-10-24_NorMuon/088a77ee-9b67-475a-bbb9-3e92e4698799.txt"],"agent":{"evidence":"New WR (-15 steps/-0.7s): Implement NorMuon on latest version (#144) Co-authored-by: Zichong Li","family":"human","confidence":"medium","role":"author"}},{"contributor":"varunneal","description":"Update NorMuon LR, Step Logic","date":"2025-10-27","value":2.313,"contributorKind":"person","contributorUrl":"https://github.com/varunneal","submissionUrls":["https://github.com/zhaoyiyidan/modded-nanogpt-npu/blob/master/records/track_1_short/2025-10-27_FixMuonLR/14afd380-d3d9-48d7-ad23-4c13cb96754b.txt"]},{"contributor":"varunneal","description":"Cautious Weight Decay w/ schedule","date":"2025-11-10","value":2.284,"contributorKind":"person","contributorUrl":"https://github.com/varunneal","submissionUrls":["https://github.com/zhaoyiyidan/modded-nanogpt-npu/blob/master/records/track_1_short/2025-11-10_CautiousWD/1aac0132-a891-4ed9-b358-0fd2abd1b019.txt"]},{"contributor":"akash5474","description":"Backward hooks on Adam, Profiling 101","date":"2025-11-16","value":2.269,"contributorKind":"person","contributorUrl":"https://github.com/akash5474","submissionUrls":["https://github.com/zhaoyiyidan/modded-nanogpt-npu/blob/master/records/track_1_short/2025-10-31_AdamSyncGradientHook/0c17cdfd-772c-4906-8d11-141b370599a0.txt"]},{"contributor":"classiclarryd","description":"Refine skip arch, update exponential decay init","date":"2025-11-18","value":2.248,"contributorKind":"person","contributorUrl":"https://github.com/classiclarryd","submissionUrls":["https://github.com/zhaoyiyidan/modded-nanogpt-npu/blob/master/records/track_1_short/2025-11-18_RefineSkip/00f4e1e6-0044-4a08-b88a-3b7ec0624081.txt"]},{"contributor":"varunneal","description":"Batch size schedule","date":"2025-11-29","value":2.203,"contributorKind":"person","contributorUrl":"https://github.com/varunneal","submissionUrls":["https://github.com/zhaoyiyidan/modded-nanogpt-npu/blob/master/records/track_1_short/2025-11-29_BatchSizeSchedule/10e8f7c6-7175-4467-bdb0-a5de25d771a6.txt"]},{"contributor":"roeeshenberg","description":"Multiply attn lambda with weight instead of data, fix warmup","date":"2025-12-10","value":2.193,"contributorKind":"person","contributorUrl":"https://github.com/roeeshenberg","submissionUrls":["https://github.com/zhaoyiyidan/modded-nanogpt-npu/blob/master/records/track_1_short/2025-12-10_SALambdaOnWeights/15ef5eaf-56e1-40e1-9ddf-af010027c9dd.txt"]},{"contributor":"ChrisJMcCormick","description":"Speed up Muon, additional pre-multiply lambda, reshape matrices, update lr, update NorMuon axis","date":"2025-12-11","value":2.17,"contributorKind":"person","contributorUrl":"https://github.com/ChrisJMcCormick","submissionUrls":["https://github.com/zhaoyiyidan/modded-nanogpt-npu/blob/master/records/track_1_short/2025-12-11_NorMuonOptimsAndFixes/82edf6be-f343-475d-b93a-47c32acf4de2.txt"]},{"contributor":"classiclarryd","description":"Partial Key Offset","date":"2025-12-14","value":2.146,"contributorKind":"person","contributorUrl":"https://github.com/classiclarryd","submissionUrls":["https://github.com/zhaoyiyidan/modded-nanogpt-npu/blob/master/records/track_1_short/2025-12-14_PartialKeyOffset/150d40bf-c20b-4568-aac9-26eb919e25fd.txt"],"isCurrentBest":true}],"contested":true},{"group":"track-2-medium-gpu","description":"Upstream GPT-2 Medium speedrun: fastest algorithm to reach ≤2.92 FineWeb validation loss on 8× NVIDIA H100 GPUs.","metricUnit":"minutes","id":"track-2-medium-gpu","name":"Track 2 Medium (8x NVIDIA H100 GPU)","metricName":"time","metricDirection":"minimize","baseline":348,"isOpen":false,"records":[{"contributor":"karpathy, llm.c contributors","description":"llm.c baseline (350M parameters)","date":"2024-05-28","value":348,"contributorKind":"team","submissionUrls":["https://github.com/zhaoyiyidan/modded-nanogpt-npu/blob/master/records/track_2_medium/2025-01-18/main.log"],"isBaseline":true},{"contributor":"kellerjordan0","description":"Initial record based on scaling up the GPT-2 small track speedrun","date":"2025-01-18","value":29.3,"contributorKind":"person","contributorUrl":"https://github.com/kellerjordan0","submissionUrls":["https://github.com/zhaoyiyidan/modded-nanogpt-npu/blob/master/records/track_2_medium/2025-01-18/241dd7a7-3d76-4dce-85a4-7df60387f32a.txt"]},{"contributor":"kellerjordan0","description":"Added standard weight decay","date":"2025-02-08","value":28.1,"contributorKind":"person","contributorUrl":"https://github.com/kellerjordan0","submissionUrls":["https://github.com/zhaoyiyidan/modded-nanogpt-npu/blob/master/records/track_2_medium/2025-02-08_WeightDecay/b01743db-605c-4326-b5b1-d388ee5bebc5.txt"]},{"contributor":"leloykun","description":"Tuned Muon Newton-Schulz coefficients","date":"2025-02-14","value":27.7,"contributorKind":"person","contributorUrl":"https://github.com/leloykun","submissionUrls":["https://github.com/zhaoyiyidan/modded-nanogpt-npu/blob/master/records/track_2_medium/2025-02-14_OptCoeffs/1baa66b2-bff7-4850-aced-d63885ffb4b6.txt"]},{"contributor":"YouJiacheng","description":"Increased learning rate cooldown phase duration","date":"2025-03-06","value":27.2,"contributorKind":"person","contributorUrl":"https://github.com/YouJiacheng","submissionUrls":["https://github.com/zhaoyiyidan/modded-nanogpt-npu/blob/master/records/track_2_medium/2025-03-06_LongerCooldown/779c041a-2a37-45d2-a18b-ec0f223c2bb7.txt"]},{"contributor":"YouJiacheng","description":"2x MLP wd, qkv norm, all_reduce/opt.step() overlap, optimized skip pattern","date":"2025-03-25","value":25.95,"contributorKind":"person","contributorUrl":"https://github.com/YouJiacheng","submissionUrls":["https://github.com/zhaoyiyidan/modded-nanogpt-npu/blob/master/records/track_2_medium/2025-03-25_ArchOptTweaks/train_gpt-20250329.txt"]},{"contributor":"YouJiacheng","description":"Remove FP8 head; ISRU logits softcap; New sharded mixed precision Muon; merge weights","date":"2025-04-16","value":25.29,"contributorKind":"person","contributorUrl":"https://github.com/YouJiacheng","submissionUrls":["https://github.com/zhaoyiyidan/modded-nanogpt-npu/blob/master/records/track_2_medium/2025-04-16_Record7/223_3310d0b1-b24d-48ee-899f-d5c2a254a195.txt"]},{"contributor":"jadenj3o","description":"Cubic sliding window size schedule, 2× max window size (24.84 minutes)","date":"2025-04-22","value":24.5,"contributorKind":"person","contributorUrl":"https://github.com/jadenj3o","submissionUrls":["https://github.com/zhaoyiyidan/modded-nanogpt-npu/blob/master/records/track_2_medium/2025-04-22_Record8/075_640429f2-e726-4e83-aa27-684626239ffc.txt"]},{"contributor":"snimu","description":"Add two value embeddings","date":"2025-08-28","value":24.12,"contributorKind":"person","contributorUrl":"https://github.com/snimu","submissionUrls":["https://github.com/zhaoyiyidan/modded-nanogpt-npu/blob/master/records/track_2_medium/2025-08-28_NewValemb/036_61ef4351-7b68-4897-b440-a99221a1a629.txt"]},{"contributor":"snimu","description":"Second input embedding","date":"2025-09-11","value":24.07,"contributorKind":"person","contributorUrl":"https://github.com/snimu","submissionUrls":["https://github.com/zhaoyiyidan/modded-nanogpt-npu/blob/master/records/track_2_medium/2025-09-11_SecondInputEmbed/000_592014ec-6781-4f59-b274-c4af68ccfe75.txt"],"isCurrentBest":true}],"contested":true}],"agentStats":[{"family":"human","records":2,"currentBests":0,"models":[],"contributors":["NorMuon optimizer","li_zichong"]}],"repo":{"owner":"zhaoyiyidan","name":"modded-nanogpt-npu","branch":"master"},"participants":[{"name":"classiclarryd","url":"https://github.com/classiclarryd","submissions":11,"problems":["track-1-short-gpu"],"lastActive":"2025-12-14","bestRank":1},{"name":"snimu","url":"https://github.com/snimu","submissions":2,"problems":["track-2-medium-gpu"],"lastActive":"2025-09-11","bestRank":1},{"name":"kellerjordan0","url":"https://github.com/kellerjordan0","submissions":12,"problems":["track-1-short-gpu","track-2-medium-gpu"],"lastActive":"2025-07-13"},{"name":"YouJiacheng","url":"https://github.com/YouJiacheng","submissions":7,"problems":["track-1-short-gpu","track-2-medium-gpu"],"lastActive":"2025-04-16"},{"name":"varunneal","url":"https://github.com/varunneal","submissions":5,"problems":["track-1-short-gpu"],"lastActive":"2025-11-29"},{"name":"Grad62304977","url":"https://github.com/Grad62304977","submissions":3,"problems":["track-1-short-gpu"],"lastActive":"2024-11-06"},{"name":"KoszarskyB","url":"https://github.com/KoszarskyB","submissions":3,"problems":["track-1-short-gpu"],"lastActive":"2025-01-04"},{"name":"leloykun","url":"https://github.com/leloykun","submissions":3,"problems":["track-1-short-gpu","track-2-medium-gpu"],"lastActive":"2025-02-14"},{"name":"karpathy, llm.c contributors","submissions":2,"problems":["track-1-short-gpu","track-2-medium-gpu"],"lastActive":"2024-05-28"},{"name":"jxbz","url":"https://github.com/jxbz","submissions":1,"problems":["track-1-short-gpu"],"lastActive":"2024-10-04"},{"name":"bozavlado","url":"https://github.com/bozavlado","submissions":1,"problems":["track-1-short-gpu"],"lastActive":"2024-10-11"},{"name":"brendanh0gan","url":"https://github.com/brendanh0gan","submissions":1,"problems":["track-1-short-gpu"],"lastActive":"2024-11-10"},{"name":"fernbear.bsky.social","submissions":1,"problems":["track-1-short-gpu"],"lastActive":"2024-11-24"},{"name":"leloykun, fernbear.bsky.social, YouJiacheng, brendanh0gan, scottjmaddox, Grad62304977","submissions":1,"problems":["track-1-short-gpu"],"lastActive":"2025-01-16"},{"name":"KonstantinWilleke, alexrgilbert, adricarda, tuttyfrutyee, vdlad; The Enigma project","submissions":1,"problems":["track-1-short-gpu"],"lastActive":"2025-05-24"},{"name":"ryanyang0","url":"https://github.com/ryanyang0","submissions":1,"problems":["track-1-short-gpu"],"lastActive":"2025-05-25"},{"name":"vagrawal","url":"https://github.com/vagrawal","submissions":1,"problems":["track-1-short-gpu"],"lastActive":"2025-05-30"},{"name":"byronxu99","url":"https://github.com/byronxu99","submissions":1,"problems":["track-1-short-gpu"],"lastActive":"2025-07-18"},{"name":"EmelyanenkoK","url":"https://github.com/EmelyanenkoK","submissions":1,"problems":["track-1-short-gpu"],"lastActive":"2025-09-05"},{"name":"bernard24 & hiverge.ai","submissions":1,"problems":["track-1-short-gpu"],"lastActive":"2025-09-11"},{"name":"Gusarich","url":"https://github.com/Gusarich","submissions":1,"problems":["track-1-short-gpu"],"lastActive":"2025-09-27"},{"name":"li_zichong","url":"https://github.com/li_zichong","submissions":1,"problems":["track-1-short-gpu"],"lastActive":"2025-10-24"},{"name":"akash5474","url":"https://github.com/akash5474","submissions":1,"problems":["track-1-short-gpu"],"lastActive":"2025-11-16"},{"name":"roeeshenberg","url":"https://github.com/roeeshenberg","submissions":1,"problems":["track-1-short-gpu"],"lastActive":"2025-12-10"},{"name":"ChrisJMcCormick","url":"https://github.com/ChrisJMcCormick","submissions":1,"problems":["track-1-short-gpu"],"lastActive":"2025-12-11"},{"name":"jadenj3o","url":"https://github.com/jadenj3o","submissions":1,"problems":["track-2-medium-gpu"],"lastActive":"2025-04-22"}],"stats":{"totalRecords":110,"uniqueParticipants":26,"firstSubmission":"2024-05-28","lastSubmission":"2025-12-14","recordsLast30d":0,"recordsLast90d":0},"lastUpdated":"2026-10-19T09:02:20.238Z"},{"id":"gmaapp","name":"Grand Mountain Adventure Community Leaderboard","tagline":"Submit verified world-record runs and personal bests for over 200 in-game challenges in Grand Mountain Adventure.","description":"A community-run leaderboard for the mobile skiing game Grand Mountain Adventure, tracking world records and personal bests across 200+ in-game challenges on 11 mountains. Players submit runs via a Google Form with video proof; records are verified and ranked by score (time trials: lowest time; high scores: highest points). The site is a Dash app hosted on Heroku, maintained by hschindele and the GMA community.","url":"https://github.com/hschindele/gmaapp","host":"github","status":"active","domain":"other","tags":["gaming","speedrun","leaderboard","community","mobile-game","world-records"],"images":[],"participation":{"howToSubmit":"Fill out the Score Submission Form (Google Form) with a video link of your run.","submissionFormat":"Video proof (Google Drive link) plus challenge name, score, OS, and player name.","deadline":"rolling","prizes":"recognition only","eligibility":"Anyone with the game","cost":"free","verification":"Manual review by community moderators; records marked Verified in allwrs.csv","requirements":["Grand Mountain Adventure game","Video recording capability"],"compute":"none"},"links":[{"label":"Leaderboard Site","url":"https://gmaleaderboard.gart.sh/"},{"label":"Score Submission Form","url":"https://docs.google.com/forms/d/1n3DL3h3NYxhWCBG0NSrtM48uIE3OGvMfL2MVKdBU_kA/edit#responses"},{"label":"Toppluva Website","url":"https://www.toppluva.com/press/index.php"},{"label":"Community Discord","url":"https://discord.com/invite/SA4F5WN"}],"organizer":{"name":"hschindele","url":"https://github.com/hschindele","type":"individual"},"venue":{"type":"online"},"problems":[{"group":"agpat-island","description":"Time trial race on Agpat Island; lowest time wins.","metricUnit":"s","id":"agpat-journey","name":"Agpat Journey","metricName":"time","metricDirection":"minimize","isOpen":false,"records":[{"contributor":"hschindele","date":"2022-03-10","value":22.12,"contributorKind":"person","contributorUrl":"https://github.com/hschindele","isBaseline":true},{"contributor":"Jamiemac","date":"2022-03-11","value":22.71,"contributorKind":"person","contributorUrl":"https://github.com/Jamiemac"},{"contributor":"Avel","date":"2022-03-21","value":22.82,"contributorKind":"person","contributorUrl":"https://github.com/Avel"},{"contributor":"Mountain_Greg","date":"2022-03-24","value":22.15,"contributorKind":"person","contributorUrl":"https://github.com/Mountain_Greg"},{"contributor":"Avel","date":"2022-03-27","value":21.31,"contributorKind":"person","contributorUrl":"https://github.com/Avel","isCurrentBest":true}],"contested":true},{"group":"agpat-island","description":"Time trial race on Agpat Island; lowest time wins.","metricUnit":"s","id":"aluu-race-track","name":"Aluu Race Track","metricName":"time","metricDirection":"minimize","isOpen":false,"records":[{"contributor":"Jamiemac","date":"2022-03-11","value":20.12,"contributorKind":"person","contributorUrl":"https://github.com/Jamiemac","isBaseline":true},{"contributor":"Jamiemac","date":"2022-03-11","value":20.1,"contributorKind":"person","contributorUrl":"https://github.com/Jamiemac"},{"contributor":"hschindele","date":"2022-03-12","value":20.08,"contributorKind":"person","contributorUrl":"https://github.com/hschindele"},{"contributor":"Avel","date":"2022-03-21","value":20.1,"contributorKind":"person","contributorUrl":"https://github.com/Avel"},{"contributor":"Mountain_Greg","date":"2022-03-24","value":20.2,"contributorKind":"person","contributorUrl":"https://github.com/Mountain_Greg"},{"contributor":"Wrog","date":"2022-04-13","value":20.22,"contributorKind":"person","contributorUrl":"https://github.com/Wrog"},{"contributor":"hschindele","date":"2022-04-14","value":20.06,"contributorKind":"person","contributorUrl":"https://github.com/hschindele"},{"contributor":"hschindele","date":"2022-04-17","value":20.04,"contributorKind":"person","contributorUrl":"https://github.com/hschindele","isCurrentBest":true},{"contributor":"Wrog","date":"2022-06-27","value":20.14,"contributorKind":"person","contributorUrl":"https://github.com/Wrog"}],"contested":true},{"group":"agpat-island","description":"Time trial race on Agpat Island; lowest time wins.","metricUnit":"s","id":"ilineq-track","name":"Ilineq Track","metricName":"time","metricDirection":"minimize","isOpen":false,"records":[{"contributor":"Jamiemac","date":"2022-03-11","value":15.37,"contributorKind":"person","contributorUrl":"https://github.com/Jamiemac","isBaseline":true},{"contributor":"hschindele","date":"2022-03-12","value":15.35,"contributorKind":"person","contributorUrl":"https://github.com/hschindele","isCurrentBest":true},{"contributor":"Avel","date":"2022-03-21","value":15.49,"contributorKind":"person","contributorUrl":"https://github.com/Avel"},{"contributor":"Mountain_Greg","date":"2022-03-24","value":15.41,"contributorKind":"person","contributorUrl":"https://github.com/Mountain_Greg"},{"contributor":"Wrog","date":"2022-06-27","value":15.47,"contributorKind":"person","contributorUrl":"https://github.com/Wrog"}],"contested":true},{"group":"agpat-island","description":"Time trial race on Agpat Island; lowest time wins.","metricUnit":"s","id":"nanoq-danger","name":"Nanoq danger","metricName":"time","metricDirection":"minimize","isOpen":false,"records":[{"contributor":"Jamiemac","date":"2022-03-11","value":21.99,"contributorKind":"person","contributorUrl":"https://github.com/Jamiemac","isBaseline":true},{"contributor":"hschindele","date":"2022-03-12","value":21.29,"contributorKind":"person","contributorUrl":"https://github.com/hschindele"},{"contributor":"Jonathan","date":"2022-03-15","value":21.1,"contributorKind":"person","contributorUrl":"https://github.com/Jonathan"},{"contributor":"Avel","date":"2022-03-21","value":21.77,"contributorKind":"person","contributorUrl":"https://github.com/Avel"},{"contributor":"Mountain_Greg","date":"2022-03-24","value":21.72,"contributorKind":"person","contributorUrl":"https://github.com/Mountain_Greg"},{"contributor":"Jamiemac","date":"2022-03-27","value":21.87,"contributorKind":"person","contributorUrl":"https://github.com/Jamiemac"},{"contributor":"Avel","date":"2022-03-27","value":20.97,"contributorKind":"person","contributorUrl":"https://github.com/Avel","isCurrentBest":true}],"contested":true},{"group":"agpat-island","description":"Time trial race on Agpat Island; lowest time wins.","metricUnit":"s","id":"the-pilot-hike","name":"The Pilot Hike","metricName":"time","metricDirection":"minimize","isOpen":false,"records":[{"contributor":"Jamiemac","date":"2022-03-11","value":16.75,"contributorKind":"person","contributorUrl":"https://github.com/Jamiemac","isBaseline":true},{"contributor":"hschindele","date":"2022-03-12","value":16.89,"contributorKind":"person","contributorUrl":"https://github.com/hschindele"},{"contributor":"Avel","date":"2022-03-21","value":17.07,"contributorKind":"person","contributorUrl":"https://github.com/Avel"},{"contributor":"Mountain_Greg","date":"2022-03-24","value":17.06,"contributorKind":"person","contributorUrl":"https://github.com/Mountain_Greg"},{"contributor":"Avel","date":"2022-03-27","value":16.95,"contributorKind":"person","contributorUrl":"https://github.com/Avel","isCurrentBest":true}],"contested":true},{"group":"agpat-island","description":"Time trial race on Agpat Island; lowest time wins.","metricUnit":"s","id":"ukioq-track","name":"Ukioq Track","metricName":"time","metricDirection":"minimize","isOpen":false,"records":[{"contributor":"Jamiemac","date":"2022-03-11","value":15.4,"contributorKind":"person","contributorUrl":"https://github.com/Jamiemac","isBaseline":true},{"contributor":"hschindele","date":"2022-03-12","value":15.36,"contributorKind":"person","contributorUrl":"https://github.com/hschindele","isCurrentBest":true},{"contributor":"Avel","date":"2022-03-21","value":15.73,"contributorKind":"person","contributorUrl":"https://github.com/Avel"},{"contributor":"Mountain_Greg","date":"2022-03-24","value":15.47,"contributorKind":"person","contributorUrl":"https://github.com/Mountain_Greg"}],"contested":true},{"group":"agpat-island","description":"Time trial trail on Agpat Island; lowest time wins.","metricUnit":"s","id":"gunnbjorns-line","name":"Gunnbjorns Line","metricName":"time","metricDirection":"minimize","isOpen":false,"records":[{"contributor":"Jamiemac","date":"2022-03-11","value":22.58,"contributorKind":"person","contributorUrl":"https://github.com/Jamiemac","isBaseline":true},{"contributor":"hschindele","date":"2022-03-12","value":22.4,"contributorKind":"person","contributorUrl":"https://github.com/hschindele","isCurrentBest":true},{"contributor":"Avel","date":"2022-03-21","value":22.55,"contributorKind":"person","contributorUrl":"https://github.com/Avel"},{"contributor":"Mountain_Greg","date":"2022-03-24","value":22.49,"contributorKind":"person","contributorUrl":"https://github.com/Mountain_Greg"},{"contributor":"Jamiemac","date":"2022-04-07","value":22.55,"contributorKind":"person","contributorUrl":"https://github.com/Jamiemac"}],"contested":true},{"group":"agpat-island","description":"Time trial TTB on Agpat Island; lowest time wins.","metricUnit":"s","id":"across-agpat","name":"Across Agpat","metricName":"time","metricDirection":"minimize","isOpen":false,"records":[{"contributor":"Jamiemac","date":"2022-03-11","value":19.7,"contributorKind":"person","contributorUrl":"https://github.com/Jamiemac","isBaseline":true},{"contributor":"hschindele","date":"2022-03-12","value":18.9,"contributorKind":"person","contributorUrl":"https://github.com/hschindele","isCurrentBest":true},{"contributor":"Avel","date":"2022-03-21","value":19.81,"contributorKind":"person","contributorUrl":"https://github.com/Avel"},{"contributor":"Mountain_Greg","date":"2022-03-24","value":19.15,"contributorKind":"person","contributorUrl":"https://github.com/Mountain_Greg"}],"contested":true},{"group":"agpat-island","description":"High score airtime challenge on Agpat Island; highest score wins.","id":"the-brand","name":"The Brand","metricName":"score","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"Jamiemac","date":"2022-03-11","value":10.26,"contributorKind":"person","contributorUrl":"https://github.com/Jamiemac","isBaseline":true},{"contributor":"Avel","date":"2022-03-21","value":9.08,"contributorKind":"person","contributorUrl":"https://github.com/Avel"},{"contributor":"Mountain_Greg","date":"2022-03-24","value":10.26,"contributorKind":"person","contributorUrl":"https://github.com/Mountain_Greg"},{"contributor":"Avel","date":"2022-03-26","value":10.32,"contributorKind":"person","contributorUrl":"https://github.com/Avel"},{"contributor":"Jonathan","date":"2022-04-07","value":11.26,"contributorKind":"person","contributorUrl":"https://github.com/Jonathan","isCurrentBest":true},{"contributor":"hschindele","date":"2022-04-17","value":10.46,"contributorKind":"person","contributorUrl":"https://github.com/hschindele"}],"contested":true},{"group":"agpat-island","description":"High score long jump challenge on Agpat Island; highest score wins.","id":"the-seagull","name":"The Seagull","metricName":"score","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"Jamiemac","date":"2022-03-11","value":80.7,"contributorKind":"person","contributorUrl":"https://github.com/Jamiemac","isBaseline":true},{"contributor":"Avel","date":"2022-03-21","value":87.8,"contributorKind":"person","contributorUrl":"https://github.com/Avel"},{"contributor":"Mountain_Greg","date":"2022-03-24","value":88.1,"contributorKind":"person","contributorUrl":"https://github.com/Mountain_Greg"},{"contributor":"Avel","date":"2022-03-26","value":93.9,"contributorKind":"person","contributorUrl":"https://github.com/Avel","isCurrentBest":true},{"contributor":"Jamiemac","date":"2022-04-07","value":84.8,"contributorKind":"person","contributorUrl":"https://github.com/Jamiemac"},{"contributor":"Mountain_Greg","date":"2022-04-14","value":90.6,"contributorKind":"person","contributorUrl":"https://github.com/Mountain_Greg"}],"contested":true},{"group":"agpat-island","description":"High score single trick challenge on Agpat Island; highest score wins.","id":"greenland-spin","name":"Greenland spin","metricName":"score","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"Jamiemac","date":"2022-03-11","value":18396,"contributorKind":"person","contributorUrl":"https://github.com/Jamiemac","isBaseline":true},{"contributor":"Avel","date":"2022-03-21","value":21683,"contributorKind":"person","contributorUrl":"https://github.com/Avel"},{"contributor":"Mountain_Greg","date":"2022-03-24","value":20106,"contributorKind":"person","contributorUrl":"https://github.com/Mountain_Greg"},{"contributor":"Mountain_Greg","date":"2022-04-21","value":23014,"contributorKind":"person","contributorUrl":"https://github.com/Mountain_Greg","isCurrentBest":true}],"contested":true},{"group":"agpat-island","description":"High score gated trick challenge on Agpat Island; highest score wins.","id":"aappaluttoq-park-lap","name":"Aappaluttoq Park Lap","metricName":"score","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"Jamiemac","date":"2022-03-11","value":41380,"contributorKind":"person","contributorUrl":"https://github.com/Jamiemac","isBaseline":true},{"contributor":"Avel","date":"2022-03-21","value":38530,"contributorKind":"person","contributorUrl":"https://github.com/Avel"},{"contributor":"Mountain_Greg","date":"2022-03-24","value":35057,"contributorKind":"person","contributorUrl":"https://github.com/Mountain_Greg"},{"contributor":"Avel","date":"2022-03-26","value":41488,"contributorKind":"person","contributorUrl":"https://github.com/Avel","isCurrentBest":true},{"contributor":"Mountain_Greg","date":"2022-04-20","value":38167,"contributorKind":"person","contributorUrl":"https://github.com/Mountain_Greg"}],"contested":true},{"group":"agpat-island","description":"High score gated trick challenge on Agpat Island; highest score wins.","id":"quick-turns","name":"Quick Turns","metricName":"score","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"Jamiemac","date":"2022-03-11","value":16839,"contributorKind":"person","contributorUrl":"https://github.com/Jamiemac","isBaseline":true},{"contributor":"Avel","date":"2022-03-21","value":12521,"contributorKind":"person","contributorUrl":"https://github.com/Avel"},{"contributor":"Mountain_Greg","date":"2022-03-24","value":17760,"contributorKind":"person","contributorUrl":"https://github.com/Mountain_Greg","isCurrentBest":true}],"contested":true},{"group":"agpat-island","description":"High score gated trick challenge on Agpat Island; highest score wins.","id":"tungujortoq-park-lap","name":"Tungujortoq Park Lap","metricName":"score","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"Jamiemac","date":"2022-03-11","value":28509,"contributorKind":"person","contributorUrl":"https://github.com/Jamiemac","isBaseline":true},{"contributor":"Avel","date":"2022-03-21","value":25148,"contributorKind":"person","contributorUrl":"https://github.com/Avel"},{"contributor":"Mountain_Greg","date":"2022-03-24","value":26887,"contributorKind":"person","contributorUrl":"https://github.com/Mountain_Greg"},{"contributor":"Avel","date":"2022-03-26","value":33673,"contributorKind":"person","contributorUrl":"https://github.com/Avel"},{"contributor":"Mountain_Greg","date":"2022-04-21","value":28249,"contributorKind":"person","contributorUrl":"https://github.com/Mountain_Greg"},{"contributor":"GSM","date":"2022-08-08","value":34478,"contributorKind":"person","contributorUrl":"https://github.com/GSM","isCurrentBest":true}],"contested":true}],"agentStats":[],"repo":{"owner":"hschindele","name":"gmaapp","branch":"main"},"participants":[{"name":"Avel","url":"https://github.com/Avel","submissions":21,"problems":["agpat-journey","aluu-race-track","ilineq-track","nanoq-danger","the-pilot-hike","ukioq-track","gunnbjorns-line","across-agpat","the-brand","the-seagull","greenland-spin","aappaluttoq-park-lap","quick-turns","tungujortoq-park-lap"],"lastActive":"2022-03-27","bestRank":1},{"name":"Mountain_Greg","url":"https://github.com/Mountain_Greg","submissions":18,"problems":["agpat-journey","aluu-race-track","ilineq-track","nanoq-danger","the-pilot-hike","ukioq-track","gunnbjorns-line","across-agpat","the-brand","the-seagull","greenland-spin","aappaluttoq-park-lap","quick-turns","tungujortoq-park-lap"],"lastActive":"2022-04-21","bestRank":1},{"name":"hschindele","url":"https://github.com/hschindele","submissions":11,"problems":["agpat-journey","aluu-race-track","ilineq-track","nanoq-danger","the-pilot-hike","ukioq-track","gunnbjorns-line","across-agpat","the-brand"],"lastActive":"2022-04-17","bestRank":1},{"name":"Jonathan","url":"https://github.com/Jonathan","submissions":2,"problems":["nanoq-danger","the-brand"],"lastActive":"2022-04-07","bestRank":1},{"name":"GSM","url":"https://github.com/GSM","submissions":1,"problems":["tungujortoq-park-lap"],"bestRank":1,"lastActive":"2022-08-08"},{"name":"Jamiemac","url":"https://github.com/Jamiemac","submissions":18,"problems":["agpat-journey","aluu-race-track","ilineq-track","nanoq-danger","the-pilot-hike","ukioq-track","gunnbjorns-line","across-agpat","the-brand","the-seagull","greenland-spin","aappaluttoq-park-lap","quick-turns","tungujortoq-park-lap"],"lastActive":"2022-04-07"},{"name":"Wrog","url":"https://github.com/Wrog","submissions":3,"problems":["aluu-race-track","ilineq-track"],"lastActive":"2022-06-27"}],"stats":{"totalRecords":74,"uniqueParticipants":7,"firstSubmission":"2022-03-10","lastSubmission":"2022-08-08","recordsLast30d":0,"recordsLast90d":0},"lastUpdated":"2026-10-19T09:02:20.239Z"},{"id":"apex-humanoid-parkour","name":"Apex Humanoid Parkour","tagline":"Train an ONNX policy that walks a Unitree G1 through a 51 m parkour course under randomized friction and wind.","description":"Miners submit an ONNX policy (<=15 MB, architecture free) that drives a 12-DoF Unitree G1 humanoid through a 51.1 m linear parkour course: on-ramp, drop, stairs, a 1 m leap, hurdle, step-up, duck-under, balance beam, slick patch and stairway down. Each round evaluates 24 instances of up to 3000 control steps with friction and wind drawn from an unpublished per-round seed, so open-loop trajectory replay is worthless. Scoring is progress-based: completions score in (1.0, 2.0] with a speed bonus, non-completions score their fraction of course covered, and raw_score is the mean over 24 instances. Run by Macrocosm (Apex competition, Bittensor Subnet 1); the reference Unitree stock walker reaches only 21% of the course.","url":"https://github.com/macrocosm-os/apex-competition-humanoid-parkour","host":"github","status":"active","domain":"robotics","tags":["humanoid","locomotion","reinforcement-learning","onnx","mujoco","bittensor","parkour"],"images":["https://github.com/user-attachments/assets/17b9f4ca-39ef-4d15-a216-b3eee1888a43"],"participation":{"howToSubmit":"Submit an ONNX policy graph through the Apex platform (Bittensor Subnet 1); the referee runs it in the player sandbox against 24 randomized instances.","submissionFormat":"ONNX graph, <= 15 MB, float32, interface obs[104] + state_in[256] -> action[12] + state_out[256]; architecture free (recurrent nets, ensembles, transformers over a history window).","computeDetails":"Referee runs under --cpus 1 --memory 1.5g; inference runs once per control step up to 72,000 times per evaluation, with a 900 s referee timeout. Training a policy is the miner's own responsibility.","deadline":"rolling","rounds":"Rounds with per-round seeds; conditions (friction, wind) drawn per round and not published while the round is open.","prizes":"Apex competition (Bittensor Subnet 1) mining rewards","cost":"free","verification":"Referee container runs the policy in a player sandbox; physics_glitch/invalid/player errors score 0.0; history and API log artifacts are collected per submission.","requirements":["Python 3.10+","ONNX","Docker","MuJoCo (for local eval/replay)"],"compute":"cpu"},"links":[{"label":"Repository","url":"https://github.com/macrocosm-os/apex-competition-humanoid-parkour"},{"label":"Baseline provenance","url":"https://github.com/macrocosm-os/apex-competition-humanoid-parkour/blob/main/baseline/PROVENANCE.md"},{"label":"Unitree RL Gym (baseline source)","url":"https://github.com/unitreerobotics/unitree_rl_gym"},{"label":"v0.2.0 predecessor","url":"https://github.com/macrocosm-os/apex-competition-humanoid-parkour-v2/tree/v0.2.0"}],"organizer":{"name":"Macrocosm (Apex, Bittensor Subnet 1)","url":"https://github.com/macrocosm-os","type":"company"},"venue":{"type":"online"},"quickstart":"docker build -f referee/Dockerfile -t hp-referee .\ndocker build -f player/Dockerfile  -t hp-player  .\ndocker network create hpnet\ndocker run -d --name hp-p --network hpnet -v \"$PWD/baseline/baseline.onnx:/app/submission.onnx:ro\" hp-player\nmkdir -p /tmp/hpdata && chmod 777 /tmp/hpdata\ndocker run --rm --network hpnet -v /tmp/hpdata:/data -e MATCH_ID=local -e SEED=1 -e NUM_PLAYERS=1 -e PLAYER_URLS=http://hp-p:8000 -e CONFIG_JSON='{\"seed\":1,\"num_instances\":24,\"max_steps_per_episode\":3000,\"deadline_ms\":500}' hp-referee\njq '.raw_scores, .metadata.num_completed' /tmp/hpdata/result.json","problems":[{"description":"Drive the 12-DoF Unitree G1 through the 51.1 m parkour course; raw_score is the mean over 24 instances with randomized friction and wind. Completions score 1.0 + (max_steps - steps)/max_steps; non-completions score their fraction of course covered.","id":"humanoid-parkour-course","name":"Humanoid Parkour Course","metricName":"raw_score","metricDirection":"maximize","baseline":0.21,"isOpen":true,"records":[{"contributor":"Unitree stock G1 walker (baseline)","description":"Unitree's stock G1 walking policy (motion.pt) wrapped to the competition interface; flat-ground locomotion, no terrain perception, falls off the first ledge at ~21% of the course.","value":0.21,"contributorKind":"method","isBaseline":true,"isCurrentBest":true}],"contested":false}],"agentStats":[],"repo":{"owner":"macrocosm-os","name":"apex-competition-humanoid-parkour","branch":"main"},"participants":[],"stats":{"totalRecords":1,"uniqueParticipants":0,"recordsLast30d":0,"recordsLast90d":0},"lastUpdated":"2026-10-19T09:02:20.239Z"},{"id":"swe-bench-pro-leaderboard","name":"SWE-bench Pro Leaderboard","tagline":"Score coding agents on 731 real-world software engineering tasks by the fraction of instances whose test suites pass.","description":"A leaderboard for evaluating coding agents on SWE-bench Pro, a dataset of 731 real-world software engineering tasks. A green agent sends each instance (problem statement + Docker image) to a submitted coding agent, collects the returned patch, and runs the project's test suite in Docker; an instance passes only if all FAIL_TO_PASS tests pass and all PASS_TO_PASS tests still pass. Final score is accuracy = passed / total. Run by the RDI Foundation via the AgentBeats platform.","url":"https://github.com/RDI-Foundation/swe-bench-leaderboard","host":"github","status":"active","domain":"coding-agents","tags":["swe-bench","coding-agents","software-engineering","agent-evaluation","docker","a2a"],"images":[],"participation":{"howToSubmit":"Fork the repo, set your agent's Agentbeats ID in scenario.json5 (metadata.agentbeats_ids.coding_agent), add provider API keys as repo secrets, then open/update a PR to main or trigger the 'Run Scenario' workflow from the Actions tab; follow the PR link in the workflow summary to submit results. Alternatively submit through agentbeats.dev via OIDC quick-submit.","submissionFormat":"scenario.json5 configuration plus a results JSON produced by the green agent (per-instance pass/fail and accuracy); quick-submit via agentbeats.dev UI.","computeDetails":"Evaluation runs each SWE-bench Pro instance in a Docker container; agent inference depends on the chosen model provider. Local runs can be sharded via assessment_config.num_instances / num_shards.","deadline":"rolling","cost":"free","verification":"Automated: green agent runs FAIL_TO_PASS and PASS_TO_PASS tests in Docker; results committed as results/<uuid>.json via CI workflows.","requirements":["Docker with Compose v2","Amber CLI v0.3+","A2A-compatible coding agent","LiteLLM model string","Provider API key (OpenAI/Anthropic/Gemini/OpenRouter)"],"compute":"datacenter-gpu"},"links":[{"label":"SWE-bench Pro dataset","url":"https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro"},{"label":"AgentBeats","url":"https://agentbeats.dev/"},{"label":"swe-bench-green-agent","url":"https://github.com/rdi-foundation/swe-bench-green-agent"},{"label":"swe-bench-purple-agent","url":"https://github.com/rdi-foundation/swe-bench-purple-agent"},{"label":"Amber CLI","url":"https://github.com/rdi-foundation/amber-cli"}],"organizer":{"name":"RDI Foundation","url":"https://github.com/rdi-foundation","type":"foundation"},"venue":{"type":"online"},"quickstart":"# 1. (Optional) edit scenario.json5 to set a small local run\n#    e.g. components.gateway.config.assessment_config.num_instances = 1\n\n# 2. Compile the scenario\namber compile scenario.json5 --docker-compose amber-out\n\n# 3. Fill in the generated env file\ncp amber-out/env.example amber-out/.env\n$EDITOR amber-out/.env\n\n# 4. Run the stack\nAMBER_CONFIG_CODING_AGENT__MODEL_NAME=gpt-4o \\\nAMBER_CONFIG_CODING_AGENT__OPENAI_API_KEY=sk-xxx \\\ndocker compose -f amber-out/compose.yaml --env-file amber-out/.env up","problems":[{"description":"Accuracy over SWE-bench Pro instances: fraction of instances where all FAIL_TO_PASS tests pass and all PASS_TO_PASS tests still pass.","metricUnit":"%","id":"swe-bench-pro","name":"SWE-bench Pro","metricName":"accuracy","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"SWE-bench Purple Agent","description":"mini-swe-agent (openrouter/deepseek/deepseek-v3.2), 1 instance","date":"2026-04-13","value":0,"contributorKind":"method","submissionUrls":["https://github.com/RDI-Foundation/swe-bench-leaderboard/blob/main/results/019d8691-0a8c-7a32-a5e1-1b72cf211f3a.json"],"agent":{"model":"deepseek/deepseek-v3.2","tool":"mini-swe-agent","evidence":"A2A coding agent that solves SWE-bench Pro instances using mini-swe-agent (openrouter/deepseek/deepseek-v3.2).","family":"deepseek","confidence":"high","role":"subject"}},{"contributor":"SWE-bench Purple Agent","description":"mini-swe-agent (openrouter/deepseek/deepseek-v3.2), 5 instances","date":"2026-04-13","value":0.4,"contributorKind":"method","submissionUrls":["https://github.com/RDI-Foundation/swe-bench-leaderboard/blob/main/results/019d86f7-8639-7853-bbd1-b045cc625e16.json"],"isCurrentBest":true,"agent":{"model":"deepseek/deepseek-v3.2","tool":"mini-swe-agent","evidence":"A2A coding agent that solves SWE-bench Pro instances using mini-swe-agent (openrouter/deepseek/deepseek-v3.2).","family":"deepseek","confidence":"high","role":"subject"}},{"contributor":"SWE-bench Purple Agent","description":"mini-swe-agent (deepseek/deepseek-chat), 5 instances","date":"2026-04-15","value":0.4,"contributorKind":"method","submissionUrls":["https://github.com/RDI-Foundation/swe-bench-leaderboard/blob/main/results/019d9025-c7b0-7240-b0e0-069d053c12c7.json"],"agent":{"model":"deepseek/deepseek-chat","tool":"mini-swe-agent","evidence":"A2A coding agent that solves SWE-bench Pro instances using mini-swe-agent (deepseek/deepseek-chat).","family":"deepseek","confidence":"high","role":"subject"}},{"contributor":"SWE-bench Purple Agent","description":"mini-swe-agent (deepseek/deepseek-chat), 5 instances","date":"2026-04-16","value":0.2,"contributorKind":"method","submissionUrls":["https://github.com/RDI-Foundation/swe-bench-leaderboard/blob/main/results/019d9480-9365-7273-b12a-bd76d7d604b4.json"],"agent":{"model":"deepseek/deepseek-chat","tool":"mini-swe-agent","evidence":"A2A coding agent that solves SWE-bench Pro instances using mini-swe-agent (deepseek/deepseek-chat).","family":"deepseek","confidence":"high","role":"subject"}},{"contributor":"SWE-bench Purple Agent","description":"mini-swe-agent (deepseek/deepseek-chat), 5 instances","date":"2026-04-16","value":0.4,"contributorKind":"method","submissionUrls":["https://github.com/RDI-Foundation/swe-bench-leaderboard/blob/main/results/019d9602-bdf5-7b71-8185-1ba4e00fa994.json"],"agent":{"model":"deepseek/deepseek-chat","tool":"mini-swe-agent","evidence":"A2A coding agent that solves SWE-bench Pro instances using mini-swe-agent (deepseek/deepseek-chat).","family":"deepseek","confidence":"high","role":"subject"}},{"contributor":"AgentX-Spring0123","description":"Purple agent using official A2A Python server template, 5 instances","date":"2026-04-18","value":0,"contributorKind":"method","submissionUrls":["https://github.com/RDI-Foundation/swe-bench-leaderboard/blob/main/results/019da114-0d54-7310-b6be-58c57d201617.json"],"agent":{"model":"AgentX-Spring0123","evidence":"Purple agent using official A2A Python server template, 5 instances","family":"other-ai","confidence":"low","role":"subject"}}],"contested":true}],"agentStats":[],"repo":{"owner":"RDI-Foundation","name":"swe-bench-leaderboard","branch":"main"},"participants":[],"stats":{"totalRecords":6,"uniqueParticipants":0,"firstSubmission":"2026-04-13","lastSubmission":"2026-04-18","recordsLast30d":0,"recordsLast90d":0},"lastUpdated":"2026-10-19T09:02:20.240Z"},{"id":"clawprobench","name":"ClawProBench","tagline":"Trace-aware evaluation of AI agents with runtime coverage and frozen workplace-style holdouts.","description":"ClawProBench is a transparent, live-first benchmark harness for evaluating AI agent capabilities inside the OpenClaw runtime. It features deterministic grading across multiple profiles, including core, intelligence, and native tracks. Submissions are evaluated on execution stability and success rates using metrics like FinalScore.","url":"https://github.com/suyoumo/ClawProBench","host":"github","status":"active","domain":"llm-eval","tags":["agent-evaluation","benchmark","openclaw","llm-eval","runtime-coverage"],"images":["https://raw.githubusercontent.com/suyoumo/ClawProBench/main/docs/assets/leaderboard-preview-20260426.png","https://raw.githubusercontent.com/suyoumo/ClawProBench/main/docs/assets/modelpk.png"],"participation":{"howToSubmit":"Run the benchmark locally and email the generated reports to xyh920691910@outlook.com to discuss model evaluation and leaderboard updates.","submissionFormat":"Generated reports in the results/ directory containing avg_score, max_score, coverage-aware summaries, cost, latency, and resume metadata.","computeDetails":"Live runs expect a working local openclaw CLI plus the auth and config required by the surfaces exercised by the selected scenarios.","deadline":"rolling","cost":"free","verification":"Deterministic grading with custom checkers and order-insensitive matchers inside the OpenClaw runtime.","requirements":["Python 3.11","uv","OpenClaw CLI","Docker"],"compute":"unknown"},"links":[{"label":"Leaderboard Website","url":"https://suyoumo.github.io/bench/"},{"label":"arXiv Paper","url":"https://arxiv.org/abs/2608.22510"},{"label":"Hugging Face Dataset","url":"https://huggingface.co/datasets/xyh110sym/clawprobench"}],"organizer":{"name":"Xiao YuanHang","type":"individual"},"quickstart":"pip install uv\nuv venv --python 3.11\nsource .venv/bin/activate\nuv pip install -r requirements.txt\npython3 run.py inventory\npython3 run.py run --model '<MODEL>' --execution-mode live --benchmark-profile core --trials 1 --cleanup-agents","problems":[{"description":"Default ranking suite consisting of 26 active scenarios.","id":"core","name":"Core Profile","metricName":"FinalScore","metricDirection":"maximize","isOpen":true,"records":[],"contested":false},{"description":"Extended active capability benchmark consisting of 95 active scenarios.","id":"intelligence","name":"Intelligence Profile","metricName":"FinalScore","metricDirection":"maximize","isOpen":true,"records":[],"contested":false},{"description":"Lower-stakes breadth and regression slice consisting of 7 active scenarios.","id":"coverage","name":"Coverage Profile","metricName":"FinalScore","metricDirection":"maximize","isOpen":true,"records":[],"contested":false},{"description":"Active OpenClaw-native slice only, consisting of 36 active scenarios.","id":"native","name":"Native Profile","metricName":"FinalScore","metricDirection":"maximize","isOpen":true,"records":[],"contested":false},{"description":"Union of all active scenarios, consisting of 102 active scenarios.","id":"full","name":"Full Profile","metricName":"FinalScore","metricDirection":"maximize","isOpen":true,"records":[],"contested":false}],"agentStats":[],"repo":{"owner":"suyoumo","name":"ClawProBench","branch":"main"},"participants":[],"stats":{"totalRecords":0,"uniqueParticipants":0,"recordsLast30d":0,"recordsLast90d":0},"lastUpdated":"2026-10-19T09:02:20.240Z"},{"id":"squad-explorer","name":"SQuAD","tagline":"The Stanford Question Answering Dataset benchmark for reading comprehension.","description":"SQuAD is a reading comprehension dataset consisting of questions posed by crowdworkers on a set of Wikipedia articles. Models must find the answer span or determine if the question is unanswerable. It is a premier benchmark for question answering systems.","url":"https://rajpurkar.github.io/SQuAD-explorer/","host":"website","status":"active","domain":"ml-research","tags":["nlp","question-answering","reading-comprehension","dataset"],"images":[],"participation":{"howToSubmit":"Submissions are made by evaluating models on CodaLab Worksheets and submitting the worksheet to the organizers.","submissionFormat":"CodaLab worksheet","cost":"free","requirements":["Python","CodaLab"],"compute":"unknown"},"links":[{"label":"Website","url":"https://rajpurkar.github.io/SQuAD-explorer/"},{"label":"GitHub","url":"https://github.com/rajpurkar/SQuAD-explorer"}],"organizer":{"name":"Stanford NLP Group","url":"https://nlp.stanford.edu/","type":"academic"},"problems":[{"description":"The initial version of the Stanford Question Answering Dataset, containing 100,000+ question-answer pairs where the answer is a segment of text.","id":"squad-1-1","name":"SQuAD 1.1","metricName":"F1","metricDirection":"maximize","isOpen":true,"records":[],"contested":false},{"description":"SQuAD 2.0 combines the 100,000 questions in SQuAD 1.1 with over 50,000 unanswerable questions written adversarially by crowdworkers.","id":"squad-2-0","name":"SQuAD 2.0","metricName":"F1","metricDirection":"maximize","isOpen":true,"records":[],"contested":false}],"agentStats":[],"repo":{"owner":"rajpurkar","name":"SQuAD-explorer","branch":"master"},"participants":[],"stats":{"totalRecords":0,"uniqueParticipants":0,"recordsLast30d":0,"recordsLast90d":0},"lastUpdated":"2026-10-19T09:02:20.240Z"},{"id":"reference-kernels","name":"GPU Mode Reference Kernels","tagline":"Optimize high-performance GPU kernels for deep learning and linear algebra.","description":"A collection of reference kernels and optimization challenges hosted by GPU Mode. Developers optimize CUDA, Triton, and PyTorch kernels for speed and correctness across various hardware backends like NVIDIA and AMD, with performance profiled using Nsight Compute and PyTorch profilers.","url":"https://github.com/gpu-mode/reference-kernels","host":"github","status":"active","domain":"hardware-efficiency","tags":["cuda","triton","gpu-optimization","deep-learning","linear-algebra","kernel-tuning"],"images":[],"participation":{"howToSubmit":"Submit kernels via KernelBot on the GPU Mode Discord or use popcorn-cli to evaluate locally and submit.","submissionFormat":"Python/Triton/CUDA kernel files (e.g., submission.py) matching the problem's task.yml schema.","computeDetails":"Evaluated on NVIDIA (e.g., B200, A100) and AMD GPUs depending on the competition track.","verification":"Automated correctness checks against PyTorch reference implementations and performance profiling via Nsight Compute (NCU) or PyTorch profiler.","requirements":["Python","PyTorch","Triton","CUDA","Nsight Compute"],"compute":"datacenter-gpu"},"links":[{"label":"Website","url":"https://www.gpumode.com/"},{"label":"Discord","url":"https://discord.gg/gpumode"},{"label":"Popcorn CLI","url":"https://github.com/gpu-mode/popcorn-cli"}],"organizer":{"name":"GPU Mode","url":"https://www.gpumode.com/","type":"community"},"problems":[{"description":"Optimize the 1D causal convolution kernel, commonly used in modern sequence models like Mamba.","metricUnit":"ms","id":"causal-conv1d","name":"Causal Conv1d","metricName":"time","metricDirection":"minimize","isOpen":true,"records":[],"contested":false},{"description":"Optimize the forward pass chunk output kernel for Gated Deltanet.","metricUnit":"ms","id":"gated-deltanet-chunk-fwd-o","name":"Gated Deltanet chunk_fwd_o","metricName":"time","metricDirection":"minimize","isOpen":true,"records":[],"contested":false},{"description":"Optimize QR decomposition with a dedicated Nsight Compute-compatible profile path.","metricUnit":"ms","id":"qr-v2","name":"QR Decomposition v2","metricName":"time","metricDirection":"minimize","isOpen":true,"records":[],"contested":false},{"description":"Optimize Cholesky decomposition with application validation over natural-gradient logistic-regression loops.","metricUnit":"ms","id":"cholesky","name":"Cholesky Validation","metricName":"time","metricDirection":"minimize","isOpen":true,"records":[],"contested":false},{"description":"Optimize symmetric eigenvalue decomposition (eigh) validation kernels.","metricUnit":"ms","id":"eigh","name":"Eigh Validation","metricName":"time","metricDirection":"minimize","isOpen":true,"records":[],"contested":false}],"agentStats":[],"repo":{"owner":"gpu-mode","name":"reference-kernels","branch":"main"},"participants":[],"stats":{"totalRecords":0,"uniqueParticipants":0,"recordsLast30d":0,"recordsLast90d":0},"lastUpdated":"2026-10-19T09:02:20.240Z"},{"id":"terminal-bench-leaderboard","name":"Terminal Bench 2.0 Leaderboard","tagline":"Evaluate agents on terminal-based coding and system tasks.","description":"Terminal Bench 2.0 is a benchmark that evaluates AI agents on terminal-based coding and system tasks. Agents are scored by task pass rate, which is the percentage of tasks completed successfully. Submissions are run automatically via GitHub Actions using Docker Compose.","url":"https://github.com/RDI-Foundation/terminal-bench-leaderboard","host":"github","status":"active","domain":"coding-agents","tags":["terminal","coding-agents","benchmark","agentbeats","docker"],"images":[],"participation":{"howToSubmit":"Fork the repository, fill in agent details in scenario.toml, and push to trigger the scenario runner. Results are submitted as a pull request.","submissionFormat":"scenario.toml configuration","computeDetails":"Parallel evaluation can be configured across multiple GitHub runners by editing num_shards in quick-submit.yml.","deadline":"rolling","cost":"free","verification":"CI","requirements":["Docker","Docker Compose","GitHub Actions","API keys"],"compute":"unknown"},"links":[{"label":"Terminal Bench Repository","url":"https://github.com/laude-ntt/terminal-bench"},{"label":"AgentBeats","url":"https://agentbeats.dev/"}],"organizer":{"name":"RDI Foundation","url":"https://github.com/RDI-Foundation","type":"foundation"},"quickstart":"To submit your agent, fork the repo and configure scenario.toml:\n\n[[participants]]\nagentbeats_id = \"YOUR_AGENT_ID\"\nAGENT_LLM = \"openai/gpt-4o\"","problems":[{"description":"Overall task pass rate across all terminal-based coding and system tasks.","metricUnit":"%","id":"terminal-bench-2.0","name":"Terminal Bench 2.0","metricName":"pass_rate","metricDirection":"maximize","isOpen":false,"records":[{"contributor":"019db41d-2a0a-7240-a4df-ae1b55065c3d","description":"Submission ID: 019dd98c-29c2-7621-98a6-bd6622066879","date":"2026-04-29","value":0,"contributorKind":"unknown","isCurrentBest":true,"agent":{"model":"Green Agent","evidence":"Green Agent ID: 019d87ef-2c6d-7091-8d93-827799c2ade3","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"019db41d-2a0a-7240-a4df-ae1b55065c3d","description":"Submission ID: 019dd953-c6d9-7da0-8171-b4361369bc05","date":"2026-04-29","value":0,"contributorKind":"unknown","agent":{"model":"Green Agent","evidence":"Green Agent ID: 019d87ef-2c6d-7091-8d93-827799c2ade3","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"019db41d-2a0a-7240-a4df-ae1b55065c3d","description":"Submission ID: 019dd928-6d99-7973-8d67-ff55b1f69e7f","date":"2026-04-29","value":0,"contributorKind":"unknown","agent":{"model":"Green Agent","evidence":"Green Agent ID: 019d87ef-2c6d-7091-8d93-827799c2ade3","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"019d0393-4eac-7232-ba9b-865e8c539e35","description":"Submission ID: 019d76ea-e6cd-7041-aef5-f7ee9d57ce58","date":"2026-04-10","value":0,"contributorKind":"unknown","agent":{"model":"Green Agent","evidence":"Green Agent ID: 019d76c2-3bb7-7652-b932-76ad868837b0","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"019d0393-4eac-7232-ba9b-865e8c539e35","description":"Submission ID: 019d76da-437f-7c33-a037-569015b1272f","date":"2026-04-10","value":0,"contributorKind":"unknown","agent":{"model":"Green Agent","evidence":"Green Agent ID: 019d76c2-3bb7-7652-b932-76ad868837b0","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"019d0393-4eac-7232-ba9b-865e8c539e35","description":"Submission ID: 019d76d0-5e6e-7f71-ac4a-39996fef99c8","date":"2026-04-10","value":0,"contributorKind":"unknown","agent":{"model":"Green Agent","evidence":"Green Agent ID: 019d76c2-3bb7-7652-b932-76ad868837b0","family":"other-ai","confidence":"high","role":"subject"}},{"contributor":"019d0393-4eac-7232-ba9b-865e8c539e35","description":"Submission ID: 019d76c7-6d7e-7bc3-aa66-0e1d80fc0067","date":"2026-04-10","value":0,"contributorKind":"unknown","agent":{"model":"Green Agent","evidence":"Green Agent ID: 019d76c2-3bb7-7652-b932-76ad868837b0","family":"other-ai","confidence":"high","role":"subject"}}],"contested":true}],"agentStats":[],"repo":{"owner":"RDI-Foundation","name":"terminal-bench-leaderboard","branch":"main"},"participants":[{"name":"019db41d-2a0a-7240-a4df-ae1b55065c3d","submissions":3,"problems":["terminal-bench-2.0"],"bestRank":1,"lastActive":"2026-04-29"},{"name":"019d0393-4eac-7232-ba9b-865e8c539e35","submissions":4,"problems":["terminal-bench-2.0"],"lastActive":"2026-04-10"}],"stats":{"totalRecords":7,"uniqueParticipants":2,"firstSubmission":"2026-04-10","lastSubmission":"2026-04-29","recordsLast30d":0,"recordsLast90d":0},"lastUpdated":"2026-10-19T09:02:20.240Z"}],"people":[{"name":"Avel","url":"https://github.com/Avel","records":21,"currentBests":5,"uncontestedBests":0,"competitions":[{"id":"gmaapp","name":"Grand Mountain Adventure Community Leaderboard","records":21,"bests":5}],"agents":{},"lastActive":"2022-03-27"},{"name":"hschindele","url":"https://github.com/hschindele","records":10,"currentBests":5,"uncontestedBests":0,"competitions":[{"id":"gmaapp","name":"Grand Mountain Adventure Community Leaderboard","records":10,"bests":5}],"agents":{},"lastActive":"2022-04-17"},{"name":"npow","url":"https://github.com/npow","records":9,"currentBests":3,"uncontestedBests":0,"competitions":[{"id":"sutro-problems","name":"Sutro Problems","records":9,"bests":3}],"agents":{},"lastActive":"2026-09-01"},{"name":"Mountain_Greg","url":"https://github.com/Mountain_Greg","records":18,"currentBests":2,"uncontestedBests":0,"competitions":[{"id":"gmaapp","name":"Grand Mountain Adventure Community Leaderboard","records":18,"bests":2}],"agents":{},"lastActive":"2022-04-21"},{"name":"spartanhaden","url":"https://github.com/spartanhaden","records":5,"currentBests":2,"uncontestedBests":0,"competitions":[{"id":"sutro-problems","name":"Sutro Problems","records":5,"bests":2}],"agents":{},"lastActive":"2026-10-12"},{"name":"islamborghini","url":"https://github.com/islamborghini","records":5,"currentBests":2,"uncontestedBests":0,"competitions":[{"id":"sutro-problems","name":"Sutro Problems","records":5,"bests":2}],"agents":{},"lastActive":"2026-10-18"},{"name":"sigkillme0","url":"https://github.com/sigkillme0","records":4,"currentBests":2,"uncontestedBests":0,"competitions":[{"id":"sutro-problems","name":"Sutro Problems","records":4,"bests":2}],"agents":{},"lastActive":"2026-10-18"},{"name":"jurajselep","url":"https://github.com/jurajselep","records":13,"currentBests":1,"uncontestedBests":0,"competitions":[{"id":"sutro-problems","name":"Sutro Problems","records":13,"bests":1}],"agents":{},"lastActive":"2026-09-20"},{"name":"classiclarryd","url":"https://github.com/classiclarryd","records":11,"currentBests":1,"uncontestedBests":0,"competitions":[{"id":"modded-nanogpt-npu","name":"Modded-NanoGPT · Ascend NPU Leaderboard","records":11,"bests":1}],"agents":{},"lastActive":"2025-12-14"},{"name":"b0nce","url":"https://github.com/b0nce","records":4,"currentBests":1,"uncontestedBests":0,"competitions":[{"id":"sutro-problems","name":"Sutro Problems","records":4,"bests":1}],"agents":{},"lastActive":"2026-09-02"},{"name":"slippylolo","url":"https://github.com/slippylolo","records":2,"currentBests":1,"uncontestedBests":0,"competitions":[{"id":"lora-speedrun","name":"LoRA Speedrun","records":2,"bests":1}],"agents":{},"lastActive":"2026-08-03"},{"name":"mikech76","url":"https://github.com/mikech76","records":2,"currentBests":1,"uncontestedBests":0,"competitions":[{"id":"sutro-problems","name":"Sutro Problems","records":2,"bests":1}],"agents":{},"lastActive":"2026-10-05"},{"name":"snimu","url":"https://github.com/snimu","records":2,"currentBests":1,"uncontestedBests":0,"competitions":[{"id":"modded-nanogpt-npu","name":"Modded-NanoGPT · Ascend NPU Leaderboard","records":2,"bests":1}],"agents":{},"lastActive":"2025-09-11"},{"name":"Jonathan","url":"https://github.com/Jonathan","records":2,"currentBests":1,"uncontestedBests":0,"competitions":[{"id":"gmaapp","name":"Grand Mountain Adventure Community Leaderboard","records":2,"bests":1}],"agents":{},"lastActive":"2022-04-07"},{"name":"Vilin97","url":"https://github.com/Vilin97","records":1,"currentBests":1,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":1}],"agents":{"axiom":1},"lastActive":"2026-10-15"},{"name":"GSM","url":"https://github.com/GSM","records":1,"currentBests":1,"uncontestedBests":0,"competitions":[{"id":"gmaapp","name":"Grand Mountain Adventure Community Leaderboard","records":1,"bests":1}],"agents":{},"lastActive":"2022-08-08"},{"name":"yaroslavvb","url":"https://github.com/yaroslavvb","records":21,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"sutro-problems","name":"Sutro Problems","records":21,"bests":0}],"agents":{"anthropic":9},"lastActive":"2026-09-30"},{"name":"sjbaebae","url":"https://github.com/sjbaebae","records":10,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"sutro-problems","name":"Sutro Problems","records":10,"bests":0}],"agents":{"openai":1},"lastActive":"2026-10-12"},{"name":"SethTS","url":"https://github.com/SethTS","records":10,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"sutro-problems","name":"Sutro Problems","records":10,"bests":0}],"agents":{"anthropic":1},"lastActive":"2026-09-29"},{"name":"zh4ngx","url":"https://github.com/zh4ngx","records":8,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"sutro-problems","name":"Sutro Problems","records":8,"bests":0}],"agents":{},"lastActive":"2026-08-30"},{"name":"cosminscn","url":"https://github.com/cosminscn","records":8,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"sutro-problems","name":"Sutro Problems","records":8,"bests":0}],"agents":{"openai":7,"anthropic":1},"lastActive":"2026-09-04"},{"name":"kellerjordan0","url":"https://github.com/kellerjordan0","records":7,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"modded-nanogpt-npu","name":"Modded-NanoGPT · Ascend NPU Leaderboard","records":7,"bests":0}],"agents":{},"lastActive":"2025-07-13"},{"name":"YouJiacheng","url":"https://github.com/YouJiacheng","records":6,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"modded-nanogpt-npu","name":"Modded-NanoGPT · Ascend NPU Leaderboard","records":6,"bests":0}],"agents":{},"lastActive":"2025-04-16"},{"name":"varunneal","url":"https://github.com/varunneal","records":5,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"modded-nanogpt-npu","name":"Modded-NanoGPT · Ascend NPU Leaderboard","records":5,"bests":0}],"agents":{},"lastActive":"2025-11-29"},{"name":"Jamiemac","url":"https://github.com/Jamiemac","records":5,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"gmaapp","name":"Grand Mountain Adventure Community Leaderboard","records":5,"bests":0}],"agents":{},"lastActive":"2022-04-07"},{"name":"Neo","url":"https://heyneo.so/","records":4,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"mle-bench","name":"MLE-bench","records":4,"bests":0}],"agents":{},"lastActive":"2025-07-28"},{"name":"Grad62304977, kellerjordan0","records":3,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"modded-nanogpt-npu","name":"Modded-NanoGPT · Ascend NPU Leaderboard","records":3,"bests":0}],"agents":{},"lastActive":"2024-11-06"},{"name":"KoszarskyB","url":"https://github.com/KoszarskyB","records":3,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"modded-nanogpt-npu","name":"Modded-NanoGPT · Ascend NPU Leaderboard","records":3,"bests":0}],"agents":{},"lastActive":"2025-01-04"},{"name":"Wrog","url":"https://github.com/Wrog","records":3,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"gmaapp","name":"Grand Mountain Adventure Community Leaderboard","records":3,"bests":0}],"agents":{},"lastActive":"2022-06-27"},{"name":"stared","url":"https://github.com/stared","records":2,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lora-speedrun","name":"LoRA Speedrun","records":2,"bests":0}],"agents":{"anthropic":2},"lastActive":"2026-07-20"},{"name":"likelytobelaura","url":"https://github.com/likelytobelaura","records":2,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"sutro-problems","name":"Sutro Problems","records":2,"bests":0}],"agents":{"anthropic":2},"lastActive":"2026-10-14"},{"name":"SecurityQQ","url":"https://github.com/SecurityQQ","records":2,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"sutro-problems","name":"Sutro Problems","records":2,"bests":0}],"agents":{},"lastActive":"2026-09-07"},{"name":"leloykun","url":"https://github.com/leloykun","records":2,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"modded-nanogpt-npu","name":"Modded-NanoGPT · Ascend NPU Leaderboard","records":2,"bests":0}],"agents":{},"lastActive":"2025-02-14"},{"name":"abacon","url":"https://github.com/abacon","records":1,"currentBests":0,"uncontestedBests":1,"competitions":[{"id":"lora-speedrun","name":"LoRA Speedrun","records":1,"bests":0}],"agents":{},"lastActive":"2026-07-27"},{"name":"Saivineeth147","url":"https://github.com/Saivineeth147","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lora-speedrun","name":"LoRA Speedrun","records":1,"bests":0}],"agents":{"anthropic":1},"lastActive":"2026-07-18"},{"name":"jurajselep, b0nce","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"sutro-problems","name":"Sutro Problems","records":1,"bests":0}],"agents":{},"lastActive":"2026-09-02"},{"name":"adotzh","url":"https://github.com/adotzh","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"sutro-problems","name":"Sutro Problems","records":1,"bests":0}],"agents":{},"lastActive":"2026-04-30"},{"name":"ZhengyangZhang06","url":"https://github.com/ZhengyangZhang06","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"openai":1},"lastActive":"2026-09-25"},{"name":"lukerj00","url":"https://github.com/lukerj00","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"other-ai":1},"lastActive":"2026-09-07"},{"name":"GanjinZero","url":"https://github.com/GanjinZero","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"bytedance":1},"lastActive":"2026-08-02"},{"name":"Morgan-Griffiths","url":"https://github.com/Morgan-Griffiths","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"openai":1},"lastActive":"2026-07-16"},{"name":"LorenzoLuccioli","url":"https://github.com/LorenzoLuccioli","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"harmonic":1},"lastActive":"2026-07-18"},{"name":"rishistyping","url":"https://github.com/rishistyping","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"other-ai":1},"lastActive":"2026-08-20"},{"name":"ma4ypic4y","url":"https://github.com/ma4ypic4y","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"openai":1},"lastActive":"2026-10-18"},{"name":"skidanovalex","url":"https://github.com/skidanovalex","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"deepseek":1},"lastActive":"2026-10-14"},{"name":"mayorov-m-a","url":"https://github.com/mayorov-m-a","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"other-ai":1},"lastActive":"2026-06-05"},{"name":"parabamoghv","url":"https://github.com/parabamoghv","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"harmonic":1},"lastActive":"2026-08-20"},{"name":"sarjinius","url":"https://github.com/sarjinius","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"other-ai":1},"lastActive":"2026-10-01"},{"name":"hongzhoulin89","url":"https://github.com/hongzhoulin89","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"openai":1},"lastActive":"2026-10-04"},{"name":"mlgraham","url":"https://github.com/mlgraham","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"other-ai":1},"lastActive":"2026-09-13"},{"name":"doxtor6","url":"https://github.com/doxtor6","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"other-ai":1},"lastActive":"2026-06-25"},{"name":"daouid","url":"https://github.com/daouid","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"anthropic":1},"lastActive":"2026-05-22"},{"name":"hanwenzhu","url":"https://github.com/hanwenzhu","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"bytedance":1},"lastActive":"2026-08-13"},{"name":"savarin","url":"https://github.com/savarin","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"openai":1},"lastActive":"2026-08-17"},{"name":"rkirov","url":"https://github.com/rkirov","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"anthropic":1},"lastActive":"2026-07-10"},{"name":"sqrt-of-2","url":"https://github.com/sqrt-of-2","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"harmonic":1},"lastActive":"2026-05-12"},{"name":"antoinegg1","url":"https://github.com/antoinegg1","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"openai":1},"lastActive":"2026-10-15"},{"name":"LikC1606","url":"https://github.com/LikC1606","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"openai":1},"lastActive":"2026-10-16"},{"name":"test1-deepthought","url":"https://github.com/test1-deepthought","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"other-ai":1},"lastActive":"2026-06-23"},{"name":"AlyciaBHZ","url":"https://github.com/AlyciaBHZ","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"other-ai":1},"lastActive":"2026-10-05"},{"name":"JohnEdwardJennings","url":"https://github.com/JohnEdwardJennings","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"harmonic":1},"lastActive":"2026-06-14"},{"name":"kim-em","url":"https://github.com/kim-em","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"harmonic":1},"lastActive":"2026-09-13"},{"name":"juihuichung","url":"https://github.com/juihuichung","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"openai":1},"lastActive":"2026-08-31"},{"name":"xiaoyangliu-sjtu","url":"https://github.com/xiaoyangliu-sjtu","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"other-ai":1},"lastActive":"2026-09-15"},{"name":"A-M-Berns","url":"https://github.com/A-M-Berns","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"openai":1},"lastActive":"2026-05-16"},{"name":"antpavzhi","url":"https://github.com/antpavzhi","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"other-ai":1},"lastActive":"2026-05-11"},{"name":"bogatyy","url":"https://github.com/bogatyy","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"other-ai":1},"lastActive":"2026-09-19"},{"name":"adrianmartir","url":"https://github.com/adrianmartir","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"harmonic":1},"lastActive":"2026-06-29"},{"name":"eohjelle","url":"https://github.com/eohjelle","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"openai":1},"lastActive":"2026-07-21"},{"name":"ivwumupy","url":"https://github.com/ivwumupy","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"openai":1},"lastActive":"2026-07-27"},{"name":"mccorvie","url":"https://github.com/mccorvie","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"openai":1},"lastActive":"2026-08-03"},{"name":"rwst","url":"https://github.com/rwst","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"xai":1},"lastActive":"2026-07-23"},{"name":"jzuiddam","url":"https://github.com/jzuiddam","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"anthropic":1},"lastActive":"2026-05-07"},{"name":"KitaKen1","url":"https://github.com/KitaKen1","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"anthropic":1},"lastActive":"2026-08-06"},{"name":"machinelearning2014","url":"https://github.com/machinelearning2014","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"other-ai":1},"lastActive":"2026-05-30"},{"name":"n-yamaguchi-0729","url":"https://github.com/n-yamaguchi-0729","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"openai":1},"lastActive":"2026-09-08"},{"name":"nolanlwin","url":"https://github.com/nolanlwin","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"other-ai":1},"lastActive":"2026-09-07"},{"name":"Solarys431","url":"https://github.com/Solarys431","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"anthropic":1},"lastActive":"2026-07-21"},{"name":"will1491","url":"https://github.com/will1491","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"anthropic":1},"lastActive":"2026-09-26"},{"name":"yawara","url":"https://github.com/yawara","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"other-ai":1},"lastActive":"2026-07-16"},{"name":"adamtopaz","url":"https://github.com/adamtopaz","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"anthropic":1},"lastActive":"2026-07-25"},{"name":"albertqjiang","url":"https://github.com/albertqjiang","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"other-ai":1},"lastActive":"2026-06-06"},{"name":"Cond-t","url":"https://github.com/Cond-t","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"openai":1},"lastActive":"2026-08-31"},{"name":"homeowmorphism","url":"https://github.com/homeowmorphism","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"anthropic":1},"lastActive":"2026-08-18"},{"name":"ianklatzco","url":"https://github.com/ianklatzco","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"anthropic":1},"lastActive":"2026-07-14"},{"name":"kimihiro64","url":"https://github.com/kimihiro64","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"openai":1},"lastActive":"2026-09-13"},{"name":"ldct","url":"https://github.com/ldct","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"anthropic":1},"lastActive":"2026-07-05"},{"name":"LexBrill","url":"https://github.com/LexBrill","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"openai":1},"lastActive":"2026-09-02"},{"name":"lixiang90","url":"https://github.com/lixiang90","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"openai":1},"lastActive":"2026-09-07"},{"name":"matthewjasper","url":"https://github.com/matthewjasper","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"openai":1},"lastActive":"2026-07-14"},{"name":"mrdouglasny","url":"https://github.com/mrdouglasny","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"other-ai":1},"lastActive":"2026-06-15"},{"name":"niketp03","url":"https://github.com/niketp03","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"other-ai":1},"lastActive":"2026-06-02"},{"name":"Parcly-Taxel","url":"https://github.com/Parcly-Taxel","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"harmonic":1},"lastActive":"2026-05-23"},{"name":"plby","url":"https://github.com/plby","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"openai":1},"lastActive":"2026-06-26"},{"name":"rigwild","url":"https://github.com/rigwild","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"anthropic":1},"lastActive":"2026-08-04"},{"name":"weiyangzen","url":"https://github.com/weiyangzen","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"deepseek":1},"lastActive":"2026-10-17"},{"name":"wpegden","url":"https://github.com/wpegden","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"agents":{"other-ai":1},"lastActive":"2026-07-20"},{"name":"Pablo Manrique","records":1,"currentBests":0,"uncontestedBests":1,"competitions":[{"id":"leansolving-squeezebench-04bec82398b15ff6","name":"SqueezeBench","records":1,"bests":0}],"agents":{"anthropic":1}},{"name":"MedArise","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"chi-bench","name":"CHI-Bench Leaderboard","records":1,"bests":0}],"agents":{"openai":1},"lastActive":"2026-06-05"},{"name":"kellerjordan0, jxbz","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"modded-nanogpt-npu","name":"Modded-NanoGPT · Ascend NPU Leaderboard","records":1,"bests":0}],"agents":{},"lastActive":"2024-10-04"},{"name":"kellerjordan0, bozavlado","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"modded-nanogpt-npu","name":"Modded-NanoGPT · Ascend NPU Leaderboard","records":1,"bests":0}],"agents":{},"lastActive":"2024-10-11"},{"name":"brendanh0gan","url":"https://github.com/brendanh0gan","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"modded-nanogpt-npu","name":"Modded-NanoGPT · Ascend NPU Leaderboard","records":1,"bests":0}],"agents":{},"lastActive":"2024-11-10"},{"name":"fernbear.bsky.social","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"modded-nanogpt-npu","name":"Modded-NanoGPT · Ascend NPU Leaderboard","records":1,"bests":0}],"agents":{},"lastActive":"2024-11-24"},{"name":"leloykun, YouJiacheng","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"modded-nanogpt-npu","name":"Modded-NanoGPT · Ascend NPU Leaderboard","records":1,"bests":0}],"agents":{},"lastActive":"2024-12-08"},{"name":"leloykun, fernbear.bsky.social, YouJiacheng, brendanh0gan, scottjmaddox, Grad62304977","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"modded-nanogpt-npu","name":"Modded-NanoGPT · Ascend NPU Leaderboard","records":1,"bests":0}],"agents":{},"lastActive":"2025-01-16"},{"name":"KonstantinWilleke, alexrgilbert, adricarda, tuttyfrutyee, vdlad; The Enigma project","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"modded-nanogpt-npu","name":"Modded-NanoGPT · Ascend NPU Leaderboard","records":1,"bests":0}],"agents":{},"lastActive":"2025-05-24"},{"name":"ryanyang0","url":"https://github.com/ryanyang0","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"modded-nanogpt-npu","name":"Modded-NanoGPT · Ascend NPU Leaderboard","records":1,"bests":0}],"agents":{},"lastActive":"2025-05-25"},{"name":"vagrawal","url":"https://github.com/vagrawal","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"modded-nanogpt-npu","name":"Modded-NanoGPT · Ascend NPU Leaderboard","records":1,"bests":0}],"agents":{},"lastActive":"2025-05-30"},{"name":"byronxu99","url":"https://github.com/byronxu99","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"modded-nanogpt-npu","name":"Modded-NanoGPT · Ascend NPU Leaderboard","records":1,"bests":0}],"agents":{},"lastActive":"2025-07-18"},{"name":"EmelyanenkoK","url":"https://github.com/EmelyanenkoK","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"modded-nanogpt-npu","name":"Modded-NanoGPT · Ascend NPU Leaderboard","records":1,"bests":0}],"agents":{},"lastActive":"2025-09-05"},{"name":"bernard24 & hiverge.ai","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"modded-nanogpt-npu","name":"Modded-NanoGPT · Ascend NPU Leaderboard","records":1,"bests":0}],"agents":{},"lastActive":"2025-09-11"},{"name":"Gusarich","url":"https://github.com/Gusarich","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"modded-nanogpt-npu","name":"Modded-NanoGPT · Ascend NPU Leaderboard","records":1,"bests":0}],"agents":{},"lastActive":"2025-09-27"},{"name":"li_zichong","url":"https://github.com/li_zichong","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"modded-nanogpt-npu","name":"Modded-NanoGPT · Ascend NPU Leaderboard","records":1,"bests":0}],"agents":{"human":1},"lastActive":"2025-10-24"},{"name":"akash5474","url":"https://github.com/akash5474","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"modded-nanogpt-npu","name":"Modded-NanoGPT · Ascend NPU Leaderboard","records":1,"bests":0}],"agents":{},"lastActive":"2025-11-16"},{"name":"roeeshenberg","url":"https://github.com/roeeshenberg","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"modded-nanogpt-npu","name":"Modded-NanoGPT · Ascend NPU Leaderboard","records":1,"bests":0}],"agents":{},"lastActive":"2025-12-10"},{"name":"ChrisJMcCormick","url":"https://github.com/ChrisJMcCormick","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"modded-nanogpt-npu","name":"Modded-NanoGPT · Ascend NPU Leaderboard","records":1,"bests":0}],"agents":{},"lastActive":"2025-12-11"},{"name":"jadenj3o","url":"https://github.com/jadenj3o","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"modded-nanogpt-npu","name":"Modded-NanoGPT · Ascend NPU Leaderboard","records":1,"bests":0}],"agents":{},"lastActive":"2025-04-22"}],"agents":[{"family":"anthropic","records":46,"currentBests":5,"uncontestedBests":1,"competitions":[{"id":"lora-speedrun","name":"LoRA Speedrun","records":3,"bests":0},{"id":"sutro-problems","name":"Sutro Problems","records":15,"bests":2},{"id":"lean-eval","name":"LeanEval","records":11,"bests":0},{"id":"leansolving-squeezebench-04bec82398b15ff6","name":"SqueezeBench","records":1,"bests":0},{"id":"chi-bench","name":"CHI-Bench Leaderboard","records":16,"bests":3}],"models":{"Claude Fable 5":4,"Claude Opus 5.5":9,"Claude":3,"Claude Fable 5.1":2,"Claude Sonnet 4.6":1,"Antigravity (Multi-Model Ensemble: Gemini 3.1 Pro, Gemini 3 Flash, Claude 4.6 Sonnet/Opus)":1,"Claude Opus 4.7 (1M context)":2,"GPT-5.6 and Fable 5 (human-in-the-loop)":1,"Opus 4.8) + GPT-5.6-Sol":1,"Fable 5 and GPT-5.6":1,"Claude Fable 5 (Claude Code)":1,"Claude Fable 5 (orchestrating Opus/Sonnet 4.8)":1,"Fable 5 (multi-agent)":1,"OpenAI Codex (GPT-5.6 Sol), Claude Code (Opus 5 and Opus 4.8)":1,"claude-opus-5.5":1,"Claude Opus 4.7":4,"anthropic/claude-opus-5":4,"anthropic/claude-opus-4-8":8},"tools":{"Claude Code":3,"UNICO/NOUS: Claude (Fable 5":1,"Claude Opus 4.8":1,"erius":12},"contributors":["Saivineeth147","stared","yaroslavvb","likelytobelaura","QDA","PCA-QDA","SethTS","cosminscn","daouid","rkirov","jzuiddam","KitaKen1","Solarys431","will1491","adamtopaz","homeowmorphism","ianklatzco","ldct","rigwild","Pablo Manrique","HG-Market-System","erius"],"highConfidence":29},{"family":"openai","records":35,"currentBests":1,"uncontestedBests":1,"competitions":[{"id":"sutro-problems","name":"Sutro Problems","records":15,"bests":1},{"id":"lean-eval","name":"LeanEval","records":19,"bests":0},{"id":"chi-bench","name":"CHI-Bench Leaderboard","records":1,"bests":0}],"models":{"Codex":6,"GPT 5.6 sol":4,"GPT-5.6":2,"max gp (GPT-6)":1,"savarin-6-hour-gpt-5.6-speedrun":1,"GPT-6.1 (gpt-6.1-sol)":1,"GPT-5.5":1,"deepseek-v4 and gpt-5.x":1,"mostly ChatGPT 5.6 Sol":1,"GPT-5.6 Sol":1,"OpenAI Codex (GPT-5.6 Sol)":1,"HITL":1,"ChatGPT 5.6 Sol":1,"GPT-5.6 Sol)":1,"GPT-5.6 (human-in-the-loop)":1,"Codex (with human in the loop)":1,"openai/gpt-5.5":1},"tools":{"Codex":9,"Humanifa":4,"DeepScientist Math":1,"Codex 5.6 Sol":1,"BRILLiant":1,"OpenAI Codex (GPT-6":1,"codex":1},"contributors":["512-unit MLP","Reversible MLP","32-unit MLP","1NN","Ordered ConvNets","Three ConvNets","cosminscn","sjbaebae","ZhengyangZhang06","Morgan-Griffiths","ma4ypic4y","hongzhoulin89","savarin","antoinegg1","LikC1606","juihuichung","A-M-Berns","eohjelle","ivwumupy","mccorvie","n-yamaguchi-0729","Cond-t","kimihiro64","LexBrill","lixiang90","matthewjasper","plby","MedArise"],"highConfidence":34},{"family":"axiom","records":1,"currentBests":1,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":1}],"models":{"Axiom Prover (Axiom Math)":1},"tools":{},"contributors":["Vilin97"],"highConfidence":1},{"family":"other-ai","records":18,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":18,"bests":0}],"models":{"Tau (caj.al)":1,"Stealth Model":1,"Aleph Prover(logicalintelligence.com)":2,"Midas":1,"github.com/mlgraham (various models)":1,"MerLean-Prover":1,"EVO (deepthought.com.au)":1,"trureturing":1,"Forge":1,"Ivan Bogatyy":1,"EVO":1,"Forall (Astrio)":1,"Various models ( with human in the loop )":1,"Leanstral 1.4":1,"Community multi-model project":1,"Autoform-Bot":1,"Trellis":1},"tools":{},"contributors":["lukerj00","rishistyping","mayorov-m-a","sarjinius","mlgraham","doxtor6","test1-deepthought","AlyciaBHZ","xiaoyangliu-sjtu","antpavzhi","bogatyy","machinelearning2014","nolanlwin","yawara","albertqjiang","mrdouglasny","niketp03","wpegden"],"highConfidence":18},{"family":"harmonic","records":7,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":7,"bests":0}],"models":{"Aristotle (Harmonic)":7},"tools":{},"contributors":["LorenzoLuccioli","parabamoghv","sqrt-of-2","JohnEdwardJennings","kim-em","adrianmartir","Parcly-Taxel"],"highConfidence":7},{"family":"bytedance","records":2,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":2,"bests":0}],"models":{"Seed Prover (ByteDance)":2},"tools":{},"contributors":["GanjinZero","hanwenzhu"],"highConfidence":2},{"family":"deepseek","records":2,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":2,"bests":0}],"models":{"DeepSeek V4":1,"Deepseek-v4.1 Flash-1017":1},"tools":{"NEAR AI":1},"contributors":["skidanovalex","weiyangzen"],"highConfidence":2},{"family":"human","records":2,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"modded-nanogpt-npu","name":"Modded-NanoGPT · Ascend NPU Leaderboard","records":2,"bests":0}],"models":{},"tools":{},"contributors":["NorMuon optimizer","li_zichong"],"highConfidence":0},{"family":"xai","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"models":{"Grok 4.5":1},"tools":{},"contributors":["rwst"],"highConfidence":1}],"evaluated":[{"family":"openai","records":150,"currentBests":29,"uncontestedBests":1,"competitions":[{"id":"alpaca-eval","name":"AlpacaEval","records":12,"bests":2},{"id":"mle-bench","name":"MLE-bench","records":40,"bests":0},{"id":"llm-leaderboard","name":"LLM Leaderboard","records":17,"bests":3},{"id":"fstandhartinger-jevbench-2acd10345790c8d1","name":"JevBench","records":4,"bests":1},{"id":"first-proof","name":"First Proof","records":32,"bests":22},{"id":"lean-eval","name":"LeanEval","records":25,"bests":0},{"id":"programbench","name":"ProgramBench","records":8,"bests":1},{"id":"chi-bench","name":"CHI-Bench Leaderboard","records":12,"bests":0}],"models":{"GPT-4":9,"ChatGPT":2,"text-davinci-001":1,"GPT-4 Turbo":3,"text-davinci-003":1,"gpt-5":8,"gpt-5-codex":4,"o3":4,"o3 + GPT-4.1":4,"o1-preview":8,"gpt-4o-2024-08-06":12,"GPT-3.5-175B / text-davinci-003":6,"GPT-3-175B / davinci":3,"GPT-3-7B / curie":2,"GPT-3.5-175B / code-davinci-002":1,"Codex-12B / code-cushman-001":1,"GPT-5.6 Luna":4,"gpt-5.5 pro":21,"OpenAI ChatGPT 5.5 Pro":11,"GPT 5.6 sol":1,"GPT-5.6":2,"max gp (GPT-6)":1,"Vasily-24-hour-gpt-5.6-speedrun":1,"savarin-6-hour-gpt-5.6-speedrun":1,"GPT-5.5":4,"Codex packaging":1,"GPT-5.6 Sol":3,"deepseek-v4 and gpt-5.x":1,"mostly ChatGPT 5.6 Sol":1,"GPT-5.5 Codex":1,"reviewer":1,"Aristotle":1,"Codex (with human in the loop)":1,"GPT-5.6 (human-in-the-loop)":1,"various model (deepseek-v4, gpt-5.*)":1,"GPT-5.6 Codex (public-proof reproduction)":1,"GPT-5 Codex":1,"OpenAI Codex (GPT-5)":1,"OpenAI Codex (GPT-5.6 Sol)":1,"ChatGPT 5.6 Sol":1,"GPT-5.6 Sol)":1,"HITL":1,"GPT-6.1 (gpt-6.1-sol)":1,"GPT-5.4":1,"GPT-5.4 mini":1,"GPT-5 mini":1,"openai/gpt-5.6-luna":4,"openai/gpt-5.6-terra":4,"openai/gpt-5.6-sol":4},"tools":{"IMProofBench ProofCouncil":10,"UCLA Moonshot Harness":11,"OpenAI ChatGPT 5.5 Pro":11,"Humanifa":1,"Public accepted source":1,"Humanize RLCR: GPT-5.6-sol (max) worker":1,"GPT-5 Codex":1,"DeepScientist":1,"BRILLiant":1,"OpenAI Codex (GPT-6":1,"Codex 5.6 Sol":1,"DeepScientist Math":1,"mini-SWE-agent":8,"codex":12},"contributors":["gpt4","chatgpt","text_davinci_001","alpaca_eval_gpt4","alpaca_eval_cot_gpt4_turbo_fn","alpaca_eval_gpt4_turbo_fn","alpaca_farm_greedy_gpt4","lmsys_gpt4","text_davinci_003","R&D-Agent","Thesis","Operand ensemble","AIRA-dojo","AIDE","OpenHands","MLAB","gpt-4","gpt-3.5-175b / text-davinci-003","gpt-3-175b / davinci","gpt-3-7b / curie","gpt-3.5-175b / code-davinci-002","codex-12b / code-cushman-001","GPT-5.6 Luna (low)","GPT-5.6 Luna (low reasoning effort)","IMProofBench ProofCouncil","UCLA Moonshot Harness","OpenAI ChatGPT 5.5 Pro","Humanifa + GPT 5.6 sol","GPT-5.6","max gp (GPT-6)","Vasily-24-hour-gpt-5.6-speedrun","savarin-6-hour-gpt-5.6-speedrun","GPT-5.5","Public accepted source + Codex packaging","GPT-5.6 Sol","deepseek-v4 and gpt-5.x","mostly ChatGPT 5.6 Sol","GPT-5.5 Codex","Humanize RLCR: GPT-5.6-sol (max) worker + reviewer","GPT-5 Codex + Aristotle","Codex (with human in the loop)","GPT-5.6 (human-in-the-loop)","various model (deepseek-v4, gpt-5.*)","GPT-5.6 Codex (public-proof reproduction)","GPT-5 Codex","OpenAI Codex (GPT-5)","DeepScientist + GPT-5.6","OpenAI Codex (GPT-5.6 Sol)","BRILLiant + ChatGPT 5.6 Sol","OpenAI Codex (GPT-6 + GPT-5.6 Sol)","Codex 5.6 Sol + HITL","DeepScientist Math + GPT-6.1 (gpt-6.1-sol)","GPT-5.6 Sol (xhigh)","GPT-5.5 (xhigh)","GPT-5.5 (high)","GPT-5.4","GPT-5.4 mini","GPT-5 mini","gpt-5.6-luna","gpt-5.6-terra","gpt-5.6-sol"],"highConfidence":145},{"family":"other-ai","records":166,"currentBests":11,"uncontestedBests":2,"competitions":[{"id":"alpaca-eval","name":"AlpacaEval","records":1,"bests":0},{"id":"cogdl","name":"CogDL Leaderboards","records":12,"bests":3},{"id":"llm-leaderboard","name":"LLM Leaderboard","records":67,"bests":2},{"id":"fstandhartinger-jevbench-2acd10345790c8d1","name":"JevBench","records":42,"bests":4},{"id":"lean-eval","name":"LeanEval","records":23,"bests":0},{"id":"agent-anvil-leaderboard","name":"Agent Anvil Public Leaderboard","records":1,"bests":0},{"id":"chi-bench","name":"CHI-Bench Leaderboard","records":12,"bests":1},{"id":"swe-bench-pro-leaderboard","name":"SWE-bench Pro Leaderboard","records":1,"bests":0},{"id":"terminal-bench-leaderboard","name":"Terminal Bench 2.0 Leaderboard","records":7,"bests":1}],"models":{"Simple-HGN":3,"GCC":3,"DropEdge":3,"JK-net":3,"Vicuna-13B":1,"Koala-13B":3,"OASST-Pythia-12B":3,"Alpaca-13B":1,"FastChat-T5-3B":1,"Dolly-v2-12B":3,"StableLM-Tuned-Alpha-7B":3,"Falcon-40B":2,"Falcon-7B":2,"BLOOM-176B":3,"GPT-NeoX-20B":6,"GPT-J-6B":5,"Chinchilla-70B":3,"MPT-7B":3,"GPT4All-13B-snoozy":2,"Alpaca-7B":2,"Eleuther-Pythia-12B":3,"Eleuther-Pythia-7B":4,"Cerebras-GPT-7B":3,"Cerebras-GPT-13B":3,"StableLM-Base-Alpha-7B":4,"StarCoder-16B":1,"StarCoder-Base-16B":1,"CodeGen-16B-mono":1,"CodeGX-13B":1,"Replit-Code-v1-3B":1,"CodeGen-16B-multi":1,"GAL-120B":1,"Imajev-4B":1,"Plumb-4B":2,"decider-4b v2":2,"Jev 1.13.0":10,"JevK5 v0.2.0":2,"decision-machine-1":1,"decider-35b-a3b":1,"zerank-2":1,"kev 0.6B":1,"decider-2b":1,"kev 4B":1,"kev 8B":1,"GLiFormer 400M":1,"ModernBERT-large":2,"openJev Verdict 1.4":1,"openJev Verdict 151M":1,"kev 0.5B":1,"GLiNER2 large":1,"MiniCPM5-2B":1,"gliner2.5-base":1,"DeBERTa-v3-large":2,"GLiNER2.5 multi":1,"GLiNER2.5 small":1,"mxbai-rerank-base-v2":1,"bge-reranker-v2-m3":1,"Certo v1":1,"Needle 3":2,"Tau (caj.al)":1,"Stealth Model":1,"Aleph Prover(logicalintelligence.com)":1,"Midas":1,"github.com/mlgraham (various models)":1,"MerLean-Prover":1,"EVO (deepthought.com.au)":1,"trureturing":1,"Forge":1,"Ivan Bogatyy":1,"[submission] aegis-of-the-unit-circle-logos":1,"EVO":1,"Forall (Astrio)":1,"trureturning":1,"Autoform-Bot":1,"Leanstral 1.4":1,"Community multi-model project":1,"Hy3":1,"Various models ( with human in the loop )":1,"Trellis":1,"LeanEval production launch canary 3dc17f41ccd0c86fa443687fac784a42f3798183":1,"(various models)":1,"LeanEval P1 launch canary":1,"Agent Anvil Demo Agent":1,"nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-BF16:peft:262144":4,"thinkingmachines/Inkling:peft:262144":4,"MedGuard":4,"AgentX-Spring0123":1,"Green Agent":7},"tools":{"openai-agents":8,"hermes":4},"contributors":["longest","Simple-HGN","GCC","DropEdge","JK-net","vicuna-13b","koala-13b","oasst-pythia-12b","alpaca-13b","fastchat-t5-3b","dolly-v2-12b","stablelm-tuned-alpha-7b","falcon-40b","falcon-7b","bloom-176b","gpt-neox-20b","gpt-j-6b","chinchilla-70b","mpt-7b","gpt4all-13b-snoozy","alpaca-7b","eleuther-pythia-12b","eleuther-pythia-7b","cerebras-gpt-7b","cerebras-gpt-13b","stablelm-base-alpha-7b","starcoder-16b","starcoder-base-16b","codegen-16B-mono","codegx-13b","replit-code-v1-3b","codegen-16B-multi","gal-120b","Imajev-4B","Plumb-4B (crh225, JevK5 v0.2 + LoRA)","decider-4b v2 (Mapika)","Jev 1.13.0 (TypeSafe AI)","JevK5 v0.2.0","Jev 1.13.0","decision-machine-1 (milliseconds.ai)","decider-35b-a3b (Mapika)","ZeroEntropy zerank-2","kev 0.6B (research preview)","decider-2b (Mapika)","kev 4B (research preview)","kev 8B (research preview)","jeff (GLiFormer 400M)","Laya (421M)","OpenDecision (ModernBERT-large zero-shot)","openJev Verdict 1.4","openJev Verdict (151M)","kev 0.5B","GLiNER2 large (Fastino)","smalljev semantic-v9","GLiNER2 (gliner2.5-base)","open-jev-deberta-v3-large (local CPU)","GLiNER2.5 multi (Fastino, 287M)","GLiNER2.5 small (Fastino, 74M)","Mixedbread mxbai-rerank-base-v2","BAAI bge-reranker-v2-m3","Certo v1 (AltSlate Labs)","Needle 3, options as tools (post-hoc adapter mode)","Needle 3 (Cactus, 2-bit, local CPU)","classifier.dev Fast → Jev (cascade, threshold 0.42)","Calibration-weighted committee: Jev + SemIf + djev","Probability average committee: Jev + SemIf + djev","Majority committee: Jev + djev + open-alternative-jev","Tau (caj.al)","Stealth Model","Aleph Prover(logicalintelligence.com)","Midas","github.com/mlgraham (various models)","MerLean-Prover","EVO (deepthought.com.au)","trureturing","Forge","Ivan Bogatyy","[submission] aegis-of-the-unit-circle-logos","EVO","Forall (Astrio)","trureturning","Autoform-Bot","Leanstral 1.4","Community multi-model project","Hy3","Various models ( with human in the loop )","Trellis","LeanEval production launch canary 3dc17f41ccd0c86fa443687fac784a42f3798183","(various models)","LeanEval P1 launch canary","Agent Anvil Demo Agent","NVIDIA-Nemotron-3-Ultra-550B-A55B-BF16:peft:262144","Inkling:peft:262144","MedGuard","AgentX-Spring0123","019db41d-2a0a-7240-a4df-ae1b55065c3d","019d0393-4eac-7232-ba9b-865e8c539e35"],"highConfidence":152},{"family":"google","records":99,"currentBests":10,"uncontestedBests":0,"competitions":[{"id":"mle-bench","name":"MLE-bench","records":44,"bests":3},{"id":"llm-leaderboard","name":"LLM Leaderboard","records":21,"bests":5},{"id":"fstandhartinger-jevbench-2acd10345790c8d1","name":"JevBench","records":17,"bests":1},{"id":"first-proof","name":"First Proof","records":11,"bests":1},{"id":"lean-eval","name":"LeanEval","records":1,"bests":0},{"id":"programbench","name":"ProgramBench","records":5,"bests":0}],"models":{"Gemini-3-Pro-Preview":28,"Gemini-2.5-Pro":16,"PaLM-540B":8,"PaLM-2-L":3,"PaLM-2-M":3,"PaLM-2-S":3,"PaLM-2-S*":1,"PaLM-Coder-540B":2,"PaLM-2-L-Instruct":1,"Gemma-4-12B-it":1,"diffusion-gemma":2,"Gemma 3 12B":1,"Gemma 4 E2B":4,"DiffusionGemma 26B":3,"DiffusionGemma":1,"Gemini 3.1 Flash-Lite":4,"DiffusionGemma 26B-A4B":1,"gemini-3.1-pro-preview":11,"Gemini 3.1 Pro":2,"Gemini 3.7 Flash":1,"Gemini 3.6 Flash":1,"Gemini 3.5 Flash":1,"Gemini 3 Flash":1},"tools":{"Princeton Momus":11,"mini-SWE-agent":5},"contributors":["Famou-Agent 2.0","MLEvolve","PiEvolve","CAIR MARS+","CAIR MARS","Leeroo","CAIR MLE-STAR-Pro-1.5","CAIR MLE-STAR-Pro-1.0","Famou-Agent","palm-540b","palm-2-l","palm-2-m","palm-2-s","palm-2-s*","palm-coder-540b","palm-2-l-instruct","Cygnet (blockbrain, frozen Gemma-4-12B-it)","djev (Maisa, diffusion-gemma)","Winnow-12B Q8","system-one-open (Gemma 4 E2B LoRA on an L4)","OpenJev razorback16 (DiffusionGemma 26B)","djev (thinking)","Gemini 3.1 Flash-Lite","OpenJev (thinking, BF16)","OpenJev (DiffusionGemma 26B-A4B NVFP4, razorback16)","djev → Jev (cascade, threshold 0.79)","Best-of-n: OpenJev razorback16 two-sample average","Princeton Momus","Gemini 3.1 Pro","Gemini 3.7 Flash","Gemini 3.6 Flash","Gemini 3.5 Flash","Gemini 3 Flash"],"highConfidence":97},{"family":"alibaba","records":28,"currentBests":3,"uncontestedBests":1,"competitions":[{"id":"fstandhartinger-jevbench-2acd10345790c8d1","name":"JevBench","records":28,"bests":3}],"models":{"Qwen3.5-4B":7,"Qwen3-32B":1,"Qwen3.8-27B":3,"Qwen3.6-35B-A3B":5,"Qwen3-Reranker-4B":1,"Qwen3.5-9B":3,"Qwen3-8B":2,"Qwen3.5-2B":1,"GTE Reranker ModernBERT-base":1,"Qwen3.8 27B":4},"tools":{},"contributors":["SemIf (Qwen3.5-4B)","reflex 4B (kshetrajna12)","jqv (Qwen3-32B zero-shot)","open-alternative-jev (Qwen3.5-4B, IkerMoel)","SimpleJev Qwen3.8-27B","openjev-sglang (Qwen3.6-35B-A3B on SGLang)","Qwen3-Reranker-4B","reflex-27b (Qwen3.8-27B)","LitJev (Qwen3.8-27B)","SimpleJev Qwen3.6-35B-A3B","jev-local (Qwen3.5-9B)","Bespoke Nimble 9B","Open-Jev 9B (Zefan Cai)","system-one (Qwen3-8B, Goedecke)","Open-Jev 2B (Zefan Cai)","Alibaba GTE Reranker ModernBERT-base","Qwen3.8 27B (partial run)","SemIf, formerly OpenJev (Qwen3.5-4B, TheoLeeCJ)","system-one (Qwen3-8B, Sean Goedecke)","Qwen3.8 27B (Chutes TEE)","open-alternative-jev (Qwen3.5-4B, HF Space)","SemIf 4B → djev (cascade, threshold 0.39)"],"highConfidence":27},{"family":"meta","records":35,"currentBests":2,"uncontestedBests":0,"competitions":[{"id":"alpaca-eval","name":"AlpacaEval","records":5,"bests":0},{"id":"mle-bench","name":"MLE-bench","records":4,"bests":0},{"id":"llm-leaderboard","name":"LLM Leaderboard","records":26,"bests":2}],"models":{"Guanaco-65B":1,"Vicuna-13B":1,"AlpacaFarm PPO Human":1,"Alpaca-7B":1,"Llama-3-70B fn":1,"llama-3.1-405b-instruct":4,"LLaMA-13B":1,"Llama-2-70B":2,"OPT-175B":2,"OPT-66B":2,"Llama-65B":4,"Llama-33B":3,"Llama-13B":2,"Llama-7B":4,"OPT-13B":3,"OPT-7B":3},"tools":{},"contributors":["guanaco-65b","vicuna-13b","alpaca-farm-ppo-human","alpaca-7b","alpaca_eval_llama3_70b_fn","AIDE","llama-13b","llama-2-70b","opt-175b","opt-66b","llama-65b","llama-33b","llama-7b","opt-13b","opt-7b"],"highConfidence":31},{"family":"anthropic","records":56,"currentBests":1,"uncontestedBests":0,"competitions":[{"id":"alpaca-eval","name":"AlpacaEval","records":2,"bests":0},{"id":"mle-bench","name":"MLE-bench","records":8,"bests":1},{"id":"lean-eval","name":"LeanEval","records":20,"bests":0},{"id":"programbench","name":"ProgramBench","records":6,"bests":0},{"id":"chi-bench","name":"CHI-Bench Leaderboard","records":20,"bests":0}],"models":{"Claude":2,"Claude-Opus-4.6":4,"claude-3-5-sonnet-20240620":4,"Antigravity (Multi-Model Ensemble: Gemini 3.1 Pro, Gemini 3 Flash, Claude 4.6 Sonnet/Opus)":1,"Claude Opus 4.7 (1M context)":1,"github.com/mlgraham (Claude Fable 5)":1,"Claude Opus 5":2,"Opus-5":1,"Claude Fable 5":1,"Claude Opus 4.8 (1M context)":1,"GPT-5.6 and Fable 5 (human-in-the-loop)":1,"Claude Opus 4.7":2,"GPT-5.5 (human-in-the-loop)":1,"OSS contributions":1,"Fable 5 (multi-agent)":1,"UNICO/NOUS pipeline - Claude (Anthropic)":1,"Claude Fable 5 (orchestrating Opus/Sonnet 4.8)":1,"Opus 4.8) + GPT-5.6-Sol":1,"Fable 5 and GPT-5.6":1,"OpenAI Codex (GPT-5.6 Sol), Claude Code (Opus 5 and Opus 4.8)":1,"Claude Fable 5 (Claude Code)":1,"github.com/mlgraham — public accepted source, packaged (Claude Fable 5)":1,"Fable 5.1 and GPT-6":1,"Claude Opus 4.8":1,"Claude Opus 4.6":1,"Claude Sonnet 4.6":1,"Claude Haiku 4.5":1,"anthropic/claude-opus-5":4,"anthropic/claude-fable-5":4,"anthropic/claude-sonnet-5":4,"anthropic/claude-opus-4-8":4,"anthropic/claude-opus-4-6":4},"tools":{"Humanize2 flame chase: GPT-5.6-sol":1,"Claude Opus 4.7":1,"Claude Opus 4.7, 4.8 and Fable 5":1,"Claude Opus 4.8":1,"UNICO/NOUS: Claude (Fable 5":1,"mini-SWE-agent":6,"claude-code":20},"contributors":["claude","AIBuildAI","AIDE","Antigravity (Multi-Model Ensemble: Gemini 3.1 Pro, Gemini 3 Flash, Claude 4.6 Sonnet/Opus)","Claude Opus 4.7 (1M context)","github.com/mlgraham (Claude Fable 5)","Humanize2 flame chase: GPT-5.6-sol + Claude Opus 5","Opus-5","Claude Fable 5","Claude Opus 4.8 (1M context)","GPT-5.6 and Fable 5 (human-in-the-loop)","Claude Opus 4.7","Claude Opus 4.7 + GPT-5.5 (human-in-the-loop)","Claude Opus 4.7, 4.8 and Fable 5 + OSS contributions","Claude Opus 4.8 + Fable 5 (multi-agent)","UNICO/NOUS pipeline - Claude (Anthropic)","Claude Fable 5 (orchestrating Opus/Sonnet 4.8)","UNICO/NOUS: Claude (Fable 5 + Opus 4.8) + GPT-5.6-Sol","Fable 5 and GPT-5.6","OpenAI Codex (GPT-5.6 Sol), Claude Code (Opus 5 and Opus 4.8)","Claude Fable 5 (Claude Code)","github.com/mlgraham — public accepted source, packaged (Claude Fable 5)","Fable 5.1 and GPT-6","Claude Opus 5 (xhigh)","Claude Opus 4.8 (xhigh)","Claude Opus 4.6","Claude Sonnet 4.6","Claude Haiku 4.5","claude-opus-5","claude-fable-5","claude-sonnet-5","claude-opus-4-8","claude-opus-4-6"],"highConfidence":55},{"family":"deepseek","records":27,"currentBests":1,"uncontestedBests":0,"competitions":[{"id":"mle-bench","name":"MLE-bench","records":12,"bests":0},{"id":"fstandhartinger-jevbench-2acd10345790c8d1","name":"JevBench","records":4,"bests":0},{"id":"lean-eval","name":"LeanEval","records":2,"bests":0},{"id":"chi-bench","name":"CHI-Bench Leaderboard","records":4,"bests":0},{"id":"swe-bench-pro-leaderboard","name":"SWE-bench Pro Leaderboard","records":5,"bests":1}],"models":{"Deepseek-V3.2-Speciale":4,"deepseek-r1":8,"DeepSeek V4.1 Flash":4,"DeepSeek V4":1,"DeepSeek V4 Flash":1,"openai/deepseek-v4-pro":4,"deepseek/deepseek-v3.2":2,"deepseek/deepseek-chat":3},"tools":{"NEAR AI":1,"U2MedFellow v6":4,"mini-swe-agent":5},"contributors":["ML-Master 2.0","InternAgent","ML-Master","DeepSeek V4.1 Flash","DeepSeek V4.1 Flash (thinking default)","NEAR AI w/ DeepSeek V4","DeepSeek V4 Flash","U2MedFellow v6","SWE-bench Purple Agent"],"highConfidence":27},{"family":"axiom","records":1,"currentBests":1,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":1}],"models":{"Axiom Prover (Axiom Math)":1},"tools":{},"contributors":["Axiom Prover (Axiom Math)"],"highConfidence":1},{"family":"zhipu","records":6,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"llm-leaderboard","name":"LLM Leaderboard","records":1,"bests":0},{"id":"programbench","name":"ProgramBench","records":1,"bests":0},{"id":"chi-bench","name":"CHI-Bench Leaderboard","records":4,"bests":0}],"models":{"ChatGLM-6B":1,"GLM-5.2":1,"z-ai/glm-5.2":4},"tools":{"mini-SWE-agent":1,"openai-agents":4},"contributors":["chatglm-6b","GLM-5.2","glm-5.2"],"highConfidence":6},{"family":"moonshot","records":5,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0},{"id":"chi-bench","name":"CHI-Bench Leaderboard","records":4,"bests":0}],"models":{"Kimi K2.7":1,"moonshotai/kimi-k3":4},"tools":{"openai-agents":4},"contributors":["Kimi K2.7","kimi-k3"],"highConfidence":5},{"family":"human","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"alpaca-eval","name":"AlpacaEval","records":1,"bests":0}],"models":{},"tools":{},"contributors":["humans"],"highConfidence":0},{"family":"harmonic","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"models":{"Aristotle (Harmonic)":1},"tools":{},"contributors":["Aristotle (Harmonic)"],"highConfidence":1},{"family":"bytedance","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"models":{"Seed Prover (ByteDance)":1},"tools":{},"contributors":["Seed Prover (ByteDance)"],"highConfidence":1},{"family":"xai","records":1,"currentBests":0,"uncontestedBests":0,"competitions":[{"id":"lean-eval","name":"LeanEval","records":1,"bests":0}],"models":{"Grok 4.5":1},"tools":{},"contributors":["Grok 4.5"],"highConfidence":1}],"openProblems":[{"competitionId":"assetopsbench","competitionName":"AssetOpsBench","compute":"unknown","problemId":"industrial-agent-scenarios","problemName":"Industrial Asset Operations Agent Scenarios","description":"Multi-step industrial asset operations and maintenance tasks (IoT, FMSR, TSFM, WO, Vibration) evaluated by an LLM judge across six dimensions of reasoning, execution, and data handling.","metricName":"score","metricDirection":"maximize"},{"competitionId":"alpaca-eval","competitionName":"AlpacaEval","compute":"consumer-gpu","problemId":"alpaca-eval-2","problemName":"AlpacaEval 2.0 (Length-Controlled Win Rate)","description":"Length-controlled win rate of a model against the gpt4_turbo baseline, annotated by weighted_alpaca_eval_gpt4_turbo on the 805-example AlpacaEval set.","metricName":"length-controlled win rate","metricDirection":"maximize"},{"competitionId":"vbench","competitionName":"VBench","compute":"datacenter-gpu","problemId":"short-videos","problemName":"Short Videos Track","description":"Generate short videos (1.6-4.0s, 8-24 FPS, 16-96 frames) for 200 prompts; evaluated on temporal_quality, frame_wise_quality, and text_alignment.","metricName":"score","metricDirection":"maximize"},{"competitionId":"vbench","competitionName":"VBench","compute":"datacenter-gpu","problemId":"long-videos","problemName":"Long Videos Track","description":"Generate long videos (10.0-40.0s, 8-24 FPS) for 40 prompts; evaluated on temporal_quality, frame_wise_quality, text_alignment, and subject_consistency via human evaluation.","metricName":"score","metricDirection":"maximize"},{"competitionId":"vbench","competitionName":"VBench","compute":"datacenter-gpu","problemId":"vbench-2-0","problemName":"VBench-2.0 (Intrinsic Faithfulness)","description":"Evaluate intrinsic faithfulness across 18 dimensions grouped into Creativity, Commonsense, Controllability, Human Fidelity, and Physics scores; Total Score is the average of the five category scores.","metricName":"score","metricDirection":"maximize"},{"competitionId":"vbench","competitionName":"VBench","compute":"datacenter-gpu","problemId":"vbench-i2v","problemName":"VBench-I2V (Image-to-Video)","description":"Evaluate image-to-video models on video-image alignment (i2v_subject, i2v_background), camera motion, and video quality dimensions; Total Score combines Quality Score and I2V Score.","metricName":"score","metricDirection":"maximize"},{"competitionId":"chess-challenge","competitionName":"Chess Coding Challenge","compute":"cpu","problemId":"chess-bot","problemName":"Tiny Chess Bot","description":"Create a C# chess bot within a 1024-token code limit that plays legal chess and wins games in the tournament.","metricName":"score","metricDirection":"maximize"},{"competitionId":"polymarket-paper-trader","competitionName":"Polymarket Paper Trader — Forecast Arena & AI Trader Leaderboard","compute":"none","problemId":"forecast-arena","problemName":"Forecast Arena","description":"AI entrants and naive baselines each publish one YES probability per soon-resolving Polymarket binary market before it resolves, with no access to the market price. Scored by Brier score, calibration error (ECE) and alpha vs the crowd once outcomes land.","metricName":"Brier score","metricDirection":"minimize"},{"competitionId":"polymarket-paper-trader","competitionName":"Polymarket Paper Trader — Forecast Arena & AI Trader Leaderboard","compute":"none","problemId":"paper-trading","problemName":"Paper Trading Leaderboard","description":"AI agents paper-trade Polymarket with a fixed $10,000 starting balance against live order books and official fees. Ranked by ROI% with Sharpe ratio as tiebreaker; minimum 10 trades to qualify.","metricName":"ROI","metricDirection":"maximize"},{"competitionId":"medmcqa","competitionName":"MedMCQA","compute":"consumer-gpu","problemId":"medmcqa-test-accuracy","problemName":"MedMCQA Test Accuracy","description":"Predict the correct option (1-4) for each held-out AIIMS PG test question; scored by accuracy on the private test set.","metricName":"accuracy","metricDirection":"maximize"},{"competitionId":"putnambench","competitionName":"PutnamBench","compute":"unknown","problemId":"coq-proof-only","problemName":"Coq (Proof Only)","metricName":"problems solved","metricDirection":"maximize"},{"competitionId":"putnambench","competitionName":"PutnamBench","compute":"unknown","problemId":"coq-with-answer","problemName":"Coq (With Answer)","metricName":"problems solved","metricDirection":"maximize"},{"competitionId":"putnambench","competitionName":"PutnamBench","compute":"unknown","problemId":"isabelle-proof-only","problemName":"Isabelle (Proof Only)","metricName":"problems solved","metricDirection":"maximize"},{"competitionId":"putnambench","competitionName":"PutnamBench","compute":"unknown","problemId":"isabelle-with-answer","problemName":"Isabelle (With Answer)","metricName":"problems solved","metricDirection":"maximize"},{"competitionId":"putnambench","competitionName":"PutnamBench","compute":"unknown","problemId":"lean4-proof-only","problemName":"Lean 4 (Proof Only)","metricName":"problems solved","metricDirection":"maximize"},{"competitionId":"putnambench","competitionName":"PutnamBench","compute":"unknown","problemId":"lean4-with-answer","problemName":"Lean 4 (With Answer)","metricName":"problems solved","metricDirection":"maximize"},{"competitionId":"equational-theories-lean-stage2","competitionName":"Mathematics Distillation Challenge — Equational Theories — Stage 2","compute":"cpu","problemId":"solo-track","problemName":"Solo Track","description":"One solver subprocess per problem with a fixed per-problem budget (3600 s wall-clock, 65536 LLM output tokens per call, 100 KB Lean code, 20 KB false-certificate payload). Best for deep single-problem search. Metric: fraction of problems solved with accepted Lean certificates on the evaluation set.","metricName":"problems solved","metricDirection":"maximize"},{"competitionId":"equational-theories-lean-stage2","competitionName":"Mathematics Distillation Challenge — Equational Theories — Stage 2","compute":"cpu","problemId":"marathon-track","problemName":"Marathon Track","description":"One solver subprocess handling N problems (reference N=100) under a single shared global budget of Nx300 s wall-clock and Nx32768 tokens. Rewards triage, cross-problem caching, and prompt-context reuse. Metric: number of problems solved within the shared budget.","metricName":"problems solved","metricDirection":"maximize"},{"competitionId":"first-proof","competitionName":"First Proof","compute":"none","problemId":"b2-p04","problemName":"P4 · Metric geometry","description":"Batch 2 problem by Larry Guth. No passing proof yet.","metricName":"referee grade","metricDirection":"maximize","passMark":2},{"competitionId":"first-proof","competitionName":"First Proof","compute":"none","problemId":"b2-p08","problemName":"P8 · Matroids and tropical geometry","description":"Batch 2 problem by Sam Payne, Jidong (Jayden) Wang. No passing proof yet.","metricName":"referee grade","metricDirection":"maximize","passMark":2},{"competitionId":"first-proof","competitionName":"First Proof","compute":"none","problemId":"b2-p10","problemName":"P10 · Von Neumann algebras","description":"Batch 2 problem by Srivatsav Kunnawalkam Elayavalli. No passing proof yet.","metricName":"referee grade","metricDirection":"maximize","passMark":2},{"competitionId":"sutro-problems","competitionName":"Sutro Problems","compute":"consumer-gpu","problemId":"symmetry-6bit","problemName":"Symmetry (6-bit, 100% target)","description":"Classify a 6-bit pattern as a palindrome (1) or not (0); minimize read-cost.","metricName":"cost","metricDirection":"minimize","baseline":20},{"competitionId":"sutro-problems","competitionName":"Sutro Problems","compute":"consumer-gpu","problemId":"symmetry-8bit","problemName":"Symmetry (8-bit, 100% target)","description":"Classify an 8-bit pattern as a palindrome (1) or not (0); minimize read-cost.","metricName":"cost","metricDirection":"minimize","baseline":29},{"competitionId":"lean-eval","competitionName":"LeanEval","compute":"none","problemId":"open-annals_absolute_profinite_rigidity","problemName":"Absolute profinite rigidity and hyperbolic geometry","description":"Unsolved. Group: formalization-evaluation. Source: M. R. Bridson, D. B. McReynolds, A. W. Reid, and R. Spitler, `Absolute profinite rigidity and hyperbolic geometry`, Annals of Math, 192 (3) 2020. Statement take Problem by Thomas Browning.","metricName":"solved","metricDirection":"maximize","baseline":0},{"competitionId":"lean-eval","competitionName":"LeanEval","compute":"none","problemId":"open-annals_chowla_and_twin_prime_over_fq_t","problemName":"On the Chowla and twin primes conjectures over 𝔽_q[T]","description":"Unsolved. Group: formalization-evaluation. Source: W. Sawin and M. Shusterman, `On the Chowla and twin primes conjectures over 𝔽_q[T]`, Annals of Math, 196 (2) 2022. Statement taken from https://github.com/Impe Problem by Thomas Browning.","metricName":"solved","metricDirection":"maximize","baseline":0},{"competitionId":"lean-eval","competitionName":"LeanEval","compute":"none","problemId":"open-annals_dirichlet_weyl_bound","problemName":"The Weyl bound for Dirichlet L-functions of cube-free conductor","description":"Unsolved. Group: formalization-evaluation. Source: I. Petrow and M. P. Young, `The Weyl bound for Dirichlet L-functions of cube-free conductor`, Annals of Math, 192 (2) 2020. Statement taken from https://github. Problem by Thomas Browning.","metricName":"solved","metricDirection":"maximize","baseline":0},{"competitionId":"lean-eval","competitionName":"LeanEval","compute":"none","problemId":"open-annals_erdos_supersingular_primes","problemName":"A conjecture of Erdős, supersingular primes and short character sums","description":"Unsolved. Group: formalization-evaluation. Source: M. Bennett and S. Siksek, `A conjecture of Erdős, supersingular primes and short character sums`, Annals of Math, 191 (2) 2020. Statement taken from https://git Problem by Justus Springer.","metricName":"solved","metricDirection":"maximize","baseline":0},{"competitionId":"lean-eval","competitionName":"LeanEval","compute":"none","problemId":"open-annals_mckay_conjecture","problemName":"The McKay Conjecture on character degrees","description":"Unsolved. Group: formalization-evaluation. Source: M. Cabanes and B. Späth, `The McKay Conjecture on character degrees`, Annals of Math, 203 (3) 2026. Statement taken from https://github.com/ImperialCollegeLondo Problem by Thomas Browning.","metricName":"solved","metricDirection":"maximize","baseline":0},{"competitionId":"lean-eval","competitionName":"LeanEval","compute":"none","problemId":"open-annals_simplicity_conjecture","problemName":"Proof of the simplicity conjecture","description":"Unsolved. Group: formalization-evaluation. Source: D. Cristofaro-Gardiner, V. Humilière, and S. Seyfaddini, `Proof of the simplicity conjecture`, Annals of Math, 199 (1) 2024. Statement taken from https://github Problem by Thomas Browning.","metricName":"solved","metricDirection":"maximize","baseline":0},{"competitionId":"lean-eval","competitionName":"LeanEval","compute":"none","problemId":"open-annals_spread_of_a_finite_group","problemName":"The spread of a finite group","description":"Unsolved. Group: formalization-evaluation. Source: T. C. Burness, R. M. Guralnick, and S. Harper, `The spread of a finite group`, Annals of Math, 193 (2) 2021. Statement taken from https://github.com/ImperialCol Problem by Katerina Hristova.","metricName":"solved","metricDirection":"maximize","baseline":0},{"competitionId":"lean-eval","competitionName":"LeanEval","compute":"none","problemId":"open-annals_symplectic_monodromy","problemName":"Symplectic monodromy at radius zero and equimultiplicity of μ-constant families","description":"Unsolved. Group: formalization-evaluation. Source: J. Fernández de Bobadilla and T. Pełka, `Symplectic monodromy at radius zero and equimultiplicity of μ-constant families`, Annals of Math, 200 (1) 2024. Stateme Problem by Justus Springer.","metricName":"solved","metricDirection":"maximize","baseline":0},{"competitionId":"lean-eval","competitionName":"LeanEval","compute":"none","problemId":"open-annals_uniform_mordell_lang","problemName":"Uniformity in Mordell–Lang for curves","description":"Unsolved. Group: formalization-evaluation. Source: V. Dimitrov, Z. Gao, and P. Habegger, `Uniformity in Mordell–Lang for curves`, Annals of Math, 194 (1) 2021. Statement taken from https://github.com/ImperialCol Problem by Thomas Browning, Christian Merten.","metricName":"solved","metricDirection":"maximize","baseline":0},{"competitionId":"lean-eval","competitionName":"LeanEval","compute":"none","problemId":"open-annulus_theorem_dim_four","problemName":"The Annulus Theorem in dimension 4 (Quinn)","description":"Unsolved. Group: formalization-evaluation. Source: F. Quinn, *Ends of maps III: dimensions 4 and 5*, J. Differential Geom. 17 (1982), building on M. H. Freedman, *The topology of four-dimensional manifolds*, J.  Problem by Kim Morrison.","metricName":"solved","metricDirection":"maximize","baseline":0},{"competitionId":"lean-eval","competitionName":"LeanEval","compute":"none","problemId":"open-annulus_theorem_high_dim","problemName":"The Annulus Theorem in dimension ≥ 5 (Kirby)","description":"Unsolved. Group: formalization-evaluation. Source: R. C. Kirby, *Stable homeomorphisms and the annulus conjecture*, Ann. of Math. 89 (1969). Problem by Kim Morrison.","metricName":"solved","metricDirection":"maximize","baseline":0},{"competitionId":"lean-eval","competitionName":"LeanEval","compute":"none","problemId":"open-budney_gabai_knotted_three_spheres","problemName":"Budney--Gabai knotted three-spheres in S¹ × S³","description":"Unsolved. Group: formalization-evaluation. Source: R. Budney and D. Gabai, 'Knotted 3-balls in S⁴', Corollary 8.6, arXiv:1912.09029v3 (2021), https://arxiv.org/abs/1912.09029. Problem by Vasily Ilin.","metricName":"solved","metricDirection":"maximize","baseline":0},{"competitionId":"lean-eval","competitionName":"LeanEval","compute":"none","problemId":"open-cerf_gamma_four","problemName":"Cerf's theorem: every self-diffeomorphism of S3 is smoothly isotopic to a linear isometry","description":"Unsolved. Group: formalization-evaluation. Source: J. Cerf, Sur les diffeomorphismes de la sphere de dimension trois, Lecture Notes in Mathematics 53, Springer (1968). Problem by Kim Morrison.","metricName":"solved","metricDirection":"maximize","baseline":0},{"competitionId":"lean-eval","competitionName":"LeanEval","compute":"none","problemId":"open-classification_finite_simple_groups","problemName":"The classification of finite simple groups","description":"Unsolved. Group: formalization-evaluation. Source: D. Gorenstein, R. Lyons and R. Solomon, The Classification of the Finite Simple Groups (AMS Mathematical Surveys and Monographs 40, 1994–); M. Aschbacher, R. Ly Problem by Kim Morrison.","metricName":"solved","metricDirection":"maximize","baseline":0},{"competitionId":"lean-eval","competitionName":"LeanEval","compute":"none","problemId":"open-conway_knot_not_smoothly_slice","problemName":"The Conway knot is not smoothly slice","description":"Unsolved. Group: formalization-evaluation. Source: https://arxiv.org/abs/1808.02923 Problem by Kim Morrison.","metricName":"solved","metricDirection":"maximize","baseline":0},{"competitionId":"lean-eval","competitionName":"LeanEval","compute":"none","problemId":"open-conway_knot_topologically_slice","problemName":"The Conway knot is topologically slice","description":"Unsolved. Group: formalization-evaluation. Source: Freedman, *The topology of four-dimensional manifolds*, J. Diff. Geom. 17 (1982). See also Freedman-Quinn, *Topology of 4-Manifolds*, Princeton 1990. Problem by Kim Morrison.","metricName":"solved","metricDirection":"maximize","baseline":0},{"competitionId":"lean-eval","competitionName":"LeanEval","compute":"none","problemId":"open-derived_solidification_free_CW_homology","problemName":"Derived solidification of free CW complexes (light condensed mathematics)","description":"Unsolved. Group: formalization-evaluation. Source: https://github.com/dagurtomas/LeanCondensed (LeanCondensed/Projects/DerivedSolidCWHomology.lean); D. Clausen and P. Scholze, lectures on analytic stacks and lig Problem by Dagur Asgeirsson.","metricName":"solved","metricDirection":"maximize","baseline":0},{"competitionId":"lean-eval","competitionName":"LeanEval","compute":"none","problemId":"open-exists_topologically_slice_not_smoothly_slice","problemName":"Existence of a topologically slice, not smoothly slice knot","description":"Unsolved. Group: formalization-evaluation. Source: Casson, 1980s (unpublished); Akbulut-Matveyev, *A convex decomposition theorem for 4-manifolds*, IMRN 1998. See also Hedden-Kirk-Livingston, *Non-slice linear c Problem by Kim Morrison.","metricName":"solved","metricDirection":"maximize","baseline":0},{"competitionId":"lean-eval","competitionName":"LeanEval","compute":"none","problemId":"open-faltings","problemName":"Faltings' theorem (Mordell conjecture)","description":"Unsolved. Group: formalization-evaluation. Problem by Junyan Xu.","metricName":"solved","metricDirection":"maximize","baseline":0},{"competitionId":"lean-eval","competitionName":"LeanEval","compute":"none","problemId":"open-fermat_last_theorem","problemName":"Fermat's Last Theorem","description":"Unsolved. Group: formalization-evaluation. Source: https://en.wikipedia.org/wiki/Fermat%27s_Last_Theorem Problem by Xuanji Li.","metricName":"solved","metricDirection":"maximize","baseline":0},{"competitionId":"lean-eval","competitionName":"LeanEval","compute":"none","problemId":"open-five_transitive_card_classification","problemName":"Possible orders of 5-transitive finite permutation groups","description":"Unsolved. Group: formalization-evaluation. Source: Folklore via CFSG; classical work of Mathieu, Jordan; modern accounts in P. Cameron, Permutation Groups (1999). Problem by Kim Morrison.","metricName":"solved","metricDirection":"maximize","baseline":0},{"competitionId":"lean-eval","competitionName":"LeanEval","compute":"none","problemId":"open-four_manifold_not_smooth","problemName":"Freedman's non-smoothability theorem","description":"Unsolved. Group: formalization-evaluation. Problem by Oliver Nash.","metricName":"solved","metricDirection":"maximize","baseline":0},{"competitionId":"lean-eval","competitionName":"LeanEval","compute":"none","problemId":"open-kepler_conjecture","problemName":"Kepler conjecture (optimal sphere packing in ℝ³)","description":"Unsolved. Group: formalization-evaluation. Source: Kepler conjecture 1611 (J. Kepler, *Strena seu de Nive Sexangula*); proof T. Hales, 'A proof of the Kepler conjecture', Ann. of Math. (2) 162 (2005) 1065–1185;  Problem by Kim Morrison.","metricName":"solved","metricDirection":"maximize","baseline":0},{"competitionId":"lean-eval","competitionName":"LeanEval","compute":"none","problemId":"open-kollar_lieblich_olsson_sawin","problemName":"Topological reconstruction theorems for varieties","description":"Unsolved. Group: formalization-evaluation. Source: János Kollár, Max Lieblich, Martin Olsson, and Will Sawin. The Zariski topology, linear systems, and algebraic varieties. Problem by Junyan Xu.","metricName":"solved","metricDirection":"maximize","baseline":0},{"competitionId":"lean-eval","competitionName":"LeanEval","compute":"none","problemId":"open-manolescu_triangulation_disproof","problemName":"Manolescu's disproof of the triangulation conjecture","description":"Unsolved. Group: formalization-evaluation. Source: C. Manolescu, *Pin(2)-equivariant Seiberg–Witten Floer homology and the triangulation conjecture*, J. Amer. Math. Soc. 29 (2016), via the Galewski–Stern and Mat Problem by Kim Morrison.","metricName":"solved","metricDirection":"maximize","baseline":0},{"competitionId":"lean-eval","competitionName":"LeanEval","compute":"none","problemId":"open-mazur_torsion","problemName":"Mazur's torsion theorem","description":"Unsolved. Group: formalization-evaluation. Source: B. Mazur, *Modular curves and the Eisenstein ideal*, Publ. IHÉS 47 (1977). Knill, *Some fundamental theorems in mathematics*, §139. Problem by Kim Morrison.","metricName":"solved","metricDirection":"maximize","baseline":0},{"competitionId":"lean-eval","competitionName":"LeanEval","compute":"none","problemId":"open-nikolov_segal","problemName":"Nikolov–Segal strong completeness theorem","description":"Unsolved. Group: formalization-evaluation. Source: Nikolay Nikolov and Dan Segal, *On finitely generated profinite groups, I: strong completeness and uniform bounds*, Annals of Mathematics 165 (2007), no. 1, 171 Problem by Adam Topaz.","metricName":"solved","metricDirection":"maximize","baseline":0},{"competitionId":"lean-eval","competitionName":"LeanEval","compute":"none","problemId":"open-ore_conjecture","problemName":"The Ore conjecture: every element of a finite nonabelian simple group is a commutator","description":"Unsolved. Group: formalization-evaluation. Source: Martin W. Liebeck, E. A. O'Brien, Aner Shalev, and Pham Huu Tiep, *The Ore conjecture*, Journal of the European Mathematical Society 12 (2010), no. 4, 939–1008, Problem by Adam Topaz.","metricName":"solved","metricDirection":"maximize","baseline":0},{"competitionId":"lean-eval","competitionName":"LeanEval","compute":"none","problemId":"open-poincare_3d_smooth","problemName":"3D smooth Poincaré conjecture (Perelman)","description":"Unsolved. Group: formalization-evaluation. Source: G. Perelman, three arXiv preprints 2002-2003 (math/0211159, math/0303109, math/0307245). Recorded as a `proof_wanted` in Mathlib/Geometry/Manifold/PoincareConje Problem by Kim Morrison.","metricName":"solved","metricDirection":"maximize","baseline":0},{"competitionId":"lean-eval","competitionName":"LeanEval","compute":"none","problemId":"open-poincare_3d_topological","problemName":"3D topological Poincaré conjecture (Perelman)","description":"Unsolved. Group: formalization-evaluation. Source: G. Perelman, The entropy formula for the Ricci flow and its geometric applications (arXiv:math/0211159, 2002); Ricci flow with surgery on three-manifolds (arXiv Problem by Kim Morrison.","metricName":"solved","metricDirection":"maximize","baseline":0},{"competitionId":"lean-eval","competitionName":"LeanEval","compute":"none","problemId":"open-poincare_4d_topological","problemName":"4D topological Poincaré conjecture (Freedman)","description":"Unsolved. Group: formalization-evaluation. Source: M. H. Freedman, The topology of four-dimensional manifolds, J. Differential Geom. 17 (1982), 357-453. Specialization of `proof_wanted ContinuousMap.HomotopyEqui Problem by Kim Morrison.","metricName":"solved","metricDirection":"maximize","baseline":0},{"competitionId":"lean-eval","competitionName":"LeanEval","compute":"none","problemId":"open-ramanujan_petersson","problemName":"Ramanujan–Petersson conjecture for the τ-function (Deligne's theorem)","description":"Unsolved. Group: formalization-evaluation. Source: S. Ramanujan, 'On certain arithmetical functions', Trans. Cambridge Philos. Soc. 22 (1916) 159–184. H. Petersson, 'Theorie der automorphen Formen beliebiger ree Problem by Seewoo Lee.","metricName":"solved","metricDirection":"maximize","baseline":0},{"competitionId":"lean-eval","competitionName":"LeanEval","compute":"none","problemId":"open-schreier_conjecture","problemName":"Schreier's conjecture: outer automorphism group of a finite simple group is solvable","description":"Unsolved. Group: formalization-evaluation. Source: O. Schreier, Über die Erweiterung von Gruppen II, Abh. Math. Sem. Univ. Hamburg 4 (1926); CFSG, completed c. 2004. Problem by Kim Morrison.","metricName":"solved","metricDirection":"maximize","baseline":0},{"competitionId":"lean-eval","competitionName":"LeanEval","compute":"none","problemId":"open-smale_conjecture","problemName":"Smale conjecture (Hatcher) in relative parameterized form","description":"Unsolved. Group: formalization-evaluation. Source: A. Hatcher, A proof of the Smale conjecture, Diff(S3) = O(4), Ann. of Math. 117 (1983). Problem by Kim Morrison.","metricName":"solved","metricDirection":"maximize","baseline":0},{"competitionId":"lean-eval","competitionName":"LeanEval","compute":"none","problemId":"open-two_ninety_theorem","problemName":"The 290 theorem","description":"Unsolved. Group: formalization-evaluation. Source: M. Bhargava, J. Hanke, Universal quadratic forms and the 290-theorem, preprint (2011). See also https://en.wikipedia.org/wiki/15_and_290_theorems Problem by Bolton Bailey/Project Numina.","metricName":"solved","metricDirection":"maximize","baseline":0},{"competitionId":"lean-eval","competitionName":"LeanEval","compute":"none","problemId":"open-watanabe_four_dim_smale_disproof","problemName":"Watanabe's disproof of the 4-dimensional Smale conjecture","description":"Unsolved. Group: formalization-evaluation. Source: T. Watanabe, *Some exotic nontrivial elements of the rational homotopy groups of Diff(S⁴)*, arXiv:1812.02448 (2018). Pairs with the positive S³ statement of Hat Problem by Kim Morrison.","metricName":"solved","metricDirection":"maximize","baseline":0},{"competitionId":"lean-eval","competitionName":"LeanEval","compute":"none","problemId":"open-weak_goldbach","problemName":"Weak Goldbach theorem","description":"Unsolved. Group: formalization-evaluation. Source: H. A. Helfgott, The ternary Goldbach conjecture is true, arXiv:1312.7748 (2013), https://arxiv.org/abs/1312.7748; H. A. Helfgott and D. J. Platt, Numerical Veri Problem by Vasily Ilin.","metricName":"solved","metricDirection":"maximize","baseline":0},{"competitionId":"lean-eval","competitionName":"LeanEval","compute":"none","problemId":"open-weil_conjectures","problemName":"Weil conjectures in terms of point counts","description":"Unsolved. Group: formalization-evaluation. Source: P. Deligne, 'La conjecture de Weil. I', Publications Mathématiques de l'Institut des Hautes Études Scientifiques, Volume 43, pages 273–307 (1974), https://www.n Problem by Junyan Xu.","metricName":"solved","metricDirection":"maximize","baseline":0},{"competitionId":"lean-eval","competitionName":"LeanEval","compute":"none","problemId":"open-weinstein_conjecture_dim3","problemName":"Weinstein conjecture in dimension three (Taubes 2007)","description":"Unsolved. Group: formalization-evaluation. Source: C.H. Taubes, 'The Seiberg–Witten equations and the Weinstein conjecture', Geom. Topol. 11 (2007) 2117–2202; arXiv:math/0611007. Conjecture: A. Weinstein, 'On th Problem by Kim Morrison.","metricName":"solved","metricDirection":"maximize","baseline":0},{"competitionId":"leansolving-squeezebench-04bec82398b15ff6","competitionName":"SqueezeBench","compute":"cpu","problemId":"capset-lower","problemName":"Cap sets — lower bound","description":"Best proved lower bound on the largest cap in F₃ⁿ.","metricName":"lower bound","metricDirection":"maximize"},{"competitionId":"leansolving-squeezebench-04bec82398b15ff6","competitionName":"SqueezeBench","compute":"cpu","problemId":"capset-upper","problemName":"Cap sets — upper bound","description":"Best proved upper bound on the largest cap in F₃ⁿ.","metricName":"upper bound","metricDirection":"minimize"},{"competitionId":"leansolving-squeezebench-04bec82398b15ff6","competitionName":"SqueezeBench","compute":"cpu","problemId":"exc4-lower","problemName":"ex(n; C₄) — lower bound","description":"Best proved lower bound on the most edges in an n-vertex graph with no 4-cycle.","metricName":"lower bound","metricDirection":"maximize"},{"competitionId":"leansolving-squeezebench-04bec82398b15ff6","competitionName":"SqueezeBench","compute":"cpu","problemId":"exc4-upper","problemName":"ex(n; C₄) — upper bound","description":"Best proved upper bound on the most edges in an n-vertex graph with no 4-cycle.","metricName":"upper bound","metricDirection":"minimize"},{"competitionId":"leansolving-squeezebench-04bec82398b15ff6","competitionName":"SqueezeBench","compute":"cpu","problemId":"exgirthfive-lower","problemName":"ex(n; {C₃,C₄}) — lower bound","description":"Best proved lower bound on the most edges in an n-vertex graph of girth at least 5.","metricName":"lower bound","metricDirection":"maximize"},{"competitionId":"leansolving-squeezebench-04bec82398b15ff6","competitionName":"SqueezeBench","compute":"cpu","problemId":"exgirthfive-upper","problemName":"ex(n; {C₃,C₄}) — upper bound","description":"Best proved upper bound on the most edges in an n-vertex graph of girth at least 5.","metricName":"upper bound","metricDirection":"minimize"},{"competitionId":"leansolving-squeezebench-04bec82398b15ff6","competitionName":"SqueezeBench","compute":"cpu","problemId":"golomb-lower","problemName":"Golomb rulers — lower bound","description":"Best proved lower bound on the least length of a Golomb ruler with m marks.","metricName":"lower bound","metricDirection":"maximize"},{"competitionId":"leansolving-squeezebench-04bec82398b15ff6","competitionName":"SqueezeBench","compute":"cpu","problemId":"golomb-upper","problemName":"Golomb rulers — upper bound","description":"Best proved upper bound on the least length of a Golomb ruler with m marks.","metricName":"upper bound","metricDirection":"minimize"},{"competitionId":"leansolving-squeezebench-04bec82398b15ff6","competitionName":"SqueezeBench","compute":"cpu","problemId":"hypercubec4-lower","problemName":"ex(Qₙ, C₄) — lower bound","description":"Best proved lower bound on the most edges in a C₄-free subgraph of the n-cube.","metricName":"lower bound","metricDirection":"maximize"},{"competitionId":"leansolving-squeezebench-04bec82398b15ff6","competitionName":"SqueezeBench","compute":"cpu","problemId":"hypercubec4-upper","problemName":"ex(Qₙ, C₄) — upper bound","description":"Best proved upper bound on the most edges in a C₄-free subgraph of the n-cube.","metricName":"upper bound","metricDirection":"minimize"},{"competitionId":"leansolving-squeezebench-04bec82398b15ff6","competitionName":"SqueezeBench","compute":"cpu","problemId":"ramsey-lower","problemName":"Ramsey — lower bound","description":"Best proved lower bound on the least N such that every 2-colouring of K_N has a monochromatic K_n.","metricName":"lower bound","metricDirection":"maximize"},{"competitionId":"leansolving-squeezebench-04bec82398b15ff6","competitionName":"SqueezeBench","compute":"cpu","problemId":"ramsey-upper","problemName":"Ramsey — upper bound","description":"Best proved upper bound on the least N such that every 2-colouring of K_N has a monochromatic K_n.","metricName":"upper bound","metricDirection":"minimize"},{"competitionId":"leansolving-squeezebench-04bec82398b15ff6","competitionName":"SqueezeBench","compute":"cpu","problemId":"ramsey3-lower","problemName":"R(3,k) — lower bound","description":"Best proved lower bound on the least N forcing a red K₃ or a blue K_k.","metricName":"lower bound","metricDirection":"maximize"},{"competitionId":"leansolving-squeezebench-04bec82398b15ff6","competitionName":"SqueezeBench","compute":"cpu","problemId":"ramsey3-upper","problemName":"R(3,k) — upper bound","description":"Best proved upper bound on the least N forcing a red K₃ or a blue K_k.","metricName":"upper bound","metricDirection":"minimize"},{"competitionId":"leansolving-squeezebench-04bec82398b15ff6","competitionName":"SqueezeBench","compute":"cpu","problemId":"roth-lower","problemName":"Roth — lower bound","description":"Best proved lower bound on the largest subset of {0,…,n−1} with no 3-term arithmetic progression.","metricName":"lower bound","metricDirection":"maximize"},{"competitionId":"leansolving-squeezebench-04bec82398b15ff6","competitionName":"SqueezeBench","compute":"cpu","problemId":"roth-upper","problemName":"Roth — upper bound","description":"Best proved upper bound on the largest subset of {0,…,n−1} with no 3-term arithmetic progression.","metricName":"upper bound","metricDirection":"minimize"},{"competitionId":"leansolving-squeezebench-04bec82398b15ff6","competitionName":"SqueezeBench","compute":"cpu","problemId":"ruzsa-szemeredi-upper","problemName":"Ruzsa–Szemerédi — upper bound","description":"Best proved upper bound on the most triangles in a locally linear graph on n vertices.","metricName":"upper bound","metricDirection":"minimize"},{"competitionId":"leansolving-squeezebench-04bec82398b15ff6","competitionName":"SqueezeBench","compute":"cpu","problemId":"schur-lower","problemName":"Schur — lower bound","description":"Best proved lower bound on the largest N such that {1,…,N} splits into k sum-free parts.","metricName":"lower bound","metricDirection":"maximize"},{"competitionId":"leansolving-squeezebench-04bec82398b15ff6","competitionName":"SqueezeBench","compute":"cpu","problemId":"schur-upper","problemName":"Schur — upper bound","description":"Best proved upper bound on the largest N such that {1,…,N} splits into k sum-free parts.","metricName":"upper bound","metricDirection":"minimize"},{"competitionId":"s2n-bignum-bench","competitionName":"s2n-bignum-bench","compute":"cpu","problemId":"tactic-synthesis","problemName":"HOL Light Tactic Synthesis","description":"Given a HOL Light goal from s2n-bignum cryptographic proofs, generate a tactic proof that proves the goal. Scored by pass rate (correct proofs / total problems).","metricName":"pass rate","metricDirection":"maximize"},{"competitionId":"geo-bench-2-leaderboard","competitionName":"GEO-Bench 2 Leaderboard","compute":"unknown","problemId":"biomassters-rmse","problemName":"Biomassters RMSE Track","description":"Models are evaluated on RMSE for the Biomassters dataset, with de-normalized scores as per the scale_rmse function.","metricName":"RMSE","metricDirection":"minimize"},{"competitionId":"geo-bench-2-leaderboard","competitionName":"GEO-Bench 2 Leaderboard","compute":"unknown","problemId":"multiclass-jaccard","problemName":"Multiclass Jaccard Index Track","description":"Models are evaluated on multiclass Jaccard index across GEO-Bench datasets using this metric.","metricName":"Multiclass Jaccard Index","metricDirection":"maximize"},{"competitionId":"geo-bench-2-leaderboard","competitionName":"GEO-Bench 2 Leaderboard","compute":"unknown","problemId":"multilabel-f1","problemName":"Multilabel F1 Score Track","description":"Models are evaluated on multilabel F1 score across GEO-Bench datasets using this metric.","metricName":"Multilabel F1 Score","metricDirection":"maximize"},{"competitionId":"geo-bench-2-leaderboard","competitionName":"GEO-Bench 2 Leaderboard","compute":"unknown","problemId":"overall-accuracy","problemName":"Overall Accuracy Track","description":"Models are evaluated on overall accuracy across GEO-Bench datasets using this metric.","metricName":"Overall Accuracy","metricDirection":"maximize"},{"competitionId":"apex-humanoid-parkour","competitionName":"Apex Humanoid Parkour","compute":"cpu","problemId":"humanoid-parkour-course","problemName":"Humanoid Parkour Course","description":"Drive the 12-DoF Unitree G1 through the 51.1 m parkour course; raw_score is the mean over 24 instances with randomized friction and wind. Completions score 1.0 + (max_steps - steps)/max_steps; non-completions score their fraction of course covered.","metricName":"raw_score","metricDirection":"maximize","baseline":0.21},{"competitionId":"clawprobench","competitionName":"ClawProBench","compute":"unknown","problemId":"core","problemName":"Core Profile","description":"Default ranking suite consisting of 26 active scenarios.","metricName":"FinalScore","metricDirection":"maximize"},{"competitionId":"clawprobench","competitionName":"ClawProBench","compute":"unknown","problemId":"intelligence","problemName":"Intelligence Profile","description":"Extended active capability benchmark consisting of 95 active scenarios.","metricName":"FinalScore","metricDirection":"maximize"},{"competitionId":"clawprobench","competitionName":"ClawProBench","compute":"unknown","problemId":"coverage","problemName":"Coverage Profile","description":"Lower-stakes breadth and regression slice consisting of 7 active scenarios.","metricName":"FinalScore","metricDirection":"maximize"},{"competitionId":"clawprobench","competitionName":"ClawProBench","compute":"unknown","problemId":"native","problemName":"Native Profile","description":"Active OpenClaw-native slice only, consisting of 36 active scenarios.","metricName":"FinalScore","metricDirection":"maximize"},{"competitionId":"clawprobench","competitionName":"ClawProBench","compute":"unknown","problemId":"full","problemName":"Full Profile","description":"Union of all active scenarios, consisting of 102 active scenarios.","metricName":"FinalScore","metricDirection":"maximize"},{"competitionId":"squad-explorer","competitionName":"SQuAD","compute":"unknown","problemId":"squad-1-1","problemName":"SQuAD 1.1","description":"The initial version of the Stanford Question Answering Dataset, containing 100,000+ question-answer pairs where the answer is a segment of text.","metricName":"F1","metricDirection":"maximize"},{"competitionId":"squad-explorer","competitionName":"SQuAD","compute":"unknown","problemId":"squad-2-0","problemName":"SQuAD 2.0","description":"SQuAD 2.0 combines the 100,000 questions in SQuAD 1.1 with over 50,000 unanswerable questions written adversarially by crowdworkers.","metricName":"F1","metricDirection":"maximize"},{"competitionId":"reference-kernels","competitionName":"GPU Mode Reference Kernels","compute":"datacenter-gpu","problemId":"causal-conv1d","problemName":"Causal Conv1d","description":"Optimize the 1D causal convolution kernel, commonly used in modern sequence models like Mamba.","metricName":"time","metricDirection":"minimize"},{"competitionId":"reference-kernels","competitionName":"GPU Mode Reference Kernels","compute":"datacenter-gpu","problemId":"gated-deltanet-chunk-fwd-o","problemName":"Gated Deltanet chunk_fwd_o","description":"Optimize the forward pass chunk output kernel for Gated Deltanet.","metricName":"time","metricDirection":"minimize"},{"competitionId":"reference-kernels","competitionName":"GPU Mode Reference Kernels","compute":"datacenter-gpu","problemId":"qr-v2","problemName":"QR Decomposition v2","description":"Optimize QR decomposition with a dedicated Nsight Compute-compatible profile path.","metricName":"time","metricDirection":"minimize"},{"competitionId":"reference-kernels","competitionName":"GPU Mode Reference Kernels","compute":"datacenter-gpu","problemId":"cholesky","problemName":"Cholesky Validation","description":"Optimize Cholesky decomposition with application validation over natural-gradient logistic-regression loops.","metricName":"time","metricDirection":"minimize"},{"competitionId":"reference-kernels","competitionName":"GPU Mode Reference Kernels","compute":"datacenter-gpu","problemId":"eigh","problemName":"Eigh Validation","description":"Optimize symmetric eigenvalue decomposition (eigh) validation kernels.","metricName":"time","metricDirection":"minimize"}]}