{ "archive_path": "archive/1789596243.573017", "base_url": "rohanbansal.com/qorl", "basename": "qorl", "bookmarked_date": "2026-09-16 22:04", "canonical": { "archive_org_path": "https://web.archive.org/web/rohanbansal.com/qorl", "dom_path": "output.html", "favicon_path": "favicon.ico", "git_path": "git/", "google_favicon_path": "https://www.google.com/s2/favicons?domain=rohanbansal.com", "headers_path": "headers.json", "htmltotext_path": "htmltotext.txt", "index_path": "index.html", "media_path": "media/", "mercury_path": "mercury/content.html", "pdf_path": "output.pdf", "readability_path": "readability/content.html", "screenshot_path": "screenshot.png", "singlefile_path": "singlefile.html", "warc_path": "warc/", "wget_path": "rohanbansal.com/qorl.html" }, "domain": "rohanbansal.com", "extension": "", "hash": "EGDQJ130XK1J4E9DM346", "history": { "archive_org": [], "dom": [], "favicon": [ { "cmd": [ "curl", "--silent", "--location", "--compressed", "--max-time", "120", "--output", "favicon.ico", "--user-agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36 ArchiveBox/0.7.4 (+https://github.com/ArchiveBox/ArchiveBox/) curl/curl 8.14.1 (x86_64-pc-linux-gnu)", "https://www.google.com/s2/favicons?domain=rohanbansal.com" ], "cmd_version": "curl 8.14.1 (x86_64-pc-linux-gnu)", "end_ts": "2026-09-16T22:04:35.622088+00:00", "index_texts": null, "output": "favicon.ico", "pwd": "/data/archive/1789596243.573017", "schema": "ArchiveResult", "start_ts": "2026-09-16T22:04:35.046855+00:00", "status": "succeeded" } ], "git": [], "headers": [ { "cmd": [ "curl", "--silent", "--location", "--compressed", "--head", "--max-time", "120", "--user-agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36 ArchiveBox/0.7.4 (+https://github.com/ArchiveBox/ArchiveBox/) curl/curl 8.14.1 (x86_64-pc-linux-gnu)", "https://rohanbansal.com/qorl" ], "cmd_version": "curl 8.14.1 (x86_64-pc-linux-gnu)", "end_ts": "2026-09-16T22:04:36.006683+00:00", "index_texts": null, "output": "headers.json", "pwd": "/data/archive/1789596243.573017", "schema": "ArchiveResult", "start_ts": "2026-09-16T22:04:35.631343+00:00", "status": "succeeded" } ], "htmltotext": [], "media": [], "mercury": [ { "cmd": [ "/app/node_modules/@postlight/parser/cli.js", "https://rohanbansal.com/qorl" ], "cmd_version": "1.0.0", "end_ts": "2026-09-16T22:04:51.248337+00:00", "index_texts": null, "output": "mercury", "pwd": "/data/archive/1789596243.573017", "schema": "ArchiveResult", "start_ts": "2026-09-16T22:04:43.754529+00:00", "status": "succeeded" } ], "pdf": [], "readability": [ { "cmd": [ "/app/node_modules/readability-extractor/readability-extractor", "/tmp/tmp3g4pebs1", "https://rohanbansal.com/qorl" ], "cmd_version": "0.0.11", "end_ts": "2026-09-16T22:04:43.743998+00:00", "index_texts": [ "How good are query optimizers, really?\nLeis et al. asked this exact question in 2015. Then, they asked it again 10 years later.\nDespite an enormous body of research spanning a decade since their original exploration, they found that query optimizers continue to leave much to be desired.\nI was surprised when I first learned about this. A Postgres database should know everything about the stuff that lives in its tables, no? How hard can it be?\nAs it turns out: enormously hard. In fact, one particular task a query optimizer needs to do, join ordering, is known to be NP-hard.\nSo query optimizers are hard. What\u2019s not as hard is verifying whether a query plan an optimizer picks is good or not. Put simply, a good query optimizer produces plans that run fast, and a bad one produces slow plans. Language models are particularly good at learning how to do tasks with easily verifiable outputs. Because there\u2019s a single axis to optimize for\u2014execution time of a query\u2014the problem beautifully reduces to reinforcing the behaviors that guide a model to produce faster query plans.\nWhat follows is a breakdown of an experiment I ran to explore the question: can a small, open-weights model be post-trained via supervised fine-tuning (SFT) and agentic reinforcement learning (RL) to produce Postgres query plans that beat Postgres\u2019s default plans?\nThe answer to our question is a resounding yes. Highlights include:\n\nAttaining a 44.7% latency reduction across 113 join-heavy queries from a 4B model initially unable to produce a query plan for 99 of them\nConstructing a Postgres measurement rig that minimizes Linux page cache contention noise across concurrent containers\nDesigning a custom GRPO variant for scoring RL rollouts in an inherently noisy environment\nSplitting RL across two machines: vLLM and the trainer on a rented 2x H100 node and four Postgres containers running on my desk\nRunning off-policy distillation across half a thousand GPT-6 Astra agent trajectories\n\nLet\u2019s start from the beginning.\nInside a query optimizer\nConsider the following slice of the IMDb dataset:\n-- An IMDb title (movie, series, episode, etc.) [~1M rows]\ntitle (\n id integer PRIMARY KEY,\n title text,\n production_year integer,\n kind_id integer -- FK -> kind_type\n)\n\n-- Movie <> company junction table [~2M rows]\nmovie_companies (\n id integer PRIMARY KEY,\n movie_id integer, -- FK -> title.id\n company_id integer, -- FK -> company_name.id\n company_type_id integer, -- FK -> company_type.id\n note text\n)\n\n-- A company's name, origin, etc. [~100k rows]\ncompany_name (\n id integer PRIMARY KEY,\n name text,\n country_code text -- '[us]', '[jp]', ...\n)\n\n-- Lookup table of company roles for a title [4 rows]\ncompany_type (\n id integer PRIMARY KEY,\n kind text -- 'production companies', 'distributors', ...\n)\n\n-- Lookup table for what a title _is_ [7 rows]\nkind_type (\n id integer PRIMARY KEY,\n kind text -- 'movie', 'tv series', 'episode', ...\n)\nLet\u2019s say I\u2019m trying to answer the question: \u201cWhich Japanese companies put out the most titles in the 2000s?\u201d We might write the following query:\nSELECT cn.name,\n COUNT(*) AS titles\nFROM title AS t,\n movie_companies AS mc,\n company_name AS cn\nWHERE t.id = mc.movie_id\n AND mc.company_id = cn.id\n AND cn.country_code = '[jp]'\n AND t.production_year BETWEEN 2000 AND 2009\nGROUP BY cn.name\nORDER BY titles DESC\nLIMIT 10;\nRunning this query outputs 10 Japanese companies with the number of titles they were associated with between 2000 and 2009, sorted from highest to lowest.\nBut how did Postgres get these results?\nThe path Postgres took to get this data for us is not a foregone conclusion, and it has everything to do with what we call selective predicates (i.e. the filtering conditions in a WHERE clause).\nTo illustrate this, let\u2019s imagine our same query without the Japanese company filter or the date range filter:\nSELECT cn.name,\n COUNT(*) AS titles\nFROM title AS t,\n movie_companies AS mc,\n company_name AS cn\nWHERE t.id = mc.movie_id\n AND mc.company_id = cn.id\nGROUP BY cn.name\nORDER BY titles DESC\nLIMIT 10;\nmc can only join with cn via mc.company_id = cn.id, and t can only join with mc via t.id = mc.movie_id.\nThese constraints produce two There are technically eight join trees if we take commutativity into account. In this case, we don\u2019t because it doesn\u2019t affect the size of the relations resulting from the joins. valid join trees:\n \u22c8 \u22c8 t cn mc (cn \u22c8 mc) \u22c8 t \u22c8 \u22c8 cn t mc (t \u22c8 mc) \u22c8 cn The two join trees for our query. The lower join runs first; the result is an input into the root join. \nThe cardinality of a table or query result is the number of rows it contains. Assume the relevant tables have the following cardinalities:\n\ncn=100kcn = 100\\text{k}\nmc=2mmc = 2\\text{m}\nt=1mt = 1\\text{m}\n\nTaking into account our joins, we get the following cardinalities:\n(cn\u22c8mc)=2m,\u00a0then\u00a0\u22c8t=2m(cn \\bowtie mc) = 2\\text{m}, \\text{ then } \\bowtie t = 2\\text{m}\n(t\u22c8mc)=2m,\u00a0then\u00a0\u22c8cn=2m(t \\bowtie mc) = 2\\text{m}, \\text{ then } \\bowtie cn = 2\\text{m}\nRegardless of the order in which these three tables are joined, the same 2m rows are always passed into the second join.\nNow let\u2019s add back our selective predicates:\n\ncn\u2032=5kcn' = 5\\text{k} (assuming 5% of our 100k companies are Japanese)\nmc=2mmc = 2\\text{m} (does not change)\nt\u2032=200kt' = 200\\text{k} (assuming 20% of our 1m titles were made in the 2000s)\n\n(cn\u2032\u22c8mc)\u2248100k,\u00a0then\u00a0\u22c8\u00a0t\u2032\u224820k(cn' \\bowtie mc) \\approx 100\\text{k}, \\text{ then } \\bowtie\\ t' \\approx 20\\text{k}\n(t\u2032\u22c8mc)\u2248400k,\u00a0then\u00a0\u22c8\u00a0cn\u2032\u224820k(t' \\bowtie mc) \\approx 400\\text{k}, \\text{ then } \\bowtie\\ cn' \\approx 20\\text{k}\nThe first join ordering filters the 2m movie_companies entries down to the 5% slice of companies that are Japanese. Assuming uniform distribution (we\u2019ll discuss later why we assume this), this join results in approximately 100k rows. Joining the result with the filtered title table keeps only the 20% of those rows from the 2000s.\nThe second join ordering filters the 2m movie_companies entries down to the 20% slice of titles that were made in the 2000s. The same uniformity assumption holds, so the first join results in 400k rows, meaning we\u2019re passing 400k rows into the second join.\nWe do 4x the work if we picked the second join ordering.\nUnfortunately, it doesn\u2019t stop there.\nA combinatorial explosion\nEach join can use any of:\n\nHash join\nMerge join\nNested-loop join\n\nFactoring commutativity back in now While commutativity doesn\u2019t change the number of rows produced, it must be considered now because it does affect performance regarding the join algorithm used. , there are 4 different outer/inner join orientations, resulting in 8 possible combinations:\n (cn\u22c8mc)\u22c8t(cn \\bowtie mc) \\bowtie t\nt\u22c8(cn\u22c8mc)t \\bowtie (cn \\bowtie mc)(mc\u22c8cn)\u22c8t(mc \\bowtie cn) \\bowtie t\nt\u22c8(mc\u22c8cn)t \\bowtie (mc \\bowtie cn)(t\u22c8mc)\u22c8cn(t \\bowtie mc) \\bowtie cn\ncn\u22c8(t\u22c8mc)cn \\bowtie (t \\bowtie mc)(mc\u22c8t)\u22c8cn(mc \\bowtie t) \\bowtie cn\ncn\u22c8(mc\u22c8t)cn \\bowtie (mc \\bowtie t) \nLastly, each table can be scanned in different ways. Considering just four types of scans:\n\nSequential\nIndex\nIndex-only\nBitmap\n\n \nThere are 4,608 different ways to run this query This is actually an undercount. Plans can run in parallel, aggregates can be hashed or sorted, etc.It\u2019s also worth noting that Postgres doesn\u2019t evaluate all of these plans. It uses dynamic programming (and a genetic algorithm for queries involving 12+ joins) to prune the search space. !\nTo make matters worse, every join combinatorially explodes the search space:\n SELECT MIN(t.title) AS movie_title\nFROM company_name AS cn,\n keyword AS k,\n movie_companies AS mc,\n movie_keyword AS mk,\n title AS t\nWHERE cn.country_code ='[de]'\n AND k.keyword ='character-name-in-title'\n AND cn.id = mc.company_id\n AND mc.movie_id = t.id\n AND t.id = mk.movie_id\n AND mk.keyword_id = k.id\n AND mc.movie_id = mk.movie_id; Click to select the number of tables being joined together. From four tables onwards, queries on the left are from JOB. On the right is a rough estimate of the size of the search space. \nEstimating, not counting\nPostgres is in a tough spot here. It would be reasonable to think it could simply count cardinalities and pick the plan that minimizes the number of rows passed through to successive joins.\nBut this would imply Postgres can count cardinalities during query planning. It can\u2019t. In order to know this, it would need to actually run each join and count the resulting rows. This defeats the whole point of a fast query optimizer. A query optimizer does not aim to be exact in its cost minimization\u2026 it aims to be good enough across many types of queries.\nInstead, Postgres uses statistics to estimate cardinalities. The planner queries the pg_statistic table, getting back common values for each column and their frequencies, and a histogram for the rest. Things get a bit more complicated when you tack on joins. Postgres doesn\u2019t know how the rows in one table are distributed over the other. To get around this, it assumes that the frequency of a given value in the first table can simply be applied over the second table. This is the uniform distribution assumption I mentioned earlier.\nAssuming a uniform distribution is fine as a heuristic, but when it fails, it fails hard. Looking back at an earlier join ordering (cn\u2032\u22c8mc)\u2248100k,\u00a0then\u00a0\u22c8\u00a0t\u2032\u224820k(cn' \\bowtie mc) \\approx 100\\text{k}, \\text{ then } \\bowtie\\ t' \\approx 20\\text{k}, we filtered 2m movie_companies entries on the assumption that 5% of them were from Japanese companies. But what if the 5% of companies that are Japanese were actually responsible for 50% of the movies? The first join would produce 1m rows! The cost model says pick the first join ordering; in reality, the second one is actually better since it only sends 400k rows through to the second join.\n Postgres assumption - 100k rows Actual - 100k rows \u22c8 \u22c8 t\u2032 cn\u2032 mc (cn\u2032 \u22c8 mc) \u22c8 t\u2032 \u22c8 \u22c8 cn\u2032 t\u2032 mc (t\u2032 \u22c8 mc) \u22c8 cn\u2032 \nShare of movie_companies rows that belong to Japanese companies:\n (uniform) Drag the slider to make Japanese companies more productive. Notice how Postgres\u2019s estimate is static while actual row counts get affected. \nOne bad estimate in an early join can cascade through the rest of the join tree, corrupting all other estimates.\nHow to steer an elephant\nPostgres always picks the plan with the lowest cost, and we can\u2019t change its cost model without modifying its source code, so how can we actually steer it to pick different plans that have higher costs?\nEnter pg_hint_plan.\npg_hint_plan is a beautifully simple third-party extension: just by adding structured \u201chints\u201d as comments above SQL statements, you can nudge Postgres towards plans that use the instructions provided in the hint. For example:\n /*+ HashJoin(a b) SeqScan(a)*/EXPLAIN SELECT * FROM pgbench_branches b JOIN pgbench_accounts a ON b.bid = a.bid ORDER BY a.aid; QUERY PLAN--------------------------------------------------------------------------------- Sort (cost=31465.84..31715.84 rows=100000 width=197) Sort Key: a.aid -> Hash Join (cost=1.02..4016.02 rows=100000 width=197) Hash Cond: (a.bid = b.bid) -> Seq Scan on pgbench_accounts a (cost=0.00..2640.00 rows=100000 width=97) -> Hash (cost=1.01..1.01 rows=1 width=100) -> Seq Scan on pgbench_branches b (cost=0.00..1.01 rows=1 width=100)(7 rows) Example from pg_hint_plan\u2019s\ndocumentation. \nThe hint mandates usage of a HashJoin for joining pgbench_accounts and pgbench_branches, and doing a sequential scan of the pgbench_accounts table; the actual query plan follows suit nicely.\nFormulating our problem\nGiven that we can influence Postgres to pick different\u2014and potentially better\u2014query plans using pg_hint_plan hints, the question we\u2019re starting with is:\n\nCan a language model learn to produce hints that result in better query plans?\n\nUseful research\nWhat might make this a worthwhile problem to solve?\nMy first idea was to give the model the query and the exact same set of information Postgres\u2019s planner has. This amounts to seeing if we could build a better cardinality estimator. I came to the conclusion this is not a worthwhile avenue to explore; we would be fighting decades of cardinality estimation research. Furthermore, the inference latency alone would far outweigh any learned usefulness compared to Postgres\u2019s ultra-fast query optimizer.\nThe second idea\u2014and what I believe is the correct formulation\u2014lies in a specific database usage pattern: heavy analytic workloads. If queries are getting run thousands of times using sub-optimal default Postgres plans, efficiency gains are being left on the table. Instead, a model could be trained to find a better way to run a specific query. The training process might require execution of that query tens to hundreds of times upfront, but the amortized cost across all runs of the query would be drastically lower.\nThe goal isn\u2019t to try and beat Postgres on the time/efficiency Pareto frontier for one-off queries, but we may be able to beat it on queries that run over and over again.\nA model and its harness\nI decided to start with a small 4B model because it would be easiest to train/inference myself on the 2x RTX 3090 rig (affectionately named FLOPper) I have at home.\nAround the time I started this project, the Qwen 3.8 family of models was released, unfortunately without a 4B variant. However, I came across a Qwen 3.8 4B distillation from a small lab in Germany called Empero and was intrigued. They used Qwen 3.8\u2019s 2.4T model as a teacher model to distill learnings into Qwen 3.5 4B, producing empero-ai/Qwen3.8-4B-Distill. This distilled model is not outright better than its base 3.5 model; it performs better on MMLU tasks and slightly worse on GSM8K tasks. In other words, this distillation performs better when evaluated on breadth of general knowledge, and slightly worse on multi-step mathematical reasoning. As to which is better for our task, I do not know; I decided to stick with the distilled model either way.\nWith the model locked in, I built a lightweight agent harness, qo-agent, that would orchestrate hint production. It was given the following six tools:\n\ninspect_relation \u2014 Lists a table\u2019s columns with types and nullability, index definitions and estimated rows and bytes\nget_column_stats \u2014 Gets Postgres planner statistics for 1-8 columns of a relation\nget_plan \u2014 Gets the default plan\u2019s estimates or a submitted candidate\u2019s stored plan\nevaluate_candidate \u2014 Validates a proposed plan action and then executes it for timing/plan diagnostics\nkeep_default \u2014 Returns Postgres\u2019s default plan itself as the candidate and ends the search\nfinish \u2014 Takes as input a submitted candidate ID or the default plan and ends the search\n\nTo take advantage of structured outputs, the agent was instructed to produce PlanAction JSON objects. Calls to evaluate_candidate then compiled PlanAction objects into hints and prepended them to the original query.\nA sample agent trajectory:\n Agent \u2192 get_plan(\"default\")Tool \u2190 Default plan: (t \u22c8 mc) \u22c8 cn, hash joins, estimated rows per node\u00a0Agent \u2192 evaluate_candidate({ \"leading\": { \"left\": { \"left\": \"cn\", \"right\": \"mc\" }, \"right\": \"t\" }, \"joins\": [{ \"relations\": [\"cn\", \"mc\"], \"force\": \"hash\" }] })Tool \u2190 c1 \u00b7 valid \u00b7 novel plan \u00b7 118 ms \u00b7 0.91\u00d7 default \u00b7 2 attempts left\u00a0Agent \u2192 evaluate_candidate({ \"leading\": { \"left\": { \"left\": \"t\", \"right\": \"mc\" }, \"right\": \"cn\" }, \"joins\": [{ \"relations\": [\"t\", \"mc\"], \"force\": \"nestloop\" }], \"scans\": [{ \"relation\": \"mc\", \"force\": \"index\" }] })Tool \u2190 c2 \u00b7 valid \u00b7 novel plan \u00b7 87 ms \u00b7 1.24\u00d7 default \u00b7 1 attempt left\u00a0Agent \u2192 evaluate_candidate({ \"leading\": { \"left\": { \"left\": \"t\", \"right\": \"mc\" }, \"right\": \"cn\" }, \"joins\": [{ \"relations\": [\"t\", \"mc\"], \"force\": \"hash\" }] })Tool \u2190 c3 \u00b7 valid \u00b7 novel plan \u00b7 100 ms \u00b7 1.08\u00d7 default \u00b7 0 attempts left\u00a0Agent \u2192 finish({ \"selected_candidate_id\": \"c2\" })Tool \u2190 Finished \u00b7 selected c2 A sample trajectory where the agent is permitted to submit up to three candidates. \nBenchmarks\nAn agent is useless without something to benchmark its performance against. Fortunately for us, the hard work of creating these benchmarks was already done.\nThe Join Order Benchmark\nLeis et al. introduced the Join Order Benchmark (JOB) in How Good Are Query Optimizers, Really?. They used it to evaluate cardinality estimation and join-order optimization using our familiar IMDb dataset.\nIt consists of 113 queries spread across 33 query templates. Query templates differ via their relational skeleton. They reference different tables and connect them with different join predicates. You can think about them as a structural family of questions that can be answered. Queries derived from templates preserve the tables used and the join graph topology but change selection predicates.\nLooking at an example:\n SELECT MIN(t.title) AS movie_title\nFROM company_name AS cn\nJOIN movie_companies AS mc ON mc.company_id = cn.id\nJOIN title AS t ON t.id = mc.movie_id\nJOIN movie_keyword AS mk ON mk.movie_id = t.id\nJOIN keyword AS k ON k.id = mk.keyword_id\nWHERE cn.country_code = :country_code\n AND k.keyword = 'character-name-in-title'; Query template 2 \u2014 \u201cWhat is the alphabetically first title of a movie associated with a company from country X and tagged with the keyword character-name-in-title?\u201d \n\u2026and here are two real queries from JOB derived from this template:\n SELECT MIN(t.title) AS movie_title\nFROM company_name AS cn,\n keyword AS k,\n movie_companies AS mc,\n movie_keyword AS mk,\n title AS t\nWHERE cn.country_code = '[de]'\n AND k.keyword = 'character-name-in-title'\n AND cn.id = mc.company_id\n AND mc.movie_id = t.id\n AND t.id = mk.movie_id\n AND mk.keyword_id = k.id\n AND mc.movie_id = mk.movie_id;Query 2a \u2014 \u201cWhat is the alphabetically first such movie title associated with a German company?\u201d SELECT MIN(t.title) AS movie_title\nFROM company_name AS cn,\n keyword AS k,\n movie_companies AS mc,\n movie_keyword AS mk,\n title AS t\nWHERE cn.country_code = '[us]'\n AND k.keyword = 'character-name-in-title'\n AND cn.id = mc.company_id\n AND mc.movie_id = t.id\n AND t.id = mk.movie_id\n AND mk.keyword_id = k.id\n AND mc.movie_id = mk.movie_id;Query 2d \u2014 \u201cWhat is the alphabetically first such movie title associated with a U.S. company?\u201d \nThe Cardinality Estimation Benchmark\nAnother relevant benchmark is the Cardinality Estimation Benchmark (CEB), introduced in Flow-loss: Learning Cardinality Estimates That Matter. It uses the same IMDb database and is a much larger benchmark consisting of ~13.6k synthetically generated queries organized across 16 query templates CEB\u2019s definition of a template is looser than JOB's. Two CEB templates can share the same join graph, differing only in their selectivity predicates. In JOB, every template's join graph is unique. .\nTrain time, test time\nDue to its size, CEB was a good fit for training the model. JOB would be used to validate the model\u2019s performance.\nYou might be wondering if it makes sense to both train and test on IMDb. If it works well, hasn\u2019t the model just learned this specific database well?\nI would argue this is precisely the point. We want our model to learn IMDb well. Given our problem formulation, if this agent is continually getting used for a company\u2019s analytic workloads across its specific databases, we need not generalize to all databases.\nThe real issue is making sure we\u2019re not overfitting to JOB query templates during training over CEB. The model should learn IMDb in a way where given any query, even for structural query families it hasn\u2019t seen before, it\u2019s still capable of producing a good plan. In practice, this means we need to prune CEB queries that have the same shape as any of the JOB queries.\nQuery topology mapping\nLet\u2019s define a query\u2019s \u201ctopology\u201d as its structural join-graph (de-aliased table names as nodes and joins as edges). The join graph excludes all selectivity predicates; we\u2019re only interested in joins here.\nCEB queries sharing a topology with a JOB query would be removed from the training set. I wrote a small script to convert all JOB and CEB queries to their topologies and checked if there was any overlap. There wasn\u2019t, so no filtering was required.\n JOB: 113 queries, 33 templates, 33 topologies\n 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 CEB: 13,646 queries, 16 templates, 12 topologies\n 1a 2a 2b 2c 3a 3b 4a 5a 6a 7a 8a 9a 9b 10a 11a 11b JOB and CEB templates displayed as an identicon of their topologies. \nHow to muffle an elephant\nBefore getting into benchmarking the agent and doing training runs, we have to talk about how Postgres was actually run, because it directly impacts the training process.\nFirst, some facts:\n\nFLOPper has a CPU with 16 physical cores, 64 GB of RAM and a 2 TB NVMe SSD\nThe slice of IMDb we\u2019re using is 8.5 GB on disk\nPostgres caches pages of data retrieved during query execution into a buffer\nThe operating system has its own filesystem cache doing the same thing one level down\n\nIf we run the exact same query on Postgres 20 times in a row, it won\u2019t take the same amount of time each run. In day-to-day work, this isn\u2019t a big deal. But the whole thesis, and the training process itself, relies on measuring whether one way of running a query is faster than the Postgres default. This means we need to do everything in our power to de-noise Postgres.\nFirst, I needed to understand just how noisy Postgres query executions are.\nI started by building a \u201ccalibration\u201d capability into my experimentation workflow. The calibration process was simple: run NN Docker containers built from a Postgres image, each given a fixed slice of CPU cores and RAM to use. I set N=4N = 4 to begin; anything lower might make future training far too slow, and anything higher might lead to more CPU contention, which means more noise. Each container was given 4 cores to use and capped at 8 GB of memory.\nOn startup, each container initialized Postgres with identical settings and loaded the IMDb data. Calibration then opened a thread pool of size four and pushed all 113 queries onto a shared queue. Whenever a container finished measuring a query, it pulled the next one off the queue.\nThe actual measurement process had two phases:\n\nRun the query a few times to \u201cwarm it up\u201d\nThen run the query 20 more times and record each execution time\n\n queuejob-01ajob-01cjob-01djob-01bjob-02ajob-02cjob-02bjob-02d +105 more container 0 \n\u2014\n warmup measure \nidle\n container 1 \n\u2014\n warmup measure \nidle\n container 2 \n\u2014\n warmup measure \nidle\n container 3 \n\u2014\n warmup measure \nidle\n Four containers pull JOB queries off a shared queue, warm each one up until its buffer counters settle, then run it 20 times. \nSo what does it mean to warm a query up? We need to bust out some OS fundamentals to understand.\nWhenever Postgres executes a query, it asks the operating system (in our case, Linux) for pages of data. Linux first checks its own filesystem cache, the page cache. If the pages are present, Linux sends them over; else it reads them from disk, stores them in its cache and then sends them over. Postgres, in turn, keeps received pages in its own shared_buffers cache for easy reuse. When shared_buffers begins to overflow, Postgres evicts pages. If it needs those pages again, it must ask Linux once more.\nEvery time there\u2019s a cache hit in shared_buffers for a page, Postgres increments a counter called \u201cshared hit blocks\u201d (SHBs). If it has to ask Linux, it increments \u201cshared read blocks\u201d (SRBs).\nPostgres conveniently reports both counters if we run EXPLAIN with the BUFFERS option. For example, running EXPLAIN (ANALYZE, TIMING OFF, BUFFERS, FORMAT JSON) outputs something like:\n{\n \"Plan\": {\n \"Node Type\": \"Aggregate\",\n \"Shared Hit Blocks\": 1800786,\n \"Shared Read Blocks\": 52990,\n ...\n },\n \"Execution Time\": 189.2,\n ...,\n}\nThese counters give us some notion of the \u201cwarmness\u201d of a query. After each warmup run, we compared its hit and read counts to the previous run\u2019s. If both were within 2% of each other (and the plan hadn\u2019t changed), we called the query warm and started measuring. A query needed at least two warmups to have something to compare, and was cut off at five regardless. The idea was that if the counters stopped moving, the data could be considered settled and cache churn would be minimized during the 20 measurements.\n Query A runs shared_buffers Postgres page cache Linux disk Query A Query B Shared hit blocks 0 Shared read blocks 0 Query A fills shared_buffers via Linux calls. Query B requires different pages, evicting Query A pages in shared_buffers along the way. When A runs again, the evicted pages count as reads. \nI set shared_buffers to a conservative 128 MB and ran the first calibration:\n 2 warmups 50 queries 48 of 50 still reading\n 3 warmups 46 queries 26 of 46 still reading\n 4 warmups 4 queries 2 of 4 still reading\n 5, capped 13 queries 13 of 13 still reading\n Still reading from Linux after warmup Fully resident in shared_buffers All 113 JOB queries in the first calibration grouped by how many warmups they needed and placed by the share of their pages still read from Linux on every run afterwards. \nHalf the queries were declared warm after only two runs. Not bad\u2026 at least until I dug deeper. The SRB counts weren\u2019t dropping to zero; rather, they were hovering steady at some large number. With only 128 MB of shared_buffers against an 8.5 GB database, Postgres was consistently missing its own cache on every execution and asking Linux for more pages. \u201cStable\u201d did not mean \u201cresident.\u201d\nLinux\u2019s page cache is fast, so this isn\u2019t the end of the world. Unfortunately, a new problem emerged when I actually looked at the 20 measurements taken for various queries. Let\u2019s look at one query in particular, job-13b:\n job-13b 128 MB shared_buffers \nrun 1 \nrun 5 \nrun 10 \nrun 15 \nrun 20 14 runs \u00b7 186\u2013204 ms 6 runs \u00b7 227\u2013253 ms job-13b's 20 measured runs at 128 MB shared_buffers. Each dot is one run. Toggle between the two buttons to see the runs first in the order they ran, and then dropped onto the x-axis, where they pile into two clumps. \n14 of the 20 landed between 186 and 204 ms. The other 6 landed between 227 and 253 ms, somewhere between 14% and 26% slower. The query wasn\u2019t even uniformly noisy, it just had two different speeds at different times, and a third of the time it ran at the slower speed.\nI initially wanted to quantify noise using the coefficient of variation:\n \nThe CV tells us the \u201cwobble\u201d of a measurement. If a query takes 100 ms and has a CV of 5%, we could say it wobbles by about 5 ms. For job-13b, the CV was 10.3%. It wasn\u2019t great. CV is also not a great measurement to use here. Because it\u2019s built on the mean, it\u2019s easily influenced by a few outlier runs.\nWe don\u2019t actually care as much about how spread out the 20 runs are. We do care about how often this causes our measurement criteria during training runs to get fooled.\nTo fool an agent\nBear with me here as I skip ahead a little bit in order to provide more color on what exactly we needed to measure.\nTo de-noise during actual agent runs, I couldn\u2019t just run the agent\u2019s proposed plan a single time. Instead, I ran three interleaved (candidate, default) pairs sequentially. Three was picked somewhat arbitrarily to provide some measure of variability while being small enough to prevent agent evaluation runs from spending most of their time in Postgres. Once the three candidate/default execution time tuples were obtained, the medians of both the three candidates and the three defaults were taken and expressed as a ratio of each other to determine the final speedup or slowdown. If the two medians differed by less than an arbitrarily declared 5%, it was a tie. Outside of that tie zone, a candidate could be declared as a speedup or a slowdown.\nNow let\u2019s go back to our earlier job-13b example. We had 14 executions in one clump, and 6 in another slower clump. The median of three strategy sounds good until you realize that if, in theory, at least two of the three measurements landed in that \u201cslower\u201d clump, the median would bias towards the less frequent slower clump.\nImagine a candidate plan that executes identically to the default. No real difference exists, so the correct reward is zero. Draw three timings for the \u201ccandidate\u201d and three for the \u201cdefault\u201d out of the 20 we observed. There are (203)=1,140\\binom{20}{3} = 1{,}140 ways to draw three from 20; for job-13b, 230 of them contain at least two slow runs, so one side\u2019s median lands in the slow clump ~20% of the time.\nThat\u2019s a totally phantom 14-26% speedup or slowdown that we would show to our model as signal ~20% of the time. Dangerous!\n job-13b 128 MB shared_buffers \u00b7 a no-op candidate (i.e. one that is identical to the default) 20 runs candidate default \u2014 drawing\u2026 0 rounds \u00b7 ties 0 \u00b7 phantom wins 0 \u00b7 phantom losses 0 \u00b7 fooled 0% A no-op candidate measured against itself. Every round draws three of job-13b's 20 runs for the candidate and three for the default, takes each side's median, and applies the 5% tie zone. Over every possible draw, the reward is fooled ~40% of the time. \nSo we can\u2019t just rely on CV as the golden number to minimize, as two queries with the exact same CV can fool the measurement reward at different rates depending on whether the spreads are a uniform blur or two clumps sitting more than 5% apart. The actual number to minimize is this fooling rate itself.\nI wrote a small script to compute the fooling rate directly from raw calibration data. It worked by sliding a window of six sequential runs across the 20. For each window, we took interleaved pairs of size two to represent an interleaved (candidate, default) pair. A window of size six gives us pairings like: (t1, t2), (t3, t4), (t5, t6). In any given pair, tnt_n and tn+1t_{n+1} can alternate roles of being the candidate query, or the default query. That means for each pair, there are two possibilities, and therefore for each window of three tuples, there are 2\u00d72\u00d72=82 \\times 2 \\times 2 = 8 possibilities. 20 measurements means we\u2019ll slide this window 15 times, so we have 15\u00d78=12015 \\times 8 = 120 total possibilities Each possibility is a binary value indicating whether or not that specific, simulated formulation of candidate/default pairs resulted in a ratio of medians between the two greater than the 5% tie-zone. for a given query.\nWe derive two metrics from these raw numbers. First, we calculate the no-op error rate for a given query as the ratio of the 120 simulated possibilities that do differ by more than 5% against the number that don\u2019t. We sum these percentages up across all 113 JOB queries and then divide by 113. This number, which we\u2019ll call the \u201cmean no-op error rate,\u201d gives us the percentage likelihood that the reward may get fooled for any JOB query when doing our three paired measurements strategy. Second, we sort the no-op error rates for all 113 queries, lowest to highest. The number that is 90% of the way to the end of this sorted list is reported as the \u201cp90 query,\u201d and gives us a measure of the fooling rate for the worst-offending queries.\nAt 128 MB for shared_buffers and four concurrent containers, the \u201cfool rate\u201d script produced the following mean no-op error rates and p90 query numbers I ran the calibration twice per config to provide a sense of how much two runs may disagree with each other. :\n \n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\nRunMean no-op error ratep90 queryMedian CV15.0%13%2.3%25.4%20%2.4% \nThe numbers aren\u2019t good. One in twenty no-op plans get rewarded, and one in ~10 queries gets fooled more than 13% of the time.\nWe can do better.\nTuning Postgres\nI focused on two memory-related settings Postgres exposes:\n\nshared_buffers decides how much of the database Postgres can keep in its own cache\nwork_mem decides how much memory a single sort/hash operation can get before spilling to disk\n\nI ran four calibrations:\n \n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\nshared_bufferswork_memNo-op error rate (run 1 / 2)p90 queryMedian CVTotal runtime128 MB4 MB5.0% / 5.4%13% / 20%2.3%95 s2 GB4 MB1.8% / 1.2%1.3% / 0%1.1%60 s128 MB32 MB7.0% / 6.6%20% / 23%2.6%94 s2 GB32 MB1.7% / 1.3%0% / 0%1.2%60 s \nSurprisingly, work_mem had no effect on noise at all, and shared_buffers carried all of the weight!\nWith 2 GB of shared_buffers, the median query ended warmup with its SRB counter at exactly zero: its working set was fully resident in Postgres\u2019s own cache. The no-op error rate dropped by roughly 4x, and the 90th percentile query went from being fooled 13%\u201320% of the time to almost never. Our two-clump query, job-13b, went from a CV of 10.3% to 0.9%, with all 20 runs landing within 7 ms of each other.\nOne neat benefit emerged that I wasn\u2019t initially chasing: the default plans themselves got faster. The summed runtime of all 113 JOB queries fell from 95 seconds to 60 seconds, just from cache residency. In other words, actually taking our measurements for both candidates and defaults would now be significantly faster, meaning the training process would take less time.\nI locked in 2 GB shared_buffers and 4 MB work_mem for the rest of the project.\nBaselines and metrics\nI used two metrics for benchmarking agent performance.\nGeometric mean speedup\n \nThe geometric mean speedup gives all queries equal weight. For example, in a two-query sample, if query 1 runs 2x faster than its baseline, and query 2 runs 0.5x faster than its baseline, then Sgeo=1.00xS_{geo} = 1.00\\text{x}. It doesn\u2019t matter if query 1\u2019s baseline took 5 minutes and our candidate took 2.5 minutes, but query 2 only regressed from 25s to 50s, as they are equally weighted.\nTotal workload speedup\n \nTotal workload speedup treats the entire query set as one batch. We simply add all the baseline times and divide by the sum of the candidate times. In our above example, Sworkload=1.4xS_{workload} = 1.4\\text{x}.\nBoth metrics tell different stories. The total workload speedup is a measure of practicality. A data analyst building out a suite of analytics queries wants to decrease the overall runtime across the batch. But from a model training standpoint, the total workload speedup could be entirely influenced by a single query plan the agent chanced upon; the rest of the batch could be degenerate. This implies the model hasn\u2019t actually learned anything interesting; it just got lucky. Because the geometric mean speedup cares not for absolutes, it gives us a measure of actual learning across the batch: values above 1x imply that the average query is executing faster.\nA frontier intelligence control\nBefore running the untrained 4B model through the qo-agent harness, I wanted to validate this problem was actually solveable by today\u2019s frontier models. If a model like GPT-6 Astra or Qwen 3.8 2.4T couldn\u2019t improve upon the default Postgres query plan, I couldn\u2019t really expect the 4B model to either.\nI took a small sample of 10 JOB queries and benchmarked them on both Astra and Qwen 3.8 2.4T running through the qo-agent harness:\n ModelCandidatesTasks scored Regressions Astra [m] 19/100.85x1.00x3 Astra [m] 510/102.54x2.12x0 Astra [m, r] 510/102.39x1.57x1 Qwen 3.8 2.4T [m] 17/102.02x1.30x1 Qwen 3.8 2.4T [m] 510/102.26x1.35x1 Evaluations of Astra and Qwen 3.8 2.4T run on the same slice of 10 JOB queries. The frontier models were benchmarked at different candidate numbers (i.e. how many candidates they were allowed to generate during a complete trajectory; either a single candidate or 5) and for Astra, whether reasoning summaries I was a little surprised to see Astra performance worsen with reasoning summaries on compared to the 5-candidate evaluation done right before it, but these evaluations were only run a single time on a small 10-query slice of JOB, so I chalked up the worse results to random variance. were enabled or not. Astra was inferenced through OpenAI\u2019s API, and Qwen 3.8 2.4T through Modal via OpenRouter. \nGiven the difference between the single-candidate scores and the 5-candidate scores, the agent was clearly capable of doing in-context learning across sequential executions of its candidates. This gave me the confidence to stick with an agentic multi-turn approach rather than try and train the 4B model to get really good at one-shotting a plan.\nDuring a run of the agent, each candidate was warmed once and then measured once. After exhausting the candidate attempts budget, the model was only presented with a single tool to call, finish, and the model was told to select the best scoring candidate (or keep the default plan). After the candidate was selected, three interleaved (candidate, default) pairs were run and passed through a clipper:\nSi=clip\u2061(median\u2061(Di)median\u2061(Ci),\u00a00.1,\u00a010)S_i = \\operatorname{clip}\\left( \\frac{\\operatorname{median}(D_i)}{\\operatorname{median}(C_i)},\\ 0.1,\\ 10 \\right)\nThe clipper constrained the result of the division between the two medians to be between [0.1,10][0.1, 10]. These clipper values were picked somewhat arbitrarily; I found they prevented the geometric mean speedup from getting overly influenced by an extreme speedup or an extreme regression.\nConclusion: frontier intelligence is capable of agentically doing query optimization.\nThe vanilla 4B baseline\nWe\u2019re now ready to evaluate the untrained 4B model on JOB and see how it does!\nThe same 5-candidate plan budget per agent trajectory configuration was employed. The results were dismal:\n \n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\nHow the trajectory endedQueriesNo valid candidate81Selection failed16Timed out1Candidate duplicated the default plan7Kept the default2Candidate measured against the default6 \nOnly the last three rows contribute to the score, leaving 15 valid trajectories out of 113. Nine of those 15 result in a score of 1.00x by construction (the plan was identical A candidate plan was determined to be equal to the default plan if their EXPLAIN outputs with cost and row estimates stripped were equivalent. to the default, or the model chose to keep the default). That left just six candidate plans that were:\n\nStructurally intact (the PlanAction the model produced was successfully compiled into a hint comment)\nValid (the resulting hint was actually valid given the schema)\nNovel (the resulting plan was distinct from Postgres\u2019s)\n\nFive of the six plans resulted in speedups of 1.02x\u20131.30x, and one of them landed at 0.05x.\nThe most common issues seen were:\n\n\nThe PlanAction was not a valid object\nJoin trees did not contain every relation exactly once\nActions were wrapped in an extraneous action key\n Leading Leading trees ended up being quite an important concept, as these are used to influence the join-ordering, which as we mentioned previously, is an NP-hard problem. trees had subtrees that were not actually connected in the query\u2019s join graph\nThe model called specific tools at the wrong time, or called tools that didn\u2019t exist\n\nNot only was the model terrible at this task, it couldn\u2019t even grok the harness wrapped around it either.\nOff-policy distillation via supervised fine-tuning\nI first needed to get the 4B model to speak the \u201clanguage\u201d of the qo-agent harness. I would make it good at query optimization after.\nWe can use supervised fine-tuning (SFT) to do this. Specifically, we can do off-policy distillation.\nOff-policy distillation is a training method by which a student model (sometimes referred to as a policy) learns to imitate outputs produced by a teacher model. It\u2019s called \u201coff-policy\u201d because the training data is not generated by the student model/policy itself. It\u2019s remarkably simple. A complete teacher trajectory (sometimes referred to as a demonstration) is shown to the student model. For every token in the trajectory, the probability the student model gave to generating that token results in a per-token loss. Averaging these per-token losses leads to a demonstration-level loss. Standard backpropagation via chain rule then lets you compute gradients for all trainable parameters in the student model, and the configured optimizer can nudge parameter values in a way where loss gets minimized in a single pass.\nWhat\u2019s super neat about off-policy distillation is that we don\u2019t need a lot of data for it to work well. Because each demonstration provides us thousands to tens of thousands of token predictions, our student model\u2019s weights adapt quickly.\nHow do we actually get these demonstrations though? We could write them all by hand, but that would take far too long. One step up would be writing a tool to randomly generate valid-looking trajectories Fun fact: I tried this initially. It actually works decently and was able to teach the 4B model the harness. However, it caused a bunch of other issues, mostly around making the model less likely to generate novel candidate plans. . But these ignore that we have the best teacher of all already available: smarter, larger models.\nThe strategy is simple: generate a bunch of trajectories by running a smart model through the qo-agent harness, and distill those trajectories into our student 4B model.\nRendering, loss masking, and unrolling\nThere are a few complexities to unpack.\nImagine we decide to use GPT-6 Astra as our teacher model. It produces trajectories in OpenAI\u2019s Responses API format. Our Qwen model doesn\u2019t understand this format; we need to transpile the human-friendly Responses API JSON format into a model-friendly token format. This is where the concept of rendering comes in. Rendering libraries can take in a trajectory\u2019s text and convert it to a raw sequence of tokens a specific model actually understands.\nAnother consideration with agent trajectories is determining which tokens our model should actually be predicting. The model never produces certain tokens in a trajectory, like the system prompt, any user prompts or the results of a tool call after a harness executes it. The model should still see these tokens when predicting the next token though; they\u2019re still part of the context, but we should only compute losses for tokens the model is responsible for predicting. We can employ a strategy called loss-masking here. A loss-masking library lets us label the parts of a teacher trajectory that are context-only, versus the parts our student model is responsible for predicting.\nFinally, when fine-tuning over agent trajectories, we generally don\u2019t include the entire trajectory as a single trainable unit. Instead, the trajectory is broken up into a set of (context, reply) pairs. The context in these pairs is additive and includes previous model replies.\n trajectory system prompt query + observation reply 1 tool result reply 2 tool result reply 3 unrolls into example 1 context context reply 1 example 2 context context context context reply 2 example 3 context context context context context context reply 3 Tokens the model is scored on\n Context only, no loss\n One trajectory with three assistant replies becomes three training examples. Each example includes as context everything preceding an assistant reply, including the model's earlier responses. \nLow-rank adaptation (LoRA)\nOur 4B model has 4.66 billion tunable parameters. If we wanted to update all of these parameters in a single pass during SFT, we would need a GPU with at least 64 GB of VRAM. I wanted to test my hypotheses first on my consumer-grade RTX 3090s, each of which carries 24 GB of VRAM, so I needed something more parameter-efficient here.\nLow-rank adapters, or LoRAs, are the canonical way to do this. At a high-level, they work by freezing the model\u2019s weights as they are, instead letting you train a much smaller pair of matrices that when multiplied together, result in an adjustment to selected weights in your original model.\n4.66 billion weights come in at 9.32 GB in bf16. The tiny LoRA I actually ended up training was only 42.5 MB and contained only 21.2 million trainable parameters.\nTeacher model selection\nWe previously learned that frontier models could operate well in qo-agent; the next decision was picking between GPT-6 Astra or Qwen 3.8 2.4T.\nLet\u2019s look back at the results from evaluating both models on 10 JOB queries, this time zooming into context length in tokens:\n \n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\nPer trajectory, 5 candidatesAstra (summaries)Qwen 3.8 2.4TModel turns, mean7.612.2Final context, mean tokens22,00945,144Final context, max tokens28,66373,608Output tokens per turn, mean1611,795Reasoning tokens per turn, mean651,417Wall clock for all 10 queries3m 13s15m 51s The same five-candidate evaluations over the 10 JOB queries mentioned earlier, this time measured by how much context each trajectory consumed. \nIt seems like Astra wins on all fronts. However, Astra\u2019s biggest drawback is that when run through the API, reasoning tokens aren\u2019t supplied. Instead, the API gives you a short summary in place of reasoning tokens. On the other hand, Qwen 3.8 2.4T is an open-weights model and is happy to provide all of its reasoning tokens.\nI was wary about training off of trajectories containing only reasoning summaries. The How to Steal Reasoning Without Reasoning Traces paper talked about this exact thing: training off reasoning summaries resulted in the student model\u2019s performance decreasing. To combat this, the authors devised a new method: trace inversion. Trace inversion calls for synthetically expanding a reasoning summary into what the raw reasoning tokens may have looked like. Although they\u2019re not exactly the ones Astra actually produced during inference, the longer reasoning blocks led to improved performance when transferred over to a smaller model. This provided some level of comfort; if I picked Astra and performance suffered, I could experiment with trace inversion.\nThe other consideration was context lengths. Given I was only doing SFT off a single RTX 3090 to start, I needed any given trajectory to not exceed ~50k tokens in sequence length. If it did, the training process might OOM given the 3090\u2019s limited 24 GB of VRAM. All Astra trajectories fit under that budget, but some of the Qwen 3.8 2.4T ones didn\u2019t.\nI decided to try out SFT on the Astra traces. If performance suffered, I could explore trace inversion; if that didn\u2019t work, I could rent beefier GPUs for training, bump our global context limit, and use Qwen 3.8 2.4T traces instead.\nI began by generating 120 Astra trajectories over a random slice of queries from CEB, with reasoning summaries enabled. 100 trajectories would be used for training, and 20 would be used as a held-out validation set. All trajectories were rendered via Prime Intellect\u2019s renderers library into Qwen format, loss-masked appropriately and unrolled and packed into usable training demonstrations. I then used Prime Intellect\u2019s prime-rl library to run SFT, training the smaller ~21 million parameter LoRA. 100 Astra trajectories became 382 training rows after unrolling and packing. I allowed training to run for just a single epoch, meaning every example was trained on exactly once, and used a batch size of one (each demonstration updated the weights). Finally, I evaluated the resulting LoRA over JOB:\n Checkpoint Valid candidate Tasks scored WinsRegressionsVanilla 4B14/11315/1130.85x0.85x311 epoch48/11344/1130.72x0.76x516 The one-epoch adapter against the untrained model on all 113 JOB queries. A query counts as scored when its trajectory ended with a measured candidate, a duplicate of the default plan, or the default itself. Wins and regressions are scored queries more than 5% faster or slower than the default. \nPromising! The adapter learned the harness.\nI could now either generate more fresh trajectories, or do more epochs over the dataset we already had. Given the latter is cheaper, I decided on more epochs.\nIt was around this point that I got impatient and wanted the training process to run even faster, so I rented a 2x H100 node on Lambda.\nNow training on an H100 Switching to the H100 meant we had 80 GB VRAM at our disposal instead of 24 GB. We could have bumped the context token limit greatly, but I decided to barrel through with the roughly ~50k token limit we set. instead of a single RTX 3090, I did two more training runs Training runs were a lot faster on the H100. The first epoch took four hours on the RTX 3090; the second took only 45 minutes on the H100. over the existing LoRA; a second epoch and then a third:\n Checkpoint Valid candidate Tasks scored WinsRegressions1 epoch48/11344/1130.72x0.76x5162 epochs85/113108/1131.08x1.04x1283 epochs57/11399/1130.82x0.89x915 Two and three epochs over the same 100 Astra trajectories, evaluated on JOB. Before epochs two and three, the qo-agent harness was upgraded to allow the model to keep the default plan after a search, which is why many more queries are scored. The two- and three-epoch rows were evaluated under identical settings. \nTwo epochs improved our results, but three epochs regressed them! This was especially interesting because validation loss didn\u2019t budge at all during the second epoch:\n Validation loss on the 20 held-out trajectories 0.485 0.305 0.306 0.322 Optimizer updates, one per packed training row Loss on the 20 held-out Astra trajectories, measured at the start and end of each epoch over the first 100-trajectory dataset. \nA very good lesson that a flat validation loss doesn\u2019t necessarily mean the model has stopped learning useful behavior.\nI still felt we had more to learn from SFT though before proceeding with RL. I did zero filtering on the training trajectory dataset, and hadn\u2019t carefully audited if I was missing any capabilities. It turns out I was, mostly around the model\u2019s ability to construct valid Leading trees.\nI generated another 320 Astra trajectories; 300 for training, and 20 for validation. I filtered out just six training trajectories where Astra opted to keep the default plan without even trying a single candidate. I did two more epochs in two separate runs:\n Checkpoint Valid candidate Tasks scored WinsRegressions2 epochs on the first 10085/113108/1131.08x1.04x128+ 1 epoch on the new 30077/113101/1131.10x1.05x2913+ 2 epochs on the new 30071/113107/1131.16x1.06x205 Continuing the two-epoch adapter on the 300 new trajectories, evaluated on JOB. \nOur 4B model not only learned the harness; it was now genuinely making good calls on various JOB queries! Fortunately, training on reasoning summaries didn\u2019t harm performance.\nMaking the 4B model good at query optimization\nThe model now spoke the \u201clanguage\u201d of the qo-agent harness, and we got some free performance gains out of SFT too. It was time to make it very good at query optimization.\nAgentic reinforcement learning\nAgentic RL differs from SFT in that we actually run the current policy over training queries inside the agent harness NN times. Each run (also referred to as a rollout) results in a final output that\u2019s scored against some verifiable criteria. Lastly, each rollout\u2019s score is then weighted relative to the other same-query rollouts. A positive \u201cadvantage\u201d is reinforced by making the model\u2019s weights more likely to produce that trajectory in future runs, and a negative advantage is penalized; the weights are updated to be less likely to produce that trajectory in future runs.\nDesigning per-rollout rewards and relative advantages\nThe initial reward algorithm was simple:\n \nThe speedup was calculated as the median of three default plan measurements divided by the median of three candidate plan measurements.\nFor every evaluated candidate that was invalid, we subtracted 0.1 from the natural log of the speedup ratio. We subtracted a further 0.05 if the rollout resulted in a plan that shared the same fingerprint as the default Postgres plan. Finally, if the trajectory ended with no valid candidate at all, a flat 3 was subtracted from the reward in lieu of any of the 0.1 or 0.05 subtractions.\nThe first few RL runs I did using this reward resulted in a model that was terrified of producing invalid plans due to the extremely harsh -3 condition. The model played it safe instead, returning Postgres\u2019s default plan over and over again, accepting the smaller 0.05 reward hits.\nGRPO exacerbated this issue. The plain GRPO algorithm converts multiple rollout rewards into relative \u201cadvantages\u201d:\n GRPO as prime-rl implements it: simply\nsubtract the group\u2019s mean reward from each rollout\u2019s reward. The GRPO paper also\ndivides by the group\u2019s standard deviation. \nLet\u2019s say we perform four rollouts for a given query resulting in the following plans, execution speeds and rewards:\n /*+ Leading((t cn) mc) */ not run t and cn never join directly, so the tree is rejected \u22123.00 \u22121.43 /*+ MergeJoin(t cn) */ not run a join method for two relations the query never joins \u22123.00 \u22121.43 /*+ NestLoop(t mc) */ 148 ms vs 118 ms \u00b7 0.80x a new plan, slower than Postgres\u2019s own \u22120.23 +1.34 /*+ HashJoin(mc cn) */ 118 ms \u00b7 1.00x Postgres already chose this; same fingerprint as the default \u22120.05 +1.52 reinforced most Four rollouts of one query under the first reward and prime-rl GRPO. Nothing in the group beat Postgres, but two rollouts still receive positive advantage because they beat the group's mean. \nWe\u2019re reinforcing bad behavior by telling the model it\u2019s okay to produce plans that end up being equivalent to Postgres\u2019s default plan!\nBoth the busted reward algorithm and GRPO needed to be swapped out for something that could actually score advantages relative to the default plan\u2019s execution time.\nThe reward was updated as follows:\n \nChanges include clipping the speedup ratio to bound scalar reward values, soft-thresholding by 0.05 to account for measurement noise, and scoring zero if the agent called keep_default or finish(default). Trajectories ending without a valid candidate incurred a flat 0.1 fee instead of the previous 3.0, and candidate plans that fingerprinted to the default plan accrued small 0.02 fees.\nprime-rl-flavored GRPO was swapped out for a custom \u201canchored\u201d variant:\n \nLet\u2019s take a look how our modified GRPO performs under the same four rollouts demonstrated above:\n /*+ Leading((t cn) mc) */ not run t and cn never join directly, so the tree is rejected none \u2014 \u22120.10 /*+ MergeJoin(t cn) */ not run a join method for two relations the query never joins none \u2014 \u22120.10 /*+ NestLoop(t mc) */ 148 ms vs 118 ms \u00b7 0.80x a new plan, slower than Postgres\u2019s own \u22120.18 +0.00 \u22120.18 /*+ HashJoin(mc cn) */ 118 ms \u00b7 1.00x Postgres already chose this; same fingerprint as the default +0.00 +0.00 \u22120.02 The same four rollouts under the reworked reward and anchored GRPO. Nothing is reinforced because none of the plans were good. \nThe modified GRPO successfully applies a negative advantage to all four of these poor rollouts, down-weighting their likelihood across the board.\nTraining commences\nWith the reward and relative advantage model locked in, I could begin training\u2026\n\u2026as soon as I built out a mechanism for Postgres measurements to happen on FLOPper and training/inference to happen on the 2x H100 node. I would have loved to keep Postgres measurements on the Lambda node; alas, I ran a bunch of Postgres calibrations on their boxes and am reasonably confident I was sharing the non-GPU bits of the box with other folks, as the noise was off the charts compared to FLOPper.\nI connected FLOPper to the Lambda node via Tailscale; the resulting process would be:\n\nQuery rollouts would begin on FLOPper and hold an available Postgres container for their full lifetime\nThe rollout would do inference via vLLM running on the first H100 to generate a trajectory\nAll measurements were run on the held Postgres container\nFinally, rollout results were sent to the second H100 to compute relative advantages and update weights\n\nNow training could start. I began by merging the final SFT adapter into the base model, creating a new base model in the process, and initialized a new adapter for RL training.\nI started with an extremely conservative learning rate of 1e-06, a batch size of 8, four rollouts per query, and did just 120 optimizer updates to prove out the mechanism.\nThe trained LoRA was evaluated against JOB, and completely flopped:\n Checkpoint Valid candidate Tasks scored WinsRegressionsSFT, starting point71/113107/1131.16x1.06x205RL, 120 updates71/113106/1131.14x0.99x217 The first RL checkpoint against the SFT checkpoint it started from, evaluated on all 113 JOB queries. \nEither the reward design wasn\u2019t good, modified GRPO wasn\u2019t working, or we simply weren\u2019t being aggressive enough.\nThe latter was easiest to test. I increased the learning rate one order-of-magnitude from 1e-06 to 1e-05, bumped the batch size to 16 and the number of rollouts per query from four to eight, and decided to do 600 optimizer updates instead of just 120.\nAround this time, I learned that for a rollout, 92% of the rollout\u2019s time was spent in vLLM inference! Given the increase in the number of optimizer updates, I needed to be smarter about this.\nI refactored the training process so that rollouts leased an available Postgres worker only for the times where measurements were needed. Because of this new async approach, I could saturate vLLM and the trainer much further, and was able to run 20 rollouts concurrently, each contending for the same four Postgres workers.\n 4 Postgres workers on FLOPper, 240 seconds of wall time t = 0 s Hold a worker for the whole rollout \u00b7 4 in flight rollouts workers \nrollouts finished \n0\n \nworkers measuring \n0%\n \nof the time\n Lease a worker only to measure \u00b7 20 in flight rollouts workers \nrollouts finished \n0\n \nworkers measuring \n0%\n \nof the time\n Requiring rollouts to hold a worker for their lifetime caps the maximum number of concurrent rollouts at the number\nof available workers. Using a worker lease strategy only when Postgres is needed lets us run 20 concurrent rollouts\nand increases the share of time Postgres workers are actually being utilized. \nI ran two stacked 600-optimizer update RL runs, back-to-back. The share of rollouts earning positive advantages steadily increased:\n Share of credited rollouts Optimizer updates Measured plan faster than Postgres Positive advantage The training signal across both 600-update runs from the anchored credit assigned to every rollout that reached the\ntrainer. Both the share of rollouts with plans beating Postgres and the share of rollouts with positive advantages\nsteadily climb throughout the run. \nAnd the results from both checkpoints against JOB:\n Checkpoint Valid candidate Tasks scored WinsRegressionsSFT, starting point71/113107/1131.16x1.06x205RL, 600 updates99/113113/1131.35x1.16x340RL, 1,200 updates101/113112/1131.41x1.29x382 Both 600-update checkpoints against the SFT checkpoint they descend from, evaluated on all 113 JOB queries. \nThe valid candidate rate rose nicely, alongside both the geometric mean speedup and total workload speedup!\nI ran the final RL checkpoint against JOB again, this time doing three rollouts per query instead of just one. In other words, each query could produce up to 15 candidates max, and the best-of-15 was picked for each query This is roughly how I would expect someone trying to tune a workload of queries to use a model specially trained on this task. They would care more about the best candidate possible sampled from many rollouts, rather than just a single rollout. :\n SelectionTasks scored WinsRegressionsModel\u2019s own choice, per trajectory339/3391.40x1.24x1197Best feedback within each trajectory339/3391.44x1.29x1291Best feedback across all three113/1131.81x1.81x680 The final 1,200-update RL checkpoint with three trajectories per JOB query. The first row averages the model\u2019s own final selections over all 339 trajectories. The second applies a fixed rule within each trajectory, choosing the candidate with the best preliminary feedback if it beat 1.05x and the default otherwise. The third applies the same rule across a query\u2019s three trajectories, so each query gets one answer chosen from up to 15 candidates. \nWhen taking the best feedback across three rollouts, we saw a 1.81x geometric mean speedup, and coincidentally a 1.81x total workload speedup too.\nWhat the model learned\nAfter all this training, what did the model actually learn?\n How searches went The investigator 295 of 337 searches that submitted candidates first inspected a relation, column statistics, or the default plan The big spender 235 of 339 searches used all five candidate attempts The reasoner On job-01d (90x speedup), its reasoning read: \"...the mi-index sequential scan, which is running a lossy filter at 575k with an 11ms timing. It looks like using a bitmap could be much faster.\" The model then actually forced a bitmap scan The model's preferred strategies Consistent favorites Of 1,347 actions, the model outputted scan hints 1,141 times, Leading trees 917 times and Parallel hints 572 times. Surprisingly, Rows corrections were used only 146 times Strong preferences Nested loops were forced over hash joins regularly, and index scans were preferred over bitmap or sequential scans Favorite settings The model regularly used enable_sort=off and random_page_cost=1.1 What actually wins Three motifs win The model's usage of Leading to rewrite the join order, making a single scan fix without changing the join order, and using Parallel resulted in much of the gains An analysis of the traces from the final evaluation's 339 searches. \nCosts\nThis project would have been free Barring the price of electricity for continually running FLOPper when both GPUs were fully utilized, which at current rates costs about ~$9/day. had it not been for my impatience to get training results faster \u201cWhat about the Astra traces?,\u201d you might ask. I did not experiment with it, but I do think the 4B model could have eventually learned the harness language through just RL alone, as DeepSeek-R1-Zero showed. It might have just taken a lot more rollouts. . I paid ~$800 to rent a 2x H100 SXM node from Lambda for ~95 hours, and ~$400 in OpenAI API fees to generate the Astra trajectory demonstrations.\nTotal cost: $1,200.\nConclusion\nVia off-policy distillation and reinforcement learning, a tiny 4B model went from not being able to understand the harness it was wrapped in, to achieving a 1.81x geometric mean speedup and a summed latency decrease of 44.7% across a workload of join-heavy SQL queries Given three attempts per query in a best-of-15 measurement, as previously mentioned. .\nIt\u2019s easy to take tiny models for granted when 5T+ parameter behemoths exist. We shouldn\u2019t.\nFrontier intelligence is extremely powerful; the distillation I did off Astra trajectories is proof enough that large models are not going anywhere.\nBut I do believe this experiment proves out a hypothesis many companies are waking up to: they have the data; it\u2019s not a far-cry to build out an RL environment and spend a small sum to train and inference open-weights models on niche, domain-specific tasks. I think small models will be increasingly used for this, as they are faster and cheaper to train.\nI architected the infra/training stack myself plus used my own GPUs as a learning exercise, but there are a growing number of out-of-the-box solutions for companies to easily train their own models.\nI\u2019m personally very excited to see how this space continues to grow!\nNext steps\nAs for what I\u2019d want to try next, just a few ideas:\n\nExplore whether structured hint sweeping in the style of Bao: Learning to Steer Query Optimizers is more effective than using a 4B model\nAttempt on-policy distillation and compare performance against off-policy distillation\nTry trace inversion and see how it influences off-policy distillation\nMeasure \u201cfooled reward\u201d noise on dedicated EC2 boxes to understand how this experiment could be scaled up\n\nCode\nAll code for this project is available here.\nCitation\nPlease cite this work as: Bansal, Rohan. \u201cTraining a 4B model to produce 81% faster query plans than Postgres\u201d. rohanbansal.com (Sep 2026). https://rohanbansal.com/qorl Or use the BibTeX citation: @article{bansal2026qorl,\n title = {Training a 4B model to produce 81% faster query plans than Postgres},\n author = {Bansal, Rohan},\n journal = {rohanbansal.com},\n year = {2026},\n month = {September},\n url = \"https://rohanbansal.com/qorl\"\n}\nReferences\n\nViktor Leis, Andrey Gubichev, Atanas Mirchev, Peter Boncz, Alfons Kemper, and Thomas Neumann. \u201cHow Good Are Query Optimizers, Really?\u201d Proceedings of the VLDB Endowment 9, no. 3 (2015): 204\u2013215. doi:10.14778/2850583.2850594.\nViktor Leis, Andrey Gubichev, Atanas Mirchev, Peter Boncz, Alfons Kemper, and Thomas Neumann. \u201cStill Asking: How Good Are Query Optimizers, Really?\u201d Proceedings of the VLDB Endowment 18, no. 12 (2025): 5531\u20135536. doi:10.14778/3750601.3760521.\nToshihide Ibaraki and Tiko Kameda. \u201cOn the Optimal Nesting Order for Computing N-Relational Joins.\u201d ACM Transactions on Database Systems 9, no. 3 (1984): 482\u2013502. doi:10.1145/1270.1498.\nParimarjan Negi, Ryan Marcus, Andreas Kipf, Hongzi Mao, Nesime Tatbul, Tim Kraska, and Mohammad Alizadeh. \u201cFlow-Loss: Learning Cardinality Estimates That Matter.\u201d Proceedings of the VLDB Endowment 14, no. 11 (2021): 2019\u20132032. doi:10.14778/3476249.3476259.\nTingwei Zhang, John X. Morris, and Vitaly Shmatikov. \u201cHow to Steal Reasoning Without Reasoning Traces.\u201d arXiv preprint arXiv:2603.07267 (2026). doi:10.48550/arXiv.2603.07267.\nZhihong Shao, Peiyi Wang, Qihao Zhu, Runxin Xu, Junxiao Song, Xiao Bi, Haowei Zhang, Mingchuan Zhang, Y. K. Li, Y. Wu, and Daya Guo. \u201cDeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models.\u201d arXiv preprint arXiv:2402.03300 (2024). doi:10.48550/arXiv.2402.03300.\nDeepSeek-AI (Daya Guo, Dejian Yang, Haowei Zhang, Junxiao Song, Peiyi Wang, Qihao Zhu, Runxin Xu, et al.). \u201cDeepSeek-R1: Incentivizing Reasoning Capability in LLMs via Reinforcement Learning.\u201d arXiv preprint arXiv:2501.12948 (2025). Published as \u201cDeepSeek-R1 Incentivizes Reasoning in LLMs Through Reinforcement Learning.\u201d Nature 645, no. 8081 (2025): 633\u2013638. doi:10.1038/s41586-025-09422-z.\nRyan Marcus, Parimarjan Negi, Hongzi Mao, Nesime Tatbul, Mohammad Alizadeh, and Tim Kraska. \u201cBao: Learning to Steer Query Optimizers.\u201d arXiv preprint arXiv:2004.03814 (2020). Published as \u201cBao: Making Learned Query Optimization Practical.\u201d Proceedings of the 2021 International Conference on Management of Data (SIGMOD \u201921) (2021): 1275\u20131288. doi:10.1145/3448016.3452838.\nKevin Lu, in collaboration with others at Thinking Machines. \u201cOn-Policy Distillation.\u201d Thinking Machines Lab (blog), October 27, 2025." ], "output": "readability", "pwd": "/data/archive/1789596243.573017", "schema": "ArchiveResult", "start_ts": "2026-09-16T22:04:37.287785+00:00", "status": "succeeded" } ], "screenshot": [], "singlefile": [], "title": [ { "cmd": [ "curl", "--silent", "--location", "--compressed", "--max-time", "120", "--user-agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36 ArchiveBox/0.7.4 (+https://github.com/ArchiveBox/ArchiveBox/) curl/curl 8.14.1 (x86_64-pc-linux-gnu)", "https://rohanbansal.com/qorl" ], "cmd_version": "curl 8.14.1 (x86_64-pc-linux-gnu)", "end_ts": "2026-09-16T22:04:37.279467+00:00", "index_texts": null, "output": "Training a 4B model to produce 81% faster query plans than Postgres - Rohan Bansal", "pwd": "/data/archive/1789596243.573017", "schema": "ArchiveResult", "start_ts": "2026-09-16T22:04:36.931488+00:00", "status": "succeeded" } ], "wget": [ { "cmd": [ "wget", "--no-verbose", "--adjust-extension", "--convert-links", "--force-directories", "--span-hosts", "--no-parent", "-e", "robots=off", "--reject=js,mjs,woff,woff2,ttf,otf,eot,mp4,m4v,mov,webm,ogv,mkv,avi,flv", "--timeout=120", "--restrict-file-names=windows", "--page-requisites", "--user-agent=Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36 ArchiveBox/0.7.4 (+https://github.com/ArchiveBox/ArchiveBox/) wget/GNU Wget 1.21.3", "--compression=auto", "--timestamping", "https://rohanbansal.com/qorl" ], "cmd_version": "GNU Wget 1.21.3", "end_ts": "2026-09-16T22:04:36.921478+00:00", "index_texts": null, "output": "rohanbansal.com/qorl.html", "pwd": "/data/archive/1789596243.573017", "schema": "ArchiveResult", "start_ts": "2026-09-16T22:04:36.016394+00:00", "status": "succeeded" } ] }, "icons": null, "is_archived": true, "is_static": false, "latest": { "archive_org": null, "dom": null, "favicon": "favicon.ico", "git": null, "media": null, "pdf": null, "screenshot": null, "singlefile": null, "title": "Training a 4B model to produce 81% faster query plans than Postgres - Rohan Bansal", "warc": null, "wget": "rohanbansal.com/qorl.html" }, "link_dir": "/data/archive/1789596243.573017", "newest_archive_date": "2026-09-16T22:04:43.754529+00:00", "num_failures": 0, "num_outputs": 6, "oldest_archive_date": "2026-09-16T22:04:35.046855+00:00", "path": "/qorl", "schema": "Link", "scheme": "https", "snapshot_id": "acf68f1f-b83d-4f02-895a-22776ec6bbeb", "sources": [ "/data/sources/1789596243-import.txt" ], "tags": null, "tags_str": "", "timestamp": "1789596243.573017", "title": "Training a 4B model to produce 81% faster query plans than Postgres - Rohan Bansal", "updated": "2026-09-16T22:04:35.046351+00:00", "updated_date": "2026-09-16 22:04", "url": "https://rohanbansal.com/qorl" }