{
    "archive_path": "archive/1791669896.613125",
    "base_url": "fzakaria.com/2026/10/07/nix-wrote-half-of-my-debugger",
    "basename": "nix-wrote-half-of-my-debugger",
    "bookmarked_date": "2026-10-10 22:04",
    "canonical": {
        "archive_org_path": "https://web.archive.org/web/fzakaria.com/2026/10/07/nix-wrote-half-of-my-debugger",
        "dom_path": "output.html",
        "favicon_path": "favicon.ico",
        "git_path": "git/",
        "google_favicon_path": "https://www.google.com/s2/favicons?domain=fzakaria.com",
        "headers_path": "headers.json",
        "htmltotext_path": "htmltotext.txt",
        "index_path": "index.html",
        "media_path": "media/",
        "mercury_path": "mercury/content.html",
        "pdf_path": "output.pdf",
        "readability_path": "readability/content.html",
        "screenshot_path": "screenshot.png",
        "singlefile_path": "singlefile.html",
        "warc_path": "warc/",
        "wget_path": "fzakaria.com/2026/10/07/nix-wrote-half-of-my-debugger.html"
    },
    "domain": "fzakaria.com",
    "extension": "",
    "hash": "14NF8E974EPC9PZ5JZWR",
    "history": {
        "archive_org": [],
        "dom": [],
        "favicon": [
            {
                "cmd": [
                    "curl",
                    "--silent",
                    "--location",
                    "--compressed",
                    "--max-time",
                    "120",
                    "--output",
                    "favicon.ico",
                    "--user-agent",
                    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36 ArchiveBox/0.7.4 (+https://github.com/ArchiveBox/ArchiveBox/) curl/curl 8.14.1 (x86_64-pc-linux-gnu)",
                    "https://www.google.com/s2/favicons?domain=fzakaria.com"
                ],
                "cmd_version": "curl 8.14.1 (x86_64-pc-linux-gnu)",
                "end_ts": "2026-10-10T22:06:06.856347+00:00",
                "index_texts": null,
                "output": "favicon.ico",
                "pwd": "/data/archive/1791669896.613125",
                "schema": "ArchiveResult",
                "start_ts": "2026-10-10T22:06:06.255103+00:00",
                "status": "succeeded"
            }
        ],
        "git": [],
        "headers": [
            {
                "cmd": [
                    "curl",
                    "--silent",
                    "--location",
                    "--compressed",
                    "--head",
                    "--max-time",
                    "120",
                    "--user-agent",
                    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36 ArchiveBox/0.7.4 (+https://github.com/ArchiveBox/ArchiveBox/) curl/curl 8.14.1 (x86_64-pc-linux-gnu)",
                    "https://fzakaria.com/2026/10/07/nix-wrote-half-of-my-debugger"
                ],
                "cmd_version": "curl 8.14.1 (x86_64-pc-linux-gnu)",
                "end_ts": "2026-10-10T22:06:07.247252+00:00",
                "index_texts": null,
                "output": "headers.json",
                "pwd": "/data/archive/1791669896.613125",
                "schema": "ArchiveResult",
                "start_ts": "2026-10-10T22:06:06.867064+00:00",
                "status": "succeeded"
            }
        ],
        "htmltotext": [],
        "media": [],
        "mercury": [
            {
                "cmd": [
                    "/app/node_modules/@postlight/parser/cli.js",
                    "https://fzakaria.com/2026/10/07/nix-wrote-half-of-my-debugger"
                ],
                "cmd_version": "1.0.0",
                "end_ts": "2026-10-10T22:06:14.412457+00:00",
                "index_texts": null,
                "output": "mercury",
                "pwd": "/data/archive/1791669896.613125",
                "schema": "ArchiveResult",
                "start_ts": "2026-10-10T22:06:10.526999+00:00",
                "status": "succeeded"
            }
        ],
        "pdf": [],
        "readability": [
            {
                "cmd": [
                    "/app/node_modules/readability-extractor/readability-extractor",
                    "/tmp/tmp8d4l0cto",
                    "https://fzakaria.com/2026/10/07/nix-wrote-half-of-my-debugger"
                ],
                "cmd_version": "0.0.11",
                "end_ts": "2026-10-10T22:06:10.518613+00:00",
                "index_texts": [
                    "A few days ago I wrote about Rewind VM,\na deterministic VM where every run of a Nix build is a pure function of its inputs,\nthread schedule included!\n\n\n\nI have been using it to find, reproduce and solve numerous race conditions in Nix builds, but it\u2019s been making me feel a little crazy. What is this superpower, and why is nobody else using it?\n\nThe tool has quickly grown a source panel, stack frames,\nbookmarks, a \u201cCompare\u201d tab, a lot more gdb support, thread lanes, which show who held\nthe CPU at every step, and \u201cCheck from here\u201d, which helps find the exact step where a race\ncondition happens.\n\nI went into each new feature thinking it would be a big code-lift but I kept running into the same thing: the hard part of each feature was already done, and Nix had done it.\n\nA debugger for instance, needs the exact inputs of the program, its debug symbols, its sources, the sources of every library under it, and a way for someone else to get all of that on their\nmachine. That\u2019s exactly what a derivation is, and Nix makes that easy.\n\n\n\n\u00a7Two tellers, one account\n\nA classic simple example of a race condition is two threads depositing into one bank account. Each deposit reads the balance, writes a line to the ledger, then stores the balance plus the deposit. If one teller runs between the other\u2019s read and store, it writes a stale balance over the other\u2019s deposits. \ud83d\udca5\n\nstatic void deposit(int teller, long amount)\n{\n  long seen = balance;\n  char line[64];\n  int n = snprintf(line, sizeof line, \n                   \"teller %d: %ld + %ld\\n\",\n                   teller, seen, amount);\n\n  if (write(1, line, n) != n)\n      return;\n  balance = seen + amount;\n}\n\n\nA teller that runs between the other\u2019s read and store writes a stale balance\nover the other\u2019s deposits.11On my 16-core laptop, bank lost money in 396 of 1,000 runs. Pinned to a single core with taskset, it lost money in none of 1,000: one core alone rarely switches threads in the middle of a deposit, which is why Rewind perturbs the schedule.\u00a0\n\n\n\n  \n    \n      \n    \n    \n      \n    \n  \n  teller 1\n  balance\n  teller 2\n  \n  \n  \n  \n  read 150\n  \n  read 150\n  \n  store 200\n  \n  store 150 + 10\n  \n  160\n  teller 1's last deposit is gone\n\n\n\nRewind\u2019s VM has one CPU, so its first run passes too. rewind check runs the\nbuild again under perturbed schedules, each asking the guest kernel to\nreschedule at different steps, and narrows the first failure down to one step:\n\n$ rewind check --where github:fzakaria/rewindvm#bank\n...\nstep 3237 decides it: a reschedule there makes the run fail\n\npassing: run 12feb5f832205a72, schedule 0\nfailing: run c2f1cfac0c288f3e, schedule 1 over steps 3237..3238\nthe two are the same run until step 3237\n\nwhere the threads were:\n        3237   139/140   deposit (bank.c:24), on the CPU at the deciding step\n        3250   139/141   deposit (bank.c:24), at the first event that differs\n\n\nRunning this check on my laptop took 11 seconds. The two runs are the same machine, exit for exit, until step 3237, where only the failing one gets a reschedule.\n\n\u00a7Free thing one: the inputs\n\nThe argument to check is a derivation and can be a flake reference. The beauty of Nix is that it knows the inputs of a derivation, so that\u2019s all we need to make sure the run is reproducible. The derivation\u2019s inputs are the source, the compiler, the libraries, the kernel, and the VM\u2019s configuration\n\nrewind nix realises the derivation\u2019s inputs, packs the closure into a read-only erofs image, and boots the VM on it.\n\nThe run\u2019s id is a hash of its inputs, the way a store path is, and rewind show prints the command that makes it again:\n\n$ rewind show 12feb5f8\nrewind nix /nix/store/cr8rl40rjb9cmmcd9sxn9mpdrhvp53jj-bank-0.1.0.drv --epoch 1791331200 --clock branches --name bank-0.1.0  # 12feb5f832205a72\n\n\nWhen the build succeeds, the guest reports each output\u2019s NAR hash, and\nrewind nix checks it against the host\u2019s copy and against every binary cache\nNix substitutes from, by fetching only the .narinfo:\n\n$ rewind nix github:fzakaria/rewindvm#bank\n...\n/nix/store/27a4q7w90jzp4wg5kaaajvb64zll6yqg-bank-0.1.0 dbf0df3de84b7a84  matches your store\n\n\nThis helps us validate that the build within the VM is the same as the build on my laptop, and that the VM\u2019s run is reproducible by Nix.\n\n\u00a7Free thing two: every symbol and every source\n\nDebugging code is much simpler when you are looking at the source. A new panel in the Rewind app or rewind where in the terminal shows the source of the program at the playhead, and the stack frames that called it.\n\n\n\nThe debugger needs to know where the sources are, and Nix gives us that for free too.\n\nnixpkgs builds the packages with separateDebugInfo, and the debug info is cached on cache.nixos.org as a debug output. We can leverage debuginfod to fetch the debug info and the sources by build ID, so we can see the source of any binary in the VM, including the Linux kernel! \ud83d\ude32\n\n$ rewind where c2f1cfac 3250\nprocess 139 (bank), thread 141, at step 3250\n#4 deposit (bank.c:24)\n      22  \tint n = snprintf(line, sizeof line, \"teller %d: %ld + %ld\\n\", teller, seen, amount);\n      23\n>     24  \tif (write(1, line, n) != n)\n      25  \t\treturn;\n      26  \tbalance = seen + amount;\ncalled from #5 teller (bank.c:34)\ncalled from #6 start_thread (pthread_create.c:454)\n\n\n\u00a7gdb, on a fork of any step\n\nSometimes though, the source is not enough, and we need to see the state of the program. rewind gdb opens gdb on a fork of the run at the playhead, with every thread of the process. The debugger can set breakpoints, watchpoints, and inspect memory, registers and variables.\n\nAt step 3249, just before teller 2 gets the CPU back, we can watch the balance variable and continue until it changes. Nothing gdb does changes the recording, and we can rewind to the same step and fork again, or fork at any other step, and gdb will see the same state.\n\n\n\nIf gdb is not enough, rewind shell --with nixpkgs#strace opens a shell in the\nVM at a step with any package from nixpkgs on its PATH. It is one more closure packed into one more image.\n\n\u00a7Compare two runs\n\nRewind makes it very easy to compare two runs. The Compare tab in the app, or rewind compare in the terminal, shows the last shared events of both runs, and the first event that differs.\n\n\n\nThe Compare tab puts both runs\u2019 events side by side from just before they part,\nwith what differs marked. Teller 2 starts from 150 in the failing run and from\n200 in the passing one:\n\n\n\n\u00a7Who had the CPU\n\nRewind\u2019s VM has one CPU, so at any step exactly one thread is running. A race\nis a question of order: which thread ran when.\n\nThe trace does not answer that directly. It records what threads did, a write, an open, a fork, an exit, a signal, but not who was running in the gaps between those events.\n\nRewind can fill in the gaps without recording anything new. Every run replays\nexactly, so it can walk any stretch of a run one step at a time and, at each\nstep, ask the guest kernel which thread is on the CPU.\n\nIn the failing run, teller 1 (Thread 140) is interrupted at step 3238, halfway through a\ndeposit: it has read the balance but not stored it. Teller 2 (Thread 141) gets the CPU for\na single step, 3239, long enough to read the balance, and then teller 1 gets\nit back and finishes.22It is a little hard to see since the event from Teller 2 is tucked underneath the Orange bar.\u00a0\n\n\n\n\u00a7Check from here\n\nFinding one bad interleaving raises the next question: how likely is it? Is the\npassing run the normal case, or did it get lucky?\n\nrewind check normally answers that by building the whole derivation again\nunder many schedules. With --run, it starts from a run you already have\ninstead, at any step you pick_. It forks the run there once per schedule, each\nfork taking a different order of threads from that step on, and counts how\nmany end differently. Everything before the step stays exactly as it was.\n\nWe can apply this technique to the bank example, starting from the step where the two runs diverged, 3,221. When we run 16 schedules from there, all 16 lose money33Schedule 0 is the run itself, which passed. Every other line is a fork, and exited:2 is make check failing because the balance came up short.\u00a0:\n\n$ rewind check --run 12feb5f8 --schedule-from 3221 --schedules 16 --all --no-narrow\nschedule   0: exited:0       4528 steps  dbf0df3de84b  run 12feb5f832205a72\nschedule   1: exited:2       3313 steps    run f6eb462913dce489\nschedule   2: exited:2       3328 steps    run 388cef73a9c545fa\nschedule   3: exited:2       3312 steps    run 4d42012072d11537\n...\nschedule  16: exited:2       3314 steps    run 63f7817972a90273\n16 of 16 perturbed schedules ended differently\n\n\nWe can do the same thing from the Rewind App with \u201cCheck from here\u201d in the context menu of the playhead. It forks the run at the playhead and runs each fork under a different schedule, counting how many end differently.\n\n\n\n\u00a7Bookmarks\n\nb bookmarks the playhead\u2019s step with a note. Bookmarks are kept with the run,\nand they travel in its .rwd export, so the person I hand the run to opens it\nwith my notes on the timeline:\n\n\n\n\u00a7Other examples to try\n\nEach one is a derivation in the flake, with one bug and one fix:\n\n\n  philosophers: the deadlock from the last post.\n  bank: the lost update above.\n  waiter: a SIGCHLD that arrives between a flag check and pause, so the\nparent sleeps until make check\u2019s ten second timeout kills it.\n  config-reload: one process rewrites a config file in place while another\nrereads it. On many cores it fails nearly every time; on one CPU only some\nschedules land the reader between the truncate and the last write.\n\n\n$ rewind check github:fzakaria/rewindvm#waiter\n\n\n\u00a7Nix is a superpower\n\nMost of the hard work of a debugger is already done by Nix. Rewind adds a little more, but the rest is already there: the inputs, the sources, the debug symbols, and a way for someone else to get all of that on their machine.\n\nWhen you start with something hermetic like a Nix derivation, you can get a debugger for free.\n\n$ nix run github:fzakaria/rewindvm -- check github:fzakaria/rewindvm#bank"
                ],
                "output": "readability",
                "pwd": "/data/archive/1791669896.613125",
                "schema": "ArchiveResult",
                "start_ts": "2026-10-10T22:06:08.775408+00:00",
                "status": "succeeded"
            }
        ],
        "screenshot": [],
        "singlefile": [],
        "title": [
            {
                "cmd": [
                    "curl",
                    "--silent",
                    "--location",
                    "--compressed",
                    "--max-time",
                    "120",
                    "--user-agent",
                    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36 ArchiveBox/0.7.4 (+https://github.com/ArchiveBox/ArchiveBox/) curl/curl 8.14.1 (x86_64-pc-linux-gnu)",
                    "https://fzakaria.com/2026/10/07/nix-wrote-half-of-my-debugger"
                ],
                "cmd_version": "curl 8.14.1 (x86_64-pc-linux-gnu)",
                "end_ts": "2026-10-10T22:06:08.767121+00:00",
                "index_texts": null,
                "output": "Nix wrote half of my debugger | Farid Zakaria\u2019s Blog",
                "pwd": "/data/archive/1791669896.613125",
                "schema": "ArchiveResult",
                "start_ts": "2026-10-10T22:06:08.746911+00:00",
                "status": "succeeded"
            }
        ],
        "wget": [
            {
                "cmd": [
                    "wget",
                    "--no-verbose",
                    "--adjust-extension",
                    "--convert-links",
                    "--force-directories",
                    "--span-hosts",
                    "--no-parent",
                    "-e",
                    "robots=off",
                    "--reject=js,mjs,woff,woff2,ttf,otf,eot,mp4,m4v,mov,webm,ogv,mkv,avi,flv",
                    "--timeout=120",
                    "--restrict-file-names=windows",
                    "--page-requisites",
                    "--user-agent=Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36 ArchiveBox/0.7.4 (+https://github.com/ArchiveBox/ArchiveBox/) wget/GNU Wget 1.21.3",
                    "--compression=auto",
                    "--timestamping",
                    "https://fzakaria.com/2026/10/07/nix-wrote-half-of-my-debugger"
                ],
                "cmd_version": "GNU Wget 1.21.3",
                "end_ts": "2026-10-10T22:06:08.737277+00:00",
                "index_texts": null,
                "output": "fzakaria.com/2026/10/07/nix-wrote-half-of-my-debugger.html",
                "pwd": "/data/archive/1791669896.613125",
                "schema": "ArchiveResult",
                "start_ts": "2026-10-10T22:06:07.256794+00:00",
                "status": "succeeded"
            }
        ]
    },
    "icons": null,
    "is_archived": true,
    "is_static": false,
    "latest": {
        "archive_org": null,
        "dom": null,
        "favicon": "favicon.ico",
        "git": null,
        "media": null,
        "pdf": null,
        "screenshot": null,
        "singlefile": null,
        "title": "Nix wrote half of my debugger | Farid Zakaria\u2019s Blog",
        "warc": null,
        "wget": "fzakaria.com/2026/10/07/nix-wrote-half-of-my-debugger.html"
    },
    "link_dir": "/data/archive/1791669896.613125",
    "newest_archive_date": "2026-10-10T22:06:10.526999+00:00",
    "num_failures": 0,
    "num_outputs": 6,
    "oldest_archive_date": "2026-10-10T22:06:06.255103+00:00",
    "path": "/2026/10/07/nix-wrote-half-of-my-debugger",
    "schema": "Link",
    "scheme": "https",
    "snapshot_id": "097c1f0f-f992-4f0b-8666-848295e3c09f",
    "sources": [
        "/data/sources/1791669896-import.txt"
    ],
    "tags": null,
    "tags_str": "",
    "timestamp": "1791669896.613125",
    "title": "Nix wrote half of my debugger | Farid Zakaria\u2019s Blog",
    "updated": "2026-10-10T22:06:06.254514+00:00",
    "updated_date": "2026-10-10 22:06",
    "url": "https://fzakaria.com/2026/10/07/nix-wrote-half-of-my-debugger"
}