{ "archive_path": "archive/1787796820.175287", "base_url": "friedkielbasa.substack.com/p/when-vibecoding-is-actually-safe", "basename": "when-vibecoding-is-actually-safe", "bookmarked_date": "2026-08-27 02:13", "canonical": { "archive_org_path": "https://web.archive.org/web/friedkielbasa.substack.com/p/when-vibecoding-is-actually-safe", "dom_path": "output.html", "favicon_path": "favicon.ico", "git_path": "git/", "google_favicon_path": "https://www.google.com/s2/favicons?domain=friedkielbasa.substack.com", "headers_path": "headers.json", "htmltotext_path": "htmltotext.txt", "index_path": "index.html", "media_path": "media/", "mercury_path": "mercury/content.html", "pdf_path": "output.pdf", "readability_path": "readability/content.html", "screenshot_path": "screenshot.png", "singlefile_path": "singlefile.html", "warc_path": "warc/", "wget_path": "friedkielbasa.substack.com/p/when-vibecoding-is-actually-safe.html" }, "domain": "friedkielbasa.substack.com", "extension": "", "hash": "1ARFRG1444BKZKKG5H73", "history": { "archive_org": [], "dom": [], "favicon": [ { "cmd": [ "curl", "--silent", "--location", "--compressed", "--max-time", "120", "--output", "favicon.ico", "--user-agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36 ArchiveBox/0.7.4 (+https://github.com/ArchiveBox/ArchiveBox/) curl/curl 8.14.1 (x86_64-pc-linux-gnu)", "https://www.google.com/s2/favicons?domain=friedkielbasa.substack.com" ], "cmd_version": "curl 8.14.1 (x86_64-pc-linux-gnu)", "end_ts": "2026-08-27T02:15:35.942240+00:00", "index_texts": null, "output": "favicon.ico", "pwd": "/data/archive/1787796820.175287", "schema": "ArchiveResult", "start_ts": "2026-08-27T02:15:35.482641+00:00", "status": "succeeded" } ], "git": [], "headers": [ { "cmd": [ "curl", "--silent", "--location", "--compressed", "--head", "--max-time", "120", "--user-agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36 ArchiveBox/0.7.4 (+https://github.com/ArchiveBox/ArchiveBox/) curl/curl 8.14.1 (x86_64-pc-linux-gnu)", "https://friedkielbasa.substack.com/p/when-vibecoding-is-actually-safe" ], "cmd_version": "curl 8.14.1 (x86_64-pc-linux-gnu)", "end_ts": "2026-08-27T02:15:36.343228+00:00", "index_texts": null, "output": "headers.json", "pwd": "/data/archive/1787796820.175287", "schema": "ArchiveResult", "start_ts": "2026-08-27T02:15:35.951089+00:00", "status": "succeeded" } ], "htmltotext": [], "media": [], "mercury": [ { "cmd": [ "/app/node_modules/@postlight/parser/cli.js", "https://friedkielbasa.substack.com/p/when-vibecoding-is-actually-safe" ], "cmd_version": "1.0.0", "end_ts": "2026-08-27T02:15:49.597512+00:00", "index_texts": null, "output": "mercury", "pwd": "/data/archive/1787796820.175287", "schema": "ArchiveResult", "start_ts": "2026-08-27T02:15:46.169057+00:00", "status": "succeeded" } ], "pdf": [], "readability": [ { "cmd": [ "/app/node_modules/readability-extractor/readability-extractor", "/tmp/tmpcra7qexo", "https://friedkielbasa.substack.com/p/when-vibecoding-is-actually-safe" ], "cmd_version": "0.0.11", "end_ts": "2026-08-27T02:15:46.158597+00:00", "index_texts": [ "The problem with vibecoded software isn\u2019t that it\u2019s bad. It\u2019s that nobody knows whether it\u2019s bad. Vibecoding trades an understanding of what the code does for massively increased speed \u2014 great for a personal project, but nobody advocates for it as a way to build quality software, because nobody has inspected the code. And all the old signals of quality, like a robust test suite or a great readme, can now be generated in minutes.There are various partial solutions to this \u2014 advisor models, doing test driven development, or providing detailed specifications, adversarial review \u2014 but they don\u2019t address the root cause. AI can take all of these guardrails and still fake correctness. We\u2019ve all asked an LLM to fix a bug and watched it modify the test cases to pass rather than changing the code.What we really want is a way to get good output while vibecoding. This is hard in general! There is, however, a subset of problems where we can verify vibecoded output to a standard we choose. Woodworkers use jigs so the fixture guarantees the cut instead of the craftsman \u2014 what we need is a jig for AI.When your goal is to functionally duplicate software \u2014 ports, migrations, refactors, performance rewrites, dependency upgrades, legacy modernization \u2014 you can use the source to verify the copy. With a source of truth you can abstract the code away, since you have a way to validate whether the application is working properly. The key is the feedback loop \u2014 with that, the AI can validate itself without human involvement.For example \u2014 I was migrating a website from WordPress to Astro. The motivation behind this was that it\u2019d be easier to throw tasks to Claude Code with the site on Astro.This was a task that AI would be great at, but obviously I don\u2019t want to review every single page on the website to ensure it\u2019s been exactly duplicated. What I used instead was the Playwright diff tool \u2014 which measures the difference between two screenshots. I had AI point this tool at both websites and then had the LLM self-correct until the difference between the two sites was within an acceptable margin. Since this was a static site I then had confidence, even without viewing the fully migrated website, that it was correctly migrated. This migration was a form of copying where functionality was all that mattered.Pixelmatch output from the migration \u2014 pink is where the site still disagreed with the originalThere\u2019s a caveat here \u2014 just because the functionality was correct for my website does not mean the code is good. We could have just generated a mountain of slop. The reason why this doesn\u2019t matter for my specific case is that since it\u2019s a static site there isn\u2019t a lot of hidden complexity that can bite me. Having unmaintainable code is also a problem, but that\u2019s now the AI\u2019s problem, not necessarily mine in this case. When the verification is durable and generation is cheap, the code becomes cattle and not pets.JustHTML by Emil Stenstr\u00f6m is another vibe-engineered example where using a test suite from an existing HTML parser became the verification jig. Actually generating the code was much less important than verifying it on the test suite. It\u2019s worth reading the full scope of his verification jig \u2014 He hooked in the 9,200 test html5lib-tests conformance suite almost from the start. There\u2019s no better way to construct a new HTML5 parser than using the test suite that the browsers themselves use.He picked the core API design himself\u2014a TagHandler base class with handle_start() etc. methods\u2014and told the model to implement that.He added a comparative benchmark to track performance compared to existing libraries like html5lib, then experimented with a Rust optimization based on those initial numbers.He threw the original code away and started from scratch as a rough port of Servo\u2019s excellent html5ever Rust library.He built a custom profiler and new benchmark and let Gemini 3 Pro loose on it, finally achieving micro-optimizations to beat the existing Pure Python libraries.He used coverage to identify and remove unnecessary code.He had his agent build a custom fuzzer to generate vast numbers of invalid HTML documents and harden the parser against them.Text from Simon Willison\u2019s postJarred Sumner recently migrated Bun from Zig to Rust using a fleet of LLM agents. Rust was simply a better fit at the scale Bun operates at \u2014 to prevent memory bugs. There was no way for him to eyeball a million lines of Rust, so he vibe-engineered the migration instead.The key was that the verification tool, i.e. the test suite, already existed. Bun\u2019s million-plus tests allowed the Rust port to be faithful without behavioral changes. One wrinkle occurred during the migration: Claude at one point started erasing functions that were causing errors and giving long explanatory comments \u2014 which is exactly the issue I pointed to earlier that we\u2019re all familiar with. Sumner\u2019s workaround was to have the reviewer models catch when an edit contained a long comment and simply reject that change.Duplication is useful \u2014 having AI offer Copying As A Service can be handy. The team at StrongDM built something called Digital Twin Universe. They decided to build replicas of Slack, Jira, Okta and various other applications their software integrates with. The replicas could then be used to test their integrations without needing to worry about rate limits or API costs.They built these replicas by using public API documentation and SDKs. Their Digital Twin Universe needed to integrate completely with those specifications and through that the StrongDM team could verify for correctness without needing to trawl through the codebase.John Regehr evaluated the Claude C compiler by using fuzzers (Csmith and YARPGen) \u2014 which revealed 34 bugs. He then pointed Codex at the compiler and had it self-repair against the fuzzing feedback, eliminating every bug the fuzzers could find \u2014 though, as Regehr is careful to note, that\u2019s not the same as correctness.Regehr makes the point that since there was no verification step for optimization, CCC had optimizations that were elaborate, yet didn\u2019t meaningfully improve its output. If an axis isn\u2019t measured, the LLM will not work to improve it.He gives the example of how adding executable oracles (Regehr\u2019s term) for the various axes on which software can be evaluated will allow for the AI to optimize those specific features \u2014 see his description below on generating dataflow transfer functions.Recently I asked Codex to start writing transfer functions. By itself, it\u2019s not bad at this, but not great. However, given access to our command-line tools for evaluating the precision and verifying the soundness of a transfer function, Codex produced results that are better than anything I\u2019ve seen either in a real compiler like LLVM, or in our own randomized synthesis results. The remaining degree of freedom that I left Codex\u2014code size\u2014allowed it to write pretty large transfer functions that explore some pretty deep case splits on the input structure, but capping the size of generated code is the easiest thing.SourceSo are we cooked? Are our jobs disappearing? No \u2014 at least not yet. ProgramBench is an eval that hands the LLM a compiled application and asks for a duplicate. The AI gets the compiled binary and its docs but can\u2019t decompile it, look up the source, or go online \u2014 so it can never close the loop against the hidden test suite. Results have been poor \u2014 across 200 tasks from tiny CLI tools to FFmpeg and SQLite, no model solves a single one outright \u2014 even the best (Claude Opus 4.7) passes 95%+ of tests on only 3%.The shape of AI-generated code no longer matters as much as the verification around it. The rest of the AI code can be a black box and we can ignore what\u2019s in it \u2014 depending on what you\u2019re building.Bun\u2019s port ended up having 19 bugs, the result of semantic differences between Zig and Rust. My static site can tolerate slop code since all that matters is the screenshot diff. Airbus, however, which is appropriately risk averse \u2014 formally verifies some of their software to ensure that it\u2019s correct. Use LLMs at your own risk!A good feedback loop to validate AI output is an underrated alternative to having a human review every line. The work doesn\u2019t disappear, however \u2014 engineering moves up a level into designing what the feedback loop should be.Vibe-engineering is only \u201cengineering\u201d if the AI\u2019s verification lives outside it and leaves no opportunity to cheat. Otherwise it\u2019s just vibecoding at scale, which is to say automated hoping.Discussion about this postReady for more?" ], "output": "readability", "pwd": "/data/archive/1787796820.175287", "schema": "ArchiveResult", "start_ts": "2026-08-27T02:15:43.960289+00:00", "status": "succeeded" } ], "screenshot": [], "singlefile": [], "title": [ { "cmd": [ "curl", "--silent", "--location", "--compressed", "--max-time", "120", "--user-agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36 ArchiveBox/0.7.4 (+https://github.com/ArchiveBox/ArchiveBox/) curl/curl 8.14.1 (x86_64-pc-linux-gnu)", "https://friedkielbasa.substack.com/p/when-vibecoding-is-actually-safe" ], "cmd_version": "curl 8.14.1 (x86_64-pc-linux-gnu)", "end_ts": "2026-08-27T02:15:43.952790+00:00", "index_texts": null, "output": "When Vibecoding Is Actually Safe - Fried Kielbasa", "pwd": "/data/archive/1787796820.175287", "schema": "ArchiveResult", "start_ts": "2026-08-27T02:15:43.926857+00:00", "status": "succeeded" } ], "wget": [ { "cmd": [ "wget", "--no-verbose", "--adjust-extension", "--convert-links", "--force-directories", "--backup-converted", "--span-hosts", "--no-parent", "-e", "robots=off", "--timeout=120", "--restrict-file-names=windows", "--warc-file=/data/archive/1787796820.175287/warc/1787796936", "--page-requisites", "--user-agent=Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36 ArchiveBox/0.7.4 (+https://github.com/ArchiveBox/ArchiveBox/) wget/GNU Wget 1.21.3", "--compression=auto", "https://friedkielbasa.substack.com/p/when-vibecoding-is-actually-safe" ], "cmd_version": "GNU Wget 1.21.3", "end_ts": "2026-08-27T02:15:43.916631+00:00", "index_texts": null, "output": "friedkielbasa.substack.com/p/when-vibecoding-is-actually-safe.html", "pwd": "/data/archive/1787796820.175287", "schema": "ArchiveResult", "start_ts": "2026-08-27T02:15:36.353469+00:00", "status": "succeeded" } ] }, "icons": null, "is_archived": true, "is_static": false, "latest": { "archive_org": null, "dom": null, "favicon": "favicon.ico", "git": null, "media": null, "pdf": null, "screenshot": null, "singlefile": null, "title": "When Vibecoding Is Actually Safe - Fried Kielbasa", "warc": null, "wget": "friedkielbasa.substack.com/p/when-vibecoding-is-actually-safe.html" }, "link_dir": "/data/archive/1787796820.175287", "newest_archive_date": "2026-08-27T02:15:46.169057+00:00", "num_failures": 0, "num_outputs": 6, "oldest_archive_date": "2026-08-27T02:15:35.482641+00:00", "path": "/p/when-vibecoding-is-actually-safe", "schema": "Link", "scheme": "https", "snapshot_id": "a008a810-4c84-456e-bdcf-71abcc4b0c39", "sources": [ "/data/sources/1787796820-import.txt" ], "tags": null, "tags_str": "", "timestamp": "1787796820.175287", "title": "When Vibecoding Is Actually Safe - Fried Kielbasa", "updated": "2026-08-27T02:15:35.482228+00:00", "updated_date": "2026-08-27 02:15", "url": "https://friedkielbasa.substack.com/p/when-vibecoding-is-actually-safe" }