{ "archive_path": "archive/1788127631.971801", "base_url": "people.kernel.org/monsieuricon/creepy-crawlies", "basename": "creepy-crawlies", "bookmarked_date": "2026-08-30 22:07", "canonical": { "archive_org_path": "https://web.archive.org/web/people.kernel.org/monsieuricon/creepy-crawlies", "dom_path": "output.html", "favicon_path": "favicon.ico", "git_path": "git/", "google_favicon_path": "https://www.google.com/s2/favicons?domain=people.kernel.org", "headers_path": "headers.json", "htmltotext_path": "htmltotext.txt", "index_path": "index.html", "media_path": "media/", "mercury_path": "mercury/content.html", "pdf_path": "output.pdf", "readability_path": "readability/content.html", "screenshot_path": "screenshot.png", "singlefile_path": "singlefile.html", "warc_path": "warc/", "wget_path": "people.kernel.org/monsieuricon/creepy-crawlies.html" }, "domain": "people.kernel.org", "extension": "", "hash": "K27CC9FQBY17M890ZA1P", "history": { "archive_org": [], "dom": [], "favicon": [ { "cmd": [ "curl", "--silent", "--location", "--compressed", "--max-time", "120", "--output", "favicon.ico", "--user-agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36 ArchiveBox/0.7.4 (+https://github.com/ArchiveBox/ArchiveBox/) curl/curl 8.14.1 (x86_64-pc-linux-gnu)", "https://www.google.com/s2/favicons?domain=people.kernel.org" ], "cmd_version": "curl 8.14.1 (x86_64-pc-linux-gnu)", "end_ts": "2026-08-30T22:08:49.192609+00:00", "index_texts": null, "output": "favicon.ico", "pwd": "/data/archive/1788127631.971801", "schema": "ArchiveResult", "start_ts": "2026-08-30T22:08:48.710282+00:00", "status": "succeeded" } ], "git": [], "headers": [ { "cmd": [ "curl", "--silent", "--location", "--compressed", "--head", "--max-time", "120", "--user-agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36 ArchiveBox/0.7.4 (+https://github.com/ArchiveBox/ArchiveBox/) curl/curl 8.14.1 (x86_64-pc-linux-gnu)", "https://people.kernel.org/monsieuricon/creepy-crawlies" ], "cmd_version": "curl 8.14.1 (x86_64-pc-linux-gnu)", "end_ts": "2026-08-30T22:08:49.660238+00:00", "index_texts": null, "output": "headers.json", "pwd": "/data/archive/1788127631.971801", "schema": "ArchiveResult", "start_ts": "2026-08-30T22:08:49.201839+00:00", "status": "succeeded" } ], "htmltotext": [], "media": [], "mercury": [ { "cmd": [ "/app/node_modules/@postlight/parser/cli.js", "https://people.kernel.org/monsieuricon/creepy-crawlies" ], "cmd_version": "1.0.0", "end_ts": "2026-08-30T22:08:56.343150+00:00", "index_texts": null, "output": "mercury", "pwd": "/data/archive/1788127631.971801", "schema": "ArchiveResult", "start_ts": "2026-08-30T22:08:53.249941+00:00", "status": "succeeded" } ], "pdf": [], "readability": [ { "cmd": [ "/app/node_modules/readability-extractor/readability-extractor", "/tmp/tmp4yl91_x7", "https://people.kernel.org/monsieuricon/creepy-crawlies" ], "cmd_version": "0.0.11", "end_ts": "2026-08-30T22:08:53.240816+00:00", "index_texts": [ "You've probably heard me complain about the \u201cAI crawlers\u201d before, but now I actually have some hard numbers I can put up to show their impact. In a few words, it's bad enough to create a constant \u201cbackground radiation\u201d of system load, permanently tying up a chunk of capacity spent on producing output that is only useful for a single purpose \u2014 feeding a learning model.\n\nTL;DR: we spend more CPU cycles rendering commits for scrapers than we spend on all other kinds of legitimate access, including git clones. At any one time, across 5 geo-distributed nodes, there are 14 CPU cores doing nothing but rendering git commits as html.\n\n\n\nWhy is git.kernel.org \u201cinteresting\u201d to crawlers\n\nLinux development happens in the open \u2014 from git repositories you can clone, to discussion archives you can follow in real time. To a large language model, this is a goldmine of learning data, because all of this is not only immediately available, but is easy to filter in order to guarantee pure unadulterated pre-AI content. Training an LLM on content produced by the LLM gives it the equivalent of a digital prion disease, so when a source is guaranteed to be LLM-free, like the entire history of kernel commits, it's worth its weight in gold as a source of training data.\n\nThe stupidest way of doing it\n\nWe make almost everything clonable, because hey \u2014 we may not be around forever, so here \u2014 clone the repos. Also, clone the archives. Grab a copy just so we're not the only ones who have it all. Seriously, it's just a \u201cgit clone\u201d away \u2014 and then you'll have the whole history.\n\nFor example, did you know you can clone the entirety of LKML and then do whatever you want with it? It's just git repos all the way down.\n\nSo, you'd think that something that pretends to be \u201cArtificial Intelligence\u201d would use the most efficient way of using our data for training purposes, right? Clone the repos, walk every commit. Done.\n\nBut no, let's in fact choose the stupidest possible way of doing it \u2014 by rendering everything as HTML commit by commit and then parsing it.\n\n\n\nAt the time of writing, linux.git is about 1.48 million commits. Oh, and we have about 922 forks of it on git.kernel.org \u2014 but don't worry, it's actually extremely efficient on the backend, since it's mostly the same objects in every fork.\n\nUnless, of course, you're a scraper, in which case you have, oh, several BILLION valid URLs you can scrape, only to get 922 duplicates of the same 1.48 million commits \u2014 which is exactly what the scrapers are doing.\n\nBut wait, it's not just commits itself. You can also ask for patches, plain renders, diffs between arbitrary commits \u2014 cgit is happy to let you, which was perfect for the times when the Internet was for humans or crawlers who obeyed robots.txt, and is AWFUL right about now, because we can generate 1.2 METRIC BAJILLION valid URLs just for a single fork of linux.git.\n\n\n\nBlock them\n\nInitially, this was the solution \u2014 look through the logs, find out which IPs are obvious scraper bots, and fail2ban them. At first, this was easy, because the bots helpfully told you who they were via their user-agent. Then, they wised up and started pretending that they were random vanilla browsers.\n\nSo, we started banning them by IP \u2014 after all, it's easy to figure out that an IP that is trying to grab every possible commit in a 8-year-old abandoned fork of linux is not really some lone Chrome on Windows user who is just furiously clicking every link that comes across their screen.\n\nThe bots then started fanning out to entire subnets, but this was still meh, because obviously an IP coming from Google Compute is just pretending to be a Firefox user. Banning the whole ASN was justified, even if this occasionally caught a random legitimate instance trying to automate link checking in commits.\n\nEnter... your TV?\n\nAnd... that's when things turned really, really ugly. Suddenly, the crawlers were coming from millions of random residential or mobile IPs, all pretending to be random modern browsers. An IP like that would make 4-5 requests and then never show up in the logs again. There was no point in banning them, because by the time you figured out that they were bots, they were already done with you. You just needlessly ballooned your firewall ruleset by adding IPs that would never be back.\n\nThey descended like swarms of locust, hit hard and fast until the system fell over and then moved on to the next target until you recovered. Then, they returned. Rinse. Repeat.\n\nThey still do that \u2014 welcome to the wonderful world of \u201cproxy SDK monetization.\u201d It's big business, and your TV is probably doing it.\n\nMake them pay\n\nWhen this first became a problem, oh, about a year ago, we naively thought that there was a way to make it stop. Just make the bots perform a task that would flip the economy of the whole thing upside-down by making them burn some cycles doing throwaway math. Like, calculate what string, when combined with their own IP and a secret we provide, would generate a sha256 sum with 4 leading zeroes.\n\nIn other words, we put Anubis in front of everything.\n\n\n\nIt was immediately extremely effective \u2014 the bots just gave up. For a few months, it was bliss: bots were blocked at the perimeter and gave up, moving on to easier targets; the users were mildly annoyed but tolerated it, and the Anubis stack was easy enough to deploy everywhere.\n\nA few months later, the bots were back, solving difficulty 4. No problem, we said, let's raise difficulty to 5.\n\nThe legitimate users were more annoyed now. Difficulty 5 takes a few seconds to solve on a mobile device, and the phone gets uncomfortably warm as it's doing the number crunching. However, it was effective and bought us a few more months of peace.\n\nThen... the bots started solving difficulty 5.\n\nWhere we are now\n\n\n\nToday, git.kernel.org receives about 6M daily requests demanding to see random commits. Of these, 66% are still immediately batted away with the Anubis challenge, but 33% are now solving the math and getting through to the main site \u2014 because apparently what we have to offer is worth spending a ton of cycles to calculate the Anubis challenge.\n\nIt's impossible to tell with certainty which of these are bots and which are real humans \u2014 but chances are, if it's asking for an old commit in a random old fork, it's probably not a real developer trying to do their work.\n\nWith a bunch of generous assumptions, legitimate requests are only about 2% of git.kernel.org traffic \u2014 everything else are scrapers.\n\nHow bad is it?\n\n\n\nAt this point, we're not quite overwhelmed \u2014 if you visit git.kernel.org, it will likely be snappy and responsive. The thing that usually takes us down are not scraper bots, but poorly designed CI systems that try to do something stupid like shallow-clone stable.git from 20 different nodes, all at the same time. (Shallow clones are awful. Run your own damn mirror if you're going to do something nasty like that.)\n\nHowever, you should know that out of the total of the 90 cores across 5 geo-distributed nodes, there are 14-16 cores that are constantly doing nothing but rendering commits for scrapers. On average, that's 20% of our entire capacity \u2014 except the swarms descend in waves and the actual graph is a lot more spiky than a 20% flatline.\n\nWhere does that leave us?\n\nUnclear. Maybe the AI bubble bursts and we suddenly have a lot fewer entities out there trying to train their models. Alternatively, maybe they smarten up and stop consuming our data in the dumbest way possible.\n\nIn terms of what we're doing, we're turning off features to reduce the number of crawlable URLs and to gate off actions that are expensive for us to run. Expect to lose some functionality, at least when accessing our resources anonymously. Trust me, we hate it just as much as you, but at this point it's a necessity.\n\nWorst of all, there are no simple solutions to the problem. Companies offering custom \u201cAI\u201d models still pop up daily, all of them hungry for training data. App makers are still looking for ways to turn a profit, so they will continue to turn your household appliances into attack vectors.\n\nThat said, we promise to still offer all of our data for download to anyone who asks. You just may have to jump through more hoops to get it.\n\nSorry. (Oblig. Canadian thing to say.)" ], "output": "readability", "pwd": "/data/archive/1788127631.971801", "schema": "ArchiveResult", "start_ts": "2026-08-30T22:08:51.776851+00:00", "status": "succeeded" } ], "screenshot": [], "singlefile": [], "title": [ { "cmd": [ "curl", "--silent", "--location", "--compressed", "--max-time", "120", "--user-agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36 ArchiveBox/0.7.4 (+https://github.com/ArchiveBox/ArchiveBox/) curl/curl 8.14.1 (x86_64-pc-linux-gnu)", "https://people.kernel.org/monsieuricon/creepy-crawlies" ], "cmd_version": "curl 8.14.1 (x86_64-pc-linux-gnu)", "end_ts": "2026-08-30T22:08:51.769328+00:00", "index_texts": null, "output": "Creepy crawlies \u2014 Konstantin Ryabitsev", "pwd": "/data/archive/1788127631.971801", "schema": "ArchiveResult", "start_ts": "2026-08-30T22:08:51.760615+00:00", "status": "succeeded" } ], "wget": [ { "cmd": [ "wget", "--no-verbose", "--adjust-extension", "--convert-links", "--force-directories", "--backup-converted", "--span-hosts", "--no-parent", "-e", "robots=off", "--timeout=120", "--restrict-file-names=windows", "--warc-file=/data/archive/1788127631.971801/warc/1788127729", "--page-requisites", "--user-agent=Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36 ArchiveBox/0.7.4 (+https://github.com/ArchiveBox/ArchiveBox/) wget/GNU Wget 1.21.3", "--compression=auto", "https://people.kernel.org/monsieuricon/creepy-crawlies" ], "cmd_version": "GNU Wget 1.21.3", "end_ts": "2026-08-30T22:08:51.752214+00:00", "index_texts": null, "output": "people.kernel.org/monsieuricon/creepy-crawlies.html", "pwd": "/data/archive/1788127631.971801", "schema": "ArchiveResult", "start_ts": "2026-08-30T22:08:49.669323+00:00", "status": "succeeded" } ] }, "icons": null, "is_archived": true, "is_static": false, "latest": { "archive_org": null, "dom": null, "favicon": "favicon.ico", "git": null, "media": null, "pdf": null, "screenshot": null, "singlefile": null, "title": "Creepy crawlies \u2014 Konstantin Ryabitsev", "warc": null, "wget": "people.kernel.org/monsieuricon/creepy-crawlies.html" }, "link_dir": "/data/archive/1788127631.971801", "newest_archive_date": "2026-08-30T22:08:53.249941+00:00", "num_failures": 0, "num_outputs": 6, "oldest_archive_date": "2026-08-30T22:08:48.710282+00:00", "path": "/monsieuricon/creepy-crawlies", "schema": "Link", "scheme": "https", "snapshot_id": "0a662ec4-55ab-4ec0-bbba-a8a860603b46", "sources": [ "/data/sources/1788127631-import.txt" ], "tags": null, "tags_str": "", "timestamp": "1788127631.971801", "title": "Creepy crawlies \u2014 Konstantin Ryabitsev", "updated": "2026-08-30T22:08:48.709814+00:00", "updated_date": "2026-08-30 22:08", "url": "https://people.kernel.org/monsieuricon/creepy-crawlies" }