{
    "archive_path": "archive/1790633269.823542",
    "base_url": "github.com/firelex/jeff",
    "basename": "jeff",
    "bookmarked_date": "2026-09-28 22:07",
    "canonical": {
        "archive_org_path": "https://web.archive.org/web/github.com/firelex/jeff",
        "dom_path": "output.html",
        "favicon_path": "favicon.ico",
        "git_path": "git/",
        "google_favicon_path": "https://www.google.com/s2/favicons?domain=github.com",
        "headers_path": "headers.json",
        "htmltotext_path": "htmltotext.txt",
        "index_path": "index.html",
        "media_path": "media/",
        "mercury_path": "mercury/content.html",
        "pdf_path": "output.pdf",
        "readability_path": "readability/content.html",
        "screenshot_path": "screenshot.png",
        "singlefile_path": "singlefile.html",
        "warc_path": "warc/",
        "wget_path": "github.com/firelex/jeff.html"
    },
    "domain": "github.com",
    "extension": "",
    "hash": "RH21WETZ051T5DJSFBR3",
    "history": {
        "archive_org": [],
        "dom": [],
        "favicon": [
            {
                "cmd": [
                    "curl",
                    "--silent",
                    "--location",
                    "--compressed",
                    "--max-time",
                    "120",
                    "--output",
                    "favicon.ico",
                    "--user-agent",
                    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36 ArchiveBox/0.7.4 (+https://github.com/ArchiveBox/ArchiveBox/) curl/curl 8.14.1 (x86_64-pc-linux-gnu)",
                    "https://www.google.com/s2/favicons?domain=github.com"
                ],
                "cmd_version": "curl 8.14.1 (x86_64-pc-linux-gnu)",
                "end_ts": "2026-09-28T22:08:48.780873+00:00",
                "index_texts": null,
                "output": "favicon.ico",
                "pwd": "/data/archive/1790633269.823542",
                "schema": "ArchiveResult",
                "start_ts": "2026-09-28T22:08:48.251301+00:00",
                "status": "succeeded"
            }
        ],
        "git": [],
        "headers": [
            {
                "cmd": [
                    "curl",
                    "--silent",
                    "--location",
                    "--compressed",
                    "--head",
                    "--max-time",
                    "120",
                    "--user-agent",
                    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36 ArchiveBox/0.7.4 (+https://github.com/ArchiveBox/ArchiveBox/) curl/curl 8.14.1 (x86_64-pc-linux-gnu)",
                    "https://github.com/firelex/jeff"
                ],
                "cmd_version": "curl 8.14.1 (x86_64-pc-linux-gnu)",
                "end_ts": "2026-09-28T22:08:49.454177+00:00",
                "index_texts": null,
                "output": "headers.json",
                "pwd": "/data/archive/1790633269.823542",
                "schema": "ArchiveResult",
                "start_ts": "2026-09-28T22:08:48.789529+00:00",
                "status": "succeeded"
            }
        ],
        "htmltotext": [],
        "media": [],
        "mercury": [
            {
                "cmd": [
                    "/app/node_modules/@postlight/parser/cli.js",
                    "https://github.com/firelex/jeff"
                ],
                "cmd_version": "1.0.0",
                "end_ts": "2026-09-28T22:09:07.230750+00:00",
                "index_texts": null,
                "output": "mercury",
                "pwd": "/data/archive/1790633269.823542",
                "schema": "ArchiveResult",
                "start_ts": "2026-09-28T22:09:02.203164+00:00",
                "status": "succeeded"
            }
        ],
        "pdf": [],
        "readability": [
            {
                "cmd": [
                    "/app/node_modules/readability-extractor/readability-extractor",
                    "/tmp/tmpruxwb10h",
                    "https://github.com/firelex/jeff"
                ],
                "cmd_version": "0.0.11",
                "end_ts": "2026-09-28T22:09:02.192839+00:00",
                "index_texts": [
                    "Jeff\nFine-tunes of Qwen3.5 and Gemma 4 for zero-shot classification: small, fast decision models you slot into your\ncode, with the same request format as Jev. You describe a situation and list the options in plain words; Jeff returns a\ncalibrated probability for each option from a single forward pass. No generated text, no parsing: about 22 ms per\ndecision on an RTX PRO 6000 and 28 ms on an Apple M4 Max (MLX).\nZero-shot means the options can be anything: support queues, user intents, moderation labels, voice commands, game\nmoves. Your categories don't need to appear in the training data; you describe them, and Jeff picks.\nWhat it is, and what it isn't. These are very small models. They make extremely fast, well-calibrated judgement\ncalls between options, and they slot easily into your local code. On benchmarks they approach, and sometimes beat, Jev;\nbut at this size their reasoning won't match Jev's, which runs on a much larger model. If zero-shot accuracy isn't\ngood enough for your purposes, a short fine-tune on your own examples takes you much further: our\nvoice-navigation fine-tune moved held-out accuracy from 31.7% to 95.8% in under half an hour on one GPU.\nBuilt entirely on local hardware. Training on one RTX PRO 6000 workstation GPU (the 0.8B trains in about 2 hours,\nthe 2B in about 3.5), all synthetic training data written by an open model (Qwen3.8-Flash-Next) on two DGX Sparks,\ntesting on a MacBook. No cloud GPUs, and no closed-model output in the training data; a closed model was used only to\nspot-check the quality of a sample of the synthetic data.\nIndependent project. Jeff uses the same request format as Jev, but it is not affiliated with or endorsed by TypeSafe, the\nmakers of Jev. Our training code starts from the open-source AutoJev recipe.\nModels on Hugging Face: Jeff-Qwen3.5-0.8B \u00b7 Jeff-Qwen3.5-2B \u00b7 Jeff-Gemma4-E2B\nQuick start\nuv sync\nuv run hf download mstrasser/Jeff-Qwen3.5-0.8B --local-dir checkpoints/jeff-0.8b\n\n# NVIDIA GPU or CPU (PyTorch)\nJEFF_CHECKPOINT=checkpoints/jeff-0.8b PORT=8765 uv run jeff-serve\n# Apple silicon (MLX, much faster on a Mac; Qwen models only)\nuv sync --extra mac\nJEFF_BACKEND=mlx JEFF_CHECKPOINT=checkpoints/jeff-0.8b PORT=8765 uv run jeff-serve\ncurl -s localhost:8765/v1/systemone -H 'content-type: application/json' -d '{\n  \"model\": \"jeff-latest\",\n  \"state\": \"Refund request: the customer says the parcel arrived crushed and wants their money back.\",\n  \"questions\": {\n    \"route\": {\"type\": \"choice\", \"instructions\": \"Which team should handle this?\",\n              \"criteria\": {\"1\": \"Refunds and payments\", \"2\": \"Damaged or lost parcels\", \"3\": \"Account and login problems\"}},\n    \"angry\": {\"type\": \"noul\", \"instructions\": \"Is the customer angry?\"}\n  }\n}'\nEach answer has a probability per option, the chosen option and a confidence. Three question types: choice (pick one\nof up to 255 options), noul (yes/no, returned as a probability) and score (a point on a scale you describe).\nSeveral independent questions in one request are answered together.\nBenchmarks\n4,599 questions from five public benchmarks, plus JevBench's public hard tier (105 items, scored separately):\n\n\n\n\nBenchmark\nQwen3.5-0.8B untrained\nJeff-Qwen3.5-0.8B\nQwen3.5-2B untrained\nJeff-Qwen3.5-2B\nGemma 4 E2B untrained\nJeff-Gemma4-E2B\nJev (published)\nAutoJev-27B (published)\n\n\n\n\nOverall (5 benchmarks)\n45.3\n79.1\n46.5\n83.1\n62.5\n81.6\n83.0\n84.9\n\n\nBBH\n39.5\n64.0\n46.0\n68.0\n51.3\n66.4\n94.3\n82.8\n\n\nFinancial PhraseBank\n36.0\n96.4\n53.4\n96.3\n86.0\n96.1\n77.0\n84.2\n\n\nJudgeBench\n56.6\n62.6\n57.4\n64.6\n46.9\n60.6\n78.6\n78.9\n\n\nRAGTruth\n49.1\n86.1\n35.9\n88.9\n63.8\n87.4\n77.3\n88.9\n\n\nWinoGrande\n49.2\n68.6\n52.2\n79.0\n51.0\n77.4\n90.7\n83.3\n\n\nJevBench hard (separate)\n36.2\n47.6\n45.7\n53.3\n41.0\n48.6\n73.3\n70.3\n\n\n\nBold: the winner of Jeff against Jev in each row. Bold italic: AutoJev-27B where it is the best of all models\nin the row (on RAGTruth, tied with Jeff-Qwen3.5-2B); it is shown for reference, since the head-to-head comparison is\nwith Jev. The published Jev and AutoJev figures were measured on a different sample of the same benchmarks. Jeff's\noverall score comes from classification and grounding, where it matches or beats the large models; on the\nreasoning-heavy benchmarks (BBH, JudgeBench, JevBench) it stays well below them, as you would expect at this size.\nGames: a zero-shot test\nTo test zero-shot performance on tasks unlike anything in the benchmarks, we had Jeff play three games. Games aren't\nthe ideal zero-shot test, since a game's state isn't typical unstructured data; but they are a common, and fun, way to\ntest a System 1 model. Each turn, the code describes the situation and the legal moves in words, and the model picks\none. The options state what each move leads to (Frogger: \"you would be hit by a car and lose a life\"; Doom: \"the\nnearest monster is a little to your left\"), but never which move is right. Each result is 20 episodes, seed 1234; \u25b6\nopens a video of the run's first episode.\nJeff-Qwen3.5-0.8B playing, zero-shot (the bold row in the table below; click a clip for the full video):\n\n\n\n\nModel\nDoom, kills (monster's direction in words)\n\nFrogger, crossings (consequences)\n\nPac-Man, pellets of 98 (consequences)\n\n\n\n\n\nRandom moves\n\u22120.05\n\n0\n\n11.2\n\n\n\nHand-coded rule bot\n6.55\n\u25b6\n10.25\n\u25b6\n94.1\n\u25b6\n\n\nQwen3.5-0.8B, untrained\n5.0\n\u25b6\n1.0\n\u25b6\n25.8\n\u25b6\n\n\nJeff-Qwen3.5-0.8B\n6.55\n\u25b6\n10.3\n\u25b6\n57.0\n\u25b6\n\n\nQwen3.5-2B, untrained\n0.55\n\u25b6\n0.05\n\u25b6\n72.1\n\u25b6\n\n\nJeff-Qwen3.5-2B\n\u22120.9\n\u25b6\n6.0\n\u25b6\n41.2\n\u25b6\n\n\nGemma 4 E2B, untrained\n\u22120.55\n\u25b6\n0\n\u25b6\n3.2\n\u25b6\n\n\nJeff-Gemma4-E2B\n0.55\n\u25b6\n0.15\n\u25b6\n53.2\n\u25b6\n\n\nJev (published, Doom)\n6.55, told the aiming rule; \u22120.60 without it\n\n\u2014\n\n\u2014\n\n\n\n\nJeff-0.8B decides in 29\u201349 ms per move on an M4 Max; Jev's published Doom run took 212 ms per call over its API. The\ntwo times were not measured on the same hardware. To play them yourself:\nuv sync --extra games\nuv run python -m jeff.games --game doom --player jeff --criteria situation --url http://127.0.0.1:8765 --video --out runs/games/doom.json\nuv run python -m jeff.games --game frogger --player jeff --criteria outcomes --url http://127.0.0.1:8765 --out runs/games/frogger.json\nuv run python -m jeff.games --game pacman --player rule --out runs/games/pacman-rule.json\nSpeed and size\nMedian time per decision over the same 200 benchmark questions (about 200 input tokens each), one question at a time,\nfrom raw text to probabilities:\n\n\n\nModel\nParameters\nWeights (16-bit)\nNVIDIA RTX PRO 6000\nApple M4 Max (MLX)\nCPU (32 threads)\n\n\n\n\nJeff-Qwen3.5-0.8B\n0.8B\n1.7 GB\n22 ms\n28 ms\n463 ms\n\n\nJeff-Qwen3.5-2B\n2B\n4.2 GB\n24 ms\n60 ms\n708 ms\n\n\nJeff-Gemma4-E2B\n2B effective (4.6B stored)\n9.3 GB\n29 ms\n\u2014 (MLX runs Qwen only)\n1.0 s\n\n\nAutoJev-27B\n27B\n~54 GB\nnot published\n\u2014\n\u2014\n\n\nJev\nnot disclosed\nAPI only\n114\u2013212 ms per call in published Doom runs, including the network\n\n\n\n\n\nUsing it well\n\nReason in code, decide with Jeff. It's a classifier, not a planner. State what each option leads to (\"this move\ngets you hit by a car\"); asked to forecast (\"a car arrives in 2 turns\"), it does no better than random.\nWording matters enormously. Describe options consistently: giving Frogger's goal option the same words as every\nother forward option took one episode from 15 crossings to 23.\nUse short option keys and descriptive text: {\"1\": \"Engagement letter\"}, not long IDs, which cost time and add\nnothing.\nAsk independent questions together in one request.\nFine-tune it if zero-shot isn't enough. A voice-navigation fine-tune on ~11k app-specific examples took about half\nan hour on one GPU and moved held-out accuracy from 31.7% to 95.8%, at about 40 ms per decision on an M4 Max:\nautojev-train --initial-checkpoint <jeff> --epochs 1 ....\nPick the size for the job. For fast option picking the 0.8B is the sweet spot: the 2B is more cautious and plays\nthe games worse, despite scoring higher on the benchmarks.\n\nTrain your own\nuv run autojev-mix ...          # build the training set (public data, synthetic data, leak filter)\nscripts/train.sh RUN data/mix/public.jsonl data/mix 5e-6 40 Qwen/Qwen3.5-0.8B <revision> --epochs 1\nuv run autojev-evaluate --data data/panel.jsonl --local --checkpoint checkpoints/RUN/selected --output runs/eval/RUN.json\nThe full pipeline (synthetic data from a local teacher, leak filter, learning-rate sweeps, dashboard) is described in\nscripts/train_all.sh, and every training source with its licence in docs/data-sources.md. Training recipe: full-weight fine-tuning, one epoch, batches of 256, cross-entropy over the\noption letters, then one fitted temperature for calibration; checkpoints are chosen on a development set, never on the\nbenchmark panel. At least half of each training family follows the panel's layout conventions (formats only; no panel\nitem is ever trained on).\nCaveats\n\nSmall models don't reason. Expect fast, calibrated choices between the options you describe, not multi-step\nreasoning. At 0.8B\u20132B parameters this holds for every model, not just Jeff.\nJeff-2B is a weaker game player than Jeff-0.8B. The untrained 2B already appears more risk-averse than the\nuntrained 0.8B, and our training seems to have made that worse. This needs more investigation.\nBenchmark scores don't predict game play. The untrained Gemma 4 E2B beats the untrained Qwen models on the\nbenchmarks yet plays the games worst: right most of the time, but not reliably. Training fixed its Pac-Man (3.2 \u2192 53.2 pellets)\nbut not its Doom or Frogger.\nPrompts matter. Jev's own Doom prompt (a raw bearing number plus an aiming rule) does not work for any of our\nmodels; options that state consequences in words do.\nEnglish and text only.\n\nHistory\nJeff began as a fork of AutoJev by Denis Yarats (MIT licence), an open recipe\nthat fine-tunes Qwen3.8-27B to return Jev-style decisions. We kept its core design (one forward pass per decision, a\ntrained answer readout, a fitted temperature for calibration) and built on it: small students (0.8B and 2B Qwen, Gemma\n4 E2B), a local synthetic-data pipeline with a leak filter, prompt layouts for domain fine-tunes, MLX serving on Apple\nsilicon, game tests and a training dashboard. The original copyright notice is kept in LICENSE.\nLicence\nCode: MIT (including AutoJev's). Model weights: Apache 2.0. Doom harness adapted from\njev-plays-doom (MIT). Training data: see the dataset card; each\nsource keeps its licence and is listed in docs/data-sources.md. We release the weights and code, not the training data; some sources are share-alike (CC BY-SA)."
                ],
                "output": "readability",
                "pwd": "/data/archive/1790633269.823542",
                "schema": "ArchiveResult",
                "start_ts": "2026-09-28T22:08:58.194284+00:00",
                "status": "succeeded"
            }
        ],
        "screenshot": [],
        "singlefile": [],
        "title": [
            {
                "cmd": [
                    "curl",
                    "--silent",
                    "--location",
                    "--compressed",
                    "--max-time",
                    "120",
                    "--user-agent",
                    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36 ArchiveBox/0.7.4 (+https://github.com/ArchiveBox/ArchiveBox/) curl/curl 8.14.1 (x86_64-pc-linux-gnu)",
                    "https://github.com/firelex/jeff"
                ],
                "cmd_version": "curl 8.14.1 (x86_64-pc-linux-gnu)",
                "end_ts": "2026-09-28T22:08:58.186728+00:00",
                "index_texts": null,
                "output": "GitHub - firelex/jeff: Fine-tunes of Qwen3.5 and Gemma 4 for zero-shot classification \u00b7 GitHub",
                "pwd": "/data/archive/1790633269.823542",
                "schema": "ArchiveResult",
                "start_ts": "2026-09-28T22:08:58.092432+00:00",
                "status": "succeeded"
            }
        ],
        "wget": [
            {
                "cmd": [
                    "wget",
                    "--no-verbose",
                    "--adjust-extension",
                    "--convert-links",
                    "--force-directories",
                    "--span-hosts",
                    "--no-parent",
                    "-e",
                    "robots=off",
                    "--reject=js,mjs,woff,woff2,ttf,otf,eot,mp4,m4v,mov,webm,ogv,mkv,avi,flv",
                    "--timeout=120",
                    "--restrict-file-names=windows",
                    "--page-requisites",
                    "--user-agent=Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36 ArchiveBox/0.7.4 (+https://github.com/ArchiveBox/ArchiveBox/) wget/GNU Wget 1.21.3",
                    "--compression=auto",
                    "--timestamping",
                    "https://github.com/firelex/jeff"
                ],
                "cmd_version": "GNU Wget 1.21.3",
                "end_ts": "2026-09-28T22:08:58.080995+00:00",
                "index_texts": null,
                "output": "github.com/firelex/jeff.html",
                "pwd": "/data/archive/1790633269.823542",
                "schema": "ArchiveResult",
                "start_ts": "2026-09-28T22:08:49.463615+00:00",
                "status": "succeeded"
            }
        ]
    },
    "icons": null,
    "is_archived": true,
    "is_static": false,
    "latest": {
        "archive_org": null,
        "dom": null,
        "favicon": "favicon.ico",
        "git": null,
        "media": null,
        "pdf": null,
        "screenshot": null,
        "singlefile": null,
        "title": "GitHub - firelex/jeff: Fine-tunes of Qwen3.5 and Gemma 4 for zero-shot classification \u00b7 GitHub",
        "warc": null,
        "wget": "github.com/firelex/jeff.html"
    },
    "link_dir": "/data/archive/1790633269.823542",
    "newest_archive_date": "2026-09-28T22:09:02.203164+00:00",
    "num_failures": 0,
    "num_outputs": 6,
    "oldest_archive_date": "2026-09-28T22:08:48.251301+00:00",
    "path": "/firelex/jeff",
    "schema": "Link",
    "scheme": "https",
    "snapshot_id": "92a81255-ffc5-4615-a422-f5932ab97820",
    "sources": [
        "/data/sources/1790633269-import.txt"
    ],
    "tags": null,
    "tags_str": "",
    "timestamp": "1790633269.823542",
    "title": "GitHub - firelex/jeff: Fine-tunes of Qwen3.5 and Gemma 4 for zero-shot classification \u00b7 GitHub",
    "updated": "2026-09-28T22:08:48.250797+00:00",
    "updated_date": "2026-09-28 22:08",
    "url": "https://github.com/firelex/jeff"
}