{ "archive_path": "archive/1790028258.552939", "base_url": "poloclub.github.io/transformer-explainer", "basename": "", "bookmarked_date": "2026-09-21 22:04", "canonical": { "archive_org_path": "https://web.archive.org/web/poloclub.github.io/transformer-explainer", "dom_path": "output.html", "favicon_path": "favicon.ico", "git_path": "git/", "google_favicon_path": "https://www.google.com/s2/favicons?domain=poloclub.github.io", "headers_path": "headers.json", "htmltotext_path": "htmltotext.txt", "index_path": "index.html", "media_path": "media/", "mercury_path": "mercury/content.html", "pdf_path": "output.pdf", "readability_path": "readability/content.html", "screenshot_path": "screenshot.png", "singlefile_path": "singlefile.html", "warc_path": "warc/", "wget_path": "poloclub.github.io/transformer-explainer/index.html" }, "domain": "poloclub.github.io", "extension": "", "hash": "1ABDY24GPB2CWYP9A1A6", "history": { "archive_org": [], "dom": [], "favicon": [ { "cmd": [ "curl", "--silent", "--location", "--compressed", "--max-time", "120", "--output", "favicon.ico", "--user-agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36 ArchiveBox/0.7.4 (+https://github.com/ArchiveBox/ArchiveBox/) curl/curl 8.14.1 (x86_64-pc-linux-gnu)", "https://www.google.com/s2/favicons?domain=poloclub.github.io" ], "cmd_version": "curl 8.14.1 (x86_64-pc-linux-gnu)", "end_ts": "2026-09-21T22:07:16.480521+00:00", "index_texts": null, "output": "favicon.ico", "pwd": "/data/archive/1790028258.552939", "schema": "ArchiveResult", "start_ts": "2026-09-21T22:07:16.125994+00:00", "status": "succeeded" } ], "git": [], "headers": [ { "cmd": [ "curl", "--silent", "--location", "--compressed", "--head", "--max-time", "120", "--user-agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36 ArchiveBox/0.7.4 (+https://github.com/ArchiveBox/ArchiveBox/) curl/curl 8.14.1 (x86_64-pc-linux-gnu)", "https://poloclub.github.io/transformer-explainer/" ], "cmd_version": "curl 8.14.1 (x86_64-pc-linux-gnu)", "end_ts": "2026-09-21T22:07:16.873057+00:00", "index_texts": null, "output": "headers.json", "pwd": "/data/archive/1790028258.552939", "schema": "ArchiveResult", "start_ts": "2026-09-21T22:07:16.488776+00:00", "status": "succeeded" } ], "htmltotext": [], "media": [], "mercury": [ { "cmd": [ "/app/node_modules/@postlight/parser/cli.js", "https://poloclub.github.io/transformer-explainer/" ], "cmd_version": "1.0.0", "end_ts": "2026-09-21T22:07:24.555706+00:00", "index_texts": null, "output": "mercury", "pwd": "/data/archive/1790028258.552939", "schema": "ArchiveResult", "start_ts": "2026-09-21T22:07:20.795795+00:00", "status": "succeeded" } ], "pdf": [], "readability": [ { "cmd": [ "/app/node_modules/readability-extractor/readability-extractor", "/tmp/tmpu8jni8aj", "https://poloclub.github.io/transformer-explainer/" ], "cmd_version": "0.0.11", "end_ts": "2026-09-21T22:07:20.785150+00:00", "index_texts": [ "What is a Transformer? Transformer is a neural network architecture that has fundamentally changed the approach to\n\t\t\tArtificial Intelligence. Transformer was first introduced in the seminal paper\n\t\t\t\"Attention is All You Need\"\n\t\t\tin 2017 and has since become the go-to architecture for deep learning models, powering text-generative\n\t\t\tmodels like OpenAI's GPT, Meta's Llama, and Google's\n\t\t\tGemini. Beyond text, Transformer is also applied in\n\t\t\taudio generation,\n\t\t\timage recognition,\n\t\t\tprotein structure prediction, and even\n\t\t\tgame playing, demonstrating its versatility across numerous domains. Fundamentally, text-generative Transformer models operate on the principle of next-token prediction: given a text prompt from the user, what is the\n\t\t\tmost probable next token (a word or part of a word) that will follow this input? The core\n\t\t\tinnovation and power of Transformers lie in their use of self-attention mechanism, which allows\n\t\t\tthem to process entire sequences and capture long-range dependencies more effectively than previous\n\t\t\tarchitectures. GPT-2 family of models are prominent examples of text-generative Transformers. Transformer\n\t\t\tExplainer is powered by the\n\t\t\tGPT-2\n\t\t\t(small) model which has 124 million parameters. While it is not the latest or most powerful Transformer\n\t\t\tmodel, it shares many of the same architectural components and principles found in the current\n\t\t\tstate-of-the-art models making it an ideal starting point for understanding the basics. Transformer Architecture Every text-generative Transformer consists of these three key components: Embedding: Text input is divided into smaller units\n\t\t\t\tcalled tokens, which can be words or subwords. These tokens are converted into numerical\n\t\t\t\tvectors called embeddings, which capture the semantic meaning of words. Transformer Block is the fundamental building block of\n\t\t\t\tthe model that processes and transforms the input data. Each block includes:\n\t\t\t\tAttention Mechanism, the core component of the Transformer block. It\n\t\t\t\t\t\tallows tokens to communicate with other tokens, capturing contextual information and\n\t\t\t\t\t\trelationships between words. MLP (Multilayer Perceptron) Layer, a feed-forward network that operates\n\t\t\t\t\t\ton each token independently. While the goal of the attention layer is to route\n\t\t\t\t\t\tinformation between tokens, the goal of the MLP is to refine each token's\n\t\t\t\t\t\trepresentation. Output Probabilities: The final linear and softmax\n\t\t\t\tlayers transform the processed embeddings into probabilities, enabling the model to make\n\t\t\t\tpredictions about the next token in a sequence. Embedding Let's say you want to generate text using a Transformer model. You add the prompt like this\n\t\t\tone: \u201cData visualization empowers users to\u201d. This input needs to be converted\n\t\t\tinto a format that the model can understand and process. That is where embedding comes in: it\n\t\t\ttransforms the text into a numerical representation that the model can work with. To convert a\n\t\t\tprompt into embedding, we need to 1) tokenize the input, 2) obtain token embeddings, 3) add\n\t\t\tpositional information, and finally 4) add up token and position encodings to get the final\n\t\t\tembedding. Let\u2019s see how each of these steps is done. Figure 1. Expanding the Embedding layer view, showing how the\n\t\t\tinput prompt is converted to a vector representation. The process involves\n\t\t\t(1) Tokenization, (2) Token Embedding, (3) Positional Encoding,\n\t\t\tand (4) Final Embedding. Step 1: Tokenization Tokenization is the process of breaking down the input text into smaller, more manageable\n\t\t\t\tpieces called tokens. These tokens can be a word or a subword. The words \"Data\"\n\t\t\t\tand \"visualization\" correspond to unique tokens, while the word\n\t\t\t\t\"empowers\"\n\t\t\t\tis split into two tokens. The full vocabulary of tokens is decided before training the model:\n\t\t\t\tGPT-2's vocabulary has 50,257 unique tokens. Now that we split our input text into\n\t\t\t\ttokens with distinct IDs, we can obtain their vector representation from embeddings. Step 2. Token Embedding GPT-2 (small) represents each token in the vocabulary as a 768-dimensional vector; the\n\t\t\t\tdimension of the vector depends on the model. These embedding vectors are stored in a matrix\n\t\t\t\tof shape (50,257, 768), containing approximately 39 million parameters! This\n\t\t\t\textensive matrix allows the model to assign semantic meaning to each token, in the sense\n\t\t\t\tthat tokens with similar usage or meaning in language are placed close together in this\n\t\t\t\thigh-dimensional space, while dissimilar tokens are farther apart. Step 3. Positional Encoding The Embedding layer also encodes information about each token's position in the input\n\t\t\t\tprompt. Different models use various methods for positional encoding. GPT-2 trains its own\n\t\t\t\tpositional encoding matrix from scratch, integrating it directly into the training process. Step 4. Final Embedding Finally, we sum the token and positional encodings to get the final embedding\n\t\t\t\trepresentation. This combined representation captures both the semantic meaning of the\n\t\t\t\ttokens and their position in the input sequence. Transformer Block The core of the Transformer's processing lies in the Transformer block, which comprises\n\t\t\tmulti-head self-attention and a Multi-Layer Perceptron layer. Most models consist of multiple\n\t\t\tsuch blocks that are stacked sequentially one after the other. The token representations\n\t\t\tevolve through layers, from the first block to the last one, allowing the model to build up an\n\t\t\tintricate understanding of each token. This layered approach leads to higher-order\n\t\t\trepresentations of the input. The GPT-2 (small) model we are examining consists of 12 such blocks. Multi-Head Self-Attention The self-attention mechanism enables the model to capture relationships among tokens in a\n\t\t\tsequence, so that each token\u2019s representation is influenced by the others. Multiple attention\n\t\t\theads allow the model to consider these relationships from different perspectives; for\n\t\t\texample, one head may capture short-range syntactic links while another tracks broader\n\t\t\tsemantic context. In the following section, we will walk through how multi-head self-attention\n\t\t\tis computed step by step. Step 1: Query, Key, and Value Matrices QKVij=(\u2211d=1768Embeddingi,d\u22c5Weightsd,j)+Biasj\n\t\tQKV_{ij} = ( \\sum_{d=1}^{768} \\text{Embedding}_{i,d} \\cdot \\text{Weights}_{d,j}) + \\text{Bias}_j\n\t\t Figure 2. Computing Query, Key, and Value matrices from the\n\t\t\t\toriginal embedding. Each token's embedding vector is transformed into three vectors:\n\t\t\t\tQuery (Q),\n\t\t\t\tKey (K), and\n\t\t\t\tValue (V). These vectors are derived by multiplying the input\n\t\t\t\tembedding matrix with learned weight matrices for\n\t\t\t\tQ,\n\t\t\t\tK, and\n\t\t\t\tV. Here's a web search analogy to help us build some intuition\n\t\t\t\tbehind these matrices: Query (Q) is the search text you type in the\n\t\t\t\t\tsearch engine bar. This is the token you want to\n\t\t\t\t\t\"find more information about\". Key (K) is the title of each web page in the search\n\t\t\t\t\tresult window. It represents the possible tokens the query can attend to. Value (V) is the actual content of web pages shown.\n\t\t\t\t\tOnce we matched the appropriate search term (Query) with the relevant results (Key), we want\n\t\t\t\t\tto get the content (Value) of the most relevant pages. By using these QKV values, the model can calculate attention scores, which determine how\n\t\t\t\tmuch focus each token should receive when generating predictions. Step 2: Multi-Head Splitting Query, key, and\n\t\t\t\tValue\n\t\t\t\tvectors are split into multiple heads\u2014in GPT-2 (small)'s case, into\n\t\t\t\t12 heads. Each head processes a segment of the embeddings independently, capturing\n\t\t\t\tdifferent syntactic and semantic relationships. This design facilitates parallel learning of\n\t\t\t\tdiverse linguistic features, enhancing the model's representational power. Step 3: Masked Self-Attention In each head, we perform masked self-attention calculations. This mechanism allows the model\n\t\t\t\tto generate sequences by focusing on relevant parts of the input while preventing access to\n\t\t\t\tfuture tokens. Figure 3. Using Query, Key, and Value matrices to calculate\n\t\t\t\tmasked self-attention. Dot Product: The dot product of\n\t\t\t\t\tQuery\n\t\t\t\t\tand Key matrices determines the\n\t\t\t\t\tattention score, producing a square matrix that reflects the relationship\n\t\t\t\t\tbetween all input tokens. Scaling \u00b7 Mask: The attention scores are scaled and a mask is applied to\n\t\t\t\t\tthe upper triangle of the attention matrix to prevent the model from accessing future\n\t\t\t\t\ttokens, setting these values to negative infinity. The model needs to learn how to predict\n\t\t\t\t\tthe next token without \u201cpeeking\u201d into the future. Softmax \u00b7 Dropout: After masking and scaling, the attention scores are\n\t\t\t\t\tconverted into probabilities by the softmax operation, then optionally regularized with\n\t\t\t\t\tdropout. Each row of the matrix sums to one and indicates the relevance of every other\n\t\t\t\t\ttoken to the left of it. Step 4: Output and Concatenation The model uses the masked self-attention scores and multiplies them with the\n\t\t\t\tValue matrix to get the\n\t\t\t\tfinal output\n\t\t\t\tof the self-attention mechanism. GPT-2 has 12 self-attention heads, each capturing\n\t\t\t\tdifferent relationships between tokens. The outputs of these heads are concatenated and passed\n\t\t\t\tthrough a linear projection. MLP: Multi-Layer Perceptron Figure 4. Using MLP layer to project the self-attention\n\t\t\trepresentations into higher dimensions to enhance the model's representational capacity. After the multiple heads of self-attention capture the diverse relationships between the input\n\t\t\ttokens, the concatenated outputs are passed through the Multilayer Perceptron (MLP) layer to\n\t\t\tenhance the model's representational capacity. The MLP block consists of two linear\n\t\t\ttransformations with a GELU activation function in between. The first linear transformation expands the dimensionality of the input four-fold from 768\n\t\t\tto\n\t\t\t3072. This expansion step allows the model to project the token representations\n\t\t\tinto a higher-dimensional space, where it can capture richer and more complex patterns that\n\t\t\tmay not be visible in the original dimension. The second linear transformation then reduces the dimensionality back to the original size of 768.This compression step brings the representations back to a manageable size while retaining\n\t\t\tthe useful nonlinear transformations introduced in the expansion step. Unlike the self-attention mechanism, which integrates information across tokens, the MLP\n\t\t\tprocesses tokens independently and simply maps each token representation from one space to\n\t\t\tanother, enriching the overall model capacity. Output Probabilities After the input has been processed through all Transformer blocks, the output is passed\n\t\t\tthrough the final linear layer to prepare it for token prediction. This layer projects the\n\t\t\tfinal representations into a 50,257\n\t\t\tdimensional space, where every token in the vocabulary has a corresponding value called\n\t\t\tlogit. Any token can be the next word, so this process allows us to simply rank\n\t\t\tthese tokens by their likelihood of being that next word. We then apply the softmax function\n\t\t\tto convert the logits into a probability distribution that sums to one. This will allow us to\n\t\t\tsample the next token based on its likelihood. Figure 5. Each token in the vocabulary is assigned a\n\t\t\tprobability based on the model's output logits. These probabilities determine the likelihood\n\t\t\tof each token being the next word in the sequence. The final step is to generate the next token by sampling from this distribution The temperature\n\t\t\thyperparameter plays a critical role in this process. Mathematically speaking, it is a very simple\n\t\t\toperation: model output logits are simply divided by the\n\t\t\ttemperature: temperature = 1: Dividing logits by one has no effect on the softmax outputs. temperature < 1: Lower temperature makes the model more confident and\n\t\t\t\tdeterministic by sharpening the probability distribution, leading to more predictable\n\t\t\t\toutputs. temperature > 1: Higher temperature creates a softer probability\n\t\t\t\tdistribution, allowing for more randomness in the generated text \u2013 what some refer to as\n\t\t\t\tmodel \u201ccreativity\u201d. In addition, the sampling process can be further refined using top-k\n\t\t\tand\n\t\t\ttop-p parameters: top-k sampling: Limits the candidate tokens to the top k tokens with the\n\t\t\t\thighest probabilities, filtering out less likely options. top-p sampling: Considers the smallest set of tokens whose cumulative\n\t\t\t\tprobability exceeds a threshold p, ensuring that only the most likely tokens contribute\n\t\t\t\twhile still allowing for diversity. By tuning temperature, top-k, and top-p, you can\n\t\t\tbalance between deterministic and diverse outputs, tailoring the model's behavior to your\n\t\t\tspecific needs. Auxiliary Architectural Features There are several auxiliary architectural features that enhance the performance of Transformer\n\t\t\tmodels. While important for the model's overall performance, they are not as important for\n\t\t\tunderstanding the core concepts of the architecture. Layer Normalization, Dropout, and\n\t\t\tResidual Connections are crucial components in Transformer models, particularly during the\n\t\t\ttraining phase. Layer Normalization stabilizes training and helps the model converge faster.\n\t\t\tDropout prevents overfitting by randomly deactivating neurons. Residual Connections allows\n\t\t\tgradients to flow directly through the network and helps to prevent the vanishing gradient\n\t\t\tproblem. Layer Normalization Layer Normalization helps to stabilize the training process and improves convergence. It\n\t\t\t\tworks by normalizing the inputs across the features, ensuring that the mean and variance of\n\t\t\t\tthe activations are consistent. This normalization helps mitigate issues related to internal\n\t\t\t\tcovariate shift, allowing the model to learn more effectively and reducing the sensitivity\n\t\t\t\tto the initial weights. Layer Normalization is applied twice in each Transformer block, once\n\t\t\t\tbefore the self-attention mechanism and once before the MLP layer. Dropout Dropout is a regularization technique used to prevent overfitting in neural networks by\n\t\t\t\trandomly setting a fraction of model weights to zero during training. This encourages the\n\t\t\t\tmodel to learn more robust features and reduces dependency on specific neurons, helping the\n\t\t\t\tnetwork generalize better to new, unseen data. During model inference, dropout is\n\t\t\t\tdeactivated. This essentially means that we are using an ensemble of the trained\n\t\t\t\tsubnetworks, which leads to a better model performance. Residual Connections Residual connections were first introduced in the ResNet model in 2015. This architectural\n\t\t\t\tinnovation revolutionized deep learning by enabling the training of very deep neural\n\t\t\t\tnetworks. Essentially, residual connections are shortcuts that bypass one or more layers,\n\t\t\t\tadding the input of a layer to its output. This helps mitigate the vanishing gradient\n\t\t\t\tproblem, making it easier to train deep networks with multiple Transformer blocks stacked on\n\t\t\t\ttop of each other. In GPT-2, residual connections are used twice within each Transformer\n\t\t\t\tblock: once before the MLP and once after, ensuring that gradients flow more easily, and\n\t\t\t\tearlier layers receive sufficient updates during backpropagation. Interactive Features Transformer Explainer is built to be interactive and allows you to explore the inner workings\n\t\t\tof the Transformer. Here are some of the interactive features you can play with: Input your own text sequence to see how the model processes it and predicts\n\t\t\t\tthe next word. Explore attention weights, intermediate computations, and see how the final output\n\t\t\t\tprobabilities are calculated. Use temperature slider to control the randomness of the model\u2019s predictions.\n\t\t\t\tExplore how you can make the model output more deterministic or more creative by changing the\n\t\t\t\ttemperature value. Select top-k and top-p sampling methods to adjust sampling behavior during inference.\n\t\t\t\tExperiment with different values and see how the probability distribution changes and influences\n\t\t\t\tthe model's predictions. Interact with attention maps to see how the model focuses on different tokens\n\t\t\t\tin the input sequence. Hover over tokens to highlight their attention weights and explore how\n\t\t\t\tthe model captures context and relationships between words. Video Tutorial How is Transformer Explainer Implemented? Transformer Explainer features a live GPT-2 (small) model running directly in the browser.\n\t\t\tThis model is derived from the PyTorch implementation of GPT by Andrej Karpathy's\n\t\t\tnanoGPT project\n\t\t\tand has been converted to\n\t\t\tONNX Runtime\n\t\t\tfor seamless in-browser execution. The interface is built using JavaScript, with\n\t\t\tSvelte\n\t\t\tas a front-end framework and\n\t\t\tD3.js\n\t\t\tfor creating dynamic visualizations. Numerical values are updated live following the user input. Who developed the Transformer Explainer? Transformer Explainer was created by\n\n\t\t\tAeree Cho,\n\t\t\tGrace C. Kim,\n\t\t\tAlexander Karpekov,\n\t\t\tAlec Helbling,\n\t\t\tJay Wang,\n\t\t\tSeongmin Lee,\n\t\t\tBenjamin Hoover, and\n\t\t\tPolo Chau\n\n\t\t\tat the Georgia Institute of Technology." ], "output": "readability", "pwd": "/data/archive/1790028258.552939", "schema": "ArchiveResult", "start_ts": "2026-09-21T22:07:18.941908+00:00", "status": "succeeded" } ], "screenshot": [], "singlefile": [], "title": [ { "cmd": [ "curl", "--silent", "--location", "--compressed", "--max-time", "120", "--user-agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36 ArchiveBox/0.7.4 (+https://github.com/ArchiveBox/ArchiveBox/) curl/curl 8.14.1 (x86_64-pc-linux-gnu)", "https://poloclub.github.io/transformer-explainer/" ], "cmd_version": "curl 8.14.1 (x86_64-pc-linux-gnu)", "end_ts": "2026-09-21T22:07:18.934540+00:00", "index_texts": null, "output": "Transformer Explainer: LLM Transformer Model Visually Explained", "pwd": "/data/archive/1790028258.552939", "schema": "ArchiveResult", "start_ts": "2026-09-21T22:07:18.915486+00:00", "status": "succeeded" } ], "wget": [ { "cmd": [ "wget", "--no-verbose", "--adjust-extension", "--convert-links", "--force-directories", "--span-hosts", "--no-parent", "-e", "robots=off", "--reject=js,mjs,woff,woff2,ttf,otf,eot,mp4,m4v,mov,webm,ogv,mkv,avi,flv", "--timeout=120", "--restrict-file-names=windows", "--page-requisites", "--user-agent=Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36 ArchiveBox/0.7.4 (+https://github.com/ArchiveBox/ArchiveBox/) wget/GNU Wget 1.21.3", "--compression=auto", "--timestamping", "https://poloclub.github.io/transformer-explainer/" ], "cmd_version": "GNU Wget 1.21.3", "end_ts": "2026-09-21T22:07:18.904328+00:00", "index_texts": null, "output": "poloclub.github.io/transformer-explainer/index.html", "pwd": "/data/archive/1790028258.552939", "schema": "ArchiveResult", "start_ts": "2026-09-21T22:07:16.882587+00:00", "status": "succeeded" } ] }, "icons": null, "is_archived": true, "is_static": false, "latest": { "archive_org": null, "dom": null, "favicon": "favicon.ico", "git": null, "media": null, "pdf": null, "screenshot": null, "singlefile": null, "title": "Transformer Explainer: LLM Transformer Model Visually Explained", "warc": null, "wget": "poloclub.github.io/transformer-explainer/index.html" }, "link_dir": "/data/archive/1790028258.552939", "newest_archive_date": "2026-09-21T22:07:20.795795+00:00", "num_failures": 0, "num_outputs": 6, "oldest_archive_date": "2026-09-21T22:07:16.125994+00:00", "path": "/transformer-explainer/", "schema": "Link", "scheme": "https", "snapshot_id": "13984380-2b16-465a-86e7-7430ea0c92b1", "sources": [ "/data/sources/1790028258-import.txt" ], "tags": null, "tags_str": "", "timestamp": "1790028258.552939", "title": "Transformer Explainer: LLM Transformer Model Visually Explained", "updated": "2026-09-21T22:07:16.125492+00:00", "updated_date": "2026-09-21 22:07", "url": "https://poloclub.github.io/transformer-explainer/" }