{
    "archive_path": "archive/1792145128.357533",
    "base_url": "orlp.net/blog/two-stack-sliding-window-aggregation",
    "basename": "",
    "bookmarked_date": "2026-10-16 10:05",
    "canonical": {
        "archive_org_path": "https://web.archive.org/web/orlp.net/blog/two-stack-sliding-window-aggregation",
        "dom_path": "output.html",
        "favicon_path": "favicon.ico",
        "git_path": "git/",
        "google_favicon_path": "https://www.google.com/s2/favicons?domain=orlp.net",
        "headers_path": "headers.json",
        "htmltotext_path": "htmltotext.txt",
        "index_path": "index.html",
        "media_path": "media/",
        "mercury_path": "mercury/content.html",
        "pdf_path": "output.pdf",
        "readability_path": "readability/content.html",
        "screenshot_path": "screenshot.png",
        "singlefile_path": "singlefile.html",
        "warc_path": "warc/",
        "wget_path": "orlp.net/blog/two-stack-sliding-window-aggregation/index.html"
    },
    "domain": "orlp.net",
    "extension": "",
    "hash": "1GV9V8NAFB9VF1HVV1BQ",
    "history": {
        "archive_org": [],
        "dom": [],
        "favicon": [
            {
                "cmd": [
                    "curl",
                    "--silent",
                    "--location",
                    "--compressed",
                    "--max-time",
                    "120",
                    "--output",
                    "favicon.ico",
                    "--user-agent",
                    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36 ArchiveBox/0.7.4 (+https://github.com/ArchiveBox/ArchiveBox/) curl/curl 8.14.1 (x86_64-pc-linux-gnu)",
                    "https://www.google.com/s2/favicons?domain=orlp.net"
                ],
                "cmd_version": "curl 8.14.1 (x86_64-pc-linux-gnu)",
                "end_ts": "2026-10-16T13:41:45.220187+00:00",
                "index_texts": null,
                "output": "favicon.ico",
                "pwd": "/data/archive/1792145128.357533",
                "schema": "ArchiveResult",
                "start_ts": "2026-10-16T13:41:44.794208+00:00",
                "status": "succeeded"
            },
            {
                "cmd": [
                    "curl",
                    "--silent",
                    "--location",
                    "--compressed",
                    "--max-time",
                    "120",
                    "--output",
                    "favicon.ico",
                    "--user-agent",
                    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36 ArchiveBox/0.7.4 (+https://github.com/ArchiveBox/ArchiveBox/) curl/curl 8.14.1 (x86_64-pc-linux-gnu)",
                    "https://www.google.com/s2/favicons?domain=orlp.net"
                ],
                "cmd_version": "curl 8.14.1 (x86_64-pc-linux-gnu)",
                "end_ts": "2026-10-16T10:06:49.312066+00:00",
                "index_texts": null,
                "output": "favicon.ico",
                "pwd": "/data/archive/1792145128.357533",
                "schema": "ArchiveResult",
                "start_ts": "2026-10-16T10:06:48.856423+00:00",
                "status": "succeeded"
            },
            {
                "cmd": [
                    "curl",
                    "--silent",
                    "--location",
                    "--compressed",
                    "--max-time",
                    "120",
                    "--output",
                    "favicon.ico",
                    "--user-agent",
                    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36 ArchiveBox/0.7.4 (+https://github.com/ArchiveBox/ArchiveBox/) curl/curl 8.14.1 (x86_64-pc-linux-gnu)",
                    "https://www.google.com/s2/favicons?domain=orlp.net"
                ],
                "cmd_version": "curl 8.14.1 (x86_64-pc-linux-gnu)",
                "end_ts": "2026-10-16T16:26:11.464099+00:00",
                "index_texts": null,
                "output": "favicon.ico",
                "pwd": "/data/archive/1792145128.357533",
                "schema": "ArchiveResult",
                "start_ts": "2026-10-16T16:26:10.997114+00:00",
                "status": "succeeded"
            }
        ],
        "git": [],
        "headers": [
            {
                "cmd": [
                    "curl",
                    "--silent",
                    "--location",
                    "--compressed",
                    "--head",
                    "--max-time",
                    "120",
                    "--user-agent",
                    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36 ArchiveBox/0.7.4 (+https://github.com/ArchiveBox/ArchiveBox/) curl/curl 8.14.1 (x86_64-pc-linux-gnu)",
                    "https://orlp.net/blog/two-stack-sliding-window-aggregation/"
                ],
                "cmd_version": "curl 8.14.1 (x86_64-pc-linux-gnu)",
                "end_ts": "2026-10-16T13:41:45.825938+00:00",
                "index_texts": null,
                "output": "headers.json",
                "pwd": "/data/archive/1792145128.357533",
                "schema": "ArchiveResult",
                "start_ts": "2026-10-16T13:41:45.229193+00:00",
                "status": "succeeded"
            },
            {
                "cmd": [
                    "curl",
                    "--silent",
                    "--location",
                    "--compressed",
                    "--head",
                    "--max-time",
                    "120",
                    "--user-agent",
                    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36 ArchiveBox/0.7.4 (+https://github.com/ArchiveBox/ArchiveBox/) curl/curl 8.14.1 (x86_64-pc-linux-gnu)",
                    "https://orlp.net/blog/two-stack-sliding-window-aggregation/"
                ],
                "cmd_version": "curl 8.14.1 (x86_64-pc-linux-gnu)",
                "end_ts": "2026-10-16T10:06:49.920848+00:00",
                "index_texts": null,
                "output": "headers.json",
                "pwd": "/data/archive/1792145128.357533",
                "schema": "ArchiveResult",
                "start_ts": "2026-10-16T10:06:49.320829+00:00",
                "status": "succeeded"
            },
            {
                "cmd": [
                    "curl",
                    "--silent",
                    "--location",
                    "--compressed",
                    "--head",
                    "--max-time",
                    "120",
                    "--user-agent",
                    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36 ArchiveBox/0.7.4 (+https://github.com/ArchiveBox/ArchiveBox/) curl/curl 8.14.1 (x86_64-pc-linux-gnu)",
                    "https://orlp.net/blog/two-stack-sliding-window-aggregation/"
                ],
                "cmd_version": "curl 8.14.1 (x86_64-pc-linux-gnu)",
                "end_ts": "2026-10-16T16:26:12.044700+00:00",
                "index_texts": null,
                "output": "headers.json",
                "pwd": "/data/archive/1792145128.357533",
                "schema": "ArchiveResult",
                "start_ts": "2026-10-16T16:26:11.471443+00:00",
                "status": "succeeded"
            }
        ],
        "htmltotext": [],
        "media": [],
        "mercury": [
            {
                "cmd": [
                    "/app/node_modules/@postlight/parser/cli.js",
                    "https://orlp.net/blog/two-stack-sliding-window-aggregation/"
                ],
                "cmd_version": "1.0.0",
                "end_ts": "2026-10-16T13:41:53.335445+00:00",
                "index_texts": null,
                "output": "mercury",
                "pwd": "/data/archive/1792145128.357533",
                "schema": "ArchiveResult",
                "start_ts": "2026-10-16T13:41:49.307962+00:00",
                "status": "succeeded"
            },
            {
                "cmd": [
                    "/app/node_modules/@postlight/parser/cli.js",
                    "https://orlp.net/blog/two-stack-sliding-window-aggregation/"
                ],
                "cmd_version": "1.0.0",
                "end_ts": "2026-10-16T10:06:56.577363+00:00",
                "index_texts": null,
                "output": "mercury",
                "pwd": "/data/archive/1792145128.357533",
                "schema": "ArchiveResult",
                "start_ts": "2026-10-16T10:06:52.536820+00:00",
                "status": "succeeded"
            },
            {
                "cmd": [
                    "/app/node_modules/@postlight/parser/cli.js",
                    "https://orlp.net/blog/two-stack-sliding-window-aggregation/"
                ],
                "cmd_version": "1.0.0",
                "end_ts": "2026-10-16T16:26:19.272411+00:00",
                "index_texts": null,
                "output": "mercury",
                "pwd": "/data/archive/1792145128.357533",
                "schema": "ArchiveResult",
                "start_ts": "2026-10-16T16:26:15.162502+00:00",
                "status": "succeeded"
            }
        ],
        "pdf": [],
        "readability": [
            {
                "cmd": [
                    "/app/node_modules/readability-extractor/readability-extractor",
                    "/tmp/tmp9zwb9yd3",
                    "https://orlp.net/blog/two-stack-sliding-window-aggregation/"
                ],
                "cmd_version": "0.0.11",
                "end_ts": "2026-10-16T13:41:49.292840+00:00",
                "index_texts": [
                    "Two-Stack Sliding-Window Aggregation\n2026-10-02\nAn aggregation is some kind of summary of a set\nof data. This can be the sum, length, minimum, etc. It is quite common to want to calculate such a\nsummary repeatedly, e.g. \u201cthe maximum noise level in dB for the past 30 seconds\u201d for a nuisance detector.\nIn such a case we say there is a sliding window over our data, and we want to aggregate over our window.\nIf our aggregation is a binary operator with an inverse, like integer sums, there is a very easy\nsolution using a double-ended queue:\nfrom collections import deque\n\nclass SlidingWindowSum:\n    def __init__(self):\n        self.sum = 0\n        self.elems = deque()\n    \n    def push(self, x):\n        self.sum += x\n        self.elems.append(x)\n    \n    def pop(self):\n        self.sum -= self.elems.popleft()\n    \n    def eval(self):\n        return self.sum\nBut what if our operator has no inverse? This is actually the case for most interesting summaries\nsuch as minimum, quantile, approximate unique count (for example using HyperLogLog), etc. In fact, even something as simple as a floating-point sum suffers from the fact\nthat floating-point addition is not invertible. For example, if you ever have a NaN in your input\ndata with the above naive algorithm your sum will forever remain NaN, even long after the bad value\nhas left your window.\nSix years ago I came up with an algorithm for maintaining just the\nminimum/maximum in a sliding window and posted it to\ncs.stackexchange.\nI now consider this algorithm pointless, because it turns out there is a simple\nand efficient algorithm that solves this problem for a very wide class of\naggregations. I\u2019m writing this blog post to spread the word, because I feel it\nshould be more widely known.\nFolklore\nI came across this algorithm while reading a far more advanced paper, Low-Latency Sliding-Window\nAggregation in Worst-Case Constant Time by Tangwongsan et al. Why is this paper titled\nlow-latency? Because it does the same as what I\u2019m about to describe, but in O(1) time for each step.\nHowever, in it they also described a \u201ctwo-stack\u201d algorithm, which does it in amortized O(1), and\nis far, far simpler.\n\nFunnily enough that paper attributes this algorithm to \u201cadamax\u201d from a 2011 Stack Overflow\npost. They in turn credit a 2001 lecture\nnote by\nD. Sleator for the inspiration. However, this lecture note does not describe a sliding window\naggregate, it describes the classical two-stack algorithm for implementing a FIFO queue and does\namortized analysis on it. Ultimately I would not be surprised to find that this algorithm was\nalready described in an obscure paper from the 1970s, seeing how simple and brilliant it is.\nTwo stacks\nLike the authors of the paper, I will generalize the two-stack algorithm to arbitrary associative\naggregation functions. By abstracting the aggregation as a set of functions, empty(), unit(x),\ncombine(x, y) and finalize(x), you can describe many possible aggregations, for example a mean:\nempty = lambda: (0, 0)\nunit = lambda x: (x, 1)\ncombine = lambda x, y: (x[0] + y[0], x[1] + y[1])\nfinalize = lambda x: x[0] / x[1] if x[1] else None\nI\u2019d like to note here that these functions have the following signatures:\nfn empty() -> Agg;\nfn unit(x: Value) -> Agg;\nfn combine(x: Agg, y: Agg) -> Agg;\nfn finalize(x: Agg) -> Out;\nI\u2019m making a distinction here between Value, Agg and Out because while they seem superficially\nsimilar for something like an integer sum, for an approximate unique count on strings you would have\n(Value, Agg, Out) = (String, HyperLogLogSketch, u64), three wildly different types.\nWithout further ado, the algorithm:\nclass TwoStackAgg:\n    def __init__(self):\n        self.values = []\n        self.values_agg = empty()\n        self.cum_aggs = []\n    \n    def push(self, x):\n        self.values.append(x)\n        self.values_agg = combine(self.values_agg, unit(x))\n    \n    def pop(self):\n        if not self.cum_aggs:\n            cum_agg = empty()\n            while self.values:\n                cum_agg = combine(unit(self.values.pop()), cum_agg)\n                self.cum_aggs.append(cum_agg)\n            self.values_agg = empty()\n        self.cum_aggs.pop()\n    \n    def eval(self):\n        return finalize(\n            combine(self.cum_aggs[-1], self.values_agg)\n            if self.cum_aggs else self.values_agg\n        )\nThat\u2019s it, the entire algorithm. There\u2019s two stacks (values and cum_aggs)\nand one more aggregate, values_agg. At any point in time values_agg\nholds the aggregate of values, and cum_aggs contains the cumulative\naggregates of all values in our window that aren\u2019t in values, in reverse\norder. From this we can get the aggregate over our entire window in constant\ntime by by combining the last value of cum_aggs with values_agg.\nThe neat part is that (assuming w is our window size) every wth operation\nwe drain all of values and maintain a running aggregate while pushing the\npartial cumulative aggregates onto cum_aggs. This is what makes it amortized\nO(1), doing O(w) internal operations every wth pop bounds the total amount\nof work per element to O(1), even though a singular operation might not be\nconstant time.\nI think this is best visualized. Suppose we sum [1, 2, ..., 10] with a\nfixed-size sliding window of four elements, then the state on each eval() call\nwould look like this (values_agg not shown as it is simply the aggregate of\nthe values):\n             cum_aggs values        out\n                   [] []            = 0\n                   [] [1]           = 1\n                   [] [1, 2]        = 1 + 2\n                   [] [1, 2, 3]     = 1 + 2 + 3\n                   [] [1, 2, 3, 4]  = 1 + 2 + 3 + 4\n[4, 3 + 4, 2 + 3 + 4] [5]           = 2 + 3 + 4 + 5\n           [4, 3 + 4] [5, 6]        = 3 + 4 + 5 + 6\n                  [4] [5, 6, 7]     = 4 + 5 + 6 + 7\n                   [] [5, 6, 7, 8]  = 5 + 6 + 7 + 8\n[8, 7 + 8, 6 + 7 + 8] [9]           = 6 + 7 + 8 + 9\n           [8, 7 + 8] [9, 10]       = 7 + 8 + 9 + 10\n                  [8] [9, 10]       = 8 + 9 + 10\n                   [] [9, 10]       = 9 + 10\n                 [10] []            = 10\n                   [] []            = 0\nIn total the memory usage is O(w), where w is your maximum window size.\nNote that for simplicity of analysis and the example I assumed a fixed-size\nwindow w, but there is nothing about the two-stack algorithm that requires\nthis. You can call push(x) and pop() as many times as you\u2019d like between\neach eval(), growing and shrinking the window size as needed.\nFloating-point non-associativity\nNote that we required above that our aggregate combine is associative, meaning:\ncombine(combine(x, y), z) = combine(x, combine(y, z))\nTechnically speaking, floating-point addition doesn\u2019t respect this. Nevertheless,\nthe above algorithm is still very useful because the results closely match\nthe expected outcome, even more so if you use a compensated summation algorithm\nlike Kahan summation.\n\nHowever, there is a second very useful property of the above algorithm. Each aggregate\nis strictly a combination of the elements in the window, and none outside the window.\nThis means if your window contains a NaN or infinity (or some other outlier), that\nvalue only poisons the windows that contain it rather than the rest of your computation.\nBut even without NaN or infinity it is useful, due to not propagating errors\nendlessly. E.g. if your sliding window starts with [1e20, 1], this is what would\nhappen with a naive rolling sum:\n>>> 1e20 + 1 - 1e20 - 1\n-1.0\nCompensated summation will reduce these effects, but not making your result depend\non values outside of the window will eliminate long-term error accumulation entirely."
                ],
                "output": "readability",
                "pwd": "/data/archive/1792145128.357533",
                "schema": "ArchiveResult",
                "start_ts": "2026-10-16T13:41:47.142422+00:00",
                "status": "succeeded"
            },
            {
                "cmd": [
                    "/app/node_modules/readability-extractor/readability-extractor",
                    "/tmp/tmpisx2uejn",
                    "https://orlp.net/blog/two-stack-sliding-window-aggregation/"
                ],
                "cmd_version": "0.0.11",
                "end_ts": "2026-10-16T10:06:52.526564+00:00",
                "index_texts": [
                    "Two-Stack Sliding-Window Aggregation\n2026-10-02\nAn aggregation is some kind of summary of a set\nof data. This can be the sum, length, minimum, etc. It is quite common to want to calculate such a\nsummary repeatedly, e.g. \u201cthe maximum noise level in dB for the past 30 seconds\u201d for a nuisance detector.\nIn such a case we say there is a sliding window over our data, and we want to aggregate over our window.\nIf our aggregation is a binary operator with an inverse, like integer sums, there is a very easy\nsolution using a double-ended queue:\nfrom collections import deque\n\nclass SlidingWindowSum:\n    def __init__(self):\n        self.sum = 0\n        self.elems = deque()\n    \n    def push(self, x):\n        self.sum += x\n        self.elems.append(x)\n    \n    def pop(self):\n        self.sum -= self.elems.popleft()\n    \n    def eval(self):\n        return self.sum\nBut what if our operator has no inverse? This is actually the case for most interesting summaries\nsuch as minimum, quantile, approximate unique count (for example using HyperLogLog), etc. In fact, even something as simple as a floating-point sum suffers from the fact\nthat floating-point addition is not invertible. For example, if you ever have a NaN in your input\ndata with the above naive algorithm your sum will forever remain NaN, even long after the bad value\nhas left your window.\nSix years ago I came up with an algorithm for maintaining just the\nminimum/maximum in a sliding window and posted it to\ncs.stackexchange.\nI now consider this algorithm pointless, because it turns out there is a simple\nand efficient algorithm that solves this problem for a very wide class of\naggregations. I\u2019m writing this blog post to spread the word, because I feel it\nshould be more widely known.\nFolklore\nI came across this algorithm while reading a far more advanced paper, Low-Latency Sliding-Window\nAggregation in Worst-Case Constant Time by Tangwongsan et al. Why is this paper titled\nlow-latency? Because it does the same as what I\u2019m about to describe, but in O(1) time for each step.\nHowever, in it they also described a \u201ctwo-stack\u201d algorithm, which does it in amortized O(1), and\nis far, far simpler.\n\nFunnily enough that paper attributes this algorithm to \u201cadamax\u201d from a 2011 Stack Overflow\npost. They in turn credit a 2001 lecture\nnote by\nD. Sleator for the inspiration. However, this lecture note does not describe a sliding window\naggregate, it describes the classical two-stack algorithm for implementing a FIFO queue and does\namortized analysis on it. Ultimately I would not be surprised to find that this algorithm was\nalready described in an obscure paper from the 1970s, seeing how simple and brilliant it is.\nTwo stacks\nLike the authors of the paper, I will generalize the two-stack algorithm to arbitrary associative\naggregation functions. By abstracting the aggregation as a set of functions, empty(), unit(x),\ncombine(x, y) and finalize(x), you can describe many possible aggregations, for example a mean:\nempty = lambda: (0, 0)\nunit = lambda x: (x, 1)\ncombine = lambda x, y: (x[0] + y[0], x[1] + y[1])\nfinalize = lambda x: x[0] / x[1] if x[1] else None\nI\u2019d like to note here that these functions have the following signatures:\nfn empty() -> Agg;\nfn unit(x: Value) -> Agg;\nfn combine(x: Agg, y: Agg) -> Agg;\nfn finalize(x: Agg) -> Out;\nI\u2019m making a distinction here between Value, Agg and Out because while they seem superficially\nsimilar for something like an integer sum, for an approximate unique count on strings you would have\n(Value, Agg, Out) = (String, HyperLogLogSketch, u64), three wildly different types.\nWithout further ado, the algorithm:\nclass TwoStackAgg:\n    def __init__(self):\n        self.values = []\n        self.values_agg = empty()\n        self.cum_aggs = []\n    \n    def push(self, x):\n        self.values.append(x)\n        self.values_agg = combine(self.values_agg, unit(x))\n    \n    def pop(self):\n        if not self.cum_aggs:\n            cum_agg = empty()\n            while self.values:\n                cum_agg = combine(unit(self.values.pop()), cum_agg)\n                self.cum_aggs.append(cum_agg)\n            self.values_agg = empty()\n        self.cum_aggs.pop()\n    \n    def eval(self):\n        return finalize(\n            combine(self.cum_aggs[-1], self.values_agg)\n            if self.cum_aggs else self.values_agg\n        )\nThat\u2019s it, the entire algorithm. There\u2019s two stacks (values and cum_aggs)\nand one more aggregate, values_agg. At any point in time values_agg\nholds the aggregate of values, and cum_aggs contains the cumulative\naggregates of all values in our window that aren\u2019t in values, in reverse\norder. From this we can get the aggregate over our entire window in constant\ntime by by combining the last value of cum_aggs with values_agg.\nThe neat part is that (assuming w is our window size) every wth operation\nwe drain all of values and maintain a running aggregate while pushing the\npartial cumulative aggregates onto cum_aggs. This is what makes it amortized\nO(1), doing O(w) internal operations every wth pop bounds the total amount\nof work per element to O(1), even though a singular operation might not be\nconstant time.\nI think this is best visualized. Suppose we sum [1, 2, ..., 10] with a\nfixed-size sliding window of four elements, then the state on each eval() call\nwould look like this (values_agg not shown as it is simply the aggregate of\nthe values):\n             cum_aggs values        out\n                   [] []            = 0\n                   [] [1]           = 1\n                   [] [1, 2]        = 1 + 2\n                   [] [1, 2, 3]     = 1 + 2 + 3\n                   [] [1, 2, 3, 4]  = 1 + 2 + 3 + 4\n[4, 3 + 4, 2 + 3 + 4] [5]           = 2 + 3 + 4 + 5\n           [4, 3 + 4] [5, 6]        = 3 + 4 + 5 + 6\n                  [4] [5, 6, 7]     = 4 + 5 + 6 + 7\n                   [] [5, 6, 7, 8]  = 5 + 6 + 7 + 8\n[8, 7 + 8, 6 + 7 + 8] [9]           = 6 + 7 + 8 + 9\n           [8, 7 + 8] [9, 10]       = 7 + 8 + 9 + 10\n                  [8] [9, 10]       = 8 + 9 + 10\n                   [] [9, 10]       = 9 + 10\n                 [10] []            = 10\n                   [] []            = 0\nIn total the memory usage is O(w), where w is your maximum window size.\nNote that for simplicity of analysis and the example I assumed a fixed-size\nwindow w, but there is nothing about the two-stack algorithm that requires\nthis. You can call push(x) and pop() as many times as you\u2019d like between\neach eval(), growing and shrinking the window size as needed.\nFloating-point non-associativity\nNote that we required above that our aggregate combine is associative, meaning:\ncombine(combine(x, y), z) = combine(x, combine(y, z))\nTechnically speaking, floating-point addition doesn\u2019t respect this. Nevertheless,\nthe above algorithm is still very useful because the results closely match\nthe expected outcome, even more so if you use a compensated summation algorithm\nlike Kahan summation.\n\nHowever, there is a second very useful property of the above algorithm. Each aggregate\nis strictly a combination of the elements in the window, and none outside the window.\nThis means if your window contains a NaN or infinity (or some other outlier), that\nvalue only poisons the windows that contain it rather than the rest of your computation.\nBut even without NaN or infinity it is useful, due to not propagating errors\nendlessly. E.g. if your sliding window starts with [1e20, 1], this is what would\nhappen with a naive rolling sum:\n>>> 1e20 + 1 - 1e20 - 1\n-1.0\nCompensated summation will reduce these effects, but not making your result depend\non values outside of the window will eliminate long-term error accumulation entirely."
                ],
                "output": "readability",
                "pwd": "/data/archive/1792145128.357533",
                "schema": "ArchiveResult",
                "start_ts": "2026-10-16T10:06:50.899343+00:00",
                "status": "succeeded"
            },
            {
                "cmd": [
                    "/app/node_modules/readability-extractor/readability-extractor",
                    "/tmp/tmppjno0tc8",
                    "https://orlp.net/blog/two-stack-sliding-window-aggregation/"
                ],
                "cmd_version": "0.0.11",
                "end_ts": "2026-10-16T16:26:15.148757+00:00",
                "index_texts": [
                    "Two-Stack Sliding-Window Aggregation\n2026-10-02\nAn aggregation is some kind of summary of a set\nof data. This can be the sum, length, minimum, etc. It is quite common to want to calculate such a\nsummary repeatedly, e.g. \u201cthe maximum noise level in dB for the past 30 seconds\u201d for a nuisance detector.\nIn such a case we say there is a sliding window over our data, and we want to aggregate over our window.\nIf our aggregation is a binary operator with an inverse, like integer sums, there is a very easy\nsolution using a double-ended queue:\nfrom collections import deque\n\nclass SlidingWindowSum:\n    def __init__(self):\n        self.sum = 0\n        self.elems = deque()\n    \n    def push(self, x):\n        self.sum += x\n        self.elems.append(x)\n    \n    def pop(self):\n        self.sum -= self.elems.popleft()\n    \n    def eval(self):\n        return self.sum\nBut what if our operator has no inverse? This is actually the case for most interesting summaries\nsuch as minimum, quantile, approximate unique count (for example using HyperLogLog), etc. In fact, even something as simple as a floating-point sum suffers from the fact\nthat floating-point addition is not invertible. For example, if you ever have a NaN in your input\ndata with the above naive algorithm your sum will forever remain NaN, even long after the bad value\nhas left your window.\nSix years ago I came up with an algorithm for maintaining just the\nminimum/maximum in a sliding window and posted it to\ncs.stackexchange.\nI now consider this algorithm pointless, because it turns out there is a simple\nand efficient algorithm that solves this problem for a very wide class of\naggregations. I\u2019m writing this blog post to spread the word, because I feel it\nshould be more widely known.\nFolklore\nI came across this algorithm while reading a far more advanced paper, Low-Latency Sliding-Window\nAggregation in Worst-Case Constant Time by Tangwongsan et al. Why is this paper titled\nlow-latency? Because it does the same as what I\u2019m about to describe, but in O(1) time for each step.\nHowever, in it they also described a \u201ctwo-stack\u201d algorithm, which does it in amortized O(1), and\nis far, far simpler.\n\nFunnily enough that paper attributes this algorithm to \u201cadamax\u201d from a 2011 Stack Overflow\npost. They in turn credit a 2001 lecture\nnote by\nD. Sleator for the inspiration. However, this lecture note does not describe a sliding window\naggregate, it describes the classical two-stack algorithm for implementing a FIFO queue and does\namortized analysis on it. Ultimately I would not be surprised to find that this algorithm was\nalready described in an obscure paper from the 1970s, seeing how simple and brilliant it is.\nTwo stacks\nLike the authors of the paper, I will generalize the two-stack algorithm to arbitrary associative\naggregation functions. By abstracting the aggregation as a set of functions, empty(), unit(x),\ncombine(x, y) and finalize(x), you can describe many possible aggregations, for example a mean:\nempty = lambda: (0, 0)\nunit = lambda x: (x, 1)\ncombine = lambda x, y: (x[0] + y[0], x[1] + y[1])\nfinalize = lambda x: x[0] / x[1] if x[1] else None\nI\u2019d like to note here that these functions have the following signatures:\nfn empty() -> Agg;\nfn unit(x: Value) -> Agg;\nfn combine(x: Agg, y: Agg) -> Agg;\nfn finalize(x: Agg) -> Out;\nI\u2019m making a distinction here between Value, Agg and Out because while they seem superficially\nsimilar for something like an integer sum, for an approximate unique count on strings you would have\n(Value, Agg, Out) = (String, HyperLogLogSketch, u64), three wildly different types.\nWithout further ado, the algorithm:\nclass TwoStackAgg:\n    def __init__(self):\n        self.values = []\n        self.values_agg = empty()\n        self.cum_aggs = []\n    \n    def push(self, x):\n        self.values.append(x)\n        self.values_agg = combine(self.values_agg, unit(x))\n    \n    def pop(self):\n        if not self.cum_aggs:\n            cum_agg = empty()\n            while self.values:\n                cum_agg = combine(unit(self.values.pop()), cum_agg)\n                self.cum_aggs.append(cum_agg)\n            self.values_agg = empty()\n        self.cum_aggs.pop()\n    \n    def eval(self):\n        return finalize(\n            combine(self.cum_aggs[-1], self.values_agg)\n            if self.cum_aggs else self.values_agg\n        )\nThat\u2019s it, the entire algorithm. There\u2019s two stacks (values and cum_aggs)\nand one more aggregate, values_agg. At any point in time values_agg\nholds the aggregate of values, and cum_aggs contains the cumulative\naggregates of all values in our window that aren\u2019t in values, in reverse\norder. From this we can get the aggregate over our entire window in constant\ntime by by combining the last value of cum_aggs with values_agg.\nThe neat part is that (assuming w is our window size) every wth operation\nwe drain all of values and maintain a running aggregate while pushing the\npartial cumulative aggregates onto cum_aggs. This is what makes it amortized\nO(1), doing O(w) internal operations every wth pop bounds the total amount\nof work per element to O(1), even though a singular operation might not be\nconstant time.\nI think this is best visualized. Suppose we sum [1, 2, ..., 10] with a\nfixed-size sliding window of four elements, then the state on each eval() call\nwould look like this (values_agg not shown as it is simply the aggregate of\nthe values):\n             cum_aggs values        out\n                   [] []            = 0\n                   [] [1]           = 1\n                   [] [1, 2]        = 1 + 2\n                   [] [1, 2, 3]     = 1 + 2 + 3\n                   [] [1, 2, 3, 4]  = 1 + 2 + 3 + 4\n[4, 3 + 4, 2 + 3 + 4] [5]           = 2 + 3 + 4 + 5\n           [4, 3 + 4] [5, 6]        = 3 + 4 + 5 + 6\n                  [4] [5, 6, 7]     = 4 + 5 + 6 + 7\n                   [] [5, 6, 7, 8]  = 5 + 6 + 7 + 8\n[8, 7 + 8, 6 + 7 + 8] [9]           = 6 + 7 + 8 + 9\n           [8, 7 + 8] [9, 10]       = 7 + 8 + 9 + 10\n                  [8] [9, 10]       = 8 + 9 + 10\n                   [] [9, 10]       = 9 + 10\n                 [10] []            = 10\n                   [] []            = 0\nIn total the memory usage is O(w), where w is your maximum window size.\nNote that for simplicity of analysis and the example I assumed a fixed-size\nwindow w, but there is nothing about the two-stack algorithm that requires\nthis. You can call push(x) and pop() as many times as you\u2019d like between\neach eval(), growing and shrinking the window size as needed.\nFloating-point non-associativity\nNote that we required above that our aggregate combine is associative, meaning:\ncombine(combine(x, y), z) = combine(x, combine(y, z))\nTechnically speaking, floating-point addition doesn\u2019t respect this. Nevertheless,\nthe above algorithm is still very useful because the results closely match\nthe expected outcome, even more so if you use a compensated summation algorithm\nlike Kahan summation.\n\nHowever, there is a second very useful property of the above algorithm. Each aggregate\nis strictly a combination of the elements in the window, and none outside the window.\nThis means if your window contains a NaN or infinity (or some other outlier), that\nvalue only poisons the windows that contain it rather than the rest of your computation.\nBut even without NaN or infinity it is useful, due to not propagating errors\nendlessly. E.g. if your sliding window starts with [1e20, 1], this is what would\nhappen with a naive rolling sum:\n>>> 1e20 + 1 - 1e20 - 1\n-1.0\nCompensated summation will reduce these effects, but not making your result depend\non values outside of the window will eliminate long-term error accumulation entirely."
                ],
                "output": "readability",
                "pwd": "/data/archive/1792145128.357533",
                "schema": "ArchiveResult",
                "start_ts": "2026-10-16T16:26:13.364630+00:00",
                "status": "succeeded"
            }
        ],
        "screenshot": [],
        "singlefile": [],
        "title": [
            {
                "cmd": [
                    "curl",
                    "--silent",
                    "--location",
                    "--compressed",
                    "--max-time",
                    "120",
                    "--user-agent",
                    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36 ArchiveBox/0.7.4 (+https://github.com/ArchiveBox/ArchiveBox/) curl/curl 8.14.1 (x86_64-pc-linux-gnu)",
                    "https://orlp.net/blog/two-stack-sliding-window-aggregation/"
                ],
                "cmd_version": "curl 8.14.1 (x86_64-pc-linux-gnu)",
                "end_ts": "2026-10-16T13:41:47.135462+00:00",
                "index_texts": null,
                "output": "Two-Stack Sliding-Window Aggregation | orlp.net",
                "pwd": "/data/archive/1792145128.357533",
                "schema": "ArchiveResult",
                "start_ts": "2026-10-16T13:41:47.117008+00:00",
                "status": "succeeded"
            },
            {
                "cmd": [
                    "curl",
                    "--silent",
                    "--location",
                    "--compressed",
                    "--max-time",
                    "120",
                    "--user-agent",
                    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36 ArchiveBox/0.7.4 (+https://github.com/ArchiveBox/ArchiveBox/) curl/curl 8.14.1 (x86_64-pc-linux-gnu)",
                    "https://orlp.net/blog/two-stack-sliding-window-aggregation/"
                ],
                "cmd_version": "curl 8.14.1 (x86_64-pc-linux-gnu)",
                "end_ts": "2026-10-16T10:06:50.892153+00:00",
                "index_texts": null,
                "output": "Two-Stack Sliding-Window Aggregation | orlp.net",
                "pwd": "/data/archive/1792145128.357533",
                "schema": "ArchiveResult",
                "start_ts": "2026-10-16T10:06:50.874624+00:00",
                "status": "succeeded"
            },
            {
                "cmd": [
                    "curl",
                    "--silent",
                    "--location",
                    "--compressed",
                    "--max-time",
                    "120",
                    "--user-agent",
                    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36 ArchiveBox/0.7.4 (+https://github.com/ArchiveBox/ArchiveBox/) curl/curl 8.14.1 (x86_64-pc-linux-gnu)",
                    "https://orlp.net/blog/two-stack-sliding-window-aggregation/"
                ],
                "cmd_version": "curl 8.14.1 (x86_64-pc-linux-gnu)",
                "end_ts": "2026-10-16T16:26:13.357445+00:00",
                "index_texts": null,
                "output": "Two-Stack Sliding-Window Aggregation | orlp.net",
                "pwd": "/data/archive/1792145128.357533",
                "schema": "ArchiveResult",
                "start_ts": "2026-10-16T16:26:13.338328+00:00",
                "status": "succeeded"
            }
        ],
        "wget": [
            {
                "cmd": [
                    "wget",
                    "--no-verbose",
                    "--adjust-extension",
                    "--convert-links",
                    "--force-directories",
                    "--span-hosts",
                    "--no-parent",
                    "-e",
                    "robots=off",
                    "--reject=js,mjs,woff,woff2,ttf,otf,eot,mp4,m4v,mov,webm,ogv,mkv,avi,flv",
                    "--timeout=120",
                    "--restrict-file-names=windows",
                    "--page-requisites",
                    "--user-agent=Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36 ArchiveBox/0.7.4 (+https://github.com/ArchiveBox/ArchiveBox/) wget/GNU Wget 1.21.3",
                    "--compression=auto",
                    "--timestamping",
                    "https://orlp.net/blog/two-stack-sliding-window-aggregation/"
                ],
                "cmd_version": "GNU Wget 1.21.3",
                "end_ts": "2026-10-16T13:41:47.107937+00:00",
                "index_texts": null,
                "output": "ArchiveError: 404 Not Found",
                "pwd": "/data/archive/1792145128.357533",
                "schema": "ArchiveResult",
                "start_ts": "2026-10-16T13:41:45.835782+00:00",
                "status": "failed"
            },
            {
                "cmd": [
                    "wget",
                    "--no-verbose",
                    "--adjust-extension",
                    "--convert-links",
                    "--force-directories",
                    "--span-hosts",
                    "--no-parent",
                    "-e",
                    "robots=off",
                    "--reject=js,mjs,woff,woff2,ttf,otf,eot,mp4,m4v,mov,webm,ogv,mkv,avi,flv",
                    "--timeout=120",
                    "--restrict-file-names=windows",
                    "--page-requisites",
                    "--user-agent=Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36 ArchiveBox/0.7.4 (+https://github.com/ArchiveBox/ArchiveBox/) wget/GNU Wget 1.21.3",
                    "--compression=auto",
                    "--timestamping",
                    "https://orlp.net/blog/two-stack-sliding-window-aggregation/"
                ],
                "cmd_version": "GNU Wget 1.21.3",
                "end_ts": "2026-10-16T10:06:50.862957+00:00",
                "index_texts": null,
                "output": "orlp.net/blog/two-stack-sliding-window-aggregation/index.html",
                "pwd": "/data/archive/1792145128.357533",
                "schema": "ArchiveResult",
                "start_ts": "2026-10-16T10:06:49.930481+00:00",
                "status": "succeeded"
            },
            {
                "cmd": [
                    "wget",
                    "--no-verbose",
                    "--adjust-extension",
                    "--convert-links",
                    "--force-directories",
                    "--span-hosts",
                    "--no-parent",
                    "-e",
                    "robots=off",
                    "--reject=js,mjs,woff,woff2,ttf,otf,eot,mp4,m4v,mov,webm,ogv,mkv,avi,flv",
                    "--timeout=120",
                    "--restrict-file-names=windows",
                    "--page-requisites",
                    "--user-agent=Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36 ArchiveBox/0.7.4 (+https://github.com/ArchiveBox/ArchiveBox/) wget/GNU Wget 1.21.3",
                    "--compression=auto",
                    "--timestamping",
                    "https://orlp.net/blog/two-stack-sliding-window-aggregation/"
                ],
                "cmd_version": "GNU Wget 1.21.3",
                "end_ts": "2026-10-16T16:26:13.322120+00:00",
                "index_texts": null,
                "output": "ArchiveError: 404 Not Found",
                "pwd": "/data/archive/1792145128.357533",
                "schema": "ArchiveResult",
                "start_ts": "2026-10-16T16:26:12.053791+00:00",
                "status": "failed"
            }
        ]
    },
    "icons": null,
    "is_archived": true,
    "is_static": false,
    "latest": {
        "archive_org": null,
        "dom": null,
        "favicon": "favicon.ico",
        "git": null,
        "media": null,
        "pdf": null,
        "screenshot": null,
        "singlefile": null,
        "title": "Two-Stack Sliding-Window Aggregation | orlp.net",
        "warc": null,
        "wget": "ArchiveError: 404 Not Found"
    },
    "link_dir": "/data/archive/1792145128.357533",
    "newest_archive_date": "2026-10-16T16:26:15.162502+00:00",
    "num_failures": 2,
    "num_outputs": 16,
    "oldest_archive_date": "2026-10-16T10:06:48.856423+00:00",
    "path": "/blog/two-stack-sliding-window-aggregation/",
    "schema": "Link",
    "scheme": "https",
    "snapshot_id": "deb9e0ff-7933-4e80-b11a-48588b7ad8a1",
    "sources": [
        "/data/sources/1792145128-import.txt"
    ],
    "tags": ",war-wiki",
    "tags_str": "war-wiki",
    "timestamp": "1792145128.357533",
    "title": "Two-Stack Sliding-Window Aggregation | orlp.net",
    "updated": "2026-10-16T16:26:10.996858+00:00",
    "updated_date": "2026-10-16 16:26",
    "url": "https://orlp.net/blog/two-stack-sliding-window-aggregation/"
}