{ "archive_path": "archive/1789337059.131441", "base_url": "pages.cs.wisc.edu/~plonka/netgear-sntp", "basename": "", "bookmarked_date": "2026-09-13 22:04", "canonical": { "archive_org_path": "https://web.archive.org/web/pages.cs.wisc.edu/~plonka/netgear-sntp", "dom_path": "output.html", "favicon_path": "favicon.ico", "git_path": "git/", "google_favicon_path": "https://www.google.com/s2/favicons?domain=pages.cs.wisc.edu", "headers_path": "headers.json", "htmltotext_path": "htmltotext.txt", "index_path": "index.html", "media_path": "media/", "mercury_path": "mercury/content.html", "pdf_path": "output.pdf", "readability_path": "readability/content.html", "screenshot_path": "screenshot.png", "singlefile_path": "singlefile.html", "warc_path": "warc/", "wget_path": "pages.cs.wisc.edu/~plonka/netgear-sntp/index.html" }, "domain": "pages.cs.wisc.edu", "extension": "", "hash": "BGBZF45BN6ZBNZGKPCT0", "history": { "archive_org": [], "dom": [], "favicon": [ { "cmd": [ "curl", "--silent", "--location", "--compressed", "--max-time", "120", "--output", "favicon.ico", "--user-agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36 ArchiveBox/0.7.4 (+https://github.com/ArchiveBox/ArchiveBox/) curl/curl 8.14.1 (x86_64-pc-linux-gnu)", "https://www.google.com/s2/favicons?domain=pages.cs.wisc.edu" ], "cmd_version": "curl 8.14.1 (x86_64-pc-linux-gnu)", "end_ts": "2026-09-13T22:05:33.166734+00:00", "index_texts": null, "output": "favicon.ico", "pwd": "/data/archive/1789337059.131441", "schema": "ArchiveResult", "start_ts": "2026-09-13T22:05:32.580228+00:00", "status": "succeeded" } ], "git": [], "headers": [ { "cmd": [ "curl", "--silent", "--location", "--compressed", "--head", "--max-time", "120", "--user-agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36 ArchiveBox/0.7.4 (+https://github.com/ArchiveBox/ArchiveBox/) curl/curl 8.14.1 (x86_64-pc-linux-gnu)", "https://pages.cs.wisc.edu/~plonka/netgear-sntp/" ], "cmd_version": "curl 8.14.1 (x86_64-pc-linux-gnu)", "end_ts": "2026-09-13T22:05:33.726355+00:00", "index_texts": null, "output": "headers.json", "pwd": "/data/archive/1789337059.131441", "schema": "ArchiveResult", "start_ts": "2026-09-13T22:05:33.177902+00:00", "status": "succeeded" } ], "htmltotext": [], "media": [], "mercury": [ { "cmd": [ "/app/node_modules/@postlight/parser/cli.js", "https://pages.cs.wisc.edu/~plonka/netgear-sntp/" ], "cmd_version": "1.0.0", "end_ts": "2026-09-13T22:05:40.169773+00:00", "index_texts": null, "output": "mercury", "pwd": "/data/archive/1789337059.131441", "schema": "ArchiveResult", "start_ts": "2026-09-13T22:05:36.968488+00:00", "status": "succeeded" } ], "pdf": [], "readability": [ { "cmd": [ "/app/node_modules/readability-extractor/readability-extractor", "/tmp/tmp2yc76emq", "https://pages.cs.wisc.edu/~plonka/netgear-sntp/" ], "cmd_version": "0.0.11", "end_ts": "2026-09-13T22:05:36.958701+00:00", "index_texts": [ "ABSTRACT\n\n\n\n In May 2003, the University of Wisconsin - Madison found that it was\n the recipient of a continuous large scale flood of inbound Internet\n traffic destined for one of the campus' public Network Time Protocol\n (NTP) servers. The flood traffic rate was hundreds-of-thousands\n of packets-per-second, and hundreds of megabits-per-second.\n\n Subsequently, we have determined the sources of this flooding to be\n literally hundreds of thousands of real Internet hosts throughout\n the world. However, rather than having originated as a malicious\n distributed denial-of-service (DDoS) attack, the root cause is\n actually a serious flaw in the design of hundreds of thousands of\n one vendor's low-cost Internet products targeted for residential\n use. The unexpected behavior of these products presents a\n significant operational problem for UW-Madison for years to come.\n\n This document includes the initial public disclosure of details of\n these products' serious design flaw. Furthermore, it discusses our\n ongoing, multifaceted approach toward the solution which involves\n the University, the products' manufacturer, the relevant Internet\n standards (RFCs), and the public Internet service and user\n communities.\n\n\n\nTable of Contents\nFlawed Routers Flood University of Wisconsin Internet Time Server\n Netgear Cooperating with University on a Resolution\n The Initial Flood\n Figure 1. The Initial Flood\n Blocking the Flood\n Background: Simple Network Time Protocol (SNTP)\n Figure 2. A SNTP Request Packet\n Figure 3. A Unicast SNTP Reply Packet\n The Flood Continues\n Figure 4. The Flood Continues: One Month Later\n Investigation\n Contacting Source Networks\n Figure 5. Email Notification to Peer Institution\n Gathering Background Information\n Examining the Netgear Code\n Contacting Netgear\n Figure 6. Email to Netgear Support\n Figure 7. Email from Netgear Support\n The Review Process\n The Flawed SNTP Client\n Impact to Netgear Customers\n Code Upgrades for Affected Netgear Products\n Figure 8. Affected Netgear Products\n Flawed Product Counts\n Figure 8a. Netgear SNTP Clients Per Day\n Suggested Fixes\n The Initial Fix: \"Instant\" Code\n Network Operational Options: To Serve or To Sever?\n Endgame A: UW-Madison Netgear Anycast Time Service\n Figure 10. A WiscNet BGP-based Anycast Time Service\n Endgame B: Attempt to Suppress the Requests\n Figure 11. Using the Global BGP Routing Table to Squelch Requests\n Endgame B: IP Resources Required\n Figure 12. IP Resources Required for BGP-based Suppression\n Inform the Internet Community\n Clarify Internet Best Current Practice and Protocol Standards\n Status, August 21, 2003\n Figure 13. The Most Recent Flood\n Afterthoughts\n Acknowledgements\n Analysis Tools\n References / Further Reading\n Frequently Asked Questions\n What is Netgear's liability for causing (however inadvertently) this denial of service for your network?\n Have you considered putting up a server which sends back fake answers to netgear clients, to cause people to upgrade?\n What is the expected life-time for these products?\n In figure 13, could that \"shark fin\" spike have anything to do with last week's power grid failure (Blackout 2003), and subsequent \"rolling\" restoration?\n Are there other devices than those mentioned which also suffer from the flaw which causes inadvertent flooding of your network?\n What was the effect of this article being slashdotted?\n Why is a traditional manufacturer recall/defect solution not a possibility?\n I'm with [the IT press], do you have some time to speak with me?\n How has this story been covered in the press?\n\n\n\nThe Initial Flood\n\nFigure 1 is a graph of inbound traffic to our campus over a 48 hour period,\ntuesday through thursday, May 13-15, 2003.\n\n\n\n\nThe first half of the graph shows typical traffic levels for our campus,\nwith peak inbound packet rates\nof about 40,000 packets-per-second. However, as you can see, our inbound\npacket-per-second rate increased dramatically\nstarting May 14 at about 8AM localtime, primarily from our commodity Internet\nService Provider, WiscNet.\nAt about 9:40AM this additional traffic began to cause problems with our\nmeasurement infrastructure and some of our legacy intra-campus routers.\nBy 11AM we had identified the inbound flood traffic by protocol and port\nnumbers. It was destined for our public time server and we\nblocked the incoming traffic upstream, at WiscNet's border routers, which\nalleviated the problem for the time being. This is a typical action for\nnetwork operators to take in reaction to malicious Denial-of-Service flood\nattacks, of which we assumed this was one.\n\n\nBlocking the Flood\n\nThe traffic in question appeared to be Network Time Protocol (NTP) queries\nin that they consisted of 76-byte IP packets destined for UDP port number 123\n(NTP).\nHowever, these packets had an unusual characteristic: although they appeared\nto come from many sources, they all had the same source port number: 23457.\nTherefore, it was possible to configure our routers to block just a subset\nof inbound queries to our NTP server, and continue to service the other\nlegitimate requests normally. We just blocked all UDP traffic sourced\nfrom port 23457 and destined for port 123 (NTP) of the NTP server in\nquestion. (Note that the number 23457 seems hand-picked, as the number\nsubsequent to 23456.) At this point we simply chalked it up to\nnaivete on the part of the \"attacker\", which we presumed was forging\nmany random source addresses, and left it at that, presuming that the\nflood would subside within hours as \"script kiddie\"-launched flood\nattacks often do.\n\n\nBackground: Simple Network Time Protocol (SNTP)\n\nParaphrased from RFC2030 by Dave Mills:\n\nThe Simple Network Time Protocol (SNTP) is an adaptation of the Network\nTime Protocol (NTP) used to synchronize computer clocks in the\nInternet. It is a simple, stateless remote-procedure call (RPC) system\nwith accuracy and reliability expectations similar to the UDP/TIME\nprotocol described in RFC-868. SNTP can be used when the ultimate\nperformance of the full NTP implementation is not necessary.\n\n\nNote that SNTP uses the same packet format as NTP.\nIn this way, SNTP clients can utilize NTP servers, even though they do\nnot implement the complexities of the full peer-to-peer NTP protocol.\n\nSNTP conversations typically follow these steps:\n\n\n\nA client that would like to know the time sends a UDP packet containing\nthe SNTP request to the well-known NTP port number 123 of an NTP\nserver, and awaits a reply.\n\n\nFigure 2. A SNTP Request Packet\n\n\n+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+\n|LI | VN |Mode | Stratum | Poll | Precision |\n| =0|= 1-4|= 3 | = 0 | = 0 | = 0 |\n+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+\n| Root Delay |\n| = 0 |\n+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+\n| Root Dispersion |\n| = 0 |\n+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+\n| Reference Identifier |\n| = 0 |\n+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+\n| |\n| Reference Timestamp (64 bits) |\n| = 0 |\n| |\n+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+\n| |\n| Originate Timestamp (64 bits) |\n| = 0 |\n| |\n+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+\n| |\n| Receive Timestamp (64 bits) |\n| = 0 |\n| |\n+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+\n| |\n| Transmit Timestamp (64 bits) |\n| = n |\n| (some number: zero, or the time of request sent by client) |\n| |\n+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+\n\n\n\n\n\n\n\n\nThe server responds with a UDP packet containing the SNTP reply from\nthe well-known NTP port number 123 to the SNTP client.\n\n\nFigure 3. A Unicast SNTP Reply Packet\n\n\n+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+\n|LI=| VN |Mode | Stratum | Poll | Precision |\n|0-2|=req.|= 4 | = 1 - 14 | (ignore) | (ignore) |\n+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+\n| Root Delay |\n| (ignore) |\n+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+\n| Root Dispersion |\n| (ignore) |\n+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+\n| Reference Identifier |\n| (ignore) |\n+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+\n| |\n| Reference Timestamp (64 bits) |\n| (ignore) |\n| |\n+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+\n| |\n| Originate Timestamp (64 bits) |\n| (copied from request Transmit Timestamp) |\n| |\n+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+\n| |\n| Receive Timestamp (64 bits) |\n| (time request was received by server) |\n| |\n+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+\n| |\n| Transmit Timestamp (64 bits) |\n| (time of reply sent by server) |\n| = n |\n| |\n+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+\n\n\n\n\n\n\n\n\nUpon receiving the response, the client optionally uses the Originate\nTimestamp from the reply to validate the response, attempting to assure\nthat it is indeed a response to this client's request. (If the reply\nwere spoofed from another source, it would be unlikely to contain the\ncorrect value as the Originate Timestamp). Then it plucks the value\nfrom the \"Transmit Timestamp\", perhaps modifying it slightly to\naccount for the estimated one-way end-to-end delay, and uses the\nresult as the current time to set its local clock.\n\n\n\nNow, back to our story...\n\nThe Flood Continues\n\nOne month later, we discovered that the flood of inbound NTP traffic\npersisted at an even more incredibly high rate, as evidenced by Figure 4\nwhich plots our router's discarded packet rates beginning in early June\n2003, for the traffic in question.\n\nIn Figure 4, note that: (1) there are slight daily fluctuations in rate\n(perhaps due to diurnal user behavior), (2) generally, the rate stays at\na value over 250,000 packets-per-second (and over 150\nmegabits-per-second), and (3) that the traffic rate increases\nthroughout the time shown. The sharp drops in traffic rate in this\nfigure are not due to the flood subsiding but rather were due to\nnetwork maintenance and a temporary block upstream from the observation\npoint.\n\n\nOnce we found that this flood was continuing and was still increasing in\nrate, we investigated further. By carefully removing the block on some\ningress interfaces, we allowed a trickle of traffic through to the\nserver and captured the packets including their payload. We learned\nthat these packets appeared to be legitimate, well-formed Simple\nNetwork Time Protocol (SNTP) version 1 queries, albeit at an\ninexplicably high rate from each client host. For instance, during one\ntrace, many clients produced about one query per second. This would be\nhighly unusual for a properly constructed SNTP client, since an\napplication which uses SNTP is merely interested in setting its own\nclock relatively accurately so that its host has some reasonable\nnotion of the current time. One query per second is ridiculous, and is\nfar from best practice for NTP client behavior.\n\n\nAlso, we discovered that many of the IP addresses could be resolved to\nDNS names and furthermore that the IP addresses all appeared to be\nvalid sources for the given ingress interface from which we removed the\nblock. This indicated that it was quite possible that the source\naddresses were not forged but instead were real Internet hosts\nrunning some very unusual SNTP client.\n\n\nAlas, none of the client source hosts were within our local campus\nnetwork. This meant we would need to recruit the help of staff at\nremote sites to aid in the investigation.\n\n\nInvestigation\nContacting Source Networks\n\nOf the top talker source IP addresses from the aforementioned packet trace,\nI selected two client hosts from other universities with talented\nnetwork staff who would be familiar with responding to such incidents.\n\n\nThe following is an email I sent to the Incident Response Team at one of\nthose institutions to which one of the client host addresses belonged.\n(To maintain a modicum of anonymity, I have replaced the real SNTP client's\nIP address with 10.42.69.10 and have also removed the email domain names.)\n\n\n\nFigure 5. Email Notification to Peer Institution\n\n\nDate: Sat, 14 Jun 2003 04:34:11 -0500\nFrom: Dave Plonka \nTo: abuse@[remotedomain]\nSubject: sntp/ntp query flood from 10.42.69.10 to ntp1.cs.wisc.edu\n\n\n[Organization] network abuse folks,\n\nSince May 14, 2003 ~0800 central time, one of our campus' NTP\nservers \"ntp1.cs.wisc.edu\" (128.105.39.11) has been the recipient of a\nlarge-scale flood of Simple Network Time Protocol (SNTP) requests -\nmuch more than it can service. This dramatic increase in inbound SNTP\nrequests inexplicably continues even now. To mitigate this flood we\nare currently blocking over over 250K pkts/sec, exceeding 150\nmegabits/sec, and it has been continuing for weeks.\n\nWe are in the process of trying to determine if this flood is potentially\nmalicious or if it is an SNTP client misconfiguration or bug.\n\nThis traffic primarily consists of 76-byte UDP packets that are SNTP\nversion 1 queries from very many source host addresses directed to\nntp1.cs.wisc.edu port 123 (NTP). Unusually, these requests all have a\nUDP source port of 23457. We have identified the host address\n10.42.69.10 as just one of the sources. (However, there are at\nleast tens of thousands of source host addresses.)\n\nI have attached a timestamped log of a packet capture from the\nafternoon of June 13, 2003 (Friday) evidencing the SNTP query packets\nfrom the host 10.42.69.10 at an unusually high rate of about one\nper second. A packet decomposition (by tethereal) and hex dump of the\nlast packet (frame 998) in the log is included as well, which shows\nthem to be valid SNTP v1 queries as described in RFC 1361,\nhttp://www.ietf.org/rfc/rfc1361.txt.\n\nCould you assist us ASAP with this investigation by identifying that\nhost's operating system and what SNTP client code may be running on\nthat host? It would be interesting to know if a process on\n10.42.69.10 currently has UDP port 23457 bound and what code that\nprocess is running.\n\nThanks,\nDave\n\nP.S. Our investigation so far has shown that Windows systems such as\n2000 and XP have an \"Internet Time\" feature which is usually configured\nto send SNTP requests to the Microsoft server \"time.windows.com\", but\nthis server can be changed. I have yet to identify any SNTP client\nthat regularly uses UDP port 23457 as its source port. (Note that\nport number seems hand-picked, as the number subsequent to 23456.)\n\n----------------------------------------------------------------------\n\n 1 2003-06-13 16:32:24.8808 10.42.69.10 -> 128.105.39.11 NTP NTP\n 7 2003-06-13 16:32:25.9611 10.42.69.10 -> 128.105.39.11 NTP NTP\n 14 2003-06-13 16:32:27.0412 10.42.69.10 -> 128.105.39.11 NTP NTP\n 21 2003-06-13 16:32:28.1215 10.42.69.10 -> 128.105.39.11 NTP NTP\n 27 2003-06-13 16:32:29.2020 10.42.69.10 -> 128.105.39.11 NTP NTP\n 33 2003-06-13 16:32:30.2821 10.42.69.10 -> 128.105.39.11 NTP NTP\n 39 2003-06-13 16:32:31.3624 10.42.69.10 -> 128.105.39.11 NTP NTP\n 45 2003-06-13 16:32:32.4427 10.42.69.10 -> 128.105.39.11 NTP NTP\n 51 2003-06-13 16:32:33.5232 10.42.69.10 -> 128.105.39.11 NTP NTP\n 56 2003-06-13 16:32:34.6049 10.42.69.10 -> 128.105.39.11 NTP NTP\n 68 2003-06-13 16:32:36.7638 10.42.69.10 -> 128.105.39.11 NTP NTP\n 74 2003-06-13 16:32:37.8441 10.42.69.10 -> 128.105.39.11 NTP NTP\n 78 2003-06-13 16:32:38.9242 10.42.69.10 -> 128.105.39.11 NTP NTP\n 84 2003-06-13 16:32:40.0050 10.42.69.10 -> 128.105.39.11 NTP NTP\n 90 2003-06-13 16:32:41.0846 10.42.69.10 -> 128.105.39.11 NTP NTP\n 96 2003-06-13 16:32:42.1647 10.42.69.10 -> 128.105.39.11 NTP NTP\n\n998 2003-06-13 16:35:13.3789 10.42.69.10 -> 128.105.39.11 NTP NTP\n\nFrame 998 (90 on wire, 90 captured)\n Arrival Time: Jun 13, 2003 16:35:13.378978000\n Time delta from previous packet: 0.524605000 seconds\n Time relative to first packet: 168.498125000 seconds\n Frame Number: 998\n Packet Length: 90 bytes\n Capture Length: 90 bytes\nEthernet II\n Destination: 00:0a:41:db:58:00 (00:0a:41:db:58:00)\n Source: 00:0a:8b:bf:70:7c (00:0a:8b:bf:70:7c)\n Type: IP (0x0800)\nInternet Protocol, Src Addr: 10.42.69.10 (10.42.69.10), Dst Addr: 128.105.39.11 (128.105.39.11)\n Version: 4\n Header length: 20 bytes\n Differentiated Services Field: 0x00 (DSCP 0x00: Default; ECN: 0x00)\n 0000 00.. = Differentiated Services Codepoint: Default (0x00)\n .... ..0. = ECN-Capable Transport (ECT): 0\n .... ...0 = ECN-CE: 0\n Total Length: 76\n Identification: 0x2cc7\n Flags: 0x00\n .0.. = Don't fragment: Not set\n ..0. = More fragments: Not set\n Fragment offset: 0\n Time to live: 243\n Protocol: UDP (0x11)\n Header checksum: 0xb335 (correct)\n Source: 10.42.69.10 (10.42.69.10)\n Destination: 128.105.39.11 (128.105.39.11)\nUser Datagram Protocol, Src Port: 23457 (23457), Dst Port: 123 (123)\n Source port: 23457 (23457)\n Destination port: 123 (123)\n Length: 56\n Checksum: 0xb0bd (correct)\nNetwork Time Protocol\n Flags: 0x0b\n 00.. .... = Leap Indicator: no warning (0)\n ..00 1... = Version number: reserved (1)\n .... .011 = Mode: client (3)\n Peer Clock Stratum: unspecified or unavailable (0)\n Peer Polling Interval: invalid (0)\n Peer Clock Precision: 1.000000 sec\n Root Delay: 0.0000 sec\n Clock Dispersion: 0.0000 sec\n Reference Clock ID: Unindentified reference source ''\n Reference Clock Update Time: NULL\n Originate Time Stamp: NULL\n Receive Time Stamp: NULL\n Transmit Time Stamp: NULL\n\n0000 00 0a 41 db 58 00 00 0a 8b bf 70 7c 08 00 45 00\n0010 00 4c 2c c7 00 00 f3 11 b3 35 0a 2a 45 0a 80 69\n0020 27 0b 5b a1 00 7b 00 38 b0 bd 0b 00 00 00 00 00\n0030 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00\n0040 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00\n0050 00 00 00 00 00 00 00 00 00 00\n\n\n\n\n\n\n\n\nNetwork staff from two universities investigated two source hosts which\nI reported as being sources of these anomalous SNTP queries. Both reported\nthat a Netgear brand router was the source of the traffic. (Specifically,\none was identified as model MR814.)\n\n\nNow things started to make some sense. Many source hosts all using the same\nsource port number could be explained by an embedded SNTP client in which\nthe programmer hard-coded the source port number (23457).\n\n\nGathering Background Information\n\nWhile searching the web for background information on Netgear products'\nacclaimed NTP support, I came across the following quote (from ICSA Labs Firewall Lab Report on the NETGEAR FR114P):\n\nThe Netgear FR114P relied on a separate NTP-based time source to set\nthe current date and time, as it did not have an internal battery and\nclock. The product is hard-coded with specific NTP time sources that\nare accessible through the public Internet. Even after configuring the\nproduct to access a specific NTP server, the product still attempted to\naccess its hard-coded NTP time sources, while simultaneously accessing\nthe time source specified\n\nConclusion\n\nThe Candidate Firewall Product met all the criteria elements in the\nBaseline and Residential modules and therefore has attained ICSA Labs\nFirewall Certification.\n\nNote that Netgear reports that the FR114P does not contain the specific\nSNTP flaws described here. I just found it interesting, since that was\nthe only NTP-related flaw about which I found information on the web.\n\nExamining the Netgear Code\n\nIn order to verify our hypothesis that the source of the flood of SNTP\nqueries are the Netgear Platinum family products and to properly\ncharacterize this problem to the vendor, the Netgear code for a number\nof their products was downloaded and investigated.\n\nSimply by using the Unix \"strings\" command, I was able to verify that indeed\nthe Netgear code seems to contain the magic number 23457 (as a port number):\n\n$ strings RP614_4_12.bin |grep 23457\n on 23457 port.\n$ strings MR814_4_11.bin |grep 23457\n on 23457 port.\n \n\n\n\nUsing a similar technique, I found the following IP addresses embedded\nas ASCII strings in RP614_4_12.bin:\n\n128.105.39.11 # ntp1.cs.wisc.edu (a.k.a. \"caesar.cs.wisc.edu\")\n192.168.1.101\n66.37.215.43\n12.234.94.14\n192.168.0.1\n \n\n(Note that I added the DNS name comments for clarity; those strings did not\noccur in the binary file.)\nLikewise, these IP addresses were embedded in MR814_4_11.bin:\n\n0.0.0.0\n12.234.94.142\n66.37.215.43\n192.168.0.102\n128.105.39.11 # ntp1.cs.wisc.edu (a.k.a. \"caesar.cs.wisc.edu\")\n192.168.0.1\n192.168.1.101\n \n\n\nOf 3 globally routable IP addresses therein, only 128.105.39.11\nappears to be used as an NTP server. One of the others was an IP\naddress previously used by the \"dyndns.org\" dynamic DNS name service.\nNetgear has reported to us that the remaining embedded globally\nroutable IP addresses are no longer used and that they are part of dead\ncode left over from debugging by one of the developers.\n\n\nContacting Netgear\n\nOn June 16, 2003 I sent the following email message to Netgear\nsupport. Since this issue is more significant than the typical customer\nsupport inquiry, I also sent it directly to some Netgear employees\n(whose email addresses were culled from the web) asking them to\ncommunicate it to the proper people in engineering and/or to have\nsomeone contact me by phone or email.\n\n\n\nFigure 6. Email to Netgear Support\n\n\nDate: Mon, 16 Jun 2003 16:00:21 -0500\nFrom: Dave Plonka \nTo: support@netgear.com\nSubject: NETGEAR products abusing University of Wisconsin time server\n\n\nNETGEAR support folks,\n\nSince May 14, 2003, the publicly-advertised Internet time server\n\"ntp1.cs.wisc.edu\" (a.k.a. \"caesar.cs.wisc.edu\", 128.105.39.11) at the\nUniversity of Wisconsin-Madison has been the recipient of a large-scale\nflood of time queries apparently from NETGEAR products deployed\nthroughout the Internet.\n\nCurrently, based on our analysis we believe that the NETGEAR \"Platinum\"\nproducts such as the RP614 and MR814 are the primary source of this\nflood of traffic. They likely will need to have their code changed to\nmitigate what is essentially an accidental Denial-of-Service flood\nagainst our NTP infrastructure. The inbound aggregate traffic rate to\nour network from NETGEAR products currently exceeds 250,000\npackets-per-second and 150 megabits-per-second at our border routers,\napparently from at least tens-of-thousands of NETGEAR sources. This\nhas also cost us numerous work hours of Internet traffic engineering,\ntroubleshooting, and abuse investigation.\n\nSpecifically, the code for the Platinum products appears to contain an\nembedded Simple Network Time Protocol (SNTP) client, which sends\nqueries from UDP port 23457 to port 123 of the IP host 128.105.39.11.\nInexplicably, NETGEAR's products code ships with our server explictly\nconfigured by IP address. We have determined that at least the\nfollowing code images explicitly contain our server's IP address:\nMR814_4_11.bin, MR814_v409.bin, RP614_4_0_0.bin, RP614_4_12.bin.\n\nWe believe this is inappropriate and not best current practice for\nload-balancing and reliability of the Internet's NTP service. In\naddition to the sheer number of deployed products, these requests often\noccur at a very fast rate from each device (for instance, one per\nsecond) and therefore put an enormous load on our NTP server.\n\nPlease contact me as soon as possible regarding your products' default\nSNTP client configration, possible SNTP client software bug, and the\nresulting incident which required us to block your customer's SNTP\nqueries to our Network Time Protocol (NTP) server.\n\nWe look forward to hearing from you soon.\n\nDave\n\nP.S. our NTP server is publicly advertised:\n\n http://www.ntp.org/\n http://www.eecis.udel.edu/~mills/ntp/servers.html\n http://www.eecis.udel.edu/~mills/ntp/clock2a.html\n\nP.P.S. NTP best practice is described in the \"Rules of Engagement\" section\nof this document:\n\n http://www.eecis.udel.edu/~mills/ntp/servers.html\n\n\n\n\n\n\nAfter receiving no response for days, I called\nNetgear's headquarters, leaving messages with two executives explaining\nthe seriousness of the situation. I also emailed members of Netgear's\nexecutive team by guessing their email addresses, based upon their\nemail naming convention. I included a \"Return-Receipt-To\" header, and\ntheir Mail-eXchanger notified me that all were delivered successfully.\nHere's a portion of that message:\n\nAt this point I have a complete write-up of this continuing incident,\nincluding traffic measurement statistics evidencing the flood and an\nanalysis of its root cause ready to be released publicly.\n\nI absolutely need to hear from responsible parties at NETGEAR\nimmediately, if NETGEAR wishes to begin a dialogue before this goes\npublic. We're not expecting an immediate solution; in fact, I'm fairly\ncertain there is no complete solution without UW-Madison's involvement.\n\n\nOn Thursday, June 19, I received a voicemail message from the director\nof support for Netgear. He confirmed that they have located some fault\nin their code. Soon afterward, I began to exchange email and voicemail with\nhim. Having now established initial contact, we proceeded to work this\nissue outside of Netgear's support system, continuing on with the\nreview process described below.\n\nNetgear's support organization was completely unresponsive. Curiously,\nI did finally receive the email message below from Netgear's\nemail-based customer support system, some 23 days after I submitted the\nproblem report on June 16.\n\n\nFigure 7. Email from Netgear Support\n\n\nDate: Wed, 09 Jul 2003 09:35:46 +1000\nFrom: support@esupport.netgear.com\nSubject: RE:NETGEAR products abusing University of Wisconsin time server [#111678]\nTo: plonka@localdomain\n\n\nThank you for your email. We apologize for the delay in responding.\n\nDue to an unexpected increase in email volume we have been unable to\nrespond in a timely manner. Your issue may have already been\nresolved. Please reply to this email if you still require assistance\nand we will respond as quickly as we can. If your issues is resolved\nyou do not need to reply and we will consider the case closed.\n\nAgain thank you for your patience and understanding.\n\n\nPlease help us serve you better by clicking here\nmailto:support@netgear.com?subject=Feedback_us if you would like to\nprovide any other valuable feedback. (Note: this feedback is not sent\nto an agent so you will not receive a reply.)\n\n\n\n\n\n\n\n\nThe Review Process\n\nShortly after beginning a dialogue with Netgear, I proposed the\nformation of a review team to discuss possible solutions. Netgear\nagreed, and a review team was formed with about fifteen members, a\nthird from each of these areas:\n\nNetgear employees\nUniversity employees\nIndependent experts from their respective fields:\n \n Regional Internet Registries\n Internet Measurement Research\n Network Time Protocol\n\n\nThe independent experts agreed to participate without prematurely\ndisclosing the details of the situation.\n\n\nA number of action items and directions were developed during the review\nprocess. These included:\n\nFix the SNTP Client\nPropose the Network Operational Options\nInform the Internet Community\nClarify Internet Best Current Practice and Protocol Standards\n\n\nThe Flawed SNTP Client\n\nThe Flawed Netgear SNTP Client implementation in the products affecting\nUW-Madison has the following characteristics:\n\n\n\nUses a hard-coded IP address for the NTP server 128.105.39.11,\nthat of ntp1.cs.wisc.edu.\n\n\n\nUses a fixed UDP source port number 23457.\n\nThis was incredibly advantageous as it allowed UW-Madison to identify\nand count the Netgear clients. However, due to the widespread use of\nNetwork Address Port Translation (NAPT, or NAT/PAT)\nupstream from some Netgear products, the SNTP request source port\nnumber is sometimes rewritten before the request packet reaches its\ndestination.\n\nNote to network operators: Please do not block UDP\ntraffic involving port 23457 nor traffic involving our NTP server's IP\naddress of 128.105.39.11. While we appreciate attempts to help, it\nmay interfere with the best possible solution to this problem.\n\n\n\n\nPolls at one second intervals until it receives a response\nfrom the NTP server, after which it uses a longer poll interval such as\none minute, ten minutes, two hours, or 24 hours,\ndepending upon product model and firmware version.\n\n\n\nImpact to Netgear Customers\n\nAs of this writing (August 2003) the University is making its best\neffort to service the Netgear time requests. As such, users\nof the affected products should not normally notice any problems due to\nthis flaw. Furthermore, based on experience so far, it seems that\nonly a small subset of the customers are even aware of the time-related\nfeatures of these products (which include logging, policy scheduling,\nand email notifications).\n\nIn parallel, Netgear has produced and continues to work on firmware\nthat does not exhibit the aforementioned problems. Customers can\nupgrade to newer firmware versions, which are available for download\nfrom Netgear's support site. At the time of this writing (August\n2003), the most current version of firmware available for the RP614v2,\nRP614, DG814, and MR814 models does not utilize UW-Madison's time\nservice nor does it poll too frequently.\n\nCode Upgrades for Affected Netgear Products\nBased on information supplied or confirmed by Netgear, the following\nproducts contained these SNTP design flaws. Where applicable, I have\nlabeled each with the earliest version of code containing a fix:\n\n\nFlawed Product Counts\n\nI have counted more than 500,000 unique Netgear sources that queried our\ntime server in one day. This measurement likely underestimates the actual\ncount because of Network Address Port Translation, which modifies the\nsource IP address and port number, and because some broadband residential\nservices drop the customer's link when the service is not in use.\n\nAs of June 30, 2003, Netgear reported a total of 707,147 affected\nproducts manufactured. Some simple math: If there are 700,000 errant\nSNTP clients each of which can generate one SNTP request per second to\nour time server, then the worst-case aggregate rate will be about\n700,000 packets per second. Since each SNTP packet is 76 bytes in\nsize, that is also 426 megabits per second of traffic.\n\nFigure 8a shows the actual number of unique NTP Netgear client IP\naddresses observed per day by a router on UW-Madison's network.\nTheoretically, counting the clients in this way could overestimate\nthe count if the clients' DHCP servers changes the client IP address\nfrequently. However, based on the number of products\nreported as having been manufactured, it seems fairly accurate.\n\n\nSuggested Fixes\nDuring the review process a number of improvements to the SNTP client\nwere suggested.\nThese included that an SNTP implementation:\n\nSHOULD use a poll interval within the range from 64 to 1024\nseconds or longer\nSHOULD use local NTP server(s) or multicast when available, as\n configured by the operator or determined by a discovery mechanism\n such as via the DHCP \"Network Time Protocol Servers Option\", which\n is defined in section 8.3 of RFC 2132.\nMAY performance exponential backoff of poll interval (within the\n aforementioned range) upon failure to receive a response from the\n NTP server(s)\nMUST NOT use a shorter poll interval upon failure to receive a\n response from the NTP server(s)\nMUST allow the operator to configure the query behavior with respect\n to whether or not it is enabled or disabled and with respect to\n which candidate time servers can be queried.\nSHOULD use the Domain Name System to determine candidate server(s)\n IP address(es), so that the NTP server's zone administrator can\n influence the client behavior.\nSHOULD resolve the server IP address via DNS before each poll/query,\n so that the pertinent DNS entries' Time-To-Live values are respected.\nSHOULD support the existing NTP access-control mechanism by, upon\n receiving a valid `kiss-of-death' packet, reporting the condition\n and discontinuing queries to the server in question until\n\t reinitialization.\nMAY use an implementation-defined fixed source port number\n\nSome of these have been implemented in the initial fix\nbut others are only under consideration.\nHopefully these suggestions will be evaluated during\nan upcoming SNTP standardization effort.\n\nThe Initial Fix: \"Instant\" Code\n\nDuring the review process, we learned that Netgear already was having\nSNTP-related code changes developed for the RP614v2 product prior to my\ninitial notification of the problems the flaw was causing to the\nUniversity.\n\nRegarding Firmware v5.13 RC7 for the RP614v2, Netgear made this new\ncode available to me on July 10. My testing found that the modified\nSNTP client had these characteristics, much as they described:\n\n\nNow requires a DNS server to be configured (or learned via DHCP) before\ngenerating any SNTP queries.\n\n\nThe code performs DNS queries for \"time-a.netgear.com\" and\n\"time-b.netgear.com\" at ten minute intervals until success, alternating\nnames if no response is received. I verified also that it supported\nresponses with CNAMEs or multiple A records as well.\n\n\nFollowing successful DNS resolution, it sends an NTP query to the resolved\nIP address and waits for a reply. If no reply comes in ten minutes, it\nagain resolves the name, and requeries. It appears to give up after five\nretries.\n\n\nWhenever any configuration change is applied via the\nweb interface, it causes the device's clock to be zeroed, the NTP\nserver to be re-resolved, and subsequently queried.\n\n\nHowever I also found these bugs:\n\n\nThe SNTP client in this code does not appear to validate the NTP\nresponse packet. It will accept any incoming packet to port\n23457 as a valid response even if the flags are set wrong (for\ninstance, indicating that it is another client query rather than a\nserver response).\n\n\nWhile the SNTP client is awaiting a response (after\nquerying either time-a or time-b) it seems to accept any UDP\nresponse packet, even if the source IP address of that UDP packet is\nnot that of the time server that it queried.\n\n\nThis code was made available for download on the Netgear web site for\nthe RP614v2 on or about July 11, 2003.\n\nNetgear continues to develop improvements to their SNTP client and\nhas vetted the design with the review team.\n\nNetwork Operational Options: To Serve or To Sever?\n\nThese flawed devices are not easily reconfigurable. Representatives\nfrom both Netgear and UW-Madison believe that it is not a viable option\nto rely on Netgear's customers to upgrade to the newer firmware (the\nfirst of which was released in July) to correct the errant behavior.\n\nOur review team has considered a number of possible options about how\nto deal with the errant Netgear time requests. While I won't discuss\nall the details here, the two primarily endgames on which we've focused\nare outlined below.\nEndgame A: UW-Madison Netgear Anycast Time Service\n\nIn this option we would deploy highly-reliable, redundant NTP servers\nat WiscNet's borders and route the inbound requests destined to\n128.104.39.11 to them using BGP anycast. (Anycast is a\ntechnique that can often be employed to route traffic for some\nstateless RPC services, such as DNS or NTP, which are based upon UDP.)\nImplementing this option would likely include placing a pair of\nrack-mount NTP servers at each of three locations within WiscNet:\nUW-Madison, UW-Milwaukee, UW-Eau Claire. These are nearest the three\ncurrent border Internet exchange points and therefore provide the most\ndiverse paths for reliability of connectivity to the global Internet.\n\nOne distinct advantage of this configuration is that UW-Madison\nretains as much control as possible over its precious IPv4 address\nallocations. Because this BGP anycast deployment resides solely within\nWiscNet (which will honor a single /32 host-address route), this option\nconsumes as little of UW-Madison's IP address space as possible - just\nthe address to which Netgear time requests were directed.\n\nEndgame A has some risk. Whether or not the servers' responses\nreach the requesting client host is not wholly within the University's\ncontrol, consequently some amount of flooding will likely continue.\nThere are many reasons other than server failure for disruptions in the\nend-to-end path between the SNTP clients and servers that could cause\nthe clients not to receive the responses and to flood requests toward\nour servers anyway. These include asymmetric routing problems,\nfirewalling policies, and disasters affecting any link between the\nclients and servers. Indeed, even while our time server is dutifully\nresponding to all netgear SNTP requests, we still regularly observe\nthat hundreds of them continue to flood. Apparently these \"zombies\"\nnever receive our responses.\n\nTo limit the possibility of the multiple servers being simultaneously\nisolated from the Internet, one could consider an even more\ngeographically diverse set of deployment locations, such as that done\nby the AS112 Project, which\neffectively mitigates the damage caused to the Internet's root name\nservers by RFC1918-related queries.\n\nFigure 10 is a diagram showing how this service would work. The\nNetgear SNTP requests heading toward UW-Madison are shown in green.\nNote that multiple NTP servers, all with the same IP address, are\nlocated in multiple locations. WiscNet's border routers divert the\ninbound SNTP requests to the nearest server. The server responses are\nshown in red. If any of the servers fail, the traffic should route\nto one of the remaining NTP servers with the same address.\n\n\nEndgame B: Attempt to Suppress the Requests\n\nTo prevent Netgear time requests from being forwarded to our network\nwould require UW-Madison to sacrifice a block of IP address space\nwithin the class B network which includes the IP address of\nntp1.cs.wisc.edu.\n\nBecause of the way the Internet's backbone routing is operated, and to\nkeep the number of routes manageable, network routes are sometimes not\nrespected unless they are sufficiently large. In today's Internet,\nthat means a route might not be considered legitimate unless it\nrepresents 2,048 or 4,096 contiguous addresses. Respectively, network\noperators would call those size \"/21\" or \"/20\" (pronounced \"slash\ntwenty\") blocks because they represent networks having netmasks of 21\nor 20 contiguous bits.\n\nFigure 11 is a diagram showing this configuration. The BGP updates\noriginating from UW-Madison's border router are shown in red. The Netgear\nSNTP Requests are shown in green. The ICMP unreachable messages returned\nto the client by BGP-aware border routers throughout the Internet are shown\nin blue. These inform the client that the network in which the NTP server\nwould reside is unreachable.\n\n\n\nEndgame B: IP Resources Required\n\nThis endgame that tries to suppress the forwarding of requests comes at\na significant cost to the University - we may have to sacrifice, likely\nfor the lifetime of the flawed products, as many as 4,096 IP\naddresses. Figure 12 shows how our existing 128.105.0.0/16 network\ncould be divided, and the one slice \"/20\" block which would be excluded\nfrom the Internet's global BGP routing table.\n\n\nThe risks of endgame B include the possibility that some portions of the\nInternet might not be able to reach legitimate campus IP addresses that\nlie near the sacrificial, unadvertised block. Input from the backbone\nnetwork operations community and real-world experience must determine\nwhich solution best serves the University and Internet community as a\nwhole.\n\nInform the Internet Community\n\nThe public release of this document is part of an effort to inform the\nInternet community of this flaw and the resulting floods, with the hope\nof minimizing the likelihood of such a mistake being repeated\nelsewhere. Note that Netgear was notified of our plan to publicly\ndisclose the details herein weeks in advance of its release.\nFurthermore, earlier revisions of this document were provided to them\nfor review.\n\nBecause of the scope of the resulting problem, with hundreds of\nthousands of ill-behaved Internet hosts distributed world-wide, and\nbecause of the scale and unexpected nature of the flooding, with\naggregate rates which could completely fill some network links, I felt\nthat it was important to inform others and solicit advice from experts.\n\nFollowing this disclosure, it's my intent to find appropriate venues to\nfurther present and review the dangers and potential solutions to this\nand similar problems.\n\nFor instance, during the review process we learned that\nthe Commonwealth Scientific & Industrial Research Organisation\n(CSIRO) in Australia is having\nsimilar trouble with about 85,000 SMC brand routers that poll the CSIRO\ntime server twice a minute when they don't receive a response.\nA story about that incident, \"Rogue routers cause havoc for CSIRO\", can be found here:\n\nhttp://australianit.news.com.au/articles/0,7204,6716567%5e15340%5e%5enbv%5e15306-15318,00.html\n\n\nWhile the scale of the CSIRO problem is orders-of-magnitude less,\nwith floods of perhaps 2,800 packets per second and 1.7 megabits per\nsecond, it is strikingly similar and perhaps not as likely to be as\nresponsibly addressed with the assistance of that manufacturer.\n\nClarify Internet Best Current Practice and Protocol Standards\n\nAlso during the review process, some members of the review team began\nwork on Internet Drafts\nto improve documentation pertinent to this issue.\nThere are at least two such efforts currently in their infancy:\n\nI am in the process of preparing an Internet Draft, currently\ntitled \"Embedding Globally Routable Internet Addresses Considered\nHarmful\", which denounces the practice of embedding unique, globally routable\nIP addresses in Internet hosts, describes some of the resulting problems,\nand considers selected alternatives.\nMembers of the NTP community have revised and reviewed the existing\nInformational RFC2030 that describes SNTP. Through their efforts and\nperhaps those of other interested parties, it may be possible to\nrevisit NTP and SNTP as a standards track protocol within the\nIETF.\n\n\nStatus, August 21, 2003\n\nI'm pleased to report that Netgear has cooperated with us on the initial\nsteps of this process and we are forging an agreement that will enable\nus to implement a suitable solution.\n\nFor the time being, UW-Madison continues to service Netgear SNTP\nrequests in spite of receiving occasional large-scale floods of traffic\nfrom Netgear products. A recent incident is shown in Figure 13. The\nshark-fin shaped anomaly on the right is a flood of inbound\nNetgear time requests which grew to about 100,000 packets per second\nbefore subsiding.\n\n\nBoth the magnitude and duration of the Netgear-caused incidents\ncontinue to present a serious operational problem for UW-Madison.\nWhile essentially involved in a game of russian roulette at the moment,\nwe are hoping to utilize the expertise of both UW-Madison and the\nInternet operator community to design and implement a good solution.\n\nAfterthoughts\nHere are some questions, offered as food for thought, that were brought\nto mind by this case study:\n\n\nWhat does this unintentional Denial-of-Service flood indicate about the\nviability of some public Internet services?\n\n\nCan the Internet routing infrastructure be improved to enable less disruptive\nsolutions to such problems?\n\n\nAre incidents such as this a likely side-effect of ubiquitous, low-cost,\nperhaps even disposable Internet hosts?\n\n\nAre the manufacturer, vendor, Internet operations, and user communities\nwilling and able to cooperate to address such problems?\n\n\n\nAcknowledgements\n\nThe following provided assistance with the data gathering, analysis,\nand people networking:\n\n\nUniversity of Wisconsin-Madison:\nJeff Bartig, Jim Gast, Michael Hare, Adam Kunen, Dave Thompson\n\n\nUniversity of Florida: Robert Bird, Greg Goddard\n\n\nHarvard University: Greg Mazzu\n\n\nk claffy, Nevil Brownlee, George Michaelson\n\n\n\nI'd also thank the members of the review team, including those remaining\nanonymous. I'm certain we'll come to a better solution because of their\nparticipation.\n\n\nAnalysis Tools\nThe following tools were used during this investigation:\n\n\nMRTG\n\n\nRRDTOOL\n\n\nRRGrapher\n\n\njunipoll\n\n\nflow-tools\n\n\nCflow\n\n\nFlowScan\n\n\ntcpdump\n\n\nethereal (now wireshark)\n\n\nsntp.pl\n\n\nstrings ;^)\n\n\n\n\nReferences / Further Reading\n\n\nRFC2030: Simple Network Time Protocol (SNTP) Version 4 for IPv4, IPv6 and OSI (status: Informational)\n\n\nRFC1305: Network Time Protocol (Version 3) Specification, Implementation and Analysis (status: DRAFT STANDARD)\n\n\nhome of the Network Time Protocol (NTP) project\n\n\nPublic NTP Time Servers\n\n\nPublic NTP Secondary (stratum 2) Time Servers\n\n\nRFC2132: DHCP Options and BOOTP Vendor Extensions\n\n\nRFC1546: Host Anycasting Service\n\n\nhttp://www.networksorcery.com/enp/protocol/sntp.htm\n\n\nan article on how to change Windows' time server configuration using regedit\n\n\nBasic Operation of the Windows Time Service\n\n\nhttp://www.microsoft.com/windows2000/docs/wintimeserv.doc\n\n\nFlawed Routers Flood University of Wisconsin Internet Time Server (NANOG 29 presentation)\n\n\nA Case Study in Internet Pathology: Flawed Routers Flood University's Network (LISA '03 talk)\n\n\n\n\nFrequently Asked Questions\n\n\nWhat is Netgear's liability for causing (however inadvertently) this denial of service for your network?\n\nMy work responsibilities are not ones that would make me a participant in such\nnegotiations. However, as I reported, an agreement is being forged.\n\nNote to others: Please do not ask me for financial or legal details.\nThis document is a technical summary of the situation and is not the vehicle\nby which to deliver such details.\n\nYou may be interested in this news article:\nhttp://www.doit.wisc.edu/news/story.asp?filename=322\n\n\n\nHave you considered putting up a server which sends back fake answers to netgear clients, to cause people to upgrade?\n\nYes, but we didn't consider it for long. Both Netgear and others\non the review team agreed that it would likely\nhave very little effect, since only a small subset of the customers\nseem to even be aware of the NTP-related features of the affected\nproducts. Besides, the proposed \"dishonest\" time server (which would\nreport the wrong time) would have to be operated at the IP address that is\ncurrently that of our well-known reliable time server. It would be\nquite rude for it to suddenly become unreliable since the whole purpose\nof the public time service is to answer with the correct time. The\nUniversity intends to provide the best possible service, regardless of\nhow it's abused.\n\n\n\nWhat is the expected life-time for these products?\n\nPersonally, I think its likely that many of them will be around until five\nto ten years from now. Its only a guess, but perhaps the half-life is five\nyears, after which we'd expect to see less than 350,000 of the affected\nproducts remaining in use.\n\n\n\nIn figure 13, could that \"shark fin\" spike have anything to do with last week's power grid failure (Blackout 2003), and subsequent \"rolling\" restoration?\n\nNo, it did not coincide with the blackout that affected the east coast\nin August 2003.\nHowever, I did look for evidence of the blackout in the netgear SNTP traffic,\nand found only slight less during the outage. Relatively little of the\nInternet was actually affected by the power outage. Early reports were\nthat only a few thousand BGP prefixes from only a couple hundred autonomous\nsystems were offline. Those that were offline were unreachable in both\ndirections, so we didn't receive requests from them until power was returned.\nThe rolling restoration served to distribute the load to our server as\nthose netgear clients came online.\n\n\n\nAre there other devices than those mentioned which also suffer from the flaw which causes inadvertent flooding of your network?\n\nA number of Netgear users have reported to me that Netgear model RO318,\nfirmware version V3.26, also utilizes our time server\nand also logs ntp errors to its security log. I have not evaluated this\nproduct, so do not know what retry times it uses, but will report this to\nNetgear.\n\n\n\nWhat was the effect of this article being slashdotted?\n\nWhile having a significant effect on the web server, overall it was an\ninsignificant level of traffic for the campus. The server handled 30\nrequests a second for a while.\nHere's a graph comparing the web server hits-per-second to the number\nof Netgear-sourced SNTP request flows-per-second, when this report was\nslashdotted. (Note that most Netgear clients were receiving replies\nfrom our server at this time, so they were not flooding requests.)\n\n\n\n\n\nWhy is a traditional manufacturer recall/defect solution not a possibility?\n\nBoth Netgear and other members of the review team felt that it was unlikely\nthat all but a very small subset of the owners would return the affected\ndevice since they appear to be working fine. Also, very few customers have\nregistered these products with the manufacturer, so it is impractical\nto contact them.\n\n\nI'm with [the IT press], do you have some time to speak with me?\n\nWhen this report was first presented, an astute audience member\nproposed that the IT press ostensibly plays an important role in\nevaluating whether or not consumer products comply with Internet\nstandards and best current practice.\n\nGiven that, if you're with the IT press and your publication does\nInternet product reviews or makes \"Editor's Choice\" awards, I am\nwilling to correspond with you by email about this story.\nSpecifically, the Internet community may benefit from our exploring the\nIT press' familiarity with Internet standards and best practices and\nthe press' assessment of its ability to evaluate products.\n\nBy tying the reporting of this story in with the product evaluation and\nrecommendation function of the IT press, I'm hopeful that the community\ncould reduce the likelihood of such flaws causing such problems.\nIt's just an idea, let me know what you think.\n\n\nHow has this story been covered in the press?\n\nWhile certainly not an exhaustive list, here is a sampling of\ncoverage.\n(Please be aware that some of these articles are\nmisleading or incorrect about the details;\nOnly a few of them contacted me to check the facts.)\n \n \n Slashdot: Netgear Routers DoS UWisc Time Server\n \n \n The Inquirer: Wisconsin Uni page describes Netgear `flawed routers'\n \n \n Wisconsin Technology Network: Flawed Routers Flood UW Server\n \n \n ZDNet: NetGear flaw triggers accidental DoS attack\n \n \n CNET News.com: NetGear flaw triggers accidental DoS attack\n \n \n ZDNet Australia: UPDATE: Netgear routers attack university\n \n Network World: Netgear router quirk perturbs college\n (also in print Network World magazine)\n \n \n TECHWORLD: NetGear sets off denial of service attack\n \n \n Network Computing: Industry Insights: Opportunity Knocks / Quality Counts\n \n \n PC World: NetGear Routers Wage War on University\n (also in print in the November 2003 issue of PC World magazine)\n \n\n\n\n\nCopyright 2003, Dave Plonka.\n\n$Id: index.wml,v 1.39 2006/07/19 15:20:28 plonka Exp $\n\n$Log: index.wml,v $\nRevision 1.39 2006/07/19 15:20:28 plonka\nupdated figure 8a\n\nRevision 1.38 2005/04/28 16:13:02 plonka\nupdated figured 8a\n\nadded NANOG and LISA talks to references\n\nadded news story url to faq entry\n\nfixed a typo\n\nRevision 1.37 2004/09/28 18:37:16 plonka\nfixed a typo\n\nRevision 1.36 2004/05/19 22:54:31 plonka\nupdated figure 8a\n\nRevision 1.35 2004/02/05 17:29:12 plonka\nadded figure 8a, Netgear SNTP Clients Per Day\n\nRevision 1.34 2003/12/04 22:30:58 plonka\nfixed a typo\n\nRevision 1.33 2003/10/16 22:30:05 plonka\nadded a faq entry\n\nRevision 1.32 2003/09/15 22:13:13 plonka\nfixed some typos and the host address counts for /20 and /21 blocks\n\nRevision 1.31 2003/09/12 18:09:49 plonka\nadded info about code upgrade for HR314\n\nRevision 1.30 2003/09/10 15:03:26 plonka\nfixed a typo\n\nRevision 1.29 2003/09/10 14:58:50 plonka\nadded using DHCP \"Network Time Protocol Servers Option\" to \"Suggested Fixes\"\n\nRevision 1.28 2003/08/30 02:22:20 plonka\nadded faq entry\n\nRevision 1.27 2003/08/27 18:42:42 plonka\nadded graph evidencing the flash crowd when this report was slashdotted\n\nRevision 1.26 2003/08/26 21:40:46 plonka\nfixed some typos and reworded a couple sentences\n\nRevision 1.25 2003/08/25 23:54:09 plonka\nadded faq entries" ], "output": "readability", "pwd": "/data/archive/1789337059.131441", "schema": "ArchiveResult", "start_ts": "2026-09-13T22:05:35.097035+00:00", "status": "succeeded" } ], "screenshot": [], "singlefile": [], "title": [ { "cmd": [ "curl", "--silent", "--location", "--compressed", "--max-time", "120", "--user-agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36 ArchiveBox/0.7.4 (+https://github.com/ArchiveBox/ArchiveBox/) curl/curl 8.14.1 (x86_64-pc-linux-gnu)", "https://pages.cs.wisc.edu/~plonka/netgear-sntp/" ], "cmd_version": "curl 8.14.1 (x86_64-pc-linux-gnu)", "end_ts": "2026-09-13T22:05:35.089577+00:00", "index_texts": null, "output": "Flawed Routers Flood University of Wisconsin Internet Time Server", "pwd": "/data/archive/1789337059.131441", "schema": "ArchiveResult", "start_ts": "2026-09-13T22:05:35.068806+00:00", "status": "succeeded" } ], "wget": [ { "cmd": [ "wget", "--no-verbose", "--adjust-extension", "--convert-links", "--force-directories", "--backup-converted", "--span-hosts", "--no-parent", "-e", "robots=off", "--timeout=120", "--restrict-file-names=windows", "--warc-file=/data/archive/1789337059.131441/warc/1789337133", "--page-requisites", "--user-agent=Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36 ArchiveBox/0.7.4 (+https://github.com/ArchiveBox/ArchiveBox/) wget/GNU Wget 1.21.3", "--compression=auto", "https://pages.cs.wisc.edu/~plonka/netgear-sntp/" ], "cmd_version": "GNU Wget 1.21.3", "end_ts": "2026-09-13T22:05:35.056018+00:00", "index_texts": null, "output": "pages.cs.wisc.edu/~plonka/netgear-sntp/index.html", "pwd": "/data/archive/1789337059.131441", "schema": "ArchiveResult", "start_ts": "2026-09-13T22:05:33.736050+00:00", "status": "succeeded" } ] }, "icons": null, "is_archived": true, "is_static": false, "latest": { "archive_org": null, "dom": null, "favicon": "favicon.ico", "git": null, "media": null, "pdf": null, "screenshot": null, "singlefile": null, "title": "Flawed Routers Flood University of Wisconsin Internet Time Server", "warc": null, "wget": "pages.cs.wisc.edu/~plonka/netgear-sntp/index.html" }, "link_dir": "/data/archive/1789337059.131441", "newest_archive_date": "2026-09-13T22:05:36.968488+00:00", "num_failures": 0, "num_outputs": 6, "oldest_archive_date": "2026-09-13T22:05:32.580228+00:00", "path": "/~plonka/netgear-sntp/", "schema": "Link", "scheme": "https", "snapshot_id": "6224e71d-1528-4104-ba60-372c339188a3", "sources": [ "/data/sources/1789337059-import.txt" ], "tags": null, "tags_str": "", "timestamp": "1789337059.131441", "title": "Flawed Routers Flood University of Wisconsin Internet Time Server", "updated": "2026-09-13T22:05:32.579753+00:00", "updated_date": "2026-09-13 22:05", "url": "https://pages.cs.wisc.edu/~plonka/netgear-sntp/" }