accept_language:nilBrowser.new; parsed into browser.accept_language.headers:nilBrowser.new; read for the Client Hints and the Cloudflare headers. The Rails helper and the middleware pass it.Browser.user_agent_size_limitBrowser::Error.Browser.accept_language_size_limitBrowser::Error.Browser.accept_language_size_limitBrowser::Error. A header listing more than 32 languages is parsed on each request, never cached.Browser.cache_size0 turns it off.Browser::Bot.matchersBrowser::Bot.botsbots.ymlBrowser.user_agent_size_limit = 4096
Browser.accept_language_size_limit = 4096A user agent or an Accept-Language header whose bytes are not valid UTF-8 (forged, or read back from a log file) does not raise: each invalid byte is read as U+FFFD. A Client Hint or a Cloudflare header that is not valid UTF-8 is ignored.
Each process holds its own cache. An attacker spreading over many IPs still costs one wait per IP: on a hot path, run the check in a background job and keep the answer in your own cache.
+The answers come from a name server the requester chooses, and versions of the resolv gem before 0.7.2 (0.3.2 in the 0.3 series) can be made to use up memory or CPU by such answers (CVE-2026-80212, CVE-2025-24294). Before turning reverse_dns: on, check bundle info resolv, or add gem "resolv", ">= 0.7.2" to your Gemfile.
Fresh ranges. The weekly bot sync rewrites bot_ip_ranges.yml from the owners’ JSON files, and each release ships them. A process that runs for weeks can fetch them again. Each process holds its own copy; a failed source raises Browser::Error and keeps the current ranges:
Browser::Bot::IPRanges.refresh # fetches every owner's JSON, swaps the ranges at once
diff --git a/lib/browser/accept_language.rb b/lib/browser/accept_language.rb
index a33729e..0c9ef35 100644
--- a/lib/browser/accept_language.rb
+++ b/lib/browser/accept_language.rb
@@ -9,6 +9,7 @@ def self.languages
def self.parse(accept_language)
return [] unless accept_language
+ accept_language = accept_language.scrub unless accept_language.valid_encoding?
languages = accept_language.split(",")
languages.map! { |string| new(string.squeeze(" ").strip) }
languages.reject! { |al| al.quality.zero? }
diff --git a/lib/browser/base.rb b/lib/browser/base.rb
index 309f03a..34a0940 100644
--- a/lib/browser/base.rb
+++ b/lib/browser/base.rb
@@ -9,7 +9,7 @@ class Base
# headers: the Rack env (request.env), for the client hints and the
# Cloudflare headers.
def initialize(ua, accept_language: nil, headers: nil)
- validate_size(:user_agent, ua.to_s)
+ Base.validate_size(:user_agent, ua.to_s)
@ua = ua
@accept_language_raw = accept_language.to_s
@@ -24,7 +24,7 @@ def meta
# Return an array with all preferred languages that this browser accepts.
def accept_language
@accept_language ||= begin
- validate_size(:accept_language, @accept_language_raw)
+ Base.validate_size(:accept_language, @accept_language_raw)
Cache.accept_language(@accept_language_raw)
end
end
@@ -288,7 +288,7 @@ def epiphany?(expected_version = nil)
@detection ||= Cache.detection(ua)
end
- private def validate_size(subject, input)
+ def self.validate_size(subject, input)
actual_bytesize = input.bytesize
size_limit =
if subject == :user_agent
diff --git a/lib/browser/bot.rb b/lib/browser/bot.rb
index 964e942..96d00c4 100644
--- a/lib/browser/bot.rb
+++ b/lib/browser/bot.rb
@@ -117,6 +117,7 @@ def self.trie_node_source(node)
private_class_method :trie_source, :trie_node_source
def self.why?(ua)
+ ua = ua.scrub unless ua.valid_encoding?
ua = ua.downcase.strip
browser = Browser.new(ua)
matchers.find { |matcher| matcher.call(ua, browser) }
diff --git a/lib/browser/browser.rb b/lib/browser/browser.rb
index 36fd5ae..adcadf0 100644
--- a/lib/browser/browser.rb
+++ b/lib/browser/browser.rb
@@ -179,6 +179,10 @@ def self.matchers
def self.new(user_agent, **)
user_agent ||= EMPTY_STRING
+ # Before the first regexp: an oversized user agent then costs nothing, and
+ # a byte that is not UTF-8 (forged, or read from a log) would make it raise.
+ Base.validate_size(:user_agent, user_agent.to_s)
+ user_agent = user_agent.scrub unless user_agent.valid_encoding?
browser_class = Cache.cached_detection(user_agent)&.browser_class
return browser_class.new(user_agent, **) if browser_class
diff --git a/lib/browser/cache.rb b/lib/browser/cache.rb
index b4f9194..1ae94ce 100644
--- a/lib/browser/cache.rb
+++ b/lib/browser/cache.rb
@@ -6,6 +6,10 @@ module Browser
# GVL a Hash read is atomic, and a thread that misses detects again. Writes
# do, for the Rubies without a GVL.
module Cache
+ # Far more than a browser sends. A longer list is parsed again each time: 2 KB of "a,a,a" makes a thousand
+ # objects, and a thousand such headers would hold 100 MB.
+ MAX_LANGUAGES_PER_CACHED_HEADER = 32
+
@mutex = Mutex.new
@detection_by_user_agent = {}
@languages_by_header = {}
@@ -28,11 +32,13 @@ def self.detection(ua)
def self.accept_language(header)
return AcceptLanguage.parse(header) if Browser.cache_size.zero?
- languages = @languages_by_header[header] || @mutex.synchronize do
- store(@languages_by_header, -header,
- AcceptLanguage.parse(header).freeze)
- end
- languages.dup
+ languages = @languages_by_header[header]
+ return languages.dup if languages
+
+ languages = AcceptLanguage.parse(header)
+ return languages if languages.size > MAX_LANGUAGES_PER_CACHED_HEADER
+
+ @mutex.synchronize { store(@languages_by_header, -header, languages.freeze) }.dup
end
# Bumped when a matcher list, the bot list or its exceptions change: the
diff --git a/lib/browser/client_hints.rb b/lib/browser/client_hints.rb
index 3f5b941..a1ab694 100644
--- a/lib/browser/client_hints.rb
+++ b/lib/browser/client_hints.rb
@@ -4,7 +4,7 @@ module Browser
# The User-Agent Client Hints of Chromium browsers, read from a Rack env.
# Safari and Firefox send none: every answer is then nil.
class ClientHints
- # A browser's hints are a few dozen bytes: longer values are ignored.
+ # A browser's hints are a few dozen bytes: longer values, and invalid UTF-8, are ignored.
HEADER_SIZE_LIMIT_BYTES = 512
PLATFORM_CLASS_BY_NAME = {
"Android" => Platform::Android,
@@ -54,7 +54,7 @@ def windows11?
private def header(headers, name)
value = headers[name]
- value if value.is_a?(String) && value.bytesize <= HEADER_SIZE_LIMIT_BYTES
+ value if value.is_a?(String) && value.bytesize <= HEADER_SIZE_LIMIT_BYTES && value.valid_encoding?
end
private def parse_brands(value)
diff --git a/lib/browser/cloudflare.rb b/lib/browser/cloudflare.rb
index b163fe2..25d90e5 100644
--- a/lib/browser/cloudflare.rb
+++ b/lib/browser/cloudflare.rb
@@ -5,7 +5,7 @@ module Browser
# sets (docs/guide.html#cloudflare) and its own CF-IPCountry. Each site names
# its headers in its rule: Browser::Cloudflare.header_names= changes them.
class Cloudflare
- # Cloudflare's values are a few bytes: a longer one is forged, and ignored.
+ # Cloudflare's values are a few bytes: a longer one, or invalid UTF-8, is forged, and ignored.
HEADER_SIZE_LIMIT_BYTES = 128
DEFAULT_HEADER_NAMES = {
verified_bot: "X-Bot-Verified",
@@ -61,7 +61,7 @@ def country
private def header(signal)
value = @headers&.[](self.class.env_key_by_signal[signal])
- value if value.is_a?(String) && value.bytesize <= HEADER_SIZE_LIMIT_BYTES
+ value if value.is_a?(String) && value.bytesize <= HEADER_SIZE_LIMIT_BYTES && value.valid_encoding?
end
end
end
diff --git a/lib/browser/detect_version.rb b/lib/browser/detect_version.rb
index 19f7c73..e4a68c8 100644
--- a/lib/browser/detect_version.rb
+++ b/lib/browser/detect_version.rb
@@ -2,6 +2,21 @@
module Browser
module DetectVersion
+ # Gem::Version.new and Gem::Version#release keep every version they build in class variables, for the life of
+ # the process: a version read from a header must not reach them, or each new one an attacker sends stays.
+ # A subclass skips the first; this #release is RubyGems' own without the second.
+ class UncachedVersion < Gem::Version
+ def release
+ return self unless prerelease?
+
+ self.class.new(segments.take_while { |segment| segment.is_a?(Integer) }.join("."))
+ end
+ end
+
+ # A browser's version is a few bytes: a longer one is not kept, so that a cached platform stays small.
+ MAX_KEPT_VERSION_BYTES = 32
+ private_constant :UncachedVersion, :MAX_KEPT_VERSION_BYTES
+
private def detect_version?(actual_version, expected_version)
return true unless expected_version
return false unless actual_version
@@ -10,11 +25,21 @@ module DetectVersion
actual_version = parse_version(actual_version)
Gem::Requirement.create(expected_version)
- .satisfied_by?(Gem::Version.create(actual_version))
+ .satisfied_by?(gem_version(actual_version))
rescue ArgumentError
false
end
+ # The last one built: a cached platform asks for the same version on each request.
+ private def gem_version(version)
+ string, gem_version = @gem_version
+ return gem_version if string == version
+
+ gem_version = UncachedVersion.create(version)
+ @gem_version = [version, gem_version] if version.bytesize <= MAX_KEPT_VERSION_BYTES
+ gem_version
+ end
+
private def parse_version(version)
version.is_a?(Numeric) ? version.to_s : version
end
diff --git a/lib/browser/middleware.rb b/lib/browser/middleware.rb
index df7ff93..55e55d9 100644
--- a/lib/browser/middleware.rb
+++ b/lib/browser/middleware.rb
@@ -60,7 +60,7 @@ def process?(request)
end
def html?(request)
- request.env["HTTP_ACCEPT"].to_s.match(ACCEPT_REGEX)
+ request.env["HTTP_ACCEPT"].to_s.scrub.match(ACCEPT_REGEX)
end
def assets?(request)
diff --git a/test/request_headers_test.rb b/test/request_headers_test.rb
index b8917eb..c5e044f 100644
--- a/test/request_headers_test.rb
+++ b/test/request_headers_test.rb
@@ -38,4 +38,20 @@ def show
assert_equal [true, "FR"], seen
end
+
+ test "the middleware and the helper read headers that are not valid UTF-8" do
+ headers = HEADERS.merge("HTTP_ACCEPT_LANGUAGE" => "fr-FR")
+ .transform_values { |value| "#{value}\xFF".force_encoding(Encoding::UTF_8) }
+ middleware = Browser::Middleware.new(->(_env) { [200, {}, []] }) do
+ redirect_to "/upgrade" if browser.chrome? && browser.accept_language.first.code == "fr"
+ end
+
+ status, = middleware.call(Rack::MockRequest.env_for("/", headers))
+ # Rails itself raises on such an Accept header.
+ _status, _headers, body = SignalsController.action(:show)
+ .call(Rack::MockRequest.env_for("/", headers.merge("HTTP_ACCEPT" => "text/html")))
+
+ assert_equal 302, status
+ assert_equal "10.0 false ", body.each.to_a.join
+ end
end
diff --git a/test/unit/cache_test.rb b/test/unit/cache_test.rb
index 40df243..55ec494 100644
--- a/test/unit/cache_test.rb
+++ b/test/unit/cache_test.rb
@@ -185,6 +185,18 @@ def cached_user_agents
Browser.new("", accept_language: "fr-FR,en;q=0.8").accept_language.first
end
+ test "does not cache an accept language listing more languages than a browser sends" do
+ header = Array.new(Browser::Cache::MAX_LANGUAGES_PER_CACHED_HEADER + 1) { |index| "x#{index}" }.join(",")
+ Browser::Cache.clear
+
+ languages = Browser.new("", accept_language: header).accept_language
+
+ assert_equal %w[x0 x1], languages.first(2).map(&:full)
+ assert_empty Browser::Cache.instance_variable_get(:@languages_by_header)
+ Browser.new("", accept_language: header.sub(/,x\d+\z/, "")).accept_language
+ assert_equal 1, Browser::Cache.instance_variable_get(:@languages_by_header).size
+ end
+
test "gives the same answers from concurrent threads" do
user_agents = Browser.user_agents.values
answers = ->(browser) do
diff --git a/test/unit/client_hints_test.rb b/test/unit/client_hints_test.rb
index 2f2236b..95ba174 100644
--- a/test/unit/client_hints_test.rb
+++ b/test/unit/client_hints_test.rb
@@ -55,6 +55,17 @@ def hints(platform: nil, platform_version: nil, mobile: nil, brands: BRANDS)
assert_nil Browser.new(WINDOWS_CHROME).client_hints.platform
end
+ test "ignores a hint that is not valid UTF-8" do
+ invalid = hints(platform: "Windows\xFF", platform_version: "19.0.0\xFF", brands: "#{BRANDS}\xFF")
+ .transform_values { |value| value.dup.force_encoding(Encoding::UTF_8) }
+ client_hints = Browser::ClientHints.new(invalid)
+
+ assert_empty client_hints.version_by_brand
+ assert_nil client_hints.platform
+ assert_nil client_hints.platform_version
+ refute Browser.new(WINDOWS_CHROME, headers: invalid).platform.windows11?
+ end
+
test "tells Windows 11 from Windows 10" do
windows11 = Browser.new(WINDOWS_CHROME, headers: hints(platform: "Windows", platform_version: "19.0.0")).platform
windows10 = Browser.new(WINDOWS_CHROME, headers: hints(platform: "Windows", platform_version: "10.0.0")).platform
diff --git a/test/unit/cloudflare_test.rb b/test/unit/cloudflare_test.rb
index e1be59a..b7735c0 100644
--- a/test/unit/cloudflare_test.rb
+++ b/test/unit/cloudflare_test.rb
@@ -57,6 +57,19 @@ def cloudflare(headers)
refute cloudflare("HTTP_X_BOT_VERIFIED" => "true#{" " * 200}").verified_bot?
end
+ test "ignores a value that is not valid UTF-8" do
+ invalid = ->(value) { "#{value}\xFF".force_encoding(Encoding::UTF_8) }
+ cloudflare = cloudflare(
+ "HTTP_X_BOT_VERIFIED" => invalid.call("true"), "HTTP_X_BOT_CATEGORY" => invalid.call("Search Engine Crawler"),
+ "HTTP_X_BOT_SCORE" => invalid.call("1"), "HTTP_CF_IPCOUNTRY" => invalid.call("US")
+ )
+
+ refute cloudflare.verified_bot?
+ assert_nil cloudflare.bot_category
+ assert_nil cloudflare.bot_score
+ assert_nil cloudflare.country
+ end
+
test "reads the header names a site chose" do
Browser::Cloudflare.header_names = {verified_bot: "X-Edge-Good-Bot", bot_score: "X-Edge-Score"}
diff --git a/test/unit/hostile_headers_test.rb b/test/unit/hostile_headers_test.rb
new file mode 100644
index 0000000..e6a7ae6
--- /dev/null
+++ b/test/unit/hostile_headers_test.rb
@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+require "test_helper"
+
+class HostileHeadersTest < Minitest::Test
+ MAC_SAFARI = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) " \
+ "Version/17.0 Safari/605.1.15"
+
+ # What a log line or a forged header holds: UTF-8 with a byte that is not.
+ def invalid_utf8(string)
+ "#{string}\xFF".force_encoding(Encoding::UTF_8)
+ end
+
+ def cached_versions
+ %i[@@all @@release].sum { |name| Gem::Version.class_variable_get(name).size }
+ end
+
+ setup do
+ Browser.cache_size = 1000
+ end
+
+ test "reads a user agent that is not valid UTF-8" do
+ browser = Browser.new(invalid_utf8(MAC_SAFARI))
+
+ assert_equal "Safari", browser.name
+ assert_equal "17", browser.version
+ assert_equal "macOS", browser.platform.name
+ refute browser.bot?
+ assert browser.ua.valid_encoding?
+ assert Browser.new(invalid_utf8("Googlebot/2.1 ")).bot?
+ end
+
+ test "Bot.why? reads a user agent that is not valid UTF-8" do
+ assert_equal Browser::Bot::KnownBotsMatcher, Browser::Bot.why?(invalid_utf8("Googlebot/2.1 "))
+ end
+
+ test "reads an Accept-Language header that is not valid UTF-8" do
+ languages = Browser.new(MAC_SAFARI, accept_language: invalid_utf8("fr-FR,en;q=0.5")).accept_language
+
+ assert_equal %w[fr-FR en], languages.map(&:full)
+ end
+
+ test "keeps none of the versions a user agent or a client hint sends" do
+ skip "RubyGems no longer caches every version" unless Gem::Version.class_variable_defined?(:@@release)
+
+ ask = lambda do |index|
+ browser = Browser.new(MAC_SAFARI.sub("10_15_7", "10_15_#{index}").sub("17.0", "17.#{index}"),
+ headers: {"HTTP_SEC_CH_UA_PLATFORM" => '"macOS"', "HTTP_SEC_CH_UA_PLATFORM_VERSION" => %("15.#{index}.a")})
+ [browser.safari?(">= 17"), browser.safari?("~> 17.0"), browser.platform.name, browser.platform.mac?("~> 15.0")]
+ end
+ ask.call(1)
+ before = cached_versions
+
+ answers = (2..21).map(&ask)
+
+ assert_equal [[true, true, "macOS", true]], answers.uniq
+ assert_equal before, cached_versions
+ end
+
+ test "compares versions as RubyGems does, prereleases included" do
+ headers = {"HTTP_SEC_CH_UA_PLATFORM" => '"macOS"', "HTTP_SEC_CH_UA_PLATFORM_VERSION" => '"15.1.a"'}
+ platform = Browser.new(MAC_SAFARI, headers:).platform
+
+ assert platform.mac?("~> 15.0")
+ assert platform.mac?("< 15.1")
+ refute platform.mac?("~> 15.1")
+ refute platform.mac?(">= 15.1")
+ end
+end