From 838a40ddb0dbe7170848022dc935c1e70c99df8b Mon Sep 17 00:00:00 2001 From: Rosa Gutierrez Date: Fri, 11 Sep 2026 15:56:47 +0200 Subject: [PATCH 1/3] Cap what an account import reads out of a zip entry The import inflated zip entries with no byte limit, so a few-megabyte archive holding one entry that expands to gigabytes exhausted memory on a jobs worker. Anyone can reach it: Account::ImportsController#create signs up the account itself. Record entries are now read against a byte budget, checked both on the size the archive declares and on the bytes that actually come out of the extractor, since a crafted archive can understate the first. Storage entries keep streaming, but a read now returns at most the length asked for rather than whatever a compressed slice inflates to. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_0186eyzivcTn6wqjEE4Wnxdt --- app/models/account/import.rb | 3 + app/models/zip_file.rb | 5 ++ app/models/zip_file/reader.rb | 44 ++++++++++- app/models/zip_file/reader/io.rb | 58 +++++++++++++-- test/jobs/account/data_import_job_test.rb | 11 +++ test/models/account/import_test.rb | 54 ++++++++++++++ test/models/zip_file_test.rb | 90 +++++++++++++++++++++++ 7 files changed, 258 insertions(+), 7 deletions(-) diff --git a/app/models/account/import.rb b/app/models/account/import.rb index 764c219352..ce3e5ede41 100644 --- a/app/models/account/import.rb +++ b/app/models/account/import.rb @@ -85,6 +85,9 @@ def cleanup end private + # Only the Disk service stages the archive on local disk. On S3 the reader + # range-requests it and blob contents upload straight back, so there is no + # local free space to preflight. def ensure_sufficient_storage_space return unless path = ZipFile.path_on_disk(file.blob) diff --git a/app/models/zip_file.rb b/app/models/zip_file.rb index 370a59a928..25b2a1cd1f 100644 --- a/app/models/zip_file.rb +++ b/app/models/zip_file.rb @@ -1,6 +1,11 @@ class ZipFile class InvalidFileError < StandardError; end + # Inherits from InvalidFileError so an oversized entry is handled everywhere a + # bad export already is: the import stops with an "invalid export" reason, and + # the job discards it rather than resuming against the same entry. + class EntryTooLargeError < InvalidFileError; end + class << self def create_for(attachment, filename:) raise ArgumentError, "No block given" unless block_given? diff --git a/app/models/zip_file/reader.rb b/app/models/zip_file/reader.rb index 12ddd6fae2..647353ac23 100644 --- a/app/models/zip_file/reader.rb +++ b/app/models/zip_file/reader.rb @@ -1,4 +1,17 @@ class ZipFile::Reader + # Entries read into memory hold one database record each, and the largest + # column any of them can fill is a rich text body; every other text column the + # export writes tops out at 64KB. Reading them with no ceiling let a + # two-megabyte upload inflate to gigabytes inside a jobs worker. + MAX_BUFFERED_ENTRY_SIZE = 2.megabytes + + # The extractor is fed compressed bytes, so how much it hands back in one call + # is the archive author's choice: a maximally compressible slice expands about + # a thousandfold. Slicing here keeps every real record a single read of the + # archive, which matters on S3 where each read is its own range request, while + # capping what one call can return at tens of megabytes. + EXTRACT_SLICE_SIZE = 64.kilobytes + def initialize(io) @io = io @reader = ZipKit::FileReader.read_zip_structure(io: io) @@ -6,7 +19,7 @@ def initialize(io) raise ZipFile::InvalidFileError, e.message end - def read(file_path) + def read(file_path, max_bytes: MAX_BUFFERED_ENTRY_SIZE) entry = @reader.find { |e| e.filename == file_path } raise ArgumentError, "File not found in zip: #{file_path}" unless entry raise ArgumentError, "Cannot read directory entry: #{file_path}" if entry.filename.end_with?("/") @@ -14,7 +27,7 @@ def read(file_path) if block_given? yield ZipFile::Reader::IO.new(entry, @io) else - entry.extractor_from(@io).extract + extract_within(entry, max_bytes) end end @@ -25,4 +38,31 @@ def glob(pattern) def exists?(file_path) @reader.any? { |e| e.filename == file_path } end + + private + def extract_within(entry, max_bytes) + ensure_within entry, entry.uncompressed_size, max_bytes + + extractor = entry.extractor_from(@io) + content = "".b + + until extractor.eof? + chunk = extractor.extract(EXTRACT_SLICE_SIZE) + break if chunk.nil? + + content << chunk + ensure_within entry, content.bytesize, max_bytes + end + + content + end + + # The size an entry declares is the archive author's word, so the bytes that + # come out of the extractor are counted as well. + def ensure_within(entry, bytes, max_bytes) + if bytes > max_bytes + raise ZipFile::EntryTooLargeError, + "#{entry.filename} expands to at least #{bytes} bytes, over the #{max_bytes} byte limit" + end + end end diff --git a/app/models/zip_file/reader/io.rb b/app/models/zip_file/reader/io.rb index cf28212aa2..e0694985ec 100644 --- a/app/models/zip_file/reader/io.rb +++ b/app/models/zip_file/reader/io.rb @@ -1,15 +1,17 @@ class ZipFile::Reader::IO + STORED = 0 + def initialize(entry, io) @entry = entry @io = io - @extractor = @entry.extractor_from(@io) + rewind end def read(length = nil, buffer = nil) - return nil if @extractor.eof? + fill_buffer_for(length) + return nil if eof? - data = @extractor.extract(length) - return nil if data.nil? + data = take(length) if buffer buffer.replace(data) @@ -20,15 +22,61 @@ def read(length = nil, buffer = nil) end def eof? - @extractor.eof? + buffered.zero? && @extractor.eof? end def rewind @extractor = @entry.extractor_from(@io) + @buffer = "".b + @consumed = 0 0 end def size @entry.uncompressed_size end + + private + def fill_buffer_for(length) + until @extractor.eof? || (length && buffered >= length) + drop_consumed + + chunk = @extractor.extract(slice_size(length)) + break if chunk.nil? + + @buffer << chunk + end + end + + # A deflated entry hands back whatever it inflates to, which is the archive + # author's choice rather than the caller's: asking for five megabytes of a + # crafted entry gets gigabytes back in one string. Read those in slices and + # keep what the caller didn't ask for. A stored entry returns exactly the + # bytes asked of it, so its reads pass straight through. + def slice_size(length) + if @entry.storage_mode == STORED + length + else + [ ZipFile::Reader::EXTRACT_SLICE_SIZE, length ].compact.min + end + end + + def take(length) + wanted = length ? [ length, buffered ].min : buffered + + @buffer.byteslice(@consumed, wanted).tap { @consumed += wanted } + end + + def buffered + @buffer.bytesize - @consumed + end + + # Handing bytes out by moving a cursor rather than trimming the front keeps a + # long run of small reads from recopying the rest of the buffer every time. + def drop_consumed + if @consumed > 0 + @buffer = @buffer.byteslice(@consumed..) + @consumed = 0 + end + end end diff --git a/test/jobs/account/data_import_job_test.rb b/test/jobs/account/data_import_job_test.rb index 59dfe6b255..ad7ed04d28 100644 --- a/test/jobs/account/data_import_job_test.rb +++ b/test/jobs/account/data_import_job_test.rb @@ -94,6 +94,17 @@ class Account::DataImportJobTest < ActiveJob::TestCase tampered_tempfile&.unlink end + test "discards the job when an export entry inflates past the size limit" do + import = Account::Import.create!(identity: identities(:david), account: Account.create!(name: "Import Test")) + Account::Import.any_instance.stubs(:check).raises(ZipFile::EntryTooLargeError) + + assert_nothing_raised do + assert_no_enqueued_jobs only: Account::DataImportJob do + Account::DataImportJob.perform_now(import) + end + end + end + private # Simulates a hand-edited export: an extra board_publications record with a # fresh id but a key copied from another record in the same export. diff --git a/test/models/account/import_test.rb b/test/models/account/import_test.rb index fba30e0cef..81092900cf 100644 --- a/test/models/account/import_test.rb +++ b/test/models/account/import_test.rb @@ -187,6 +187,37 @@ class Account::ImportTest < ActiveSupport::TestCase tempfile&.unlink end + test "check sets failure_reason to invalid_export when a record entry inflates past the size limit" do + source_account = accounts("37s") + exporter = users(:david) + identity = exporter.identity + + export = Account::Export.create!(account: source_account, user: exporter) + export.build + + export_tempfile = Tempfile.new([ "export", ".zip" ]) + export.file.open { |f| FileUtils.cp(f.path, export_tempfile.path) } + + bomb_tempfile = with_oversized_tag(export_tempfile) + + source_account.destroy! + + target_account = Account.create_with_owner(account: { name: "Import Test" }, owner: { identity: identity, name: exporter.name }) + import = Account::Import.create!(identity: identity, account: target_account) + Current.set(account: target_account) do + import.file.attach(io: File.open(bomb_tempfile.path), filename: "export.zip", content_type: "application/zip") + end + + assert_raises(ZipFile::EntryTooLargeError) { import.check } + + assert import.reload.failed_due_to_invalid_export? + ensure + export_tempfile&.close + export_tempfile&.unlink + bomb_tempfile&.close + bomb_tempfile&.unlink + end + test "check sets failure_reason to conflict when records already exist" do source_account = accounts("37s") exporter = users(:david) @@ -311,6 +342,29 @@ class Account::ImportTest < ActiveSupport::TestCase end private + def with_oversized_tag(export_tempfile) + bomb = Tempfile.new([ "oversized_export", ".zip" ]) + bomb.binmode + + File.open(export_tempfile.path, "rb") do |file| + reader = ZipFile::Reader.new(file) + writer = ZipFile::Writer.new(bomb) + + target = reader.glob("data/tags/*.json").first + tag = JSON.parse(reader.read(target)) + + reader.glob("**/*").reject { |name| name.end_with?("/") || name == target }.each do |entry| + writer.add_file(entry, reader.read(entry)) + end + + writer.add_file target, tag.merge("title" => "a" * (ZipFile::Reader::MAX_BUFFERED_ENTRY_SIZE + 1)).to_json + writer.close + end + + bomb.rewind + bomb + end + def import_with_attached_zip account = Account.create!(name: "Disk Check") import = Account::Import.create!(account: account, identity: identities(:david)) diff --git a/test/models/zip_file_test.rb b/test/models/zip_file_test.rb index 3484f9f923..7c5a2a2e34 100644 --- a/test/models/zip_file_test.rb +++ b/test/models/zip_file_test.rb @@ -119,6 +119,72 @@ class ZipFileTest < ActiveSupport::TestCase end end + test "reader reads an entry at the size limit" do + content = "a" * ZipFile::Reader::MAX_BUFFERED_ENTRY_SIZE + tempfile = create_test_zip("data/tags/big.json" => content) + + reader = ZipFile::Reader.new(tempfile) + + assert_equal content.bytesize, reader.read("data/tags/big.json").bytesize + end + + test "reader raises EntryTooLargeError for an entry over the size limit" do + content = "a" * (ZipFile::Reader::MAX_BUFFERED_ENTRY_SIZE + 1) + tempfile = create_test_zip("data/tags/bomb.json" => content) + + reader = ZipFile::Reader.new(tempfile) + + assert_raises(ZipFile::EntryTooLargeError) { reader.read("data/tags/bomb.json") } + end + + test "reader raises EntryTooLargeError when the zip understates an entry's size" do + content = "a" * (4 * ZipFile::Reader::MAX_BUFFERED_ENTRY_SIZE) + tempfile = understate_declared_size(create_test_zip("data/tags/bomb.json" => content), "data/tags/bomb.json", 1024) + + reader = ZipFile::Reader.new(tempfile) + + assert_equal 1024, reader.instance_variable_get(:@reader).find { |e| e.filename == "data/tags/bomb.json" }.uncompressed_size + assert_raises(ZipFile::EntryTooLargeError) { reader.read("data/tags/bomb.json") } + end + + test "reader streams an entry over the size limit when given a block" do + content = "a" * (2 * ZipFile::Reader::MAX_BUFFERED_ENTRY_SIZE) + tempfile = create_test_zip("storage/blob_key" => content) + + reader = ZipFile::Reader.new(tempfile) + streamed = 0 + reader.read("storage/blob_key") do |io| + streamed += io.read.bytesize until io.eof? + end + + assert_equal content.bytesize, streamed + end + + test "reader io returns no more than the length asked for from a deflated entry" do + content = "a" * 2.megabytes + tempfile = create_test_zip("storage/blob_key" => content) + + reader = ZipFile::Reader.new(tempfile) + reader.read("storage/blob_key") do |io| + assert_equal 1024, io.read(1024).bytesize + end + end + + test "reader io streams a deflated entry intact in small reads" do + content = ("fizzy" * 200_000).b + tempfile = create_test_zip("storage/blob_key" => content) + + reader = ZipFile::Reader.new(tempfile) + streamed = "".b + reader.read("storage/blob_key") do |io| + while chunk = io.read(7919) + streamed << chunk + end + end + + assert_equal content, streamed + end + test "reader raises InvalidFileError for non-zip file" do tempfile = Tempfile.new([ "not_a_zip", ".zip" ]) tempfile.write("this is not a zip file at all") @@ -131,6 +197,30 @@ class ZipFileTest < ActiveSupport::TestCase end private + # Rewrites the uncompressed size the central directory declares for one + # entry, the way a crafted archive would. + def understate_declared_size(tempfile, path, size) + bytes = File.binread(tempfile.path) + offset = 0 + + while offset = bytes.index("PK\x01\x02".b, offset) + name_length = bytes[offset + 28, 2].unpack1("v") + + if bytes[offset + 46, name_length] == path.b + bytes[offset + 24, 4] = [ size ].pack("V") + break + end + + offset += 4 + end + + Tempfile.new([ "understated", ".zip" ]).tap do |patched| + patched.binmode + patched.write(bytes) + patched.rewind + end + end + def create_test_zip(files) tempfile = Tempfile.new([ "test", ".zip" ]) tempfile.binmode From a9348ca65b2efdbd206499540b09670c0eb45bc0 Mon Sep 17 00:00:00 2001 From: Rosa Gutierrez Date: Fri, 11 Sep 2026 16:00:00 +0200 Subject: [PATCH 2/3] Stop reading an entry whose deflate stream is truncated An entry whose compressed data runs out before the deflate stream ends leaves the extractor short of eof while it has nothing left to give, so take its first nil as the end rather than asking again forever. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_0186eyzivcTn6wqjEE4Wnxdt --- app/models/zip_file/reader/io.rb | 19 +++++++++++++++---- test/models/zip_file_test.rb | 31 ++++++++++++++++++++++--------- 2 files changed, 37 insertions(+), 13 deletions(-) diff --git a/app/models/zip_file/reader/io.rb b/app/models/zip_file/reader/io.rb index e0694985ec..26c9e040b3 100644 --- a/app/models/zip_file/reader/io.rb +++ b/app/models/zip_file/reader/io.rb @@ -22,13 +22,14 @@ def read(length = nil, buffer = nil) end def eof? - buffered.zero? && @extractor.eof? + buffered.zero? && drained? end def rewind @extractor = @entry.extractor_from(@io) @buffer = "".b @consumed = 0 + @drained = false 0 end @@ -38,16 +39,26 @@ def size private def fill_buffer_for(length) - until @extractor.eof? || (length && buffered >= length) + until drained? || (length && buffered >= length) drop_consumed chunk = @extractor.extract(slice_size(length)) - break if chunk.nil? - @buffer << chunk + if chunk.nil? + @drained = true + else + @buffer << chunk + end end end + # An entry whose compressed data runs out before the deflate stream ends + # leaves the extractor short of eof while it has nothing left to give, so + # take its first nil as the end too rather than asking again forever. + def drained? + @drained || @extractor.eof? + end + # A deflated entry hands back whatever it inflates to, which is the archive # author's choice rather than the caller's: asking for five megabytes of a # crafted entry gets gigabytes back in one string. Read those in slices and diff --git a/test/models/zip_file_test.rb b/test/models/zip_file_test.rb index 7c5a2a2e34..88496779fa 100644 --- a/test/models/zip_file_test.rb +++ b/test/models/zip_file_test.rb @@ -185,6 +185,18 @@ class ZipFileTest < ActiveSupport::TestCase assert_equal content, streamed end + test "reader io stops at a truncated deflate stream" do + tempfile = understate_declared_size(create_test_zip("storage/blob_key" => "a" * 64.kilobytes), "storage/blob_key", 16, offset: 20) + + reader = ZipFile::Reader.new(tempfile) + reads = 0 + reader.read("storage/blob_key") do |io| + reads += 1 while io.read(1024) + end + + assert_operator reads, :<, 100 + end + test "reader raises InvalidFileError for non-zip file" do tempfile = Tempfile.new([ "not_a_zip", ".zip" ]) tempfile.write("this is not a zip file at all") @@ -197,21 +209,22 @@ class ZipFileTest < ActiveSupport::TestCase end private - # Rewrites the uncompressed size the central directory declares for one - # entry, the way a crafted archive would. - def understate_declared_size(tempfile, path, size) + # Rewrites a size the central directory declares for one entry, the way a + # crafted archive would: offset 24 is the uncompressed size, 20 the + # compressed one. + def understate_declared_size(tempfile, path, size, offset: 24) bytes = File.binread(tempfile.path) - offset = 0 + cdir = 0 - while offset = bytes.index("PK\x01\x02".b, offset) - name_length = bytes[offset + 28, 2].unpack1("v") + while cdir = bytes.index("PK\x01\x02".b, cdir) + name_length = bytes[cdir + 28, 2].unpack1("v") - if bytes[offset + 46, name_length] == path.b - bytes[offset + 24, 4] = [ size ].pack("V") + if bytes[cdir + 46, name_length] == path.b + bytes[cdir + offset, 4] = [ size ].pack("V") break end - offset += 4 + cdir += 4 end Tempfile.new([ "understated", ".zip" ]).tap do |patched| From c9b005ff79e42c99a090d1a1bb28358230ecf1e9 Mon Sep 17 00:00:00 2001 From: Rosa Gutierrez Date: Fri, 11 Sep 2026 16:29:31 +0200 Subject: [PATCH 3/3] Budget what an import reads against the size of the archive it came from Capping a buffered entry left the streaming path open: a crafted deflated storage entry returns a bounded amount per read but no bounded amount in total, so a small archive could still feed gigabytes into blob.upload. Every byte a reader hands out now counts against a budget set from the archive's own size, with a floor so a small archive holding one large rich text body still imports. A real export measures 0.76 bytes out per byte of archive, and its most compressible entry 2:1, because entries are deflated individually and records are small. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_0186eyzivcTn6wqjEE4Wnxdt --- app/models/zip_file.rb | 8 +++++--- app/models/zip_file/reader.rb | 25 ++++++++++++++++++++++++- app/models/zip_file/reader/io.rb | 4 +++- test/models/zip_file_test.rb | 27 +++++++++++++++++++++++++++ 4 files changed, 59 insertions(+), 5 deletions(-) diff --git a/app/models/zip_file.rb b/app/models/zip_file.rb index 25b2a1cd1f..437df993ad 100644 --- a/app/models/zip_file.rb +++ b/app/models/zip_file.rb @@ -1,10 +1,12 @@ class ZipFile class InvalidFileError < StandardError; end - # Inherits from InvalidFileError so an oversized entry is handled everywhere a - # bad export already is: the import stops with an "invalid export" reason, and - # the job discards it rather than resuming against the same entry. + # Both inherit from InvalidFileError so an archive that busts a limit is + # handled everywhere a bad export already is: the import stops with an + # "invalid export" reason, and the job discards it rather than resuming + # against the same entry. class EntryTooLargeError < InvalidFileError; end + class ArchiveTooLargeError < InvalidFileError; end class << self def create_for(attachment, filename:) diff --git a/app/models/zip_file/reader.rb b/app/models/zip_file/reader.rb index 647353ac23..87fc97f19c 100644 --- a/app/models/zip_file/reader.rb +++ b/app/models/zip_file/reader.rb @@ -12,9 +12,21 @@ class ZipFile::Reader # capping what one call can return at tens of megabytes. EXTRACT_SLICE_SIZE = 64.kilobytes + # Deflating each entry on its own is what makes an archive's total expansion a + # usable signal: a real export barely shrinks, since records are small and per + # entry overhead eats the savings, while a crafted one expands a thousandfold. + # Budgeting everything read out of an archive against what was actually + # uploaded bounds the streaming path too, where an entry's own declared size is + # the archive author's word. The floor keeps a small archive holding one large + # rich text body importable. + MAX_TOTAL_EXPANSION = 100 + MIN_EXPANSION_BUDGET = 64.megabytes + def initialize(io) @io = io @reader = ZipKit::FileReader.read_zip_structure(io: io) + @expanded = 0 + @budget = [ io.size * MAX_TOTAL_EXPANSION, MIN_EXPANSION_BUDGET ].max rescue ZipKit::FileReader::ReadError, ZipKit::FileReader::MissingEOCD, ZipKit::FileReader::UnsupportedFeature => e raise ZipFile::InvalidFileError, e.message end @@ -25,7 +37,7 @@ def read(file_path, max_bytes: MAX_BUFFERED_ENTRY_SIZE) raise ArgumentError, "Cannot read directory entry: #{file_path}" if entry.filename.end_with?("/") if block_given? - yield ZipFile::Reader::IO.new(entry, @io) + yield ZipFile::Reader::IO.new(entry, @io, self) else extract_within(entry, max_bytes) end @@ -39,6 +51,16 @@ def exists?(file_path) @reader.any? { |e| e.filename == file_path } end + # Called for every byte handed out, buffered or streamed. + def count_expanded(bytes) + @expanded += bytes + + if @expanded > @budget + raise ZipFile::ArchiveTooLargeError, + "archive has produced #{@expanded} bytes, over the #{@budget} byte limit for its size" + end + end + private def extract_within(entry, max_bytes) ensure_within entry, entry.uncompressed_size, max_bytes @@ -50,6 +72,7 @@ def extract_within(entry, max_bytes) chunk = extractor.extract(EXTRACT_SLICE_SIZE) break if chunk.nil? + count_expanded chunk.bytesize content << chunk ensure_within entry, content.bytesize, max_bytes end diff --git a/app/models/zip_file/reader/io.rb b/app/models/zip_file/reader/io.rb index 26c9e040b3..c1efe3fe22 100644 --- a/app/models/zip_file/reader/io.rb +++ b/app/models/zip_file/reader/io.rb @@ -1,9 +1,10 @@ class ZipFile::Reader::IO STORED = 0 - def initialize(entry, io) + def initialize(entry, io, reader) @entry = entry @io = io + @reader = reader rewind end @@ -47,6 +48,7 @@ def fill_buffer_for(length) if chunk.nil? @drained = true else + @reader.count_expanded chunk.bytesize @buffer << chunk end end diff --git a/test/models/zip_file_test.rb b/test/models/zip_file_test.rb index 88496779fa..46a1d285ed 100644 --- a/test/models/zip_file_test.rb +++ b/test/models/zip_file_test.rb @@ -197,6 +197,33 @@ class ZipFileTest < ActiveSupport::TestCase assert_operator reads, :<, 100 end + test "reader raises ArchiveTooLargeError when a streamed entry expands past what the archive can account for" do + tempfile = Tempfile.new([ "bomb", ".zip" ]) + tempfile.binmode + writer = ZipFile::Writer.new(tempfile) + writer.add_file("storage/blob_key") do |sink| + chunk = "a" * 1.megabyte + 80.times { sink.write(chunk) } + end + writer.close + tempfile.rewind + + reader = ZipFile::Reader.new(tempfile) + streamed = 0 + + error = assert_raises(ZipFile::ArchiveTooLargeError) do + reader.read("storage/blob_key") do |io| + streamed += io.read(64.kilobytes).bytesize until io.eof? + end + end + + assert_operator streamed, :<, 80.megabytes + assert_match(/over the \d+ byte limit/, error.message) + ensure + tempfile&.close + tempfile&.unlink + end + test "reader raises InvalidFileError for non-zip file" do tempfile = Tempfile.new([ "not_a_zip", ".zip" ]) tempfile.write("this is not a zip file at all")