From 686a3b5b464a8857f2b3dfb4df9e52864ad4023c Mon Sep 17 00:00:00 2001 From: Markus Stange Date: Wed, 28 Oct 2026 10:06:06 -0400 Subject: [PATCH 1/5] Validate tar headers strictly in untar.ts - Parse octal fields with a stricter parser: Compared to parseInt, this parser won't accept negative values or numbers with trailing garbage. - Use the octal parser for the checksum and device number fields. These were incorrectly parsed as decimal. - Verify the header checksum. - Fix end-of-archive detection: The end is marked by a full zero block. We were trusting the size written in the header and not detecting truncated input. --- src/test/unit/untar.test.ts | 163 ++++++++++++++++++++++++++++++++++++ src/utils/untar.ts | 111 +++++++++++++++++++----- 2 files changed, 251 insertions(+), 23 deletions(-) diff --git a/src/test/unit/untar.test.ts b/src/test/unit/untar.test.ts index b98afc2b3e..b51fa2c8ec 100644 --- a/src/test/unit/untar.test.ts +++ b/src/test/unit/untar.test.ts @@ -4,6 +4,97 @@ import { UntarFileStream } from 'firefox-profiler/utils/untar'; +type HeaderSpec = { + name: string; + type?: string; + // 'ustar' is POSIX UStar / pax, 'gnu' is the old GNU format used by cargo. + format?: 'ustar' | 'gnu'; + prefix?: string; + uid?: string; + // Raw bytes written at the given offset before the checksum is computed. + patches?: Array<{ offset: number; bytes: string | Uint8Array }>; + // Corrupt the checksum after it is computed. + badChecksum?: boolean; +}; + +const encoder = new TextEncoder(); + +function toBytes(data: string | Uint8Array): Uint8Array { + return typeof data === 'string' ? encoder.encode(data) : data; +} + +function makeHeader(spec: HeaderSpec, size: number): Uint8Array { + const header = new Uint8Array(512); + const write = (offset: number, data: string | Uint8Array) => + header.set(toBytes(data), offset); + write(0, spec.name); + write(100, '0000644\0'); + write(108, spec.uid ?? '0000000\0'); + write(116, '0000000\0'); + write(124, size.toString(8).padStart(11, '0') + '\0'); + write(136, '00000000000\0'); + write(156, spec.type ?? '0'); + if (spec.format === 'gnu') { + write(257, 'ustar \0'); + } else { + write(257, 'ustar\x0000'); + } + if (spec.prefix !== undefined) { + write(345, spec.prefix); + } + for (const { offset, bytes } of spec.patches ?? []) { + write(offset, bytes); + } + write(148, ' '); + let checksum = header.reduce((a, b) => a + b, 0); + if (spec.badChecksum) { + checksum++; + } + write(148, checksum.toString(8).padStart(6, '0') + '\0 '); + return header; +} + +function makeEntry(spec: HeaderSpec, content: string | Uint8Array = '') { + const data = toBytes(content); + const paddedSize = Math.ceil(data.length / 512) * 512; + const entry = new Uint8Array(512 + paddedSize); + entry.set(makeHeader(spec, data.length), 0); + entry.set(data, 512); + return entry; +} + +function makeArchive( + entries: Uint8Array[], + { trailer = true }: { trailer?: boolean } = {} +): ArrayBuffer { + if (trailer) { + entries = [...entries, new Uint8Array(1024)]; + } + const totalLength = entries.reduce((a, e) => a + e.length, 0); + const archive = new Uint8Array(totalLength); + let offset = 0; + for (const entry of entries) { + archive.set(entry, offset); + offset += entry.length; + } + return archive.buffer; +} + +function listEntries(archive: ArrayBuffer) { + const decoder = new TextDecoder(); + const stream = new UntarFileStream(archive); + const result = []; + while (stream.hasNext()) { + const { name, type, buffer } = stream.next(); + result.push({ + name, + type, + content: buffer === null ? null : decoder.decode(buffer), + }); + } + return result; +} + describe('untar', function () { it('lists all files in example tar', function () { const fs = require('fs'); @@ -151,4 +242,76 @@ describe('untar', function () { ] `); }); + + it('reads a simple UStar archive', function () { + const archive = makeArchive([ + makeEntry({ name: 'dir', type: '5' }), + makeEntry({ name: 'file.txt' }, 'hello'), + makeEntry({ name: 'big.txt' }, 'x'.repeat(1000)), + ]); + expect(listEntries(archive)).toEqual([ + { name: 'dir', type: '5', content: null }, + { name: 'file.txt', type: '0', content: 'hello' }, + { name: 'big.txt', type: '0', content: 'x'.repeat(1000) }, + ]); + }); + + it('accepts an archive without the trailing zero blocks', function () { + const archive = makeArchive([makeEntry({ name: 'a' }, 'a')], { + trailer: false, + }); + expect(listEntries(archive)).toEqual([ + { name: 'a', type: '0', content: 'a' }, + ]); + }); + + it('stops at the first zero block', function () { + const archive = makeArchive([ + makeEntry({ name: 'a' }, 'a'), + new Uint8Array(512), + makeEntry({ name: 'b' }, 'b'), + ]); + expect(listEntries(archive).map((e) => e.name)).toEqual(['a']); + }); + + it('rejects a header with a bad checksum', function () { + const archive = makeArchive([ + makeEntry({ name: 'a', badChecksum: true }, 'a'), + ]); + expect(() => listEntries(archive)).toThrow('checksum'); + }); + + it('rejects data that is not a tar archive', function () { + const archive = encoder.encode('not a tar file'.repeat(100)).buffer; + expect(() => listEntries(archive)).toThrow(); + }); + + it('rejects malformed numeric fields', function () { + for (const size of [ + '-0000000001\0', + '00000000012x', + '\x80' + '\0'.repeat(11), + ]) { + const archive = makeArchive([ + makeEntry({ name: 'a', patches: [{ offset: 124, bytes: size }] }), + ]); + expect(() => listEntries(archive)).toThrow('Invalid numeric field'); + } + }); + + it('treats an empty numeric field as missing', function () { + const archive = makeArchive([ + makeEntry({ name: 'a', uid: '\0'.repeat(8) }, 'a'), + ]); + const stream = new UntarFileStream(archive); + expect(stream.next().uid).toBe(null); + }); + + it('rejects a truncated archive', function () { + const archive = makeArchive([makeEntry({ name: 'a' }, 'x'.repeat(600))], { + trailer: false, + }); + expect(() => listEntries(archive.slice(0, 1000))).toThrow('Unexpected end'); + expect(() => listEntries(archive.slice(0, 300))).toThrow('Unexpected end'); + }); }); diff --git a/src/utils/untar.ts b/src/utils/untar.ts index 1578e472f6..902790f4a7 100644 --- a/src/utils/untar.ts +++ b/src/utils/untar.ts @@ -30,7 +30,14 @@ class ByteReader { : this._asciiDecoder.decode(bytes.subarray(0, nulBytePos)); } + readOctal(byteCount: number): number | null { + return _parseOctal(this.readBuffer(byteCount)); + } + readBuffer(byteCount: number): Uint8Array { + if (this.position() + byteCount > this.size()) { + throw new Error('Unexpected end of tar archive'); + } const buf = this._uint8Array.subarray( this.position(), this.position() + byteCount @@ -47,8 +54,11 @@ class ByteReader { this._position = newpos; } - peekUint32(): number { - return this._bufferView.getUint32(this.position(), true); + peekBuffer(byteCount: number): Uint8Array { + return this._uint8Array.subarray( + this.position(), + this.position() + byteCount + ); } position(): number { @@ -65,6 +75,61 @@ type FieldEntry = { value: string | number | null; }; +const BLOCK_SIZE = 512; + +// The checksum field covers bytes 148..156 of the header. +const CHECKSUM_OFFSET = 148; +const CHECKSUM_LENGTH = 8; + +/** + * Parse a numeric header field. These fields contain octal digits, optionally + * preceded by spaces or NULs and followed by spaces or NULs. Anything else, + * including GNU base-256 numbers, is rejected. Returns null if the field + * contains no digits at all. + */ +function _parseOctal(bytes: Uint8Array): number | null { + const isPadding = (b: number) => b === 0x00 || b === 0x20; + let i = 0; + while (i < bytes.length && isPadding(bytes[i])) { + i++; + } + let value = 0; + let digitCount = 0; + for (; i < bytes.length && bytes[i] >= 0x30 && bytes[i] <= 0x37; i++) { + value = value * 8 + (bytes[i] - 0x30); + digitCount++; + } + for (; i < bytes.length; i++) { + if (!isPadding(bytes[i])) { + throw new Error('Invalid numeric field in tar header'); + } + } + return digitCount === 0 ? null : value; +} + +function _isZeroBlock(block: Uint8Array): boolean { + return block.every((b) => b === 0); +} + +/** + * Check the header checksum: the unsigned sum of all header bytes, with the + * checksum field itself counted as if it contained spaces. + */ +function _verifyChecksum(header: Uint8Array): void { + const expected = _parseOctal( + header.subarray(CHECKSUM_OFFSET, CHECKSUM_OFFSET + CHECKSUM_LENGTH) + ); + let sum = 0x20 * CHECKSUM_LENGTH; + for (let i = 0; i < header.length; i++) { + if (i < CHECKSUM_OFFSET || i >= CHECKSUM_OFFSET + CHECKSUM_LENGTH) { + sum += header[i]; + } + } + if (expected !== sum) { + throw new Error('Tar header checksum mismatch'); + } +} + function _parseIntStrict(s: string, base: number): number { const val = parseInt(s, base); if (isNaN(val)) { @@ -73,13 +138,6 @@ function _parseIntStrict(s: string, base: number): number { return val; } -function _parseOptionalInt(s: string, base: number): number | null { - if (s === '') { - return null; - } - return _parseIntStrict(s, base); -} - /** * A PaxHeader is a set of fields with names and values. The fields get applied * to a file entry. @@ -188,7 +246,6 @@ export type TarFileEntry = { uid: number | null; gid: number | null; mtime: number | null; - checksum: number | null; linkname: string; ustarFormat: string; version: string | null; @@ -208,10 +265,12 @@ export class UntarFileStream { } hasNext(): boolean { - // A tar file ends with 4 zero bytes + // A tar archive ends with two all-zero blocks. We stop at the first one, + // and also accept an archive that ends right after its last member. + const reader = this._reader; return ( - this._reader.position() + 4 < this._reader.size() && - this._reader.peekUint32() !== 0 + reader.position() < reader.size() && + !_isZeroBlock(reader.peekBuffer(BLOCK_SIZE)) ); } @@ -223,15 +282,22 @@ export class UntarFileStream { const stream = this._reader; const headerBeginPos = stream.position(); + if (headerBeginPos + BLOCK_SIZE > stream.size()) { + throw new Error('Unexpected end of tar archive'); + } + _verifyChecksum(stream.peekBuffer(BLOCK_SIZE)); // Read header let name = stream.readString(100); const mode = stream.readString(8); - const uid = _parseOptionalInt(stream.readString(8), 8); - const gid = _parseOptionalInt(stream.readString(8), 8); - const size = _parseIntStrict(stream.readString(12), 8); - const mtime = _parseOptionalInt(stream.readString(12), 8); - const checksum = _parseOptionalInt(stream.readString(8), 10); + const uid = stream.readOctal(8); + const gid = stream.readOctal(8); + const size = stream.readOctal(12); + if (size === null) { + throw new Error('Missing size in tar header'); + } + const mtime = stream.readOctal(12); + stream.seekBy(CHECKSUM_LENGTH); const type = stream.readString(1); const linkname = stream.readString(100); const ustarFormat = stream.readString(6); @@ -246,8 +312,8 @@ export class UntarFileStream { version = stream.readString(2); uname = stream.readString(32); gname = stream.readString(32); - devmajor = _parseOptionalInt(stream.readString(8), 10); - devminor = _parseOptionalInt(stream.readString(8), 10); + devmajor = stream.readOctal(8); + devminor = stream.readOctal(8); namePrefix = stream.readString(155); if (namePrefix.length > 0) { @@ -256,9 +322,9 @@ export class UntarFileStream { } // The size is padded to be aligned with 512 bytes. - const alignedSize = ((size - 1) | 511) + 1; + const alignedSize = Math.ceil(size / BLOCK_SIZE) * BLOCK_SIZE; - const dataBeginPos = headerBeginPos + 512; + const dataBeginPos = headerBeginPos + BLOCK_SIZE; const dataEndPos = dataBeginPos + alignedSize; stream.seekTo(dataBeginPos); @@ -307,7 +373,6 @@ export class UntarFileStream { gid, size, mtime, - checksum, linkname, ustarFormat, version, From 5570aed0ca93675b5115ec67bec8f98ec6cc7395 Mon Sep 17 00:00:00 2001 From: Markus Stange Date: Wed, 28 Oct 2026 10:07:59 -0400 Subject: [PATCH 2/5] Decode tar header names as UTF-8 in untar.ts TextDecoder's 'ascii' label actually means windows-1252, so UTF-8 names written by cargo were garbled and never matched the requested path. With fatal: true, some non-ASCII bytes also made the whole archive fail. --- src/test/unit/untar.test.ts | 9 +++++++++ src/utils/untar.ts | 8 +++++--- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/src/test/unit/untar.test.ts b/src/test/unit/untar.test.ts index b51fa2c8ec..5fed5dec64 100644 --- a/src/test/unit/untar.test.ts +++ b/src/test/unit/untar.test.ts @@ -314,4 +314,13 @@ describe('untar', function () { expect(() => listEntries(archive.slice(0, 1000))).toThrow('Unexpected end'); expect(() => listEntries(archive.slice(0, 300))).toThrow('Unexpected end'); }); + + it('decodes names as UTF-8', function () { + const archive = makeArchive([ + makeEntry({ name: 'mycrate-1.0.0/é.rs', prefix: 'prefix-ü' }, 'a'), + ]); + expect(listEntries(archive).map((e) => e.name)).toEqual([ + 'prefix-ü/mycrate-1.0.0/é.rs', + ]); + }); }); diff --git a/src/utils/untar.ts b/src/utils/untar.ts index 902790f4a7..4a8e5cd5bb 100644 --- a/src/utils/untar.ts +++ b/src/utils/untar.ts @@ -15,7 +15,9 @@ class ByteReader { _uint8Array: Uint8Array; _bufferView: DataView; _position = 0; - _asciiDecoder = new TextDecoder('ascii', { fatal: true }); + // Names in tar headers are raw bytes; in practice they are UTF-8. Bytes that + // aren't valid UTF-8 become U+FFFD, so such names can't match any real path. + _textDecoder = new TextDecoder('utf-8'); constructor(arrayBuffer: ArrayBufferLike) { this._uint8Array = new Uint8Array(arrayBuffer); @@ -26,8 +28,8 @@ class ByteReader { const bytes = this.readBuffer(byteCount); const nulBytePos = bytes.indexOf(0); return nulBytePos === -1 - ? this._asciiDecoder.decode(bytes) - : this._asciiDecoder.decode(bytes.subarray(0, nulBytePos)); + ? this._textDecoder.decode(bytes) + : this._textDecoder.decode(bytes.subarray(0, nulBytePos)); } readOctal(byteCount: number): number | null { From cba70202fceb10d83c9e09a33228fd75952a8720 Mon Sep 17 00:00:00 2001 From: Markus Stange Date: Wed, 28 Oct 2026 10:10:20 -0400 Subject: [PATCH 3/5] Resolve pax headers before framing in untar.ts Replace the recursive _readNextFile with a loop that collects a pending pax 'x' header and then resolves the next member's name, linkname and size before reading its data. Previously, the pax records were patched onto the entry after it had been read, so: - a pax 'size' record changed entry.size but not the buffer or the position of the next header, desynchronizing the parse; - with two stacked 'x' headers, the earlier one won; - a tombstoned 'path' left entry.name undefined; - every record was copied onto the entry object, so 'buffer=1' replaced the file contents, and all-digit paths became numbers; - many extension headers in a row overflowed the stack. Now only the 'path', 'linkpath' and 'size' keywords are recognized and other records are ignored. A second 'x' header for the same member, an empty resolved name, an 'x' header at the end of the archive, duplicate records and malformed records are rejected. Global 'g' headers may not contain any of the recognized keywords, so their records don't need to be kept. Since hasNext() can no longer know whether a member follows without parsing extension headers, next() now returns null at the end of the archive. TarFileEntry is reduced to the fields that are resolved properly and that callers can use. --- src/test/unit/untar.test.ts | 176 ++++++++++--- src/utils/fetch-source.ts | 3 +- src/utils/untar.ts | 481 ++++++++++++++---------------------- 3 files changed, 320 insertions(+), 340 deletions(-) diff --git a/src/test/unit/untar.test.ts b/src/test/unit/untar.test.ts index 5fed5dec64..f1f2889d90 100644 --- a/src/test/unit/untar.test.ts +++ b/src/test/unit/untar.test.ts @@ -11,6 +11,8 @@ type HeaderSpec = { format?: 'ustar' | 'gnu'; prefix?: string; uid?: string; + // Overrides the size field, which defaults to the content length. + size?: number; // Raw bytes written at the given offset before the checksum is computed. patches?: Array<{ offset: number; bytes: string | Uint8Array }>; // Corrupt the checksum after it is computed. @@ -58,11 +60,37 @@ function makeEntry(spec: HeaderSpec, content: string | Uint8Array = '') { const data = toBytes(content); const paddedSize = Math.ceil(data.length / 512) * 512; const entry = new Uint8Array(512 + paddedSize); - entry.set(makeHeader(spec, data.length), 0); + entry.set(makeHeader(spec, spec.size ?? data.length), 0); entry.set(data, 512); return entry; } +function concatBytes( + parts: Array +): Uint8Array { + const arrays = parts.map(toBytes); + const result = new Uint8Array(arrays.reduce((a, b) => a + b.length, 0)); + let offset = 0; + for (const array of arrays) { + result.set(array, offset); + offset += array.length; + } + return result; +} + +// Build the payload of a pax header from [keyword, value] pairs. +function paxRecords(records: Array<[string, string | Uint8Array]>): Uint8Array { + return concatBytes( + records.map(([keyword, value]) => { + const rest = concatBytes([' ' + keyword + '=', value, '\n']); + // The length prefix counts its own digits. + let length = rest.length + String(rest.length).length; + length = rest.length + String(length).length; + return concatBytes([String(length), rest]); + }) + ); +} + function makeArchive( entries: Uint8Array[], { trailer = true }: { trailer?: boolean } = {} @@ -70,22 +98,15 @@ function makeArchive( if (trailer) { entries = [...entries, new Uint8Array(1024)]; } - const totalLength = entries.reduce((a, e) => a + e.length, 0); - const archive = new Uint8Array(totalLength); - let offset = 0; - for (const entry of entries) { - archive.set(entry, offset); - offset += entry.length; - } - return archive.buffer; + return concatBytes(entries).buffer; } function listEntries(archive: ArrayBuffer) { const decoder = new TextDecoder(); const stream = new UntarFileStream(archive); const result = []; - while (stream.hasNext()) { - const { name, type, buffer } = stream.next(); + for (let entry = stream.next(); entry !== null; entry = stream.next()) { + const { name, type, buffer } = entry; result.push({ name, type, @@ -107,12 +128,11 @@ describe('untar', function () { const fileEntries = []; - while (stream.hasNext()) { - const { name, buffer, size, uid } = stream.next(); + for (let entry = stream.next(); entry !== null; entry = stream.next()) { + const { name, buffer, size } = entry; fileEntries.push({ name, size, - uid, offset: buffer ? buffer.byteOffset : null, }); } @@ -123,121 +143,101 @@ describe('untar', function () { "name": "addr2line-0.17.0/.cargo_vcs_info.json", "offset": 512, "size": 74, - "uid": null, }, Object { "name": "addr2line-0.17.0/.gitignore", "offset": 1536, "size": 33, - "uid": 0, }, Object { "name": "addr2line-0.17.0/CHANGELOG.md", "offset": 2560, "size": 7347, - "uid": 0, }, Object { "name": "addr2line-0.17.0/Cargo.lock", "offset": 10752, "size": 11506, - "uid": null, }, Object { "name": "addr2line-0.17.0/Cargo.toml", "offset": 23040, "size": 2792, - "uid": null, }, Object { "name": "addr2line-0.17.0/Cargo.toml.orig", "offset": 26624, "size": 2334, - "uid": 0, }, Object { "name": "addr2line-0.17.0/LICENSE-APACHE", "offset": 29696, "size": 10847, - "uid": 0, }, Object { "name": "addr2line-0.17.0/LICENSE-MIT", "offset": 41472, "size": 1069, - "uid": 0, }, Object { "name": "addr2line-0.17.0/README.md", "offset": 43520, "size": 2550, - "uid": 0, }, Object { "name": "addr2line-0.17.0/bench.plot.r", "offset": 46592, "size": 1000, - "uid": 0, }, Object { "name": "addr2line-0.17.0/benchmark.sh", "offset": 48128, "size": 3084, - "uid": 0, }, Object { "name": "addr2line-0.17.0/coverage.sh", "offset": 52224, "size": 177, - "uid": 0, }, Object { "name": "addr2line-0.17.0/examples/addr2line.rs", "offset": 53248, "size": 9447, - "uid": 0, }, Object { "name": "addr2line-0.17.0/rustfmt.toml", "offset": 63488, "size": 1, - "uid": 0, }, Object { "name": "addr2line-0.17.0/src/function.rs", "offset": 64512, "size": 19078, - "uid": 0, }, Object { "name": "addr2line-0.17.0/src/lazy.rs", "offset": 84480, "size": 919, - "uid": 0, }, Object { "name": "addr2line-0.17.0/src/lib.rs", "offset": 86016, "size": 41209, - "uid": 0, }, Object { "name": "addr2line-0.17.0/tests/correctness.rs", "offset": 128000, "size": 2756, - "uid": 0, }, Object { "name": "addr2line-0.17.0/tests/output_equivalence.rs", "offset": 131584, "size": 3905, - "uid": 0, }, Object { "name": "addr2line-0.17.0/tests/parse.rs", "offset": 136192, "size": 3035, - "uid": 0, }, ] `); @@ -299,12 +299,11 @@ describe('untar', function () { } }); - it('treats an empty numeric field as missing', function () { + it('accepts an empty optional numeric field', function () { const archive = makeArchive([ makeEntry({ name: 'a', uid: '\0'.repeat(8) }, 'a'), ]); - const stream = new UntarFileStream(archive); - expect(stream.next().uid).toBe(null); + expect(listEntries(archive).map((e) => e.name)).toEqual(['a']); }); it('rejects a truncated archive', function () { @@ -323,4 +322,107 @@ describe('untar', function () { 'prefix-ü/mycrate-1.0.0/é.rs', ]); }); + + it('applies pax path and linkpath records', function () { + const longName = 'dir/'.repeat(40) + 'é\nfile.rs'; + const archive = makeArchive([ + makeEntry( + { name: 'PaxHeaders/x', type: 'x' }, + paxRecords([ + ['path', longName], + ['linkpath', 'target'], + ]) + ), + makeEntry({ name: 'truncated', prefix: 'ignored', type: '2' }), + makeEntry({ name: 'next' }, 'n'), + ]); + const stream = new UntarFileStream(archive); + expect(stream.next()).toMatchObject({ + name: longName, + linkname: 'target', + }); + // The pax header only applies to one member. + expect(stream.next()).toMatchObject({ name: 'next', linkname: '' }); + }); + + it('keeps all-digit pax paths as strings', function () { + const archive = makeArchive([ + makeEntry({ name: 'x', type: 'x' }, paxRecords([['path', '1234']])), + makeEntry({ name: 'a' }, 'a'), + ]); + expect(listEntries(archive)[0].name).toBe('1234'); + }); + + it('uses the pax size for framing', function () { + const archive = makeArchive([ + makeEntry({ name: 'x', type: 'x' }, paxRecords([['size', '600']])), + makeEntry({ name: 'a', size: 0 }, 'a'.repeat(600)), + makeEntry({ name: 'b' }, 'b'), + ]); + expect(listEntries(archive)).toEqual([ + { name: 'a', type: '0', content: 'a'.repeat(600) }, + { name: 'b', type: '0', content: 'b' }, + ]); + }); + + it('ignores unknown pax records', function () { + const archive = makeArchive([ + makeEntry( + { name: 'x', type: 'x' }, + paxRecords([ + ['buffer', '1'], + ['type', '5'], + ['SCHILY.xattr.user.foo', new Uint8Array([0xff, 0x00, 0x0a])], + ['mtime', '1234.5'], + ]) + ), + makeEntry({ name: 'a' }, 'a'), + ]); + expect(listEntries(archive)).toEqual([ + { name: 'a', type: '0', content: 'a' }, + ]); + }); + + it('ignores global pax headers without recognized records', function () { + const archive = makeArchive([ + makeEntry({ name: 'g', type: 'g' }, paxRecords([['comment', 'hi']])), + makeEntry({ name: 'a' }, 'a'), + ]); + expect(listEntries(archive).map((e) => e.name)).toEqual(['a']); + }); + + it('rejects invalid pax header sequences', function () { + const a = makeEntry({ name: 'a' }, 'a'); + const x = (records: Array<[string, string]>) => + makeEntry({ name: 'x', type: 'x' }, paxRecords(records)); + const g = (records: Array<[string, string]>) => + makeEntry({ name: 'g', type: 'g' }, paxRecords(records)); + const cases: Array<[Uint8Array[], string]> = [ + [[x([['path', 'b']]), x([['path', 'c']]), a], 'Multiple pax headers'], + [[x([['path', '']]), a], 'empty name'], + [[x([['size', '']]), a], 'Invalid pax size'], + [[x([['size', '-1']]), a], 'Invalid pax size'], + [[x([['size', '1e3']]), a], 'Invalid pax size'], + [[g([['size', '1']]), a], 'global pax header'], + [[g([['path', 'b']]), a], 'global pax header'], + [[x([['path', 'b']])], 'end of the tar archive'], + [ + [ + x([ + ['path', 'b'], + ['path', 'c'], + ]), + a, + ], + 'Duplicate pax record', + ], + [[makeEntry({ name: 'x', type: 'x' }, '99 path=a\n'), a], 'pax'], + [[makeEntry({ name: 'x', type: 'x' }, '3 path=a\n'), a], 'pax'], + [[makeEntry({ name: 'x', type: 'x' }, '07 a=b\n'), a], 'pax'], + [[makeEntry({ name: 'x', type: 'x' }, '5 =b\n'), a], 'pax'], + ]; + for (const [entries, message] of cases) { + expect(() => listEntries(makeArchive(entries))).toThrow(message); + } + }); }); diff --git a/src/utils/fetch-source.ts b/src/utils/fetch-source.ts index 4ad15da0db..c0c86ff95d 100644 --- a/src/utils/fetch-source.ts +++ b/src/utils/fetch-source.ts @@ -165,8 +165,7 @@ export async function fetchSource( const stream = new UntarFileStream(bytes.buffer); const textDecoder = new TextDecoder(); - while (stream.hasNext()) { - const entry = stream.next(); + for (let entry = stream.next(); entry !== null; entry = stream.next()) { if (entry.name === pathInArchive && entry.buffer !== null) { const source = textDecoder.decode(entry.buffer); return { type: 'SUCCESS', source }; diff --git a/src/utils/untar.ts b/src/utils/untar.ts index 4a8e5cd5bb..3e5e7ace70 100644 --- a/src/utils/untar.ts +++ b/src/utils/untar.ts @@ -2,86 +2,44 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -// This code was originally from https://github.com/InvokIT/js-untar/blob/master/src/untar-worker.js (MIT). -// It was modernized, the Worker requirement was removed, and some Flow types were added. -// Tar is an uncompressed format. If you have the tar bytes in a buffer, you can -// access the contained files simply by creating a new view around that same buffer, -// with the correct offsets. The parsing of the tar header is fast; the slow part -// would be the copy of the contained bytes to and from the worker. So having -// everything on the main thread is both simpler and faster, because it skips the copy. +// A tar parser which works in place on an ArrayBuffer. Tar is an uncompressed +// format, so the contained files are returned as views into that same buffer, +// without copies. +// +// The parser is deliberately strict: it rejects malformed headers and unusual +// combinations of extension headers rather than guessing, so that it is less +// likely to disagree with other tar implementations about the contents of an +// archive. See https://astral.sh/blog/tar-codec for background. +// +// References: +// https://pubs.opengroup.org/onlinepubs/9699919799/utilities/pax.html +// https://www.gnu.org/software/tar/manual/html_node/Standard.html -// TODO: Consider unifying this ByteReader with the one in art-trace.js -class ByteReader { - _uint8Array: Uint8Array; - _bufferView: DataView; - _position = 0; - // Names in tar headers are raw bytes; in practice they are UTF-8. Bytes that - // aren't valid UTF-8 become U+FFFD, so such names can't match any real path. - _textDecoder = new TextDecoder('utf-8'); - - constructor(arrayBuffer: ArrayBufferLike) { - this._uint8Array = new Uint8Array(arrayBuffer); - this._bufferView = new DataView(arrayBuffer); - } - - readString(byteCount: number): string { - const bytes = this.readBuffer(byteCount); - const nulBytePos = bytes.indexOf(0); - return nulBytePos === -1 - ? this._textDecoder.decode(bytes) - : this._textDecoder.decode(bytes.subarray(0, nulBytePos)); - } - - readOctal(byteCount: number): number | null { - return _parseOctal(this.readBuffer(byteCount)); - } - - readBuffer(byteCount: number): Uint8Array { - if (this.position() + byteCount > this.size()) { - throw new Error('Unexpected end of tar archive'); - } - const buf = this._uint8Array.subarray( - this.position(), - this.position() + byteCount - ); - this.seekBy(byteCount); - return buf; - } - - seekBy(byteCount: number): void { - this._position += byteCount; - } - - seekTo(newpos: number): void { - this._position = newpos; - } - - peekBuffer(byteCount: number): Uint8Array { - return this._uint8Array.subarray( - this.position(), - this.position() + byteCount - ); - } - - position(): number { - return this._position; - } +const BLOCK_SIZE = 512; - size(): number { - return this._bufferView.byteLength; - } +// Byte ranges of the header fields we use, as [offset, length]. +type Field = [number, number]; +const NAME: Field = [0, 100]; +const SIZE: Field = [124, 12]; +const CHECKSUM: Field = [148, 8]; +const TYPEFLAG: Field = [156, 1]; +const LINKNAME: Field = [157, 100]; +const MAGIC: Field = [257, 6]; +const PREFIX: Field = [345, 155]; + +// Names in tar headers are raw bytes; in practice they are UTF-8. Bytes that +// aren't valid UTF-8 become U+FFFD, so such names can't match any real path. +const textDecoder = new TextDecoder('utf-8'); + +function _field(header: Uint8Array, [offset, length]: Field): Uint8Array { + return header.subarray(offset, offset + length); } -type FieldEntry = { - name: string; - value: string | number | null; -}; - -const BLOCK_SIZE = 512; - -// The checksum field covers bytes 148..156 of the header. -const CHECKSUM_OFFSET = 148; -const CHECKSUM_LENGTH = 8; +// Decode a NUL-terminated (or NUL-padded) string. +function _parseString(bytes: Uint8Array): string { + const nulPos = bytes.indexOf(0); + return textDecoder.decode(nulPos === -1 ? bytes : bytes.subarray(0, nulPos)); +} /** * Parse a numeric header field. These fields contain octal digits, optionally @@ -118,12 +76,11 @@ function _isZeroBlock(block: Uint8Array): boolean { * checksum field itself counted as if it contained spaces. */ function _verifyChecksum(header: Uint8Array): void { - const expected = _parseOctal( - header.subarray(CHECKSUM_OFFSET, CHECKSUM_OFFSET + CHECKSUM_LENGTH) - ); - let sum = 0x20 * CHECKSUM_LENGTH; + const [checksumOffset, checksumLength] = CHECKSUM; + const expected = _parseOctal(_field(header, CHECKSUM)); + let sum = 0x20 * checksumLength; for (let i = 0; i < header.length; i++) { - if (i < CHECKSUM_OFFSET || i >= CHECKSUM_OFFSET + CHECKSUM_LENGTH) { + if (i < checksumOffset || i >= checksumOffset + checksumLength) { sum += header[i]; } } @@ -132,263 +89,185 @@ function _verifyChecksum(header: Uint8Array): void { } } -function _parseIntStrict(s: string, base: number): number { - const val = parseInt(s, base); - if (isNaN(val)) { - throw new Error('Parsing number failed'); - } - return val; -} +// The pax keywords we use. Records with other keywords are ignored. +const PAX_KEYWORDS = new Set(['path', 'linkpath', 'size']); /** - * A PaxHeader is a set of fields with names and values. The fields get applied - * to a file entry. - * When reading file entries, the header is read first, then the file, and then - * the header is applied to the file. + * Parse the records of a pax extended header. Each record has the form + * "%d %s=%s\n" (, , ), where is the decimal + * length of the whole record in bytes, including the trailing newline. + * An empty value means that the field is deleted. */ -class PaxHeader { - _fields: FieldEntry[]; - - static parse(buffer: Uint8Array): PaxHeader { - // https://pubs.opengroup.org/onlinepubs/9699919799/utilities/pax.html#tag_20_92_13_03 - // An extended header shall consist of one or more records, each constructed as follows: - // "%d %s=%s\n", , , - - // The extended header records shall be encoded according to the ISO/IEC10646-1:2000 standard (UTF-8). - // The field, , equals sign, and shown shall be limited to the portable character set, as - // encoded in UTF-8. The and fields can be any UTF-8 characters. The field shall be the - // decimal length of the extended header record in octets, including the trailing . - - let bytes = new Uint8Array(buffer); - const fields = []; - - const textDecoder = new TextDecoder('utf-8', { fatal: true }); - - while (bytes.length > 0) { - // Decode bytes up to the first space character; that is the total field length - const spacePos = bytes.indexOf(0x20); - if (spacePos === -1) { - throw new Error('Expected space character'); +function _parsePaxRecords(data: Uint8Array): Map { + const records = new Map(); + let pos = 0; + while (pos < data.length) { + const spacePos = data.indexOf(0x20, pos); + const lengthString = + spacePos === -1 ? '' : textDecoder.decode(data.subarray(pos, spacePos)); + if (!/^[1-9][0-9]*$/.test(lengthString)) { + throw new Error('Invalid pax record length'); + } + const end = pos + Number(lengthString); + const equalsPos = data.indexOf(0x3d, spacePos + 1); + if ( + end > data.length || + data[end - 1] !== 0x0a || + equalsPos === -1 || + equalsPos >= end || + equalsPos === spacePos + 1 + ) { + throw new Error('Invalid pax record'); + } + const keyword = textDecoder.decode(data.subarray(spacePos + 1, equalsPos)); + if (PAX_KEYWORDS.has(keyword)) { + if (records.has(keyword)) { + throw new Error(`Duplicate pax record: ${keyword}`); } - const fieldLength = _parseIntStrict( - textDecoder.decode(bytes.subarray(0, spacePos)), - 10 + records.set( + keyword, + textDecoder.decode(data.subarray(equalsPos + 1, end - 1)) ); - const fieldText = textDecoder.decode(bytes.subarray(0, fieldLength)); - const fieldMatch = fieldText.match(/^\d+ ([^=]+)=(.*)\n$/); - - if (fieldMatch === null) { - throw new Error('Invalid PAX header data format.'); - } - - const fieldName = fieldMatch[1]; - let fieldValue: string | number | null = fieldMatch[2]; - - if (typeof fieldValue === 'string' && fieldValue.length === 0) { - fieldValue = null; - } else if (typeof fieldValue === 'string' && /^\d+$/.test(fieldValue)) { - // If it's an integer field, parse it as int - fieldValue = _parseIntStrict(fieldValue, 10); - } - // Don't parse float values since precision is lost - - const field = { - name: fieldName, - value: fieldValue, - }; - - fields.push(field); - - bytes = bytes.subarray(fieldLength); // Cut off the parsed field data } - - return new PaxHeader(fields); - } - - constructor(fields: FieldEntry[]) { - this._fields = fields; + pos = end; } + return records; +} - applyHeader(file: any): void { - // Apply fields to the file - // If a field is of value null, it should be deleted from the file - // https://pubs.opengroup.org/onlinepubs/9699919799/utilities/pax.html#tag_20_92_13_03 - - for (const field of this._fields) { - let fieldName = field.name; - const fieldValue = field.value; - - if (fieldName === 'path') { - // This overrides the name and prefix fields in the following header block. - fieldName = 'name'; - - if (file.namePrefix !== undefined) { - delete file.namePrefix; - } - } else if (fieldName === 'linkpath') { - // This overrides the linkname field in the following header block. - fieldName = 'linkname'; - } - - if (fieldValue === null) { - delete file[fieldName]; - } else { - file[fieldName] = fieldValue; - } - } +function _parsePaxSize(value: string): number { + const size = Number(value); + if (!/^[0-9]+$/.test(value) || !Number.isSafeInteger(size)) { + throw new Error('Invalid pax size'); } + return size; } export type TarFileEntry = { name: string; + // The typeflag, e.g. '0' for a regular file or '5' for a directory. Old + // archives use '' for regular files. type: string; size: number; + // The contents of a regular file, as a view into the archive buffer. + // null for all other entry types. buffer: Uint8Array | null; - mode: string; - uid: number | null; - gid: number | null; - mtime: number | null; linkname: string; - ustarFormat: string; - version: string | null; - uname: string | null; - gname: string | null; - devmajor: number | null; - devminor: number | null; - namePrefix: string | null; }; export class UntarFileStream { - _reader: ByteReader; - _globalPaxHeader: PaxHeader | null = null; + _bytes: Uint8Array; + _position = 0; constructor(arrayBuffer: ArrayBufferLike) { - this._reader = new ByteReader(arrayBuffer); - } - - hasNext(): boolean { - // A tar archive ends with two all-zero blocks. We stop at the first one, - // and also accept an archive that ends right after its last member. - const reader = this._reader; - return ( - reader.position() < reader.size() && - !_isZeroBlock(reader.peekBuffer(BLOCK_SIZE)) - ); + this._bytes = new Uint8Array(arrayBuffer); } - next(): TarFileEntry { - return this._readNextFile(); - } - - _readNextFile(): TarFileEntry { - const stream = this._reader; - - const headerBeginPos = stream.position(); - if (headerBeginPos + BLOCK_SIZE > stream.size()) { - throw new Error('Unexpected end of tar archive'); - } - _verifyChecksum(stream.peekBuffer(BLOCK_SIZE)); - - // Read header - let name = stream.readString(100); - const mode = stream.readString(8); - const uid = stream.readOctal(8); - const gid = stream.readOctal(8); - const size = stream.readOctal(12); - if (size === null) { - throw new Error('Missing size in tar header'); - } - const mtime = stream.readOctal(12); - stream.seekBy(CHECKSUM_LENGTH); - const type = stream.readString(1); - const linkname = stream.readString(100); - const ustarFormat = stream.readString(6); - let version = null; - let uname = null; - let gname = null; - let devmajor = null; - let devminor = null; - let namePrefix = null; - - if (ustarFormat.indexOf('ustar') > -1) { - version = stream.readString(2); - uname = stream.readString(32); - gname = stream.readString(32); - devmajor = stream.readOctal(8); - devminor = stream.readOctal(8); - namePrefix = stream.readString(155); - - if (namePrefix.length > 0) { - name = namePrefix + '/' + name; + /** + * Return the next member of the archive, or null at the end of the archive. + * Throws if the archive is malformed. + */ + next(): TarFileEntry | null { + // The records of a pax 'x' header, which apply to the next member. + let localPax: Map | null = null; + + while (true) { + const header = this._readHeader(); + if (header === null) { + if (localPax !== null) { + throw new Error('Pax header at the end of the tar archive'); + } + return null; } - } - // The size is padded to be aligned with 512 bytes. - const alignedSize = Math.ceil(size / BLOCK_SIZE) * BLOCK_SIZE; + const type = _parseString(_field(header, TYPEFLAG)); + const headerSize = _parseOctal(_field(header, SIZE)); + if (headerSize === null) { + throw new Error('Missing size in tar header'); + } - const dataBeginPos = headerBeginPos + BLOCK_SIZE; - const dataEndPos = dataBeginPos + alignedSize; + if (type === 'g') { + // Global pax header. Its records apply to all following members. We + // reject all the keywords we'd otherwise use: overriding the path or + // size of every member isn't meaningful. So there's nothing to keep. + const records = _parsePaxRecords(this._readData(headerSize)); + if (records.size !== 0) { + throw new Error('Unsupported record in global pax header'); + } + continue; + } - stream.seekTo(dataBeginPos); + if (type === 'x') { + if (localPax !== null) { + throw new Error('Multiple pax headers for one tar member'); + } + localPax = _parsePaxRecords(this._readData(headerSize)); + continue; + } - // Handle the various file types. + // This is a real member. Resolve its fields, with pax records taking + // precedence over the header. A pax record with an empty value deletes + // the field. + let headerName = _parseString(_field(header, NAME)); + if (_parseString(_field(header, MAGIC)).startsWith('ustar')) { + const prefix = _parseString(_field(header, PREFIX)); + if (prefix !== '') { + headerName = prefix + '/' + headerName; + } + } + const name = localPax?.get('path') ?? headerName; + const linkname = + localPax?.get('linkpath') ?? _parseString(_field(header, LINKNAME)); + const paxSize = localPax?.get('size'); + const size = paxSize === undefined ? headerSize : _parsePaxSize(paxSize); + if (name === '') { + throw new Error('Tar member with an empty name'); + } - // Derived from https://www.mkssoftware.com/docs/man4/pax.4.asp - // and https://www.ibm.com/support/knowledgecenter/en/SSLTBW_2.3.0/com.ibm.zos.v2r3.bpxa500/pxarchfm.htm - // Also see https://en.wikipedia.org/wiki/Tar_(computing)#File_format + const data = this._readData(size); - if (type === 'g') { - // Global PAX header - this._globalPaxHeader = PaxHeader.parse(stream.readBuffer(size)); - stream.seekTo(dataEndPos); + // Regular files have the type '0', or '' in old archives. Type '7' + // (contiguous file) is usually treated like a regular file. + const isNormalFile = type === '0' || type === '' || type === '7'; - // Read the next entry. It will apply the global PAX header to itself. - return this._readNextFile(); + return { + name, + type, + size, + buffer: isNormalFile ? data : null, + linkname, + }; } + } - if (type === 'x') { - // PAX header - const paxHeader = PaxHeader.parse(stream.readBuffer(size)); - stream.seekTo(dataEndPos); - - // Read the next entry, and then apply this PAX header to it. - const file = this._readNextFile(); - paxHeader.applyHeader(file); - return file; + // Read a header block, or return null at the end of the archive. A tar + // archive ends with two all-zero blocks. We stop at the first one, and also + // accept an archive that ends right after its last member. + _readHeader(): Uint8Array | null { + if (this._position >= this._bytes.length) { + return null; } + const header = this._read(BLOCK_SIZE); + if (_isZeroBlock(header)) { + return null; + } + _verifyChecksum(header); + return header; + } - // Normal file is either "0" or "\0". - // In case of "\0", readString returns an empty string, that is "". - // In addition, according to wikipedia, type 7 is usually handled like type 0. - const isNormalFile = type === '0' || type === '' || type === '7'; - - const buffer = isNormalFile ? stream.readBuffer(size) : null; - - stream.seekTo(dataEndPos); - - const file = { - name, - type, - buffer, - mode, - uid, - gid, - size, - mtime, - linkname, - ustarFormat, - version, - uname, - gname, - devmajor, - devminor, - namePrefix, - }; + // Read the data that follows a header, and skip its padding. + _readData(size: number): Uint8Array { + const data = this._read(size); + this._position += Math.ceil(size / BLOCK_SIZE) * BLOCK_SIZE - size; + return data; + } - if (this._globalPaxHeader !== null) { - this._globalPaxHeader.applyHeader(file); + _read(byteCount: number): Uint8Array { + const end = this._position + byteCount; + if (end > this._bytes.length) { + throw new Error('Unexpected end of tar archive'); } - - return file; + const bytes = this._bytes.subarray(this._position, end); + this._position = end; + return bytes; } } From 8300ae9de474233183847ffa7612c8b064438c65 Mon Sep 17 00:00:00 2001 From: Markus Stange Date: Wed, 28 Oct 2026 10:11:58 -0400 Subject: [PATCH 4/5] Distinguish GNU from UStar headers in untar.ts The parser treated any header containing 'ustar' in its magic field as UStar, including GNU headers. GNU headers don't have a name prefix field; those bytes hold atime, ctime and other fields, so an archive with atime set would produce names like '/'. Classify each header by its exact magic and version as UStar, GNU or v7, and only read the prefix for UStar. Reject unknown formats, archives that mix formats, and pax headers in non-UStar archives. --- src/test/unit/untar.test.ts | 55 +++++++++++++++++++++++++++++++++++-- src/utils/untar.ts | 40 +++++++++++++++++++++++++-- 2 files changed, 90 insertions(+), 5 deletions(-) diff --git a/src/test/unit/untar.test.ts b/src/test/unit/untar.test.ts index f1f2889d90..7ea633e9d0 100644 --- a/src/test/unit/untar.test.ts +++ b/src/test/unit/untar.test.ts @@ -7,8 +7,9 @@ import { UntarFileStream } from 'firefox-profiler/utils/untar'; type HeaderSpec = { name: string; type?: string; - // 'ustar' is POSIX UStar / pax, 'gnu' is the old GNU format used by cargo. - format?: 'ustar' | 'gnu'; + // 'ustar' is POSIX UStar / pax, 'gnu' is the old GNU format used by cargo, + // 'v7' has no magic. + format?: 'ustar' | 'gnu' | 'v7'; prefix?: string; uid?: string; // Overrides the size field, which defaults to the content length. @@ -38,7 +39,7 @@ function makeHeader(spec: HeaderSpec, size: number): Uint8Array { write(156, spec.type ?? '0'); if (spec.format === 'gnu') { write(257, 'ustar \0'); - } else { + } else if (spec.format !== 'v7') { write(257, 'ustar\x0000'); } if (spec.prefix !== undefined) { @@ -425,4 +426,52 @@ describe('untar', function () { expect(() => listEntries(makeArchive(entries))).toThrow(message); } }); + + it('does not read a name prefix from GNU headers', function () { + // In GNU headers, the UStar prefix bytes contain atime and ctime. + const archive = makeArchive([ + makeEntry( + { + name: 'crate/a.rs', + format: 'gnu', + patches: [{ offset: 345, bytes: '14751573214\x0014751573214\x00' }], + }, + 'a' + ), + ]); + expect(listEntries(archive).map((e) => e.name)).toEqual(['crate/a.rs']); + }); + + it('reads v7 archives', function () { + const archive = makeArchive([ + makeEntry({ name: 'a', format: 'v7', prefix: 'ignored' }, 'a'), + ]); + expect(listEntries(archive).map((e) => e.name)).toEqual(['a']); + }); + + it('rejects unknown header formats', function () { + const archive = makeArchive([ + makeEntry({ name: 'a', patches: [{ offset: 263, bytes: '01' }] }, 'a'), + ]); + expect(() => listEntries(archive)).toThrow('Unknown tar header format'); + }); + + it('rejects mixed header formats', function () { + const archive = makeArchive([ + makeEntry({ name: 'a', format: 'gnu' }, 'a'), + makeEntry({ name: 'b', format: 'ustar' }, 'b'), + ]); + expect(() => listEntries(archive)).toThrow('Mixed header formats'); + }); + + it('rejects pax headers in GNU archives', function () { + const archive = makeArchive([ + makeEntry( + { name: 'x', type: 'x', format: 'gnu' }, + paxRecords([['path', 'b']]) + ), + makeEntry({ name: 'a', format: 'gnu' }, 'a'), + ]); + expect(() => listEntries(archive)).toThrow('Pax header in a gnu'); + }); }); diff --git a/src/utils/untar.ts b/src/utils/untar.ts index 3e5e7ace70..94583365b1 100644 --- a/src/utils/untar.ts +++ b/src/utils/untar.ts @@ -24,7 +24,8 @@ const SIZE: Field = [124, 12]; const CHECKSUM: Field = [148, 8]; const TYPEFLAG: Field = [156, 1]; const LINKNAME: Field = [157, 100]; -const MAGIC: Field = [257, 6]; +// The magic and version fields together. +const MAGIC: Field = [257, 8]; const PREFIX: Field = [345, 155]; // Names in tar headers are raw bytes; in practice they are UTF-8. Bytes that @@ -134,6 +135,27 @@ function _parsePaxRecords(data: Uint8Array): Map { return records; } +// Archives use one of these header formats: +// - 'ustar': POSIX UStar, possibly with pax extended headers ('x' and 'g'). +// - 'gnu': The old GNU format, which cargo uses. It has no name prefix field; +// those bytes contain other fields. +// - 'v7': The original tar format, without any extensions. +type TarFormat = 'ustar' | 'gnu' | 'v7'; + +function _getFormat(header: Uint8Array): TarFormat { + const magic = String.fromCharCode(..._field(header, MAGIC)); + switch (magic) { + case 'ustar\x0000': + return 'ustar'; + case 'ustar \x00': + return 'gnu'; + case '\x00'.repeat(8): + return 'v7'; + default: + throw new Error('Unknown tar header format'); + } +} + function _parsePaxSize(value: string): number { const size = Number(value); if (!/^[0-9]+$/.test(value) || !Number.isSafeInteger(size)) { @@ -157,6 +179,9 @@ export type TarFileEntry = { export class UntarFileStream { _bytes: Uint8Array; _position = 0; + // Mixing header formats in one archive is not allowed. This is the format + // of the first header. + _format: TarFormat | null = null; constructor(arrayBuffer: ArrayBufferLike) { this._bytes = new Uint8Array(arrayBuffer); @@ -185,6 +210,17 @@ export class UntarFileStream { throw new Error('Missing size in tar header'); } + const format = _getFormat(header); + if (this._format === null) { + this._format = format; + } else if (format !== this._format) { + throw new Error('Mixed header formats in tar archive'); + } + + if ((type === 'g' || type === 'x') && format !== 'ustar') { + throw new Error(`Pax header in a ${format} tar archive`); + } + if (type === 'g') { // Global pax header. Its records apply to all following members. We // reject all the keywords we'd otherwise use: overriding the path or @@ -208,7 +244,7 @@ export class UntarFileStream { // precedence over the header. A pax record with an empty value deletes // the field. let headerName = _parseString(_field(header, NAME)); - if (_parseString(_field(header, MAGIC)).startsWith('ustar')) { + if (format === 'ustar') { const prefix = _parseString(_field(header, PREFIX)); if (prefix !== '') { headerName = prefix + '/' + headerName; From e5ea81f469780c17d697aee78b964c8c5325303a Mon Sep 17 00:00:00 2001 From: Markus Stange Date: Wed, 28 Oct 2026 10:12:32 -0400 Subject: [PATCH 5/5] Support GNU long names in untar.ts cargo writes crates in the GNU tar format. In that format, names longer than 100 bytes are stored in a preceding 'L' entry, and long link names in a 'K' entry. We treated these as ordinary entries and returned the following member with its name truncated to 100 bytes, so files with long paths, such as 95 of the files in aws-sdk-iam-1.132.0, could never be found in the archive. Apply the payload of 'L' and 'K' entries to the next member. Reject more than one of each per member, and 'L'/'K' entries in non-GNU archives. --- src/test/fixtures/synthetic-gnu.tar.gz | Bin 0 -> 286 bytes src/test/fixtures/synthetic-pax.tar.gz | Bin 0 -> 326 bytes src/test/unit/untar.test.ts | 86 +++++++++++++++++++++++++ src/utils/untar.ts | 45 ++++++++++--- 4 files changed, 122 insertions(+), 9 deletions(-) create mode 100644 src/test/fixtures/synthetic-gnu.tar.gz create mode 100644 src/test/fixtures/synthetic-pax.tar.gz diff --git a/src/test/fixtures/synthetic-gnu.tar.gz b/src/test/fixtures/synthetic-gnu.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..d59e36747eff5e979ade2a187ee9e27d2237f633 GIT binary patch literal 286 zcmV+(0pb21iwFP!00000|LxXMZi65g2XKzU1&V-DU-v%mV2Gs~nZ*R%k{+q&=*3#s zWGPLWW=Vl2{(mo`KKO-?Kf(%DyfuUSXsr7paqx?KS1$SbZ+v-P*QzA6iBwPEdz`eV zG{vW0(Qe;)J(-nYobh7xU7_0_XCOP2l1G|8`tfQVBw3ip7ciUr+Jp;HRet@^Syw k?fFmf66e1ZemkDK2mk;800000004;M1j7792LLDl0OKr?Bme*a literal 0 HcmV?d00001 diff --git a/src/test/fixtures/synthetic-pax.tar.gz b/src/test/fixtures/synthetic-pax.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..5eaf0525b89f1e8dc5fa7f27c1301677d9c34bd4 GIT binary patch literal 326 zcmV-M0lEGkiwFP!00000|LxdKPJ}QR2H;tH3NA3Ee~25S%UOB@(lFI&WWbb;nK_c@ z@M4xRF#(-SOrRyweySAsIYU-C_YpgC^<1cHq*W*8W zHkBr}y@|aXixZ;kT~psmd9nW~WBvXYObFlqL3(oG{ofTV>B5)c^nh literal 0 HcmV?d00001 diff --git a/src/test/unit/untar.test.ts b/src/test/unit/untar.test.ts index 7ea633e9d0..ca48375f66 100644 --- a/src/test/unit/untar.test.ts +++ b/src/test/unit/untar.test.ts @@ -474,4 +474,90 @@ describe('untar', function () { ]); expect(() => listEntries(archive)).toThrow('Pax header in a gnu'); }); + + // These fixtures were generated with Python's tarfile module, as an + // independent implementation. The GNU one matches the output of cargo. + // Each contains a member with a long name, a member with a non-ASCII name, + // a symlink with a long target, and a member with a short name. + for (const format of ['gnu', 'pax']) { + it(`reads a ${format} archive generated by Python`, function () { + const fs = require('fs'); + const zlib = require('zlib'); + const buffer = zlib.gunzipSync( + fs.readFileSync(`src/test/fixtures/synthetic-${format}.tar.gz`) + ); + const longName = + 'mycrate-1.0.0/src/' + 'a_very_long_directory_name/'.repeat(4); + const stream = new UntarFileStream(buffer.buffer); + const entries = []; + for (let entry = stream.next(); entry !== null; entry = stream.next()) { + const { name, type, buffer, linkname } = entry; + entries.push({ + name, + type, + linkname, + content: buffer === null ? null : new TextDecoder().decode(buffer), + }); + } + expect(entries).toEqual([ + { + name: longName + 'file.rs', + type: '0', + linkname: '', + content: 'long name', + }, + { + name: 'mycrate-1.0.0/é.rs', + type: '0', + linkname: '', + content: 'non-ascii', + }, + { + name: 'mycrate-1.0.0/link', + type: '2', + linkname: longName + 'file.rs', + content: null, + }, + { + name: 'mycrate-1.0.0/short.rs', + type: '0', + linkname: '', + content: 'short', + }, + ]); + }); + } + + it('rejects invalid GNU long name sequences', function () { + const a = makeEntry({ name: 'a', format: 'gnu' }, 'a'); + const longEntry = (type: string, format: 'gnu' | 'ustar' = 'gnu') => + makeEntry({ name: '././@LongLink', type, format }, 'b\0'); + const cases: Array<[Uint8Array[], string]> = [ + [[longEntry('L'), longEntry('L'), a], 'Multiple GNU long names'], + [[longEntry('K'), longEntry('K'), a], 'Multiple GNU long link names'], + [[longEntry('L')], 'end of the tar archive'], + [ + [ + longEntry('L'), + makeEntry( + { name: 'x', type: 'x', format: 'gnu' }, + paxRecords([['path', 'c']]) + ), + a, + ], + 'Pax header in a gnu', + ], + [ + [longEntry('L', 'ustar'), makeEntry({ name: 'a' }, 'a')], + 'GNU long name header in a ustar', + ], + [ + [makeEntry({ name: '././@LongLink', type: 'L', format: 'gnu' }), a], + 'empty name', + ], + ]; + for (const [entries, message] of cases) { + expect(() => listEntries(makeArchive(entries))).toThrow(message); + } + }); }); diff --git a/src/utils/untar.ts b/src/utils/untar.ts index 94583365b1..effa8d78f3 100644 --- a/src/utils/untar.ts +++ b/src/utils/untar.ts @@ -138,7 +138,8 @@ function _parsePaxRecords(data: Uint8Array): Map { // Archives use one of these header formats: // - 'ustar': POSIX UStar, possibly with pax extended headers ('x' and 'g'). // - 'gnu': The old GNU format, which cargo uses. It has no name prefix field; -// those bytes contain other fields. +// those bytes contain other fields. Long names are stored in separate 'L' +// and 'K' entries. // - 'v7': The original tar format, without any extensions. type TarFormat = 'ustar' | 'gnu' | 'v7'; @@ -192,14 +193,17 @@ export class UntarFileStream { * Throws if the archive is malformed. */ next(): TarFileEntry | null { - // The records of a pax 'x' header, which apply to the next member. + // Extension headers which apply to the next member: the records of a pax + // 'x' header, and the GNU long name ('L') and long link name ('K'). let localPax: Map | null = null; + let longName: string | null = null; + let longLinkname: string | null = null; while (true) { const header = this._readHeader(); if (header === null) { - if (localPax !== null) { - throw new Error('Pax header at the end of the tar archive'); + if (localPax !== null || longName !== null || longLinkname !== null) { + throw new Error('Extension header at the end of the tar archive'); } return null; } @@ -221,6 +225,26 @@ export class UntarFileStream { throw new Error(`Pax header in a ${format} tar archive`); } + if ((type === 'L' || type === 'K') && format !== 'gnu') { + throw new Error(`GNU long name header in a ${format} tar archive`); + } + + if (type === 'L') { + if (longName !== null) { + throw new Error('Multiple GNU long names for one tar member'); + } + longName = _parseString(this._readData(headerSize)); + continue; + } + + if (type === 'K') { + if (longLinkname !== null) { + throw new Error('Multiple GNU long link names for one tar member'); + } + longLinkname = _parseString(this._readData(headerSize)); + continue; + } + if (type === 'g') { // Global pax header. Its records apply to all following members. We // reject all the keywords we'd otherwise use: overriding the path or @@ -240,9 +264,10 @@ export class UntarFileStream { continue; } - // This is a real member. Resolve its fields, with pax records taking - // precedence over the header. A pax record with an empty value deletes - // the field. + // This is a real member. Resolve its fields, with extension headers + // taking precedence over the header. A pax record with an empty value + // deletes the field. Only one kind of extension header can be present, + // because the format check above ties pax to UStar and L/K to GNU. let headerName = _parseString(_field(header, NAME)); if (format === 'ustar') { const prefix = _parseString(_field(header, PREFIX)); @@ -250,9 +275,11 @@ export class UntarFileStream { headerName = prefix + '/' + headerName; } } - const name = localPax?.get('path') ?? headerName; + const name = localPax?.get('path') ?? longName ?? headerName; const linkname = - localPax?.get('linkpath') ?? _parseString(_field(header, LINKNAME)); + localPax?.get('linkpath') ?? + longLinkname ?? + _parseString(_field(header, LINKNAME)); const paxSize = localPax?.get('size'); const size = paxSize === undefined ? headerSize : _parsePaxSize(paxSize); if (name === '') {