diff --git a/src/test/fixtures/synthetic-gnu.tar.gz b/src/test/fixtures/synthetic-gnu.tar.gz new file mode 100644 index 0000000000..d59e36747e Binary files /dev/null and b/src/test/fixtures/synthetic-gnu.tar.gz differ diff --git a/src/test/fixtures/synthetic-pax.tar.gz b/src/test/fixtures/synthetic-pax.tar.gz new file mode 100644 index 0000000000..5eaf0525b8 Binary files /dev/null and b/src/test/fixtures/synthetic-pax.tar.gz differ diff --git a/src/test/unit/untar.test.ts b/src/test/unit/untar.test.ts index b98afc2b3e..ca48375f66 100644 --- a/src/test/unit/untar.test.ts +++ b/src/test/unit/untar.test.ts @@ -4,6 +4,119 @@ import { UntarFileStream } from 'firefox-profiler/utils/untar'; +type HeaderSpec = { + name: string; + type?: string; + // 'ustar' is POSIX UStar / pax, 'gnu' is the old GNU format used by cargo, + // 'v7' has no magic. + format?: 'ustar' | 'gnu' | 'v7'; + prefix?: string; + uid?: string; + // Overrides the size field, which defaults to the content length. + size?: number; + // Raw bytes written at the given offset before the checksum is computed. + patches?: Array<{ offset: number; bytes: string | Uint8Array }>; + // Corrupt the checksum after it is computed. + badChecksum?: boolean; +}; + +const encoder = new TextEncoder(); + +function toBytes(data: string | Uint8Array): Uint8Array { + return typeof data === 'string' ? encoder.encode(data) : data; +} + +function makeHeader(spec: HeaderSpec, size: number): Uint8Array { + const header = new Uint8Array(512); + const write = (offset: number, data: string | Uint8Array) => + header.set(toBytes(data), offset); + write(0, spec.name); + write(100, '0000644\0'); + write(108, spec.uid ?? '0000000\0'); + write(116, '0000000\0'); + write(124, size.toString(8).padStart(11, '0') + '\0'); + write(136, '00000000000\0'); + write(156, spec.type ?? '0'); + if (spec.format === 'gnu') { + write(257, 'ustar \0'); + } else if (spec.format !== 'v7') { + write(257, 'ustar\x0000'); + } + if (spec.prefix !== undefined) { + write(345, spec.prefix); + } + for (const { offset, bytes } of spec.patches ?? []) { + write(offset, bytes); + } + write(148, ' '); + let checksum = header.reduce((a, b) => a + b, 0); + if (spec.badChecksum) { + checksum++; + } + write(148, checksum.toString(8).padStart(6, '0') + '\0 '); + return header; +} + +function makeEntry(spec: HeaderSpec, content: string | Uint8Array = '') { + const data = toBytes(content); + const paddedSize = Math.ceil(data.length / 512) * 512; + const entry = new Uint8Array(512 + paddedSize); + entry.set(makeHeader(spec, spec.size ?? data.length), 0); + entry.set(data, 512); + return entry; +} + +function concatBytes( + parts: Array +): Uint8Array { + const arrays = parts.map(toBytes); + const result = new Uint8Array(arrays.reduce((a, b) => a + b.length, 0)); + let offset = 0; + for (const array of arrays) { + result.set(array, offset); + offset += array.length; + } + return result; +} + +// Build the payload of a pax header from [keyword, value] pairs. +function paxRecords(records: Array<[string, string | Uint8Array]>): Uint8Array { + return concatBytes( + records.map(([keyword, value]) => { + const rest = concatBytes([' ' + keyword + '=', value, '\n']); + // The length prefix counts its own digits. + let length = rest.length + String(rest.length).length; + length = rest.length + String(length).length; + return concatBytes([String(length), rest]); + }) + ); +} + +function makeArchive( + entries: Uint8Array[], + { trailer = true }: { trailer?: boolean } = {} +): ArrayBuffer { + if (trailer) { + entries = [...entries, new Uint8Array(1024)]; + } + return concatBytes(entries).buffer; +} + +function listEntries(archive: ArrayBuffer) { + const decoder = new TextDecoder(); + const stream = new UntarFileStream(archive); + const result = []; + for (let entry = stream.next(); entry !== null; entry = stream.next()) { + const { name, type, buffer } = entry; + result.push({ + name, + type, + content: buffer === null ? null : decoder.decode(buffer), + }); + } + return result; +} + describe('untar', function () { it('lists all files in example tar', function () { const fs = require('fs'); @@ -16,12 +129,11 @@ describe('untar', function () { const fileEntries = []; - while (stream.hasNext()) { - const { name, buffer, size, uid } = stream.next(); + for (let entry = stream.next(); entry !== null; entry = stream.next()) { + const { name, buffer, size } = entry; fileEntries.push({ name, size, - uid, offset: buffer ? buffer.byteOffset : null, }); } @@ -32,123 +144,420 @@ describe('untar', function () { "name": "addr2line-0.17.0/.cargo_vcs_info.json", "offset": 512, "size": 74, - "uid": null, }, Object { "name": "addr2line-0.17.0/.gitignore", "offset": 1536, "size": 33, - "uid": 0, }, Object { "name": "addr2line-0.17.0/CHANGELOG.md", "offset": 2560, "size": 7347, - "uid": 0, }, Object { "name": "addr2line-0.17.0/Cargo.lock", "offset": 10752, "size": 11506, - "uid": null, }, Object { "name": "addr2line-0.17.0/Cargo.toml", "offset": 23040, "size": 2792, - "uid": null, }, Object { "name": "addr2line-0.17.0/Cargo.toml.orig", "offset": 26624, "size": 2334, - "uid": 0, }, Object { "name": "addr2line-0.17.0/LICENSE-APACHE", "offset": 29696, "size": 10847, - "uid": 0, }, Object { "name": "addr2line-0.17.0/LICENSE-MIT", "offset": 41472, "size": 1069, - "uid": 0, }, Object { "name": "addr2line-0.17.0/README.md", "offset": 43520, "size": 2550, - "uid": 0, }, Object { "name": "addr2line-0.17.0/bench.plot.r", "offset": 46592, "size": 1000, - "uid": 0, }, Object { "name": "addr2line-0.17.0/benchmark.sh", "offset": 48128, "size": 3084, - "uid": 0, }, Object { "name": "addr2line-0.17.0/coverage.sh", "offset": 52224, "size": 177, - "uid": 0, }, Object { "name": "addr2line-0.17.0/examples/addr2line.rs", "offset": 53248, "size": 9447, - "uid": 0, }, Object { "name": "addr2line-0.17.0/rustfmt.toml", "offset": 63488, "size": 1, - "uid": 0, }, Object { "name": "addr2line-0.17.0/src/function.rs", "offset": 64512, "size": 19078, - "uid": 0, }, Object { "name": "addr2line-0.17.0/src/lazy.rs", "offset": 84480, "size": 919, - "uid": 0, }, Object { "name": "addr2line-0.17.0/src/lib.rs", "offset": 86016, "size": 41209, - "uid": 0, }, Object { "name": "addr2line-0.17.0/tests/correctness.rs", "offset": 128000, "size": 2756, - "uid": 0, }, Object { "name": "addr2line-0.17.0/tests/output_equivalence.rs", "offset": 131584, "size": 3905, - "uid": 0, }, Object { "name": "addr2line-0.17.0/tests/parse.rs", "offset": 136192, "size": 3035, - "uid": 0, }, ] `); }); + + it('reads a simple UStar archive', function () { + const archive = makeArchive([ + makeEntry({ name: 'dir', type: '5' }), + makeEntry({ name: 'file.txt' }, 'hello'), + makeEntry({ name: 'big.txt' }, 'x'.repeat(1000)), + ]); + expect(listEntries(archive)).toEqual([ + { name: 'dir', type: '5', content: null }, + { name: 'file.txt', type: '0', content: 'hello' }, + { name: 'big.txt', type: '0', content: 'x'.repeat(1000) }, + ]); + }); + + it('accepts an archive without the trailing zero blocks', function () { + const archive = makeArchive([makeEntry({ name: 'a' }, 'a')], { + trailer: false, + }); + expect(listEntries(archive)).toEqual([ + { name: 'a', type: '0', content: 'a' }, + ]); + }); + + it('stops at the first zero block', function () { + const archive = makeArchive([ + makeEntry({ name: 'a' }, 'a'), + new Uint8Array(512), + makeEntry({ name: 'b' }, 'b'), + ]); + expect(listEntries(archive).map((e) => e.name)).toEqual(['a']); + }); + + it('rejects a header with a bad checksum', function () { + const archive = makeArchive([ + makeEntry({ name: 'a', badChecksum: true }, 'a'), + ]); + expect(() => listEntries(archive)).toThrow('checksum'); + }); + + it('rejects data that is not a tar archive', function () { + const archive = encoder.encode('not a tar file'.repeat(100)).buffer; + expect(() => listEntries(archive)).toThrow(); + }); + + it('rejects malformed numeric fields', function () { + for (const size of [ + '-0000000001\0', + '00000000012x', + '\x80' + '\0'.repeat(11), + ]) { + const archive = makeArchive([ + makeEntry({ name: 'a', patches: [{ offset: 124, bytes: size }] }), + ]); + expect(() => listEntries(archive)).toThrow('Invalid numeric field'); + } + }); + + it('accepts an empty optional numeric field', function () { + const archive = makeArchive([ + makeEntry({ name: 'a', uid: '\0'.repeat(8) }, 'a'), + ]); + expect(listEntries(archive).map((e) => e.name)).toEqual(['a']); + }); + + it('rejects a truncated archive', function () { + const archive = makeArchive([makeEntry({ name: 'a' }, 'x'.repeat(600))], { + trailer: false, + }); + expect(() => listEntries(archive.slice(0, 1000))).toThrow('Unexpected end'); + expect(() => listEntries(archive.slice(0, 300))).toThrow('Unexpected end'); + }); + + it('decodes names as UTF-8', function () { + const archive = makeArchive([ + makeEntry({ name: 'mycrate-1.0.0/é.rs', prefix: 'prefix-ü' }, 'a'), + ]); + expect(listEntries(archive).map((e) => e.name)).toEqual([ + 'prefix-ü/mycrate-1.0.0/é.rs', + ]); + }); + + it('applies pax path and linkpath records', function () { + const longName = 'dir/'.repeat(40) + 'é\nfile.rs'; + const archive = makeArchive([ + makeEntry( + { name: 'PaxHeaders/x', type: 'x' }, + paxRecords([ + ['path', longName], + ['linkpath', 'target'], + ]) + ), + makeEntry({ name: 'truncated', prefix: 'ignored', type: '2' }), + makeEntry({ name: 'next' }, 'n'), + ]); + const stream = new UntarFileStream(archive); + expect(stream.next()).toMatchObject({ + name: longName, + linkname: 'target', + }); + // The pax header only applies to one member. + expect(stream.next()).toMatchObject({ name: 'next', linkname: '' }); + }); + + it('keeps all-digit pax paths as strings', function () { + const archive = makeArchive([ + makeEntry({ name: 'x', type: 'x' }, paxRecords([['path', '1234']])), + makeEntry({ name: 'a' }, 'a'), + ]); + expect(listEntries(archive)[0].name).toBe('1234'); + }); + + it('uses the pax size for framing', function () { + const archive = makeArchive([ + makeEntry({ name: 'x', type: 'x' }, paxRecords([['size', '600']])), + makeEntry({ name: 'a', size: 0 }, 'a'.repeat(600)), + makeEntry({ name: 'b' }, 'b'), + ]); + expect(listEntries(archive)).toEqual([ + { name: 'a', type: '0', content: 'a'.repeat(600) }, + { name: 'b', type: '0', content: 'b' }, + ]); + }); + + it('ignores unknown pax records', function () { + const archive = makeArchive([ + makeEntry( + { name: 'x', type: 'x' }, + paxRecords([ + ['buffer', '1'], + ['type', '5'], + ['SCHILY.xattr.user.foo', new Uint8Array([0xff, 0x00, 0x0a])], + ['mtime', '1234.5'], + ]) + ), + makeEntry({ name: 'a' }, 'a'), + ]); + expect(listEntries(archive)).toEqual([ + { name: 'a', type: '0', content: 'a' }, + ]); + }); + + it('ignores global pax headers without recognized records', function () { + const archive = makeArchive([ + makeEntry({ name: 'g', type: 'g' }, paxRecords([['comment', 'hi']])), + makeEntry({ name: 'a' }, 'a'), + ]); + expect(listEntries(archive).map((e) => e.name)).toEqual(['a']); + }); + + it('rejects invalid pax header sequences', function () { + const a = makeEntry({ name: 'a' }, 'a'); + const x = (records: Array<[string, string]>) => + makeEntry({ name: 'x', type: 'x' }, paxRecords(records)); + const g = (records: Array<[string, string]>) => + makeEntry({ name: 'g', type: 'g' }, paxRecords(records)); + const cases: Array<[Uint8Array[], string]> = [ + [[x([['path', 'b']]), x([['path', 'c']]), a], 'Multiple pax headers'], + [[x([['path', '']]), a], 'empty name'], + [[x([['size', '']]), a], 'Invalid pax size'], + [[x([['size', '-1']]), a], 'Invalid pax size'], + [[x([['size', '1e3']]), a], 'Invalid pax size'], + [[g([['size', '1']]), a], 'global pax header'], + [[g([['path', 'b']]), a], 'global pax header'], + [[x([['path', 'b']])], 'end of the tar archive'], + [ + [ + x([ + ['path', 'b'], + ['path', 'c'], + ]), + a, + ], + 'Duplicate pax record', + ], + [[makeEntry({ name: 'x', type: 'x' }, '99 path=a\n'), a], 'pax'], + [[makeEntry({ name: 'x', type: 'x' }, '3 path=a\n'), a], 'pax'], + [[makeEntry({ name: 'x', type: 'x' }, '07 a=b\n'), a], 'pax'], + [[makeEntry({ name: 'x', type: 'x' }, '5 =b\n'), a], 'pax'], + ]; + for (const [entries, message] of cases) { + expect(() => listEntries(makeArchive(entries))).toThrow(message); + } + }); + + it('does not read a name prefix from GNU headers', function () { + // In GNU headers, the UStar prefix bytes contain atime and ctime. + const archive = makeArchive([ + makeEntry( + { + name: 'crate/a.rs', + format: 'gnu', + patches: [{ offset: 345, bytes: '14751573214\x0014751573214\x00' }], + }, + 'a' + ), + ]); + expect(listEntries(archive).map((e) => e.name)).toEqual(['crate/a.rs']); + }); + + it('reads v7 archives', function () { + const archive = makeArchive([ + makeEntry({ name: 'a', format: 'v7', prefix: 'ignored' }, 'a'), + ]); + expect(listEntries(archive).map((e) => e.name)).toEqual(['a']); + }); + + it('rejects unknown header formats', function () { + const archive = makeArchive([ + makeEntry({ name: 'a', patches: [{ offset: 263, bytes: '01' }] }, 'a'), + ]); + expect(() => listEntries(archive)).toThrow('Unknown tar header format'); + }); + + it('rejects mixed header formats', function () { + const archive = makeArchive([ + makeEntry({ name: 'a', format: 'gnu' }, 'a'), + makeEntry({ name: 'b', format: 'ustar' }, 'b'), + ]); + expect(() => listEntries(archive)).toThrow('Mixed header formats'); + }); + + it('rejects pax headers in GNU archives', function () { + const archive = makeArchive([ + makeEntry( + { name: 'x', type: 'x', format: 'gnu' }, + paxRecords([['path', 'b']]) + ), + makeEntry({ name: 'a', format: 'gnu' }, 'a'), + ]); + expect(() => listEntries(archive)).toThrow('Pax header in a gnu'); + }); + + // These fixtures were generated with Python's tarfile module, as an + // independent implementation. The GNU one matches the output of cargo. + // Each contains a member with a long name, a member with a non-ASCII name, + // a symlink with a long target, and a member with a short name. + for (const format of ['gnu', 'pax']) { + it(`reads a ${format} archive generated by Python`, function () { + const fs = require('fs'); + const zlib = require('zlib'); + const buffer = zlib.gunzipSync( + fs.readFileSync(`src/test/fixtures/synthetic-${format}.tar.gz`) + ); + const longName = + 'mycrate-1.0.0/src/' + 'a_very_long_directory_name/'.repeat(4); + const stream = new UntarFileStream(buffer.buffer); + const entries = []; + for (let entry = stream.next(); entry !== null; entry = stream.next()) { + const { name, type, buffer, linkname } = entry; + entries.push({ + name, + type, + linkname, + content: buffer === null ? null : new TextDecoder().decode(buffer), + }); + } + expect(entries).toEqual([ + { + name: longName + 'file.rs', + type: '0', + linkname: '', + content: 'long name', + }, + { + name: 'mycrate-1.0.0/é.rs', + type: '0', + linkname: '', + content: 'non-ascii', + }, + { + name: 'mycrate-1.0.0/link', + type: '2', + linkname: longName + 'file.rs', + content: null, + }, + { + name: 'mycrate-1.0.0/short.rs', + type: '0', + linkname: '', + content: 'short', + }, + ]); + }); + } + + it('rejects invalid GNU long name sequences', function () { + const a = makeEntry({ name: 'a', format: 'gnu' }, 'a'); + const longEntry = (type: string, format: 'gnu' | 'ustar' = 'gnu') => + makeEntry({ name: '././@LongLink', type, format }, 'b\0'); + const cases: Array<[Uint8Array[], string]> = [ + [[longEntry('L'), longEntry('L'), a], 'Multiple GNU long names'], + [[longEntry('K'), longEntry('K'), a], 'Multiple GNU long link names'], + [[longEntry('L')], 'end of the tar archive'], + [ + [ + longEntry('L'), + makeEntry( + { name: 'x', type: 'x', format: 'gnu' }, + paxRecords([['path', 'c']]) + ), + a, + ], + 'Pax header in a gnu', + ], + [ + [longEntry('L', 'ustar'), makeEntry({ name: 'a' }, 'a')], + 'GNU long name header in a ustar', + ], + [ + [makeEntry({ name: '././@LongLink', type: 'L', format: 'gnu' }), a], + 'empty name', + ], + ]; + for (const [entries, message] of cases) { + expect(() => listEntries(makeArchive(entries))).toThrow(message); + } + }); }); diff --git a/src/utils/fetch-source.ts b/src/utils/fetch-source.ts index 4ad15da0db..c0c86ff95d 100644 --- a/src/utils/fetch-source.ts +++ b/src/utils/fetch-source.ts @@ -165,8 +165,7 @@ export async function fetchSource( const stream = new UntarFileStream(bytes.buffer); const textDecoder = new TextDecoder(); - while (stream.hasNext()) { - const entry = stream.next(); + for (let entry = stream.next(); entry !== null; entry = stream.next()) { if (entry.name === pathInArchive && entry.buffer !== null) { const source = textDecoder.decode(entry.buffer); return { type: 'SUCCESS', source }; diff --git a/src/utils/untar.ts b/src/utils/untar.ts index 1578e472f6..effa8d78f3 100644 --- a/src/utils/untar.ts +++ b/src/utils/untar.ts @@ -2,326 +2,335 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -// This code was originally from https://github.com/InvokIT/js-untar/blob/master/src/untar-worker.js (MIT). -// It was modernized, the Worker requirement was removed, and some Flow types were added. -// Tar is an uncompressed format. If you have the tar bytes in a buffer, you can -// access the contained files simply by creating a new view around that same buffer, -// with the correct offsets. The parsing of the tar header is fast; the slow part -// would be the copy of the contained bytes to and from the worker. So having -// everything on the main thread is both simpler and faster, because it skips the copy. - -// TODO: Consider unifying this ByteReader with the one in art-trace.js -class ByteReader { - _uint8Array: Uint8Array; - _bufferView: DataView; - _position = 0; - _asciiDecoder = new TextDecoder('ascii', { fatal: true }); - - constructor(arrayBuffer: ArrayBufferLike) { - this._uint8Array = new Uint8Array(arrayBuffer); - this._bufferView = new DataView(arrayBuffer); - } - - readString(byteCount: number): string { - const bytes = this.readBuffer(byteCount); - const nulBytePos = bytes.indexOf(0); - return nulBytePos === -1 - ? this._asciiDecoder.decode(bytes) - : this._asciiDecoder.decode(bytes.subarray(0, nulBytePos)); - } - - readBuffer(byteCount: number): Uint8Array { - const buf = this._uint8Array.subarray( - this.position(), - this.position() + byteCount - ); - this.seekBy(byteCount); - return buf; - } - - seekBy(byteCount: number): void { - this._position += byteCount; - } +// A tar parser which works in place on an ArrayBuffer. Tar is an uncompressed +// format, so the contained files are returned as views into that same buffer, +// without copies. +// +// The parser is deliberately strict: it rejects malformed headers and unusual +// combinations of extension headers rather than guessing, so that it is less +// likely to disagree with other tar implementations about the contents of an +// archive. See https://astral.sh/blog/tar-codec for background. +// +// References: +// https://pubs.opengroup.org/onlinepubs/9699919799/utilities/pax.html +// https://www.gnu.org/software/tar/manual/html_node/Standard.html + +const BLOCK_SIZE = 512; + +// Byte ranges of the header fields we use, as [offset, length]. +type Field = [number, number]; +const NAME: Field = [0, 100]; +const SIZE: Field = [124, 12]; +const CHECKSUM: Field = [148, 8]; +const TYPEFLAG: Field = [156, 1]; +const LINKNAME: Field = [157, 100]; +// The magic and version fields together. +const MAGIC: Field = [257, 8]; +const PREFIX: Field = [345, 155]; + +// Names in tar headers are raw bytes; in practice they are UTF-8. Bytes that +// aren't valid UTF-8 become U+FFFD, so such names can't match any real path. +const textDecoder = new TextDecoder('utf-8'); + +function _field(header: Uint8Array, [offset, length]: Field): Uint8Array { + return header.subarray(offset, offset + length); +} - seekTo(newpos: number): void { - this._position = newpos; - } +// Decode a NUL-terminated (or NUL-padded) string. +function _parseString(bytes: Uint8Array): string { + const nulPos = bytes.indexOf(0); + return textDecoder.decode(nulPos === -1 ? bytes : bytes.subarray(0, nulPos)); +} - peekUint32(): number { - return this._bufferView.getUint32(this.position(), true); +/** + * Parse a numeric header field. These fields contain octal digits, optionally + * preceded by spaces or NULs and followed by spaces or NULs. Anything else, + * including GNU base-256 numbers, is rejected. Returns null if the field + * contains no digits at all. + */ +function _parseOctal(bytes: Uint8Array): number | null { + const isPadding = (b: number) => b === 0x00 || b === 0x20; + let i = 0; + while (i < bytes.length && isPadding(bytes[i])) { + i++; } - - position(): number { - return this._position; + let value = 0; + let digitCount = 0; + for (; i < bytes.length && bytes[i] >= 0x30 && bytes[i] <= 0x37; i++) { + value = value * 8 + (bytes[i] - 0x30); + digitCount++; } - - size(): number { - return this._bufferView.byteLength; + for (; i < bytes.length; i++) { + if (!isPadding(bytes[i])) { + throw new Error('Invalid numeric field in tar header'); + } } + return digitCount === 0 ? null : value; } -type FieldEntry = { - name: string; - value: string | number | null; -}; - -function _parseIntStrict(s: string, base: number): number { - const val = parseInt(s, base); - if (isNaN(val)) { - throw new Error('Parsing number failed'); - } - return val; +function _isZeroBlock(block: Uint8Array): boolean { + return block.every((b) => b === 0); } -function _parseOptionalInt(s: string, base: number): number | null { - if (s === '') { - return null; +/** + * Check the header checksum: the unsigned sum of all header bytes, with the + * checksum field itself counted as if it contained spaces. + */ +function _verifyChecksum(header: Uint8Array): void { + const [checksumOffset, checksumLength] = CHECKSUM; + const expected = _parseOctal(_field(header, CHECKSUM)); + let sum = 0x20 * checksumLength; + for (let i = 0; i < header.length; i++) { + if (i < checksumOffset || i >= checksumOffset + checksumLength) { + sum += header[i]; + } + } + if (expected !== sum) { + throw new Error('Tar header checksum mismatch'); } - return _parseIntStrict(s, base); } +// The pax keywords we use. Records with other keywords are ignored. +const PAX_KEYWORDS = new Set(['path', 'linkpath', 'size']); + /** - * A PaxHeader is a set of fields with names and values. The fields get applied - * to a file entry. - * When reading file entries, the header is read first, then the file, and then - * the header is applied to the file. + * Parse the records of a pax extended header. Each record has the form + * "%d %s=%s\n" (, , ), where is the decimal + * length of the whole record in bytes, including the trailing newline. + * An empty value means that the field is deleted. */ -class PaxHeader { - _fields: FieldEntry[]; - - static parse(buffer: Uint8Array): PaxHeader { - // https://pubs.opengroup.org/onlinepubs/9699919799/utilities/pax.html#tag_20_92_13_03 - // An extended header shall consist of one or more records, each constructed as follows: - // "%d %s=%s\n", , , - - // The extended header records shall be encoded according to the ISO/IEC10646-1:2000 standard (UTF-8). - // The field, , equals sign, and shown shall be limited to the portable character set, as - // encoded in UTF-8. The and fields can be any UTF-8 characters. The field shall be the - // decimal length of the extended header record in octets, including the trailing . - - let bytes = new Uint8Array(buffer); - const fields = []; - - const textDecoder = new TextDecoder('utf-8', { fatal: true }); - - while (bytes.length > 0) { - // Decode bytes up to the first space character; that is the total field length - const spacePos = bytes.indexOf(0x20); - if (spacePos === -1) { - throw new Error('Expected space character'); +function _parsePaxRecords(data: Uint8Array): Map { + const records = new Map(); + let pos = 0; + while (pos < data.length) { + const spacePos = data.indexOf(0x20, pos); + const lengthString = + spacePos === -1 ? '' : textDecoder.decode(data.subarray(pos, spacePos)); + if (!/^[1-9][0-9]*$/.test(lengthString)) { + throw new Error('Invalid pax record length'); + } + const end = pos + Number(lengthString); + const equalsPos = data.indexOf(0x3d, spacePos + 1); + if ( + end > data.length || + data[end - 1] !== 0x0a || + equalsPos === -1 || + equalsPos >= end || + equalsPos === spacePos + 1 + ) { + throw new Error('Invalid pax record'); + } + const keyword = textDecoder.decode(data.subarray(spacePos + 1, equalsPos)); + if (PAX_KEYWORDS.has(keyword)) { + if (records.has(keyword)) { + throw new Error(`Duplicate pax record: ${keyword}`); } - const fieldLength = _parseIntStrict( - textDecoder.decode(bytes.subarray(0, spacePos)), - 10 + records.set( + keyword, + textDecoder.decode(data.subarray(equalsPos + 1, end - 1)) ); - const fieldText = textDecoder.decode(bytes.subarray(0, fieldLength)); - const fieldMatch = fieldText.match(/^\d+ ([^=]+)=(.*)\n$/); - - if (fieldMatch === null) { - throw new Error('Invalid PAX header data format.'); - } - - const fieldName = fieldMatch[1]; - let fieldValue: string | number | null = fieldMatch[2]; - - if (typeof fieldValue === 'string' && fieldValue.length === 0) { - fieldValue = null; - } else if (typeof fieldValue === 'string' && /^\d+$/.test(fieldValue)) { - // If it's an integer field, parse it as int - fieldValue = _parseIntStrict(fieldValue, 10); - } - // Don't parse float values since precision is lost - - const field = { - name: fieldName, - value: fieldValue, - }; - - fields.push(field); - - bytes = bytes.subarray(fieldLength); // Cut off the parsed field data } - - return new PaxHeader(fields); + pos = end; } + return records; +} - constructor(fields: FieldEntry[]) { - this._fields = fields; +// Archives use one of these header formats: +// - 'ustar': POSIX UStar, possibly with pax extended headers ('x' and 'g'). +// - 'gnu': The old GNU format, which cargo uses. It has no name prefix field; +// those bytes contain other fields. Long names are stored in separate 'L' +// and 'K' entries. +// - 'v7': The original tar format, without any extensions. +type TarFormat = 'ustar' | 'gnu' | 'v7'; + +function _getFormat(header: Uint8Array): TarFormat { + const magic = String.fromCharCode(..._field(header, MAGIC)); + switch (magic) { + case 'ustar\x0000': + return 'ustar'; + case 'ustar \x00': + return 'gnu'; + case '\x00'.repeat(8): + return 'v7'; + default: + throw new Error('Unknown tar header format'); } +} - applyHeader(file: any): void { - // Apply fields to the file - // If a field is of value null, it should be deleted from the file - // https://pubs.opengroup.org/onlinepubs/9699919799/utilities/pax.html#tag_20_92_13_03 - - for (const field of this._fields) { - let fieldName = field.name; - const fieldValue = field.value; - - if (fieldName === 'path') { - // This overrides the name and prefix fields in the following header block. - fieldName = 'name'; - - if (file.namePrefix !== undefined) { - delete file.namePrefix; - } - } else if (fieldName === 'linkpath') { - // This overrides the linkname field in the following header block. - fieldName = 'linkname'; - } - - if (fieldValue === null) { - delete file[fieldName]; - } else { - file[fieldName] = fieldValue; - } - } +function _parsePaxSize(value: string): number { + const size = Number(value); + if (!/^[0-9]+$/.test(value) || !Number.isSafeInteger(size)) { + throw new Error('Invalid pax size'); } + return size; } export type TarFileEntry = { name: string; + // The typeflag, e.g. '0' for a regular file or '5' for a directory. Old + // archives use '' for regular files. type: string; size: number; + // The contents of a regular file, as a view into the archive buffer. + // null for all other entry types. buffer: Uint8Array | null; - mode: string; - uid: number | null; - gid: number | null; - mtime: number | null; - checksum: number | null; linkname: string; - ustarFormat: string; - version: string | null; - uname: string | null; - gname: string | null; - devmajor: number | null; - devminor: number | null; - namePrefix: string | null; }; export class UntarFileStream { - _reader: ByteReader; - _globalPaxHeader: PaxHeader | null = null; + _bytes: Uint8Array; + _position = 0; + // Mixing header formats in one archive is not allowed. This is the format + // of the first header. + _format: TarFormat | null = null; constructor(arrayBuffer: ArrayBufferLike) { - this._reader = new ByteReader(arrayBuffer); + this._bytes = new Uint8Array(arrayBuffer); } - hasNext(): boolean { - // A tar file ends with 4 zero bytes - return ( - this._reader.position() + 4 < this._reader.size() && - this._reader.peekUint32() !== 0 - ); - } + /** + * Return the next member of the archive, or null at the end of the archive. + * Throws if the archive is malformed. + */ + next(): TarFileEntry | null { + // Extension headers which apply to the next member: the records of a pax + // 'x' header, and the GNU long name ('L') and long link name ('K'). + let localPax: Map | null = null; + let longName: string | null = null; + let longLinkname: string | null = null; + + while (true) { + const header = this._readHeader(); + if (header === null) { + if (localPax !== null || longName !== null || longLinkname !== null) { + throw new Error('Extension header at the end of the tar archive'); + } + return null; + } - next(): TarFileEntry { - return this._readNextFile(); - } + const type = _parseString(_field(header, TYPEFLAG)); + const headerSize = _parseOctal(_field(header, SIZE)); + if (headerSize === null) { + throw new Error('Missing size in tar header'); + } - _readNextFile(): TarFileEntry { - const stream = this._reader; - - const headerBeginPos = stream.position(); - - // Read header - let name = stream.readString(100); - const mode = stream.readString(8); - const uid = _parseOptionalInt(stream.readString(8), 8); - const gid = _parseOptionalInt(stream.readString(8), 8); - const size = _parseIntStrict(stream.readString(12), 8); - const mtime = _parseOptionalInt(stream.readString(12), 8); - const checksum = _parseOptionalInt(stream.readString(8), 10); - const type = stream.readString(1); - const linkname = stream.readString(100); - const ustarFormat = stream.readString(6); - let version = null; - let uname = null; - let gname = null; - let devmajor = null; - let devminor = null; - let namePrefix = null; - - if (ustarFormat.indexOf('ustar') > -1) { - version = stream.readString(2); - uname = stream.readString(32); - gname = stream.readString(32); - devmajor = _parseOptionalInt(stream.readString(8), 10); - devminor = _parseOptionalInt(stream.readString(8), 10); - namePrefix = stream.readString(155); - - if (namePrefix.length > 0) { - name = namePrefix + '/' + name; + const format = _getFormat(header); + if (this._format === null) { + this._format = format; + } else if (format !== this._format) { + throw new Error('Mixed header formats in tar archive'); } - } - // The size is padded to be aligned with 512 bytes. - const alignedSize = ((size - 1) | 511) + 1; + if ((type === 'g' || type === 'x') && format !== 'ustar') { + throw new Error(`Pax header in a ${format} tar archive`); + } - const dataBeginPos = headerBeginPos + 512; - const dataEndPos = dataBeginPos + alignedSize; + if ((type === 'L' || type === 'K') && format !== 'gnu') { + throw new Error(`GNU long name header in a ${format} tar archive`); + } - stream.seekTo(dataBeginPos); + if (type === 'L') { + if (longName !== null) { + throw new Error('Multiple GNU long names for one tar member'); + } + longName = _parseString(this._readData(headerSize)); + continue; + } - // Handle the various file types. + if (type === 'K') { + if (longLinkname !== null) { + throw new Error('Multiple GNU long link names for one tar member'); + } + longLinkname = _parseString(this._readData(headerSize)); + continue; + } - // Derived from https://www.mkssoftware.com/docs/man4/pax.4.asp - // and https://www.ibm.com/support/knowledgecenter/en/SSLTBW_2.3.0/com.ibm.zos.v2r3.bpxa500/pxarchfm.htm - // Also see https://en.wikipedia.org/wiki/Tar_(computing)#File_format + if (type === 'g') { + // Global pax header. Its records apply to all following members. We + // reject all the keywords we'd otherwise use: overriding the path or + // size of every member isn't meaningful. So there's nothing to keep. + const records = _parsePaxRecords(this._readData(headerSize)); + if (records.size !== 0) { + throw new Error('Unsupported record in global pax header'); + } + continue; + } - if (type === 'g') { - // Global PAX header - this._globalPaxHeader = PaxHeader.parse(stream.readBuffer(size)); - stream.seekTo(dataEndPos); + if (type === 'x') { + if (localPax !== null) { + throw new Error('Multiple pax headers for one tar member'); + } + localPax = _parsePaxRecords(this._readData(headerSize)); + continue; + } - // Read the next entry. It will apply the global PAX header to itself. - return this._readNextFile(); - } + // This is a real member. Resolve its fields, with extension headers + // taking precedence over the header. A pax record with an empty value + // deletes the field. Only one kind of extension header can be present, + // because the format check above ties pax to UStar and L/K to GNU. + let headerName = _parseString(_field(header, NAME)); + if (format === 'ustar') { + const prefix = _parseString(_field(header, PREFIX)); + if (prefix !== '') { + headerName = prefix + '/' + headerName; + } + } + const name = localPax?.get('path') ?? longName ?? headerName; + const linkname = + localPax?.get('linkpath') ?? + longLinkname ?? + _parseString(_field(header, LINKNAME)); + const paxSize = localPax?.get('size'); + const size = paxSize === undefined ? headerSize : _parsePaxSize(paxSize); + if (name === '') { + throw new Error('Tar member with an empty name'); + } + + const data = this._readData(size); - if (type === 'x') { - // PAX header - const paxHeader = PaxHeader.parse(stream.readBuffer(size)); - stream.seekTo(dataEndPos); + // Regular files have the type '0', or '' in old archives. Type '7' + // (contiguous file) is usually treated like a regular file. + const isNormalFile = type === '0' || type === '' || type === '7'; - // Read the next entry, and then apply this PAX header to it. - const file = this._readNextFile(); - paxHeader.applyHeader(file); - return file; + return { + name, + type, + size, + buffer: isNormalFile ? data : null, + linkname, + }; } + } - // Normal file is either "0" or "\0". - // In case of "\0", readString returns an empty string, that is "". - // In addition, according to wikipedia, type 7 is usually handled like type 0. - const isNormalFile = type === '0' || type === '' || type === '7'; - - const buffer = isNormalFile ? stream.readBuffer(size) : null; - - stream.seekTo(dataEndPos); - - const file = { - name, - type, - buffer, - mode, - uid, - gid, - size, - mtime, - checksum, - linkname, - ustarFormat, - version, - uname, - gname, - devmajor, - devminor, - namePrefix, - }; - - if (this._globalPaxHeader !== null) { - this._globalPaxHeader.applyHeader(file); + // Read a header block, or return null at the end of the archive. A tar + // archive ends with two all-zero blocks. We stop at the first one, and also + // accept an archive that ends right after its last member. + _readHeader(): Uint8Array | null { + if (this._position >= this._bytes.length) { + return null; } + const header = this._read(BLOCK_SIZE); + if (_isZeroBlock(header)) { + return null; + } + _verifyChecksum(header); + return header; + } - return file; + // Read the data that follows a header, and skip its padding. + _readData(size: number): Uint8Array { + const data = this._read(size); + this._position += Math.ceil(size / BLOCK_SIZE) * BLOCK_SIZE - size; + return data; + } + + _read(byteCount: number): Uint8Array { + const end = this._position + byteCount; + if (end > this._bytes.length) { + throw new Error('Unexpected end of tar archive'); + } + const bytes = this._bytes.subarray(this._position, end); + this._position = end; + return bytes; } }