From 4132d806fbd15b4985f50998947bae67886649aa Mon Sep 17 00:00:00 2001 From: Cody Tapscott Date: Wed, 7 Oct 2026 22:40:54 -0400 Subject: [PATCH 1/2] Skip auxiliary records when iterating COFF symbols COFF symbol table entries may be followed by auxiliary records, which `scan_symbol_offsets()` found but `Symbols` never used: iterating a `COFFSymbols` listed every record, auxiliary or not, as a symbol. It also stopped one record short, dropping a final symbol without any. Index through the scanned (non-auxiliary) symbol table indices instead. `symbol_number()` remains the symbol's index within the symbol table. Co-Authored-By: Claude Opus 5.5 --- src/COFF/COFFSymbol.jl | 63 +++++++++++++++++++++--------------------- test/runtests.jl | 9 ++++++ 2 files changed, 40 insertions(+), 32 deletions(-) diff --git a/src/COFF/COFFSymbol.jl b/src/COFF/COFFSymbol.jl index 2d7d938..da7b2e8 100644 --- a/src/COFF/COFFSymbol.jl +++ b/src/COFF/COFFSymbol.jl @@ -5,61 +5,60 @@ export COFFSymbols, COFFSymtabEntry, COFFSymbolRef COFF symbol table, contains the list of symbols defined within the object file. -Note that because COFF Symbols are variable-length, we store a table of offsets -at which the (non-auxilliary) symbols can be found. +Note that because COFF Symbols are followed by a variable number of auxiliary +records within the symbol table, we store the (1-based) symbol table index at +which each (non-auxiliary) symbol can be found. """ struct COFFSymbols{H<:COFFHandle} <: Symbols{H} handle::H - symbol_offsets::Vector{UInt64} + symbol_indices::Vector{UInt32} end """ - scan_symbol_offsets(oh::COFFHandle) + scan_symbol_indices(oh::COFFHandle) -Find the offsets for each Symbol within a COFFHandle, skipping over auxilliary -symbols as necessary. +Find the symbol table index of each Symbol within a COFFHandle, skipping over +auxiliary records as necessary. """ -function scan_symbol_offsets(oh::H) where {H <: COFFHandle} - # num_syms represents the number of total symbols, but we're only +function scan_symbol_indices(oh::H) where {H <: COFFHandle} + # num_syms represents the number of total records, but we're only # interested in the non-auxiliary symbols num_syms = num_symbols(header(oh)) - offsets = UInt64[] + indices = UInt32[] - curr_idx = 1 idx = 1 - while idx < num_syms - # Load in the next symbol and store its offset + while idx <= num_syms + # Load in the next symbol and store its index seek(oh, symtab_entry_offset(oh) + symtab_entry_size(oh)*(idx-1)) + sym = unpack(oh, symtab_entry_type(oh)) + push!(indices, idx) - sym = unpack(oh, COFFSymtabEntry{H}) - push!(offsets, idx) - - # Move our offset forward by 1 (because we just read in a SymtabEntry) - # and again as many times as needed to skip over the auxilliary symbols + # Move our index forward by 1 (because we just read in a SymtabEntry) + # and again as many times as needed to skip over the auxiliary records idx += 1 + sym.NumberOfAuxSymbols - curr_idx += 1 end - return offsets + return indices end function Symbols(oh::H) where {H <: COFFHandle} - symbol_offsets = scan_symbol_offsets(oh) - return COFFSymbols(oh, symbol_offsets) + return COFFSymbols(oh, scan_symbol_indices(oh)) end handle(syms::COFFSymbols) = syms.handle -# function next(syms::COFFSymbols, idx) -# # We skip over auxiliary symbols here -# next_idx = idx + deref(syms[idx]).NumberOfAuxSymbols + 1 -# return (syms[idx], next_idx) -# end -lastindex(syms::COFFSymbols) = num_symbols(header(handle(syms))) - -# We override `iteratorsize` so that list comprehensions and collect() calls on -# our Symbols don't have a bunch of #undef entries at the end of the array -# because it tries to pre-allocate an array of the proper size. -# iteratorsize(::Type{H}) where {H <: COFFSymbols} = SizeUnknown() +lastindex(syms::COFFSymbols) = length(syms.symbol_indices) + +# Index into the symbols, skipping over auxiliary records. The resulting +# `symbol_number()` is the symbol's index within the symbol table. +function getindex(syms::COFFSymbols, idx) + if !(0 < idx <= length(syms)) + throw(BoundsError(syms, idx)) + end + oh = handle(syms) + table_idx = syms.symbol_indices[idx] + seek(oh, symtab_entry_offset(oh) + symtab_entry_size(oh)*(table_idx-1)) + return SymbolRef(syms, unpack(oh, symtab_entry_type(oh)), table_idx) +end @io struct COFFSymtabEntry{H <: COFFHandle} <: SymtabEntry{H} Name::fixed_string{UInt64} diff --git a/test/runtests.jl b/test/runtests.jl index f30b965..8bd104d 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -139,6 +139,15 @@ function test_libfoo_and_fooifier(fooifier_path, libfoo_path) @test !islocal(syms_exe[main_idx_exe]) @test !islocal(syms_lib[foo_idx_lib]) + # COFF symbols are followed by auxiliary records, which are skipped + if isa(oh_exe, COFFHandle) + for (oh, syms) in ((oh_exe, syms_exe), (oh_lib, syms_lib)) + naux = [ObjectFile.deref(sym).NumberOfAuxSymbols for sym in syms] + @test length(syms) + sum(naux) == header(oh).NumberOfSymbols + @test ObjectFile.symbol_number(syms[end]) + naux[end] == header(oh).NumberOfSymbols + end + end + # Global detection doesn't seem to be working on OSX... if !isa(oh_exe, MachOHandle) @test isglobal(syms_exe[foo_idx_exe]) From e82bece788f8220f855cc9c7d2ef168d88902838 Mon Sep 17 00:00:00 2001 From: Cody Tapscott Date: Wed, 7 Oct 2026 22:44:16 -0400 Subject: [PATCH 2/2] Read COFF object files Object files (e.g. `foo.obj`, or the members of `foo.lib`) have no MZ stub, PE signature or optional header: they start directly with their COFF header. `readmeta(io, COFFHandle)` now falls back to reading one, identifying it by its `Machine` (which is all the magic an object has) and a header consistent with an object of its size. Such handles have no `opt_header`, and no `DynamicLinks`. Also: - Add the ARM64EC and ARM64X machine types. - Only prefix symbols with an underscore on 32-bit x86, not on all 32-bit architectures (e.g. ARMv7). Co-Authored-By: Claude Opus 5.5 --- src/COFF/COFFDynamicLink.jl | 4 ++ src/COFF/COFFHandle.jl | 71 +++++++++++++++++++++++++++++------- src/COFF/COFFHeader.jl | 2 + src/COFF/constants.jl | 4 +- test/coff/build.sh | 14 +++++++ test/coff/foo_aarch64.obj | Bin 0 -> 422 bytes test/coff/foo_i686.obj | Bin 0 -> 420 bytes test/coff/foo_thumbv7.obj | Bin 0 -> 400 bytes test/coff/foo_x86_64.obj | Bin 0 -> 418 bytes test/runtests.jl | 39 +++++++++++++++++++- 10 files changed, 119 insertions(+), 15 deletions(-) create mode 100755 test/coff/build.sh create mode 100644 test/coff/foo_aarch64.obj create mode 100644 test/coff/foo_i686.obj create mode 100644 test/coff/foo_thumbv7.obj create mode 100644 test/coff/foo_x86_64.obj diff --git a/src/COFF/COFFDynamicLink.jl b/src/COFF/COFFDynamicLink.jl index 6c9e129..48ddc98 100644 --- a/src/COFF/COFFDynamicLink.jl +++ b/src/COFF/COFFDynamicLink.jl @@ -34,6 +34,10 @@ function find_section_for_rva(oh::H, rva) where {H <: COFFHandle} end function COFFDynamicLinks(oh::H) where {H <: COFFHandle} + if oh.opt_header === nothing + return COFFDynamicLinks(oh, COFFDynamicLink{H}[]) + end + # Start by finding the virtual address of the import table import_table_rva = oh.opt_header.directories.ImportTable.VirtualAddress diff --git a/src/COFF/COFFHandle.jl b/src/COFF/COFFHandle.jl index f4ff1d8..b5d1f50 100644 --- a/src/COFF/COFFHandle.jl +++ b/src/COFF/COFFHandle.jl @@ -17,8 +17,9 @@ struct COFFHandle{T<:IO} <: ObjectHandle # The location of the header (because of MZ confusion, we must store this) header_offset::UInt32 - # The "Optional" header, which isn't actually optional - opt_header::COFFOptionalHeader + # The "Optional" header, which isn't actually optional for images, but is + # absent from object files + opt_header::Union{Nothing,COFFOptionalHeader} # The path of the file this was created with path::String @@ -35,22 +36,32 @@ function readmeta(io::IO, ::Type{H}) where {H <: COFFHandle} # Check to see if this is an 'MZ' file; if it is, we have to jump forward # into the file a bit to get to the PE header - magic = [read(io, UInt8) for idx in 1:2] - if all(magic .== MZ_magic) + magic = read(io, 2) + if magic == MZ_magic # Skip ahead to the PE header offset skip(io, 58) header_offset = read(io, UInt32) seek(io, start + header_offset) else - # If it's not, assume the PE header starts at the beginning of the file - skip(io, -2) + # If it's not, the PE header may start at the beginning of the file + seek(io, start) end - magic = [read(io, UInt8) for idx in 1:4] - if any(magic .!= PE_magic) + magic = read(io, 4) + if magic != PE_magic + # An object file has no PE header (nor an MZ stub), it starts directly + # with its COFF header. + if magic[1:min(2, end)] != MZ_magic + seek(io, start) + header = read_object_header(io) + if header !== nothing + return [COFFHandle(io, Int64(start), header, UInt32(0), nothing, path(io))] + end + end + msg = """ Magic Number 0x$(join(string.(magic, base=16, pad=2),"")) does not match expected PE - magic number 0x$(join(string.(PE_magic, base=16, pad=2),"")) + magic number 0x$(join(string.(PE_magic, base=16, pad=2),"")), nor is this a COFF object """ throw(MagicMismatch(replace(strip(msg), "\n" => " "))) end @@ -66,6 +77,40 @@ function readmeta(io::IO, ::Type{H}) where {H <: COFFHandle} return [COFFHandle(io, Int64(start), header, header_offset, opt_header, path(io))] end +""" + read_object_header(io::IO) + +Read the `COFFHeader` of an object file (e.g. `foo.obj`) at the current position +of `io`, returning `nothing` if it does not look like one. Object files have no +magic number beyond the `Machine` field of their header, so we check that the +rest of the header is consistent with an object file of that size. +""" +function read_object_header(io::IO) + start = position(io) + header = unpack(io, COFFHeader) + # Find out how much room there is for this object, at most + seekend(io) + available = position(io) - start + seek(io, start) + + if !haskey(IMAGE_FILE_MACHINE, header.Machine) || header.Machine == IMAGE_FILE_MACHINE_UNKNOWN + return nothing + end + # Only images have an optional header, or are executables/DLLs + if header.SizeOfOptionalHeader != 0 || + (header.Characteristics & (IMAGE_FILE_EXECUTABLE_IMAGE | IMAGE_FILE_DLL)) != 0 + return nothing + end + # The section and symbol tables must fit within the object + if sizeof(COFFHeader) + header.NumberOfSections * packed_sizeof(COFFSection{COFFHandle}) > available + return nothing + end + if header.PointerToSymbolTable != 0 && + header.PointerToSymbolTable + header.NumberOfSymbols * packed_sizeof(COFFSymtabEntry{COFFHandle}) > available + return nothing + end + return header +end ## IOStream-like operations: startaddr(oh::COFFHandle) = oh.start @@ -82,11 +127,11 @@ islibrary(oh::COFFHandle) = islibrary(header(oh)) isdynamic(oh::COFFHandle) = !isempty(findall(Sections(oh), [".idata"])) mangle_section_name(oh::COFFHandle, name::AbstractString) = string(".", name) function mangle_symbol_name(oh::COFFHandle, name::AbstractString) - # sob - if is64bit(oh) - return name - else + # sob; only 32-bit x86 prefixes C symbols with an underscore + if header(oh).Machine == IMAGE_FILE_MACHINE_I386 return string("_", name) + else + return name end end format_string(::Type{H}) where {H <: COFFHandle} = "COFF" diff --git a/src/COFF/COFFHeader.jl b/src/COFF/COFFHeader.jl index 577d1a2..2d7d3d8 100644 --- a/src/COFF/COFFHeader.jl +++ b/src/COFF/COFFHeader.jl @@ -16,6 +16,8 @@ function coff_header_is64bit(h::COFFHeader) wide_machines = [ IMAGE_FILE_MACHINE_AMD64, IMAGE_FILE_MACHINE_ARM64, + IMAGE_FILE_MACHINE_ARM64EC, + IMAGE_FILE_MACHINE_ARM64X, IMAGE_FILE_MACHINE_IA64, ] return h.Machine in wide_machines diff --git a/src/COFF/constants.jl b/src/COFF/constants.jl index d1569bd..c9be3e0 100644 --- a/src/COFF/constants.jl +++ b/src/COFF/constants.jl @@ -6,6 +6,8 @@ const IMAGE_FILE_MACHINE_ARM = 0x1c0 # ARM little endian const IMAGE_FILE_MACHINE_ARMNT = 0x1c4 # ARMv7 (or higher) Thumb mode only const IMAGE_FILE_MACHINE_ARM64 = 0xaa64 # ARMv8 in 64-bit mode + const IMAGE_FILE_MACHINE_ARM64EC = 0xa641 # ARMv8 in 64-bit mode, with the x64 ABI + const IMAGE_FILE_MACHINE_ARM64X = 0xa64e # ARM64 and ARM64EC code together const IMAGE_FILE_MACHINE_EBC = 0xebc # EFI byte code const IMAGE_FILE_MACHINE_I386 = 0x14c # Intel 386 or later processors and compatible processors const IMAGE_FILE_MACHINE_IA64 = 0x200 # Intel Itanium processor family @@ -31,7 +33,7 @@ function coff_machine_to_arch(machine::UInt16) return "x86_64" elseif machine ∈ (IMAGE_FILE_MACHINE_ARM, IMAGE_FILE_MACHINE_ARMNT, IMAGE_FILE_MACHINE_THUMB) return "armv7l" - elseif machine ∈ (IMAGE_FILE_MACHINE_ARM64,) + elseif machine ∈ (IMAGE_FILE_MACHINE_ARM64, IMAGE_FILE_MACHINE_ARM64EC, IMAGE_FILE_MACHINE_ARM64X) return "aarch64" elseif machine ∈ (IMAGE_FILE_MACHINE_POWERPC,) return "ppc64le" diff --git a/test/coff/build.sh b/test/coff/build.sh new file mode 100755 index 0000000..fea0291 --- /dev/null +++ b/test/coff/build.sh @@ -0,0 +1,14 @@ +#!/bin/bash +# Regenerates the COFF object fixtures in this directory. Requires `clang`. +set -euo pipefail +cd "$(dirname "$0")" +tmp="$(mktemp -d)" +trap 'rm -rf "${tmp}"' EXIT + +echo 'int foo(int x) { return x + 1; }' > "${tmp}/foo.c" + +# (`-mno-incremental-linker-compatible` keeps timestamps out of COFF objects) +for arch in x86_64 i686 aarch64 thumbv7; do + clang --target="${arch}-pc-windows-msvc" -O2 -mno-incremental-linker-compatible \ + -c -o "foo_${arch}.obj" "${tmp}/foo.c" +done diff --git a/test/coff/foo_aarch64.obj b/test/coff/foo_aarch64.obj new file mode 100644 index 0000000000000000000000000000000000000000..b32bf13c4ac4744c987dd0dfa7c12080202be72c GIT binary patch literal 422 zcmYdU#liprdw`e+%wk~BD@m;=fwDmq2M}*T;)9q93Ea=r69oaC%hjd0`v=#8<~K7 zW{5CYj)4Ir0#t+KK4u^vRSif40w5s;vL0OxM8+X4HL*m`zyPHB|9_wwhP3>Ah*luU t$e;kUPcJPqClxC852z-c5vWEl8LkS%5CUSooSd@U_{5ZyqT$2-L{T2y!O~XoPt(gG7KnL2@AzkPj6G z%P}y3M1X3LT*nOLqpAUkKma717#Kh_sv3xlLt1KLiJpN0$mak5fod4y)AIA7ni&`w s6oB^WrDf)%LWTYT)ub~5mFOkI6oF|WAgPy=QKACK{t+8?4(KHn z7bA;dV>bZR>6>86pqt0QA;8e<$aompjf_BT%#09!Ffc@E%a(#ffc`*o9}`#&SQtux zM1X3L+{O&#Gefk3<$ydW4G9Af2V@JX8i>6PX{m`NdIkm{HUIyEbf)F!gE(No$e_T$ q#Gsd!nUe|?`3F>!&InYamkbqwP(naTFDIugH$E{XrKmVFodE#s{3a6s literal 0 HcmV?d00001 diff --git a/test/runtests.jl b/test/runtests.jl index 8bd104d..af43569 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -366,7 +366,7 @@ end ("libfoo_darwin64.a", :bsd64, MachOHandle, member_names), ("libfoo_bsd_nosymtab.a", :bsd, MachOHandle, member_names), # `llvm-lib` orders members by name - ("foo.lib", :coff, nothing, reverse(member_names)), + ("foo.lib", :coff, COFFHandle, reverse(member_names)), ) ah = readmeta(open(joinpath("./archives", file), "r"), ArchiveHandle) @test archive_kind(ah) == kind @@ -392,6 +392,9 @@ end syms = symbol_name.(Symbols(oh)) fn = startswith(member.name, "foo") ? "foo" : "bar" @test mangle_symbol_name(oh, fn) in syms + # `bar()` calls `foo()`, which another member defines + foo_sym = only(sym for sym in Symbols(oh) if symbol_name(sym) == mangle_symbol_name(oh, "foo")) + @test isundef(foo_sym) == (fn == "bar") # ... and reads the same as it would on its own @test oh.header == only(readmeta(IOBuffer(data))).header end @@ -481,3 +484,37 @@ end end end end + +# Fixtures are generated by `./coff/build.sh` +@testset "COFF objects" begin + @testset "$(arch)" for (arch, platform) in ( + ("x86_64", Platform("x86_64", "windows")), + ("i686", Platform("i686", "windows")), + ("aarch64", Platform("aarch64", "windows")), + ("thumbv7", Platform("armv7l", "windows")), + ) + oh = only(readmeta(open("./coff/foo_$(arch).obj", "r"))) + @test oh isa COFFHandle + @test platforms_match(Platform(oh), platform) + @test is64bit(oh) == (arch in ("x86_64", "aarch64")) + @test isrelocatable(oh) + @test !isexecutable(oh) + @test !islibrary(oh) + @test isempty(DynamicLinks(oh)) + @test ".text" in section_name.(Sections(oh)) + foo_sym = only(sym for sym in Symbols(oh) if symbol_name(sym) == mangle_symbol_name(oh, "foo")) + @test isglobal(foo_sym) + @test !isundef(foo_sym) + end + + @testset "not an object" begin + not_obj(data) = readmeta(IOBuffer(data), COFFHandle) + @test_throws MagicMismatch not_obj(b"hello, world, this is not an object file") + # An x86_64 `Machine`, but with an optional header + @test_throws MagicMismatch not_obj(vcat(b"d\x86", zeros(UInt8, 14), [0x08, 0x00], zeros(UInt8, 2))) + # ... or with more sections than fit in the file + @test_throws MagicMismatch not_obj(vcat(b"d\x86", [0xff, 0x00], zeros(UInt8, 16))) + # An unknown `Machine` + @test_throws MagicMismatch not_obj(vcat(b"zz", zeros(UInt8, 18))) + end +end