Add support for .obj files in COFF

src/COFF/COFFDynamicLink.jl

@@ -34,6 +34,10 @@ function find_section_for_rva(oh::H, rva) where {H <: COFFHandle}
 end
 
 function COFFDynamicLinks(oh::H) where {H <: COFFHandle}
+    if oh.opt_header === nothing
+        return COFFDynamicLinks(oh, COFFDynamicLink{H}[])
+    end
+
     # Start by finding the virtual address of the import table
     import_table_rva = oh.opt_header.directories.ImportTable.VirtualAddress
 

src/COFF/COFFHandle.jl

@@ -17,8 +17,9 @@ struct COFFHandle{T<:IO} <: ObjectHandle
     # The location of the header (because of MZ confusion, we must store this)
     header_offset::UInt32
 
-    # The "Optional" header, which isn't actually optional
-    opt_header::COFFOptionalHeader
+    # The "Optional" header, which isn't actually optional for images, but is
+    # absent from object files
+    opt_header::Union{Nothing,COFFOptionalHeader}
 
     # The path of the file this was created with
     path::String
@@ -35,22 +36,32 @@ function readmeta(io::IO, ::Type{H}) where {H <: COFFHandle}
 
     # Check to see if this is an 'MZ' file; if it is, we have to jump forward
     # into the file a bit to get to the PE header
-    magic = [read(io, UInt8) for idx in 1:2]
-    if all(magic .== MZ_magic)
+    magic = read(io, 2)
+    if magic == MZ_magic
         # Skip ahead to the PE header offset
         skip(io, 58)
         header_offset = read(io, UInt32)
         seek(io, start + header_offset)
     else
-        # If it's not, assume the PE header starts at the beginning of the file
-        skip(io, -2)
+        # If it's not, the PE header may start at the beginning of the file
+        seek(io, start)
     end
 
-    magic = [read(io, UInt8) for idx in 1:4]
-    if any(magic .!= PE_magic)
+    magic = read(io, 4)
+    if magic != PE_magic
+        # An object file has no PE header (nor an MZ stub), it starts directly
+        # with its COFF header.
+        if magic[1:min(2, end)] != MZ_magic
+            seek(io, start)
+            header = read_object_header(io)
+            if header !== nothing
+                return [COFFHandle(io, Int64(start), header, UInt32(0), nothing, path(io))]
+            end
+        end
+
         msg = """
         Magic Number 0x$(join(string.(magic, base=16, pad=2),"")) does not match expected PE
-        magic number 0x$(join(string.(PE_magic, base=16, pad=2),""))
+        magic number 0x$(join(string.(PE_magic, base=16, pad=2),"")), nor is this a COFF object
         """
         throw(MagicMismatch(replace(strip(msg), "\n" => " ")))
     end
@@ -66,6 +77,40 @@ function readmeta(io::IO, ::Type{H}) where {H <: COFFHandle}
     return [COFFHandle(io, Int64(start), header, header_offset, opt_header, path(io))]
 end
 
+"""
+    read_object_header(io::IO)
+
+Read the `COFFHeader` of an object file (e.g. `foo.obj`) at the current position
+of `io`, returning `nothing` if it does not look like one.  Object files have no
+magic number beyond the `Machine` field of their header, so we check that the
+rest of the header is consistent with an object file of that size.
+"""
+function read_object_header(io::IO)
+    start = position(io)
+    header = unpack(io, COFFHeader)
+    # Find out how much room there is for this object, at most
+    seekend(io)
+    available = position(io) - start
+    seek(io, start)
+
+    if !haskey(IMAGE_FILE_MACHINE, header.Machine) || header.Machine == IMAGE_FILE_MACHINE_UNKNOWN
+        return nothing
+    end
+    # Only images have an optional header, or are executables/DLLs
+    if header.SizeOfOptionalHeader != 0 ||
+       (header.Characteristics & (IMAGE_FILE_EXECUTABLE_IMAGE | IMAGE_FILE_DLL)) != 0
+        return nothing
+    end
+    # The section and symbol tables must fit within the object
+    if sizeof(COFFHeader) + header.NumberOfSections * packed_sizeof(COFFSection{COFFHandle}) > available
+        return nothing
+    end
+    if header.PointerToSymbolTable != 0 &&
+       header.PointerToSymbolTable + header.NumberOfSymbols * packed_sizeof(COFFSymtabEntry{COFFHandle}) > available
+        return nothing
+    end
+    return header
+end
 
 ## IOStream-like operations:
 startaddr(oh::COFFHandle) = oh.start
@@ -82,11 +127,11 @@ islibrary(oh::COFFHandle) = islibrary(header(oh))
 isdynamic(oh::COFFHandle) = !isempty(findall(Sections(oh), [".idata"]))
 mangle_section_name(oh::COFFHandle, name::AbstractString) = string(".", name)
 function mangle_symbol_name(oh::COFFHandle, name::AbstractString)
-    # sob
-    if is64bit(oh)
-        return name
-    else
+    # sob; only 32-bit x86 prefixes C symbols with an underscore
+    if header(oh).Machine == IMAGE_FILE_MACHINE_I386
         return string("_", name)
+    else
+        return name
     end
 end
 format_string(::Type{H}) where {H <: COFFHandle} = "COFF"

src/COFF/COFFHeader.jl

@@ -16,6 +16,8 @@ function coff_header_is64bit(h::COFFHeader)
     wide_machines = [
         IMAGE_FILE_MACHINE_AMD64,
         IMAGE_FILE_MACHINE_ARM64,
+        IMAGE_FILE_MACHINE_ARM64EC,
+        IMAGE_FILE_MACHINE_ARM64X,
         IMAGE_FILE_MACHINE_IA64,
     ]
     return h.Machine in wide_machines

src/COFF/COFFSymbol.jl

@@ -5,61 +5,60 @@ export COFFSymbols, COFFSymtabEntry, COFFSymbolRef
 
 COFF symbol table, contains the list of symbols defined within the object file.
 
-Note that because COFF Symbols are variable-length, we store a table of offsets
-at which the (non-auxilliary) symbols can be found.
+Note that because COFF Symbols are followed by a variable number of auxiliary
+records within the symbol table, we store the (1-based) symbol table index at
+which each (non-auxiliary) symbol can be found.
 """
 struct COFFSymbols{H<:COFFHandle} <: Symbols{H}
     handle::H
-    symbol_offsets::Vector{UInt64}
+    symbol_indices::Vector{UInt32}
 end
 
 """
-    scan_symbol_offsets(oh::COFFHandle)
+    scan_symbol_indices(oh::COFFHandle)
 
-Find the offsets for each Symbol within a COFFHandle, skipping over auxilliary
-symbols as necessary.
+Find the symbol table index of each Symbol within a COFFHandle, skipping over
+auxiliary records as necessary.
 """
-function scan_symbol_offsets(oh::H) where {H <: COFFHandle}
-    # num_syms represents the number of total symbols, but we're only
+function scan_symbol_indices(oh::H) where {H <: COFFHandle}
+    # num_syms represents the number of total records, but we're only
     # interested in the non-auxiliary symbols
     num_syms = num_symbols(header(oh))
-    offsets = UInt64[]
+    indices = UInt32[]
 
-    curr_idx = 1
     idx = 1
-    while idx < num_syms
-        # Load in the next symbol and store its offset
+    while idx <= num_syms
+        # Load in the next symbol and store its index
         seek(oh, symtab_entry_offset(oh) + symtab_entry_size(oh)*(idx-1))
+        sym = unpack(oh, symtab_entry_type(oh))
+        push!(indices, idx)
 
-        sym = unpack(oh, COFFSymtabEntry{H})
-        push!(offsets, idx)
-
-        # Move our offset forward by 1 (because we just read in a SymtabEntry)
-        # and again as many times as needed to skip over the auxilliary symbols
+        # Move our index forward by 1 (because we just read in a SymtabEntry)
+        # and again as many times as needed to skip over the auxiliary records
         idx += 1 + sym.NumberOfAuxSymbols
-        curr_idx += 1
     end
 
-    return offsets
+    return indices
 end
 
 function Symbols(oh::H) where {H <: COFFHandle}
-    symbol_offsets = scan_symbol_offsets(oh)
-    return COFFSymbols(oh, symbol_offsets)
+    return COFFSymbols(oh, scan_symbol_indices(oh))
 end
 
 handle(syms::COFFSymbols) = syms.handle
-# function next(syms::COFFSymbols, idx)
-#     # We skip over auxiliary symbols here
-#     next_idx = idx + deref(syms[idx]).NumberOfAuxSymbols + 1
-#     return (syms[idx], next_idx)
-# end
-lastindex(syms::COFFSymbols) = num_symbols(header(handle(syms)))
-
-# We override `iteratorsize` so that list comprehensions and collect() calls on
-# our Symbols don't have a bunch of #undef entries at the end of the array
-# because it tries to pre-allocate an array of the proper size.
-# iteratorsize(::Type{H}) where {H <: COFFSymbols} = SizeUnknown()
+lastindex(syms::COFFSymbols) = length(syms.symbol_indices)
+
+# Index into the symbols, skipping over auxiliary records.  The resulting
+# `symbol_number()` is the symbol's index within the symbol table.
+function getindex(syms::COFFSymbols, idx)
+    if !(0 < idx <= length(syms))
+        throw(BoundsError(syms, idx))
+    end
+    oh = handle(syms)
+    table_idx = syms.symbol_indices[idx]
+    seek(oh, symtab_entry_offset(oh) + symtab_entry_size(oh)*(table_idx-1))
+    return SymbolRef(syms, unpack(oh, symtab_entry_type(oh)), table_idx)
+end
 
 @io struct COFFSymtabEntry{H <: COFFHandle} <: SymtabEntry{H}
     Name::fixed_string{UInt64}

src/COFF/constants.jl

@@ -6,6 +6,8 @@
     const IMAGE_FILE_MACHINE_ARM            = 0x1c0     #   ARM little endian
     const IMAGE_FILE_MACHINE_ARMNT          = 0x1c4     #   ARMv7 (or higher) Thumb mode only
     const IMAGE_FILE_MACHINE_ARM64          = 0xaa64    #  ARMv8 in 64-bit mode
+    const IMAGE_FILE_MACHINE_ARM64EC        = 0xa641    #  ARMv8 in 64-bit mode, with the x64 ABI
+    const IMAGE_FILE_MACHINE_ARM64X         = 0xa64e    #  ARM64 and ARM64EC code together
     const IMAGE_FILE_MACHINE_EBC            = 0xebc     #   EFI byte code
     const IMAGE_FILE_MACHINE_I386           = 0x14c     #   Intel 386 or later processors and compatible processors
     const IMAGE_FILE_MACHINE_IA64           = 0x200     #   Intel Itanium processor family
@@ -31,7 +33,7 @@ function coff_machine_to_arch(machine::UInt16)
         return "x86_64"
     elseif machine ∈ (IMAGE_FILE_MACHINE_ARM, IMAGE_FILE_MACHINE_ARMNT, IMAGE_FILE_MACHINE_THUMB)
         return "armv7l"
-    elseif machine ∈ (IMAGE_FILE_MACHINE_ARM64,)
+    elseif machine ∈ (IMAGE_FILE_MACHINE_ARM64, IMAGE_FILE_MACHINE_ARM64EC, IMAGE_FILE_MACHINE_ARM64X)
         return "aarch64"
     elseif machine ∈ (IMAGE_FILE_MACHINE_POWERPC,)
         return "ppc64le"

test/coff/build.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+# Regenerates the COFF object fixtures in this directory.  Requires `clang`.
+set -euo pipefail
+cd "$(dirname "$0")"
+tmp="$(mktemp -d)"
+trap 'rm -rf "${tmp}"' EXIT
+
+echo 'int foo(int x) { return x + 1; }' > "${tmp}/foo.c"
+
+# (`-mno-incremental-linker-compatible` keeps timestamps out of COFF objects)
+for arch in x86_64 i686 aarch64 thumbv7; do
+    clang --target="${arch}-pc-windows-msvc" -O2 -mno-incremental-linker-compatible \
+          -c -o "foo_${arch}.obj" "${tmp}/foo.c"
+done

test/runtests.jl

@@ -139,6 +139,15 @@ function test_libfoo_and_fooifier(fooifier_path, libfoo_path)
             @test !islocal(syms_exe[main_idx_exe])
             @test !islocal(syms_lib[foo_idx_lib])
 
+            # COFF symbols are followed by auxiliary records, which are skipped
+            if isa(oh_exe, COFFHandle)
+                for (oh, syms) in ((oh_exe, syms_exe), (oh_lib, syms_lib))
+                    naux = [ObjectFile.deref(sym).NumberOfAuxSymbols for sym in syms]
+                    @test length(syms) + sum(naux) == header(oh).NumberOfSymbols
+                    @test ObjectFile.symbol_number(syms[end]) + naux[end] == header(oh).NumberOfSymbols
+                end
+            end
+
             # Global detection doesn't seem to be working on OSX...
             if !isa(oh_exe, MachOHandle)
                 @test isglobal(syms_exe[foo_idx_exe])
@@ -357,7 +366,7 @@ end
             ("libfoo_darwin64.a", :bsd64, MachOHandle, member_names),
             ("libfoo_bsd_nosymtab.a", :bsd, MachOHandle, member_names),
             # `llvm-lib` orders members by name
-            ("foo.lib", :coff, nothing, reverse(member_names)),
+            ("foo.lib", :coff, COFFHandle, reverse(member_names)),
         )
         ah = readmeta(open(joinpath("./archives", file), "r"), ArchiveHandle)
         @test archive_kind(ah) == kind
@@ -383,6 +392,9 @@ end
                 syms = symbol_name.(Symbols(oh))
                 fn = startswith(member.name, "foo") ? "foo" : "bar"
                 @test mangle_symbol_name(oh, fn) in syms
+                # `bar()` calls `foo()`, which another member defines
+                foo_sym = only(sym for sym in Symbols(oh) if symbol_name(sym) == mangle_symbol_name(oh, "foo"))
+                @test isundef(foo_sym) == (fn == "bar")
                 # ... and reads the same as it would on its own
                 @test oh.header == only(readmeta(IOBuffer(data))).header
             end
@@ -472,3 +484,37 @@ end
         end
     end
 end
+
+# Fixtures are generated by `./coff/build.sh`
+@testset "COFF objects" begin
+    @testset "$(arch)" for (arch, platform) in (
+            ("x86_64", Platform("x86_64", "windows")),
+            ("i686", Platform("i686", "windows")),
+            ("aarch64", Platform("aarch64", "windows")),
+            ("thumbv7", Platform("armv7l", "windows")),
+        )
+        oh = only(readmeta(open("./coff/foo_$(arch).obj", "r")))
+        @test oh isa COFFHandle
+        @test platforms_match(Platform(oh), platform)
+        @test is64bit(oh) == (arch in ("x86_64", "aarch64"))
+        @test isrelocatable(oh)
+        @test !isexecutable(oh)
+        @test !islibrary(oh)
+        @test isempty(DynamicLinks(oh))
+        @test ".text" in section_name.(Sections(oh))
+        foo_sym = only(sym for sym in Symbols(oh) if symbol_name(sym) == mangle_symbol_name(oh, "foo"))
+        @test isglobal(foo_sym)
+        @test !isundef(foo_sym)
+    end
+
+    @testset "not an object" begin
+        not_obj(data) = readmeta(IOBuffer(data), COFFHandle)
+        @test_throws MagicMismatch not_obj(b"hello, world, this is not an object file")
+        # An x86_64 `Machine`, but with an optional header
+        @test_throws MagicMismatch not_obj(vcat(b"d\x86", zeros(UInt8, 14), [0x08, 0x00], zeros(UInt8, 2)))
+        # ... or with more sections than fit in the file
+        @test_throws MagicMismatch not_obj(vcat(b"d\x86", [0xff, 0x00], zeros(UInt8, 16)))
+        # An unknown `Machine`
+        @test_throws MagicMismatch not_obj(vcat(b"zz", zeros(UInt8, 18)))
+    end
+end