0.11.1: dist-apk packs native libraries as the Android Gradle plugin does; dist-appimage takes an SVG icon (#27)

A HuxerUI application packed by dist-apk and by Gradle differed in its native
libraries: dist-apk shipped them with their symbol tables and debug information
(libhuxerui.so 8.1 MB against 5.5 MB, the NDK's libc++_shared.so 9.1 MB against
1.3 MB), and always compressed, so a manifest stating
android:extractNativeLibs="false" produced a package the platform refuses.

Each native library is now written by an `llvm-strip --strip-unneeded` action
with the build's own tool, beside the compiler mcpp::toolchain_dir() reports;
options::keep_debug_symbols packs them as staged. A manifest stating
extractNativeLibs="false" gets lib/ stored uncompressed and aligned to 16 KB
pages; one stating nothing is packed compressed as before.

dist-appimage's options::icon may be an SVG: the staged icon keeps the supplied
extension, a stale file of the other format is removed, any other format is
refused through mcpp::warning, and the payload floor ignores the SVG.

Tests: check-apk-features.sh (k) stripped, compressed by default, stored and
16 KB-aligned when loaded in place; (l) keep_debug_symbols. appimage-consumer
packs an SVG icon and refuses a .jpg.

Author: Sunrisepeak

.github/workflows/ci.yml

@@ -1164,6 +1164,29 @@ jobs:
           fi
           echo "ok: one AppImage, and it printed the program's output"
 
+          # AN SVG ICON KEEPS ITS FORMAT (0.11.1): the desktop entry names the
+          # icon without an extension and appimagetool finds it by the file's,
+          # so the image carries `<name>.svg` with the supplied bytes and no
+          # PNG of that name. A format no desktop entry reads is refused.
+          APPIMAGE_CONSUMER_ICON="$PWD/assets/icon.svg" "$MCPP" pack --format appimage | tee pack-svg.log
+          img=$(find target -name '*.AppImage' | head -1)
+          rm -rf squashfs-root
+          APPIMAGE_EXTRACT_AND_RUN=1 "$img" --appimage-extract > /dev/null
+          cmp assets/icon.svg squashfs-root/AppimageConsumer.svg \
+            || { echo "FAIL: the image does not carry the SVG icon"; ls -la squashfs-root; exit 1; }
+          test ! -e squashfs-root/AppimageConsumer.png \
+            || { echo "FAIL: the placeholder PNG is still beside the SVG"; exit 1; }
+          grep -qx 'Icon=AppimageConsumer' squashfs-root/AppimageConsumer.desktop \
+            || { echo "FAIL: the desktop entry does not name the icon"; exit 1; }
+          cp assets/icon.svg assets/icon.jpg
+          set +e
+          APPIMAGE_CONSUMER_ICON="$PWD/assets/icon.jpg" "$MCPP" pack --format appimage > pack-jpg.log 2>&1
+          set -e
+          grep -q 'neither a .png nor an .svg' pack-jpg.log \
+            || { cat pack-jpg.log; echo "FAIL: a .jpg icon was not refused by name"; exit 1; }
+          rm -f assets/icon.jpg
+          echo "ok: an SVG icon is packed as an SVG, and a .jpg is refused"
+
       # THE iOS ROW OF `dist-apple`, AT THE PLAN LEVEL (#622 B1). No runner
       # in this workflow has an Apple SDK, so `--target aarch64-ios-sim`
       # refuses before `build.mcpp` even runs -- see

README.md

@@ -215,6 +215,14 @@ struct options {
     // things.
     bool sign = true;
 
+    // `true` packs the native libraries as the engine staged them (0.11.1).
+    // By default each is stripped with the build's own `llvm-strip
+    // --strip-unneeded`, which keeps the dynamic symbols the loader reads and
+    // drops the symbol table and the debug information -- what the Android
+    // Gradle plugin does to every library it packages, and what `mcpp pack`
+    // reports it did.
+    bool keep_debug_symbols = false;
+
     // LEVEL 1, KOTLIN (0.11.0). One or more directories of `.kt` sources,
     // compiled by `kotlinc` with every Java root as its reference sources,
     // before `javac` compiles the Java against the Kotlin classes; the Kotlin
@@ -1336,6 +1344,19 @@ inline std::vector<std::string> shared_objects_in(const fs::path& dir) {
     return out;
 }
 
+// Does the manifest ask for the native libraries to be loaded from the APK in
+// place (`<application android:extractNativeLibs="false">`, 0.11.1)? The
+// platform can map a library only from an entry stored uncompressed on a page
+// boundary, so such a package is laid out that way; a compressed library in it
+// fails to install.
+inline bool loads_native_libraries_in_place(const std::string& manifest) {
+    xml::node root;
+    std::string error;
+    if (!xml::parse(manifest, root, error) || root.name != "manifest") return false;
+    const xml::node* application = find_child(root, "application");
+    return application && xml::attr_of(*application, "android:extractNativeLibs") == "false";
+}
+
 // ─── Plan ──────────────────────────────────────────────────────────────────
 
 inline plan plan_for(options opt = {}) {
@@ -1975,6 +1996,22 @@ inline plan plan_for(options opt = {}) {
         return refuse(p, "cannot write AndroidManifest.xml", std::format(
             "mcpp.dist.apk: cannot write {}.", manifestPath));
     }
+    const bool inPlaceLibraries = loads_native_libraries_in_place(manifestBytes);
+
+    // THE BUILD'S OWN llvm-strip (0.11.1): the one beside the compiler mcpp
+    // resolved for this row, the NDK's.
+    std::string llvmStrip;
+    if (!opt.keep_debug_symbols) {
+        const std::string toolchain = mcpp::toolchain_dir();
+        const fs::path candidate = fs::path(toolchain) / "bin" / "llvm-strip";
+        if (!toolchain.empty() && is_file(candidate.string())) {
+            llvmStrip = candidate.string();
+        } else {
+            mcpp::warning(std::format(
+                "mcpp.dist.apk: no llvm-strip beside the toolchain ({}); the native libraries are packed "
+                "with their debug information.", toolchain.empty() ? "none reported" : toolchain).c_str());
+        }
+    }
 
     // ── the temporary staging tree: lib/<abi>/, assets/ ─────────────────
     //
@@ -1986,9 +2023,43 @@ inline plan plan_for(options opt = {}) {
     const fs::path work = outDir / "stage";
     { std::error_code ec; fs::remove_all(work, ec); }
     std::vector<std::string> libInputs;
+    // A library reaches `lib/<abi>/` stripped, as an action of its own, or
+    // copied as it is when the debug information is kept. A file that is not
+    // an ELF object -- an archive can carry anything under `jni/` -- is copied
+    // as it is with a warning, as the Android Gradle plugin packs a library it
+    // cannot strip.
+    const auto is_elf = [](const std::string& path) {
+        std::ifstream in(path, std::ios::binary);
+        char magic[4] = {};
+        return in.read(magic, 4) && magic[0] == 0x7f && magic[1] == 'E' && magic[2] == 'L' && magic[3] == 'F';
+    };
+    const auto place_library = [&](const std::string& so, const std::string& abi) {
+        const fs::path dst = work / "lib" / abi / fs::path(so).filename();
+        if (llvmStrip.empty()) {
+            collect_tree(so, dst, libInputs);
+            return;
+        }
+        if (!is_elf(so)) {
+            mcpp::warning(std::format(
+                "mcpp.dist.apk: {} is not an ELF object, so it is packed without being stripped.", so).c_str());
+            collect_tree(so, dst, libInputs);
+            return;
+        }
+        std::error_code ec;
+        fs::create_directories(dst.parent_path(), ec);
+        step strip;
+        strip.id          = std::format("{}:strip:{}:{}", bundle ? "aab" : "apk", abi, fs::path(so).filename().string());
+        strip.role        = "artifact";
+        strip.description = "LLVM-STRIP " + abi + "/" + fs::path(so).filename().string();
+        strip.output      = dst.string();
+        strip.argv        = { llvmStrip, "--strip-unneeded", "-o", strip.output, so };
+        strip.inputs      = { so };
+        p.steps.push_back(strip);
+        libInputs.push_back(strip.output);
+    };
     for (auto const& leg : legs)
         for (auto const& so : leg.libraries)
-            collect_tree(so, work / "lib" / leg.abi / fs::path(so).filename(), libInputs);
+            place_library(so, leg.abi);
 
     // An AAR's native libraries, for every ABI this package carries.
     for (auto const& c : contributions) {
@@ -2002,7 +2073,7 @@ inline plan plan_for(options opt = {}) {
                 continue;
             }
             for (auto const& so : shared_objects_in(abiDir))
-                collect_tree(so, work / "lib" / leg.abi / fs::path(so).filename(), libInputs);
+                place_library(so, leg.abi);
         }
     }
 
@@ -2049,17 +2120,21 @@ inline plan plan_for(options opt = {}) {
     // dex with `jar`: aapt2 has no flag for native libraries.
     // `--dex <dir>` adds every `classes*.dex` d8 wrote there: a package whose
     // classes pass the 64K-method limit of one dex gets `classes2.dex` and on,
-    // and which of them exist is known only when d8 has run.
+    // and which of them exist is known only when d8 has run. `--stored <dir>
+    // <entry>` adds an entry uncompressed, which native libraries loaded in
+    // place must be.
     const std::string copyThenJar = helper("copy-then-jar.sh",
         "#!/bin/sh\n"
         "# mcpp.dist.apk helper. Do not edit.\n"
-        "# copy-then-jar.sh <src> <dst> <jar> [--dex <dir>] <jar update arguments>...\n"
+        "# copy-then-jar.sh <src> <dst> <jar> [--dex <dir>] [--stored <dir> <entry>] <jar update arguments>...\n"
         "set -e\n"
         "src=\"$1\"; dst=\"$2\"; jar=\"$3\"; shift 3\n"
-        "dex=\"\"\n"
+        "dex=\"\"; stored_dir=\"\"; stored=\"\"\n"
         "if [ \"${1:-}\" = --dex ]; then dex=\"$2\"; shift 2; fi\n"
+        "if [ \"${1:-}\" = --stored ]; then stored_dir=\"$2\"; stored=\"$3\"; shift 3; fi\n"
         "cp \"$src\" \"$dst\"\n"
-        "\"$jar\" uf \"$dst\" \"$@\"\n"
+        "if [ -n \"$stored\" ]; then \"$jar\" --update --no-compress --file \"$dst\" -C \"$stored_dir\" \"$stored\"; fi\n"
+        "if [ \"$#\" -gt 0 ]; then \"$jar\" uf \"$dst\" \"$@\"; fi\n"
         "if [ -n \"$dex\" ]; then (cd \"$dex\" && \"$jar\" uf \"$dst\" classes*.dex); fi\n");
     const std::string runAndStamp = helper("run-and-stamp.sh",
         "#!/bin/sh\n"
@@ -2408,15 +2483,19 @@ inline plan plan_for(options opt = {}) {
     libs.role = "artifact";
     libs.description = "APK LIBS+ASSETS";
     libs.output = (outDir / "withlibs.apk").string();
-    libs.argv = { copyThenJar, link.output, libs.output, jar,
-                 "-C", work.string(), "lib",
-                 "-C", work.string(), "assets" };
+    libs.argv = { copyThenJar, link.output, libs.output, jar };
     if (!javaOutputs.empty()) {
         // After the three fixed operands and before the `jar` arguments: see
         // `copy-then-jar.sh`.
-        libs.argv.insert(libs.argv.begin() + 4, (outDir / "dex").string());
-        libs.argv.insert(libs.argv.begin() + 4, "--dex");
+        libs.argv.push_back("--dex");
+        libs.argv.push_back((outDir / "dex").string());
+    }
+    if (inPlaceLibraries) {
+        libs.argv.insert(libs.argv.end(), { "--stored", work.string(), "lib" });
+    } else {
+        libs.argv.insert(libs.argv.end(), { "-C", work.string(), "lib" });
     }
+    libs.argv.insert(libs.argv.end(), { "-C", work.string(), "assets" });
     libs.inputs = { link.output };
     for (auto const& f : libInputs)   libs.inputs.push_back(f);
     for (auto const& f : assetInputs) libs.inputs.push_back(f);
@@ -2435,7 +2514,12 @@ inline plan plan_for(options opt = {}) {
     // output already exists ("Output file '...' exists"), which every
     // rebuild after the first hits, because ninja does not delete a stale
     // output before an edge reruns it.
-    align.argv = { zipalign, "-f", "-p", "4", libs.output, align.output };
+    // A library loaded in place is aligned to a 16 KB page, which a 4 KB
+    // device reads as well; `apksigner sign` aligns a stored library to the
+    // same 16 KB by default, so a signed package keeps it.
+    align.argv = inPlaceLibraries
+        ? std::vector<std::string>{ zipalign, "-f", "-P", "16", "4", libs.output, align.output }
+        : std::vector<std::string>{ zipalign, "-f", "-p", "4", libs.output, align.output };
     align.inputs = { libs.output };
     p.steps.push_back(align);
     if (!opt.sign) {

dist/apk.cppm

@@ -80,10 +80,14 @@ struct options {
     // here is visible only to a desktop launcher, never to a build.
     bool terminal = true;
 
-    // A PNG a project supplies. Empty uses the built-in placeholder, which
-    // exists so that an AppImage can be produced with nothing declared:
-    // appimagetool requires an icon and refuses without one, and a member whose
-    // first use needs a graphic asset is a member nobody tries.
+    // A PNG or, from 0.11.1, an SVG a project supplies. The desktop entry
+    // names the icon without an extension and a reader -- appimagetool, a
+    // desktop's icon lookup -- finds it by the file's, so the file keeps the
+    // one it was supplied with; any other is refused. Empty uses the built-in
+    // placeholder, which exists so that an AppImage can be produced with
+    // nothing declared: appimagetool requires an icon and refuses without one,
+    // and a member whose first use needs a graphic asset is a member nobody
+    // tries.
     std::string icon;
 
     // An explicit `appimagetool` wins over discovery. Set it to pin a build
@@ -365,12 +369,33 @@ inline plan plan_for(options opt = {}) {
     const auto launcher_rel = mcpp::plugins::names::relative_to(launcher, stage);
 
     // ── The three files AppImage requires, written into the staged tree ────
+    std::string iconExtension = ".png";
+    if (!opt.icon.empty()) {
+        iconExtension = std::filesystem::path(opt.icon).extension().string();
+        for (char& c : iconExtension) c = static_cast<char>(std::tolower(static_cast<unsigned char>(c)));
+        if (iconExtension != ".png" && iconExtension != ".svg") {
+            // Also a warning: the engine discards a build program's stderr when it exits 0.
+            const std::string message = std::format(
+                "mcpp.dist.appimage: the icon {} is neither a .png nor an .svg, the two "
+                "formats a desktop entry's icon is read in.", opt.icon);
+            std::cerr << message << '\n';
+            mcpp::warning(message.c_str());
+            p.reason = "icon format";
+            return p;
+        }
+    }
     const std::string name = app_name_for(opt);
     const auto stagePath   = std::filesystem::path(stage);
     const auto desktop     = stagePath / (name + ".desktop");
-    const auto icon        = stagePath / (name + ".png");
+    const auto icon        = stagePath / (name + iconExtension);
     const auto dirIcon     = stagePath / ".DirIcon";
     const auto runFile     = stagePath / "AppRun";
+    // The other format's file from an earlier pack of this tree: appimagetool
+    // takes a PNG over an SVG of the same name.
+    {
+        std::error_code ec;
+        std::filesystem::remove(stagePath / (name + (iconExtension == ".png" ? ".svg" : ".png")), ec);
+    }
 
     std::string iconBytes;
     if (!opt.icon.empty()) {
@@ -481,7 +506,7 @@ inline bool submit(const plan& p) {
         const auto name = e.path().filename().string();
         if (e.path().parent_path() == std::filesystem::path(p.appdir)
             && (name == "AppRun" || name == ".DirIcon"
-                || name.ends_with(".desktop") || name.ends_with(".png")))
+                || name.ends_with(".desktop") || name.ends_with(".png") || name.ends_with(".svg")))
             continue;
         ++carried;
     }

dist/appimage.cppm

@@ -1,7 +1,7 @@
 [package]
 name        = "plugins"
 namespace   = "mcpp"
-version     = "0.11.0"
+version     = "0.11.1"
 description = "Official mcpp build plugins: rule packages under mcpp.rules.*, build-time utilities under mcpp.tools.*, each member selected by a feature"
 license     = "Apache-2.0"
 authors     = ["mcpp-community"]

mcpp.toml

@@ -49,7 +49,7 @@ export namespace mcpp::plugins {
 //
 // One package, one version: the number lives in mcpp.toml, and the CI step
 // `the collection states its own version` compares the two.
-inline constexpr std::string_view version = "0.11.0";
+inline constexpr std::string_view version = "0.11.1";
 
 } // namespace mcpp::plugins
 

src/plugins.cppm

@@ -72,6 +72,10 @@ int main() {
         opt.sign     = false;
         opt.keystore = "xim:android-debug-keystore";
     }
+    // (l) the native libraries as the engine staged them, symbol table and all.
+    if (const char* keep = std::getenv("APK_CONSUMER_KEEP_DEBUG_SYMBOLS"); keep && *keep) {
+        opt.keep_debug_symbols = true;
+    }
 
     return mcpp::dist::apk::generate(opt) ? 0 : 1;
 }

tests/apk-consumer/build.mcpp

@@ -258,3 +258,53 @@ export APK_CONSUMER_SIGN_CONFLICT=1
 grep -q 'opposite things' pack-j.log || fail "the refusal does not say the two options contradict" pack-j.log
 unset APK_CONSUMER_SIGN_CONFLICT
 echo "ok: sign = false with a keystore is refused"
+
+# ── (k),(l) 0.11.1: native libraries as the Android Gradle plugin packs them ─
+#
+# (k) A packed library is stripped with the build's own llvm-strip
+# (`--strip-unneeded`: no symbol table and no debug information, the dynamic
+# symbols kept), and a package whose manifest states nothing stores it
+# compressed, as before. A manifest stating `android:extractNativeLibs="false"`
+# gets it stored uncompressed and aligned to a 16 KB page, which the platform
+# needs to load it from the APK in place. (l) `keep_debug_symbols` packs the
+# library as the engine staged it.
+LIB=lib/x86_64/libapk-consumer.so
+method_of() { unzip -v "$1" | awk -v name="$LIB" '$NF == name { print $2 }'; }
+
+echo "== (k) a stripped library, stored and 16 KB-aligned when loaded in place =="
+rm -rf target k
+unset APK_CONSUMER_TEMPLATE APK_CONSUMER_KEEP_DEBUG_SYMBOLS || true
+"$MCPP" pack --format apk --target "$TARGET" > pack-k1.log 2>&1 || fail "pack failed" pack-k1.log
+APK=$(find target -name 'apk-consumer.apk' | head -1)
+[ -n "$APK" ] || fail "no apk-consumer.apk" pack-k1.log
+mkdir -p k && unzip -q -o "$APK" "$LIB" -d k
+readelf -S "k/$LIB" > sections-k1.log
+if grep -qE '\.symtab|\.debug_' sections-k1.log; then fail "the packed library keeps its symbol table or debug information" sections-k1.log; fi
+readelf --dyn-syms -W "k/$LIB" > dynsym-k1.log
+grep -q 'ANativeActivity_onCreate' dynsym-k1.log || fail "stripping dropped the entry point the platform calls" dynsym-k1.log
+[ "$(method_of "$APK")" != Stored ] || fail "a manifest stating nothing got its library stored uncompressed" pack-k1.log
+
+rm -rf target
+export APK_CONSUMER_TEMPLATE=manifest-template-in-place.xml
+"$MCPP" pack --format apk --target "$TARGET" > pack-k2.log 2>&1 || fail "pack failed" pack-k2.log
+APK=$(find target -name 'apk-consumer.apk' | head -1)
+[ -n "$APK" ] || fail "no apk-consumer.apk" pack-k2.log
+[ "$(method_of "$APK")" = Stored ] || fail "a library loaded in place is stored compressed" pack-k2.log
+"$BT/zipalign" -c -P 16 4 "$APK" > align-k2.log 2>&1 || fail "the library is not aligned to a 16 KB page" align-k2.log
+"$AAPT2" dump xmltree "$APK" --file AndroidManifest.xml > xmltree-k2.log 2>&1
+grep -q 'extractNativeLibs.*=false' xmltree-k2.log || fail "the manifest does not state extractNativeLibs=false" xmltree-k2.log
+unset APK_CONSUMER_TEMPLATE
+echo "ok: the library is stripped, compressed by default, and stored on a 16 KB page when loaded in place"
+
+echo "== (l) keep_debug_symbols =="
+rm -rf target k
+export APK_CONSUMER_KEEP_DEBUG_SYMBOLS=1
+"$MCPP" pack --format apk --target "$TARGET" > pack-l.log 2>&1 || fail "pack failed" pack-l.log
+APK=$(find target -name 'apk-consumer.apk' | head -1)
+[ -n "$APK" ] || fail "no apk-consumer.apk" pack-l.log
+mkdir -p k && unzip -q -o "$APK" "$LIB" -d k
+readelf -S "k/$LIB" > sections-l.log
+grep -q '\.symtab' sections-l.log || fail "keep_debug_symbols packed a stripped library" sections-l.log
+unset APK_CONSUMER_KEEP_DEBUG_SYMBOLS
+rm -rf k
+echo "ok: keep_debug_symbols packs the library with its symbol table"

tests/apk-consumer/check-apk-features.sh

@@ -0,0 +1,19 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!-- Fixture: a project manifest that asks for the native libraries to be
+     loaded from the APK in place, as the Android Gradle plugin's manifest
+     does from minSdk 23; criterion (k) checks the libraries are then stored
+     uncompressed on a 16 KB page. -->
+<manifest xmlns:android="http://schemas.android.com/apk/res/android"
+    android:versionName="{{version_name}}" android:versionCode="{{version_code}}"
+    package="{{application_id}}">
+    <uses-sdk android:minSdkVersion="{{min_sdk}}" android:targetSdkVersion="{{target_sdk}}"/>
+    <application android:label="{{label}}" android:hasCode="false" android:extractNativeLibs="false">
+        <activity android:name="{{activity}}" android:exported="true">
+            <meta-data android:name="android.app.lib_name" android:value="{{lib_name}}"/>
+            <intent-filter>
+                <action android:name="android.intent.action.MAIN"/>
+                <category android:name="android.intent.category.LAUNCHER"/>
+            </intent-filter>
+        </activity>
+    </application>
+</manifest>