feat(build): link plugins into the server binary via a generated registry

.github/workflows/test.yml

@@ -56,6 +56,11 @@ jobs:
         cd backend
         go mod tidy
         git diff --exit-code -- go.mod go.sum
+    - name: Check plugins/all is up to date
+      run: |
+        cd backend
+        go run ./tools/pluginsgen
+        git diff --exit-code -- plugins/all
     - name: Install mockery 3.8.1
       run: backend/scripts/install-mockery.sh
     - name: Build Python

Makefile

@@ -71,10 +71,13 @@ swag:
 build-plugin:
 	make build-plugin -C backend
 
+plugins-registry:
+	make plugins-registry -C backend
+
 build-server:
 	make build-server -C backend
 
-build: build-plugin build-server
+build: build-server
 
 all: build
 

backend/Dockerfile

@@ -168,18 +168,22 @@ ARG DEBUG=
 ENV LD_LIBRARY_PATH=/app/libs
 RUN mkdir -p /app/libs
 COPY --from=build /usr/local/deps/target/lib/*.so* /app/libs
-COPY --from=build /go/bin /usr/bin
 RUN ldconfig -vn /app/libs
 
 # apps
-COPY --from=build /app/bin /app/bin
+# The single binary already contains every plugin; the .so files that
+# `make build-plugin` produces stay in the builder stage and are never copied.
+COPY --from=build /app/bin/lake /app/bin/lake
 COPY --from=build /app/resources /app/resources
 
 ENV PATH="/app/bin:${PATH}"
 ENV DEBUG="$DEBUG"
 
 USER root
-RUN chgrp -R 0 /app && \
+# Empty mount point for sideloaded plugins: dropping a .so here registers it on
+# top of the compiled-in set at startup; see PLUGIN_DIR.
+RUN mkdir -p /app/bin/plugins && \
+    chgrp -R 0 /app && \
     chmod -R g=u /app
 USER devlake
 

backend/Dockerfile.local

@@ -72,15 +72,12 @@ RUN --mount=type=cache,target=/root/.cache/go-build \
     --mount=type=cache,target=/go/pkg/mod \
     make swag
 
-# Build plugins (parallel, with persistent Go build cache)
+# Build the server. `build` is `build-server`: the framework, the API and every
+# plugin in the generated registry link into this one binary. No .so files are
+# produced for the release image.
 RUN --mount=type=cache,target=/root/.cache/go-build \
     --mount=type=cache,target=/go/pkg/mod \
-    make build-plugin
-
-# Build server
-RUN --mount=type=cache,target=/root/.cache/go-build \
-    --mount=type=cache,target=/go/pkg/mod \
-    VERSION=${TAG}@${SHA} make build-server
+    VERSION=${TAG}@${SHA} make build
 
 FROM debian:bookworm-slim
 
@@ -102,7 +99,10 @@ RUN ldconfig
 
 WORKDIR /app
 
-COPY --from=builder /app/bin ./bin
+COPY --from=builder /app/bin/lake ./bin/lake
+
+# Empty mount point for sideloaded plugins
+RUN mkdir -p bin/plugins
 
 # Create empty python plugins dir to avoid startup error
 RUN mkdir -p python/plugins

backend/Makefile

@@ -54,27 +54,42 @@ dep: go-dep python-dep
 swag: mock
 	scripts/swag.sh
 
+# Regenerate plugins/all, the package that links the Go plugins into the server
+# binary. Run it after adding, removing or renaming a plugin; CI checks that the
+# checked-in file matches the tree.
+#
+# GOOS/GOARCH are cleared because `backend/Dockerfile` exports them to
+# cross-compile the server: `go run` would otherwise build the generator for the
+# target platform and fail with "exec format error" on the build host. Same
+# reason as MOCKERY above.
+plugins-registry:
+	GOOS= GOARCH= CGO_ENABLED=0 go run ./tools/pluginsgen
+
 build-plugin:
 	scripts/build-plugins.sh
 
 build-plugin-debug:
 	DEVLAKE_DEBUG=1 scripts/build-plugins.sh
 
-build-server: swag
+build-server: plugins-registry swag
 	VERSION=$(VERSION) scripts/build-server.sh
 
 build-python: #don't mix this with the other build commands
 	scripts/build-python.sh
 
-build: build-plugin build-server
+# `build` produces the only artifact the release ships: one binary with the
+# framework, the API and every plugin in the generated registry compiled in.
+# Use build-plugin when you want shared objects instead, either to sideload
+# them into an existing binary or to ship a core-only server.
+build: build-server
 
 run:
 	go run server/main.go
 
-dev: build-plugin build-python run
+dev: build-python run
 
 godev:
-	DISABLED_REMOTE_PLUGINS=true make build-plugin run
+	DISABLED_REMOTE_PLUGINS=true make run
 
 debug: build-plugin-debug
 	dlv debug server/main.go

backend/core/plugin/hub.go

@@ -43,6 +43,27 @@ func RegisterPlugin(name string, plugin PluginMeta) errors.Error {
 	return nil
 }
 
+// MustRegister registers p under the name it reports, panicking if the plugin
+// cannot be identified.
+//
+// The generated plugins/all package calls it for every plugin that is compiled
+// into the binary, which makes the plugin set a build-time decision. Registering
+// a name twice overwrites the previous entry, so a plugin that is both compiled
+// in and loaded from a shared object keeps working: the loader registers the
+// same value again.
+func MustRegister(p PluginMeta) {
+	if p == nil {
+		panic("plugin: MustRegister called with a nil plugin")
+	}
+	name := p.Name()
+	if name == "" {
+		panic("plugin: MustRegister called with an empty plugin name")
+	}
+	if err := RegisterPlugin(name, p); err != nil {
+		panic(err)
+	}
+}
+
 func GetPlugin(name string) (PluginMeta, errors.Error) {
 	if plugins == nil {
 		return nil, errors.Default.New("RegisterPlugin have never been called.")

backend/core/runner/loader.go

@@ -20,6 +20,7 @@ package runner
 import (
 	"fmt"
 	"io/fs"
+	"os"
 	"path/filepath"
 	goplugin "plugin"
 	"strings"
@@ -50,44 +51,71 @@ func LoadPlugins(basicRes context.BasicRes) errors.Error {
 
 func LoadGoPlugins(basicRes context.BasicRes) errors.Error {
 	pluginsDir := basicRes.GetConfig("PLUGIN_DIR")
+	if pluginsDir == "" {
+		return nil
+	}
+	// The official binary already has every official plugin compiled in, so a
+	// missing directory simply means "nothing to sideload".
+	if _, statErr := os.Stat(pluginsDir); os.IsNotExist(statErr) {
+		basicRes.GetLogger().Info("plugin directory %s does not exist, no plugins to sideload", pluginsDir)
+		return nil
+	}
+
 	var wg sync.WaitGroup
 	walkErr := filepath.WalkDir(pluginsDir, func(path string, d fs.DirEntry, err error) error {
 		if err != nil {
 			return err
 		}
 		fileName := d.Name()
-		if strings.HasSuffix(fileName, ".so") && fileName != ".so" {
-			pluginName := fileName[0 : len(d.Name())-3]
-			plug, loadErr := goplugin.Open(path)
-			if loadErr != nil {
-				return loadErr
-			}
-			symPluginEntry, pluginEntryError := plug.Lookup("PluginEntry")
-			if pluginEntryError != nil {
-				return pluginEntryError
-			}
-			pluginMeta, ok := symPluginEntry.(plugin.PluginMeta)
-			if !ok {
-				return errors.Default.New(fmt.Sprintf("%s PluginEntry must implement PluginMeta interface", pluginName))
-			}
-			wg.Add(1)
-			go func(pluginName string, pluginMeta plugin.PluginMeta) {
-				defer func() {
-					wg.Done()
-				}()
-				err = plugin.RegisterPlugin(pluginName, pluginMeta)
-				if err != nil {
-					panic(err)
-				}
-				basicRes.GetLogger().Info(`plugin loaded %s`, pluginName)
-			}(pluginName, pluginMeta)
+		if !strings.HasSuffix(fileName, ".so") || fileName == ".so" {
+			return nil
+		}
+
+		pluginName := strings.TrimSuffix(fileName, ".so")
+		pluginMeta, loadErr := openGoPlugin(path)
+		if loadErr != nil {
+			// Sideloaded plugins are third-party code built outside this repo. One
+			// incompatible or broken file must not keep the server from starting,
+			// so it is skipped loudly instead. The usual cause is a toolchain or
+			// dependency mismatch with the binary that is loading it.
+			basicRes.GetLogger().Error(loadErr, "skipping plugin %s from %s", pluginName, path)
+			return nil
+		}
+		if _, err := plugin.GetPlugin(pluginName); err == nil {
+			basicRes.GetLogger().Warn(nil, "plugin %s is already registered, the shared object %s takes precedence", pluginName, path)
 		}
+
+		wg.Add(1)
+		go func(name string, meta plugin.PluginMeta) {
+			defer wg.Done()
+			if err := plugin.RegisterPlugin(name, meta); err != nil {
+				panic(err)
+			}
+			basicRes.GetLogger().Info("plugin loaded %s", name)
+		}(pluginName, pluginMeta)
 		return nil
 	})
 	wg.Wait()
 	return errors.Convert(walkErr)
 }
 
+// openGoPlugin opens a shared object and reads its exported entry point.
+func openGoPlugin(path string) (plugin.PluginMeta, error) {
+	plug, err := goplugin.Open(path)
+	if err != nil {
+		return nil, err
+	}
+	symPluginEntry, err := plug.Lookup("PluginEntry")
+	if err != nil {
+		return nil, err
+	}
+	pluginMeta, ok := symPluginEntry.(plugin.PluginMeta)
+	if !ok {
+		return nil, fmt.Errorf("%s: PluginEntry must implement plugin.PluginMeta", filepath.Base(path))
+	}
+	return pluginMeta, nil
+}
+
 func LoadRemotePlugins(basicRes context.BasicRes) errors.Error {
 	remotePluginDir := basicRes.GetConfig("REMOTE_PLUGIN_DIR")
 	if remotePluginDir != "" {