diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 21dfc96530d..86e2ea95e32 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -56,6 +56,11 @@ jobs: cd backend go mod tidy git diff --exit-code -- go.mod go.sum + - name: Check plugins/all is up to date + run: | + cd backend + go run ./tools/pluginsgen + git diff --exit-code -- plugins/all - name: Install mockery 3.8.1 run: backend/scripts/install-mockery.sh - name: Build Python diff --git a/Makefile b/Makefile index 8639ce2dc6f..cd74395f9bd 100644 --- a/Makefile +++ b/Makefile @@ -71,10 +71,13 @@ swag: build-plugin: make build-plugin -C backend +plugins-registry: + make plugins-registry -C backend + build-server: make build-server -C backend -build: build-plugin build-server +build: build-server all: build diff --git a/backend/Dockerfile b/backend/Dockerfile index b85a5ec8d1a..2c31a3d53fc 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -168,18 +168,22 @@ ARG DEBUG= ENV LD_LIBRARY_PATH=/app/libs RUN mkdir -p /app/libs COPY --from=build /usr/local/deps/target/lib/*.so* /app/libs -COPY --from=build /go/bin /usr/bin RUN ldconfig -vn /app/libs # apps -COPY --from=build /app/bin /app/bin +# The single binary already contains every plugin; the .so files that +# `make build-plugin` produces stay in the builder stage and are never copied. +COPY --from=build /app/bin/lake /app/bin/lake COPY --from=build /app/resources /app/resources ENV PATH="/app/bin:${PATH}" ENV DEBUG="$DEBUG" USER root -RUN chgrp -R 0 /app && \ +# Empty mount point for sideloaded plugins: dropping a .so here registers it on +# top of the compiled-in set at startup; see PLUGIN_DIR. +RUN mkdir -p /app/bin/plugins && \ + chgrp -R 0 /app && \ chmod -R g=u /app USER devlake diff --git a/backend/Dockerfile.local b/backend/Dockerfile.local index 8651900d5ab..e2046db3bd2 100644 --- a/backend/Dockerfile.local +++ b/backend/Dockerfile.local @@ -72,15 +72,12 @@ RUN --mount=type=cache,target=/root/.cache/go-build \ --mount=type=cache,target=/go/pkg/mod \ make swag -# Build plugins (parallel, with persistent Go build cache) +# Build the server. `build` is `build-server`: the framework, the API and every +# plugin in the generated registry link into this one binary. No .so files are +# produced for the release image. RUN --mount=type=cache,target=/root/.cache/go-build \ --mount=type=cache,target=/go/pkg/mod \ - make build-plugin - -# Build server -RUN --mount=type=cache,target=/root/.cache/go-build \ - --mount=type=cache,target=/go/pkg/mod \ - VERSION=${TAG}@${SHA} make build-server + VERSION=${TAG}@${SHA} make build FROM debian:bookworm-slim @@ -102,7 +99,10 @@ RUN ldconfig WORKDIR /app -COPY --from=builder /app/bin ./bin +COPY --from=builder /app/bin/lake ./bin/lake + +# Empty mount point for sideloaded plugins +RUN mkdir -p bin/plugins # Create empty python plugins dir to avoid startup error RUN mkdir -p python/plugins diff --git a/backend/Makefile b/backend/Makefile index 3e40d4c008c..e8bb9128ec8 100644 --- a/backend/Makefile +++ b/backend/Makefile @@ -54,27 +54,42 @@ dep: go-dep python-dep swag: mock scripts/swag.sh +# Regenerate plugins/all, the package that links the Go plugins into the server +# binary. Run it after adding, removing or renaming a plugin; CI checks that the +# checked-in file matches the tree. +# +# GOOS/GOARCH are cleared because `backend/Dockerfile` exports them to +# cross-compile the server: `go run` would otherwise build the generator for the +# target platform and fail with "exec format error" on the build host. Same +# reason as MOCKERY above. +plugins-registry: + GOOS= GOARCH= CGO_ENABLED=0 go run ./tools/pluginsgen + build-plugin: scripts/build-plugins.sh build-plugin-debug: DEVLAKE_DEBUG=1 scripts/build-plugins.sh -build-server: swag +build-server: plugins-registry swag VERSION=$(VERSION) scripts/build-server.sh build-python: #don't mix this with the other build commands scripts/build-python.sh -build: build-plugin build-server +# `build` produces the only artifact the release ships: one binary with the +# framework, the API and every plugin in the generated registry compiled in. +# Use build-plugin when you want shared objects instead, either to sideload +# them into an existing binary or to ship a core-only server. +build: build-server run: go run server/main.go -dev: build-plugin build-python run +dev: build-python run godev: - DISABLED_REMOTE_PLUGINS=true make build-plugin run + DISABLED_REMOTE_PLUGINS=true make run debug: build-plugin-debug dlv debug server/main.go diff --git a/backend/core/plugin/hub.go b/backend/core/plugin/hub.go index d043600c177..12d89ab1706 100644 --- a/backend/core/plugin/hub.go +++ b/backend/core/plugin/hub.go @@ -43,6 +43,27 @@ func RegisterPlugin(name string, plugin PluginMeta) errors.Error { return nil } +// MustRegister registers p under the name it reports, panicking if the plugin +// cannot be identified. +// +// The generated plugins/all package calls it for every plugin that is compiled +// into the binary, which makes the plugin set a build-time decision. Registering +// a name twice overwrites the previous entry, so a plugin that is both compiled +// in and loaded from a shared object keeps working: the loader registers the +// same value again. +func MustRegister(p PluginMeta) { + if p == nil { + panic("plugin: MustRegister called with a nil plugin") + } + name := p.Name() + if name == "" { + panic("plugin: MustRegister called with an empty plugin name") + } + if err := RegisterPlugin(name, p); err != nil { + panic(err) + } +} + func GetPlugin(name string) (PluginMeta, errors.Error) { if plugins == nil { return nil, errors.Default.New("RegisterPlugin have never been called.") diff --git a/backend/core/runner/loader.go b/backend/core/runner/loader.go index f05a78632dc..a19ba329fe0 100644 --- a/backend/core/runner/loader.go +++ b/backend/core/runner/loader.go @@ -20,6 +20,7 @@ package runner import ( "fmt" "io/fs" + "os" "path/filepath" goplugin "plugin" "strings" @@ -50,44 +51,71 @@ func LoadPlugins(basicRes context.BasicRes) errors.Error { func LoadGoPlugins(basicRes context.BasicRes) errors.Error { pluginsDir := basicRes.GetConfig("PLUGIN_DIR") + if pluginsDir == "" { + return nil + } + // The official binary already has every official plugin compiled in, so a + // missing directory simply means "nothing to sideload". + if _, statErr := os.Stat(pluginsDir); os.IsNotExist(statErr) { + basicRes.GetLogger().Info("plugin directory %s does not exist, no plugins to sideload", pluginsDir) + return nil + } + var wg sync.WaitGroup walkErr := filepath.WalkDir(pluginsDir, func(path string, d fs.DirEntry, err error) error { if err != nil { return err } fileName := d.Name() - if strings.HasSuffix(fileName, ".so") && fileName != ".so" { - pluginName := fileName[0 : len(d.Name())-3] - plug, loadErr := goplugin.Open(path) - if loadErr != nil { - return loadErr - } - symPluginEntry, pluginEntryError := plug.Lookup("PluginEntry") - if pluginEntryError != nil { - return pluginEntryError - } - pluginMeta, ok := symPluginEntry.(plugin.PluginMeta) - if !ok { - return errors.Default.New(fmt.Sprintf("%s PluginEntry must implement PluginMeta interface", pluginName)) - } - wg.Add(1) - go func(pluginName string, pluginMeta plugin.PluginMeta) { - defer func() { - wg.Done() - }() - err = plugin.RegisterPlugin(pluginName, pluginMeta) - if err != nil { - panic(err) - } - basicRes.GetLogger().Info(`plugin loaded %s`, pluginName) - }(pluginName, pluginMeta) + if !strings.HasSuffix(fileName, ".so") || fileName == ".so" { + return nil + } + + pluginName := strings.TrimSuffix(fileName, ".so") + pluginMeta, loadErr := openGoPlugin(path) + if loadErr != nil { + // Sideloaded plugins are third-party code built outside this repo. One + // incompatible or broken file must not keep the server from starting, + // so it is skipped loudly instead. The usual cause is a toolchain or + // dependency mismatch with the binary that is loading it. + basicRes.GetLogger().Error(loadErr, "skipping plugin %s from %s", pluginName, path) + return nil + } + if _, err := plugin.GetPlugin(pluginName); err == nil { + basicRes.GetLogger().Warn(nil, "plugin %s is already registered, the shared object %s takes precedence", pluginName, path) } + + wg.Add(1) + go func(name string, meta plugin.PluginMeta) { + defer wg.Done() + if err := plugin.RegisterPlugin(name, meta); err != nil { + panic(err) + } + basicRes.GetLogger().Info("plugin loaded %s", name) + }(pluginName, pluginMeta) return nil }) wg.Wait() return errors.Convert(walkErr) } +// openGoPlugin opens a shared object and reads its exported entry point. +func openGoPlugin(path string) (plugin.PluginMeta, error) { + plug, err := goplugin.Open(path) + if err != nil { + return nil, err + } + symPluginEntry, err := plug.Lookup("PluginEntry") + if err != nil { + return nil, err + } + pluginMeta, ok := symPluginEntry.(plugin.PluginMeta) + if !ok { + return nil, fmt.Errorf("%s: PluginEntry must implement plugin.PluginMeta", filepath.Base(path)) + } + return pluginMeta, nil +} + func LoadRemotePlugins(basicRes context.BasicRes) errors.Error { remotePluginDir := basicRes.GetConfig("REMOTE_PLUGIN_DIR") if remotePluginDir != "" { diff --git a/backend/plugins/all/all.go b/backend/plugins/all/all.go new file mode 100644 index 00000000000..7fd68c5a114 --- /dev/null +++ b/backend/plugins/all/all.go @@ -0,0 +1,131 @@ +//go:build !devlake_external_plugins + +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by tools/pluginsgen. DO NOT EDIT. + +// Package all links the official Go plugins into the server binary. +// +// Importing this package registers every plugin listed below and fixes the +// official plugin set at build time. It does not close the door on extensions: +// any .so found in PLUGIN_DIR is still opened at startup and registered on +// top of these. Build with the devlake_external_plugins tag to leave this +// registry empty and ship a core-only binary instead. +package all + +import ( + "github.com/apache/devlake/core/plugin" + + ae "github.com/apache/devlake/plugins/ae/impl" + argocd "github.com/apache/devlake/plugins/argocd/impl" + asana "github.com/apache/devlake/plugins/asana/impl" + azuredevops_go "github.com/apache/devlake/plugins/azuredevops_go/impl" + bamboo "github.com/apache/devlake/plugins/bamboo/impl" + bitbucket "github.com/apache/devlake/plugins/bitbucket/impl" + bitbucket_server "github.com/apache/devlake/plugins/bitbucket_server/impl" + circleci "github.com/apache/devlake/plugins/circleci/impl" + claude_code "github.com/apache/devlake/plugins/claude_code/impl" + clickup "github.com/apache/devlake/plugins/clickup/impl" + cursor "github.com/apache/devlake/plugins/cursor/impl" + customize "github.com/apache/devlake/plugins/customize/impl" + dora "github.com/apache/devlake/plugins/dora/impl" + feishu "github.com/apache/devlake/plugins/feishu/impl" + gh_copilot "github.com/apache/devlake/plugins/gh-copilot/impl" + gitee "github.com/apache/devlake/plugins/gitee/impl" + gitextractor "github.com/apache/devlake/plugins/gitextractor/impl" + github "github.com/apache/devlake/plugins/github/impl" + github_graphql "github.com/apache/devlake/plugins/github_graphql/impl" + gitlab "github.com/apache/devlake/plugins/gitlab/impl" + grafana_irm "github.com/apache/devlake/plugins/grafana_irm/impl" + icla "github.com/apache/devlake/plugins/icla/impl" + incidentio "github.com/apache/devlake/plugins/incidentio/impl" + issue_trace "github.com/apache/devlake/plugins/issue_trace/impl" + jenkins "github.com/apache/devlake/plugins/jenkins/impl" + jira "github.com/apache/devlake/plugins/jira/impl" + kiro "github.com/apache/devlake/plugins/kiro/impl" + linear "github.com/apache/devlake/plugins/linear/impl" + linker "github.com/apache/devlake/plugins/linker/impl" + monorepo "github.com/apache/devlake/plugins/monorepo/impl" + opsgenie "github.com/apache/devlake/plugins/opsgenie/impl" + org "github.com/apache/devlake/plugins/org/impl" + pagerduty "github.com/apache/devlake/plugins/pagerduty/impl" + refdiff "github.com/apache/devlake/plugins/refdiff/impl" + rootly "github.com/apache/devlake/plugins/rootly/impl" + slack "github.com/apache/devlake/plugins/slack/impl" + sonarqube "github.com/apache/devlake/plugins/sonarqube/impl" + starrocks "github.com/apache/devlake/plugins/starrocks/impl" + taiga "github.com/apache/devlake/plugins/taiga/impl" + tapd "github.com/apache/devlake/plugins/tapd/impl" + teambition "github.com/apache/devlake/plugins/teambition/impl" + tempo "github.com/apache/devlake/plugins/tempo/impl" + testmo "github.com/apache/devlake/plugins/testmo/impl" + trello "github.com/apache/devlake/plugins/trello/impl" + webhook "github.com/apache/devlake/plugins/webhook/impl" + youtrack "github.com/apache/devlake/plugins/youtrack/impl" + zentao "github.com/apache/devlake/plugins/zentao/impl" +) + +func init() { + plugin.MustRegister(ae.AE{}) + plugin.MustRegister(argocd.ArgoCD{}) + plugin.MustRegister(asana.Asana{}) + plugin.MustRegister(azuredevops_go.Azuredevops{}) + plugin.MustRegister(bamboo.Bamboo{}) + plugin.MustRegister(bitbucket.Bitbucket{}) + plugin.MustRegister(bitbucket_server.BitbucketServer{}) + plugin.MustRegister(circleci.Circleci{}) + plugin.MustRegister(claude_code.ClaudeCode{}) + plugin.MustRegister(clickup.ClickUp{}) + plugin.MustRegister(cursor.Cursor{}) + plugin.MustRegister(customize.Customize{}) + plugin.MustRegister(dora.Dora{}) + plugin.MustRegister(feishu.Feishu{}) + plugin.MustRegister(gh_copilot.GhCopilot{}) + plugin.MustRegister(gitee.Gitee{}) + plugin.MustRegister(gitextractor.GitExtractor{}) + plugin.MustRegister(github.Github{}) + plugin.MustRegister(github_graphql.GithubGraphql{}) + plugin.MustRegister(gitlab.Gitlab{}) + plugin.MustRegister(grafana_irm.GrafanaIrm{}) + plugin.MustRegister(icla.Icla{}) + plugin.MustRegister(incidentio.Incidentio{}) + plugin.MustRegister(issue_trace.IssueTrace{}) + plugin.MustRegister(jenkins.Jenkins{}) + plugin.MustRegister(jira.Jira{}) + plugin.MustRegister(kiro.Kiro{}) + plugin.MustRegister(linear.Linear{}) + plugin.MustRegister(linker.Linker{}) + plugin.MustRegister(monorepo.Monorepo{}) + plugin.MustRegister(opsgenie.Opsgenie{}) + plugin.MustRegister(org.Org{}) + plugin.MustRegister(pagerduty.PagerDuty{}) + plugin.MustRegister(refdiff.RefDiff{}) + plugin.MustRegister(rootly.Rootly{}) + plugin.MustRegister(slack.Slack{}) + plugin.MustRegister(sonarqube.Sonarqube{}) + plugin.MustRegister(starrocks.StarRocks{}) + plugin.MustRegister(taiga.Taiga{}) + plugin.MustRegister(tapd.Tapd{}) + plugin.MustRegister(teambition.Teambition{}) + plugin.MustRegister(tempo.Tempo{}) + plugin.MustRegister(testmo.Testmo{}) + plugin.MustRegister(trello.Trello{}) + plugin.MustRegister(webhook.Webhook{}) + plugin.MustRegister(youtrack.Youtrack{}) + plugin.MustRegister(zentao.Zentao{}) +} diff --git a/backend/plugins/all/external.go b/backend/plugins/all/external.go new file mode 100644 index 00000000000..995ed338915 --- /dev/null +++ b/backend/plugins/all/external.go @@ -0,0 +1,30 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +//go:build devlake_external_plugins + +// Package all is empty in a core-only build. +// +// A core-only binary compiles in the framework and the API but no plugin, which +// is useful for a minimal image: every plugin is then sideloaded as a shared +// object from PLUGIN_DIR. The default build links the official plugins in and +// still accepts extra .so files from PLUGIN_DIR on top. +// +// A sideloaded plugin must be built with the same Go toolchain, build flags and +// dependency versions as the binary loading it, so build it in the matching +// builder image rather than on the host. +package all diff --git a/backend/scripts/build-server.sh b/backend/scripts/build-server.sh index b390185c3f0..479e1fcfa40 100755 --- a/backend/scripts/build-server.sh +++ b/backend/scripts/build-server.sh @@ -22,9 +22,13 @@ echo build-server.sh VERSION: $VERSION ROOT_DIR=$(dirname $(dirname "$0")) VERSION=${VERSION:-$(git describe --tags --always --dirty || true)} EXTRA="" +# Strip the symbol table and DWARF from release builds: they roughly halve the +# binary and nothing needs them at runtime. A debug build keeps them for delve. +LDFLAGS="-s -w" if [ -n "$DEVLAKE_DEBUG" ]; then EXTRA="-gcflags='all=-N -l'" + LDFLAGS="" fi -go build $EXTRA -ldflags "-X 'github.com/apache/devlake/core/version.Version=$VERSION'" -o $ROOT_DIR/bin/lake $ROOT_DIR/server/ \ No newline at end of file +go build $EXTRA -trimpath -ldflags "$LDFLAGS -X 'github.com/apache/devlake/core/version.Version=$VERSION'" -o $ROOT_DIR/bin/lake $ROOT_DIR/server/ \ No newline at end of file diff --git a/backend/server/main.go b/backend/server/main.go index 7a52c6597fa..836b605e745 100644 --- a/backend/server/main.go +++ b/backend/server/main.go @@ -21,6 +21,9 @@ import ( "github.com/apache/devlake/core/config" "github.com/apache/devlake/core/plugin" _ "github.com/apache/devlake/core/version" + // Compile the official plugins into this binary and register them at startup. + // Extra plugins can still be sideloaded from PLUGIN_DIR as shared objects. + _ "github.com/apache/devlake/plugins/all" "github.com/apache/devlake/server/api" ) diff --git a/backend/tools/pluginsgen/main.go b/backend/tools/pluginsgen/main.go new file mode 100644 index 00000000000..719b2fcb492 --- /dev/null +++ b/backend/tools/pluginsgen/main.go @@ -0,0 +1,282 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Command pluginsgen generates plugins/all, the package that links the Go +// plugins into the server binary. +// +// It scans plugins/*/impl for the type implementing plugin.PluginMeta and emits +// one import plus one plugin.MustRegister call per plugin. Keeping the registry +// generated means adding a plugin needs no edit to a shared list, and CI can +// assert that the checked-in file matches the tree. +package main + +import ( + "bytes" + "flag" + "fmt" + "go/ast" + "go/format" + "go/parser" + "go/token" + "os" + "path/filepath" + "sort" + "strings" +) + +const modulePath = "github.com/apache/devlake" + +// licenseHeader is emitted verbatim; license-eye checks generated files too. +const licenseHeader = `/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +` + +// pluginMetaMethods are the methods declared by plugin.PluginMeta. Requiring +// all of them keeps the scan from picking an unrelated Name method declared on +// some other type in the same package. +var pluginMetaMethods = []string{"Name", "Description", "RootPkgPath"} + +type plugin struct { + name string // directory name, e.g. gh-copilot + alias string // import alias, e.g. gh_copilot + typ string // type implementing plugin.PluginMeta +} + +func main() { + root := flag.String("root", ".", "backend module root") + out := flag.String("o", "plugins/all/all.go", "output file, relative to root") + subset := flag.String("plugins", os.Getenv("DEVLAKE_PLUGINS"), "comma-separated plugin names to include (default: $DEVLAKE_PLUGINS, otherwise all)") + flag.Parse() + + if err := run(*root, *out, *subset); err != nil { + fmt.Fprintf(os.Stderr, "pluginsgen: %v\n", err) + os.Exit(1) + } +} + +func run(root, out, subset string) error { + plugins, err := scan(root, subset) + if err != nil { + return err + } + + src, err := render(plugins) + if err != nil { + return err + } + + target := out + if !filepath.IsAbs(target) { + target = filepath.Join(root, out) + } + if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil { + return err + } + return os.WriteFile(target, src, 0o644) +} + +// scan returns the plugins to link, sorted by name. +func scan(root, subset string) ([]plugin, error) { + dirs, err := filepath.Glob(filepath.Join(root, "plugins", "*", "impl")) + if err != nil { + return nil, err + } + if len(dirs) == 0 { + return nil, fmt.Errorf("no plugins found under %s/plugins/*/impl", root) + } + + selected, explicit := parseSubset(subset) + + var plugins []plugin + for _, dir := range dirs { + name := filepath.Base(filepath.Dir(dir)) + if !selected(name) { + continue + } + typ, err := pluginMetaType(dir) + if err != nil { + return nil, fmt.Errorf("%s: %w", name, err) + } + plugins = append(plugins, plugin{ + name: name, + alias: strings.ReplaceAll(name, "-", "_"), + typ: typ, + }) + } + + // An explicit selection may legitimately match nothing ("none" builds a + // core with no compiled-in plugins); an unfiltered scan must not. + if len(plugins) == 0 && !explicit { + return nil, fmt.Errorf("no plugin found under %s/plugins/*/impl", root) + } + sort.Slice(plugins, func(i, j int) bool { return plugins[i].name < plugins[j].name }) + return plugins, nil +} + +// parseSubset builds the selection predicate from DEVLAKE_PLUGINS. An empty +// value selects every plugin; "none" selects none, mirroring build-plugins.sh. +// The second result reports whether a selection was requested at all. +func parseSubset(subset string) (func(string) bool, bool) { + subset = strings.TrimSpace(subset) + switch subset { + case "": + return func(string) bool { return true }, false + case "none": + return func(string) bool { return false }, true + } + + wanted := map[string]struct{}{} + for _, name := range strings.Split(subset, ",") { + if name = strings.TrimSpace(name); name != "" { + wanted[name] = struct{}{} + } + } + return func(name string) bool { + _, ok := wanted[name] + return ok + }, true +} + +// pluginMetaType finds the type in dir whose method set covers +// plugin.PluginMeta. +func pluginMetaType(dir string) (string, error) { + entries, err := os.ReadDir(dir) + if err != nil { + return "", err + } + + fset := token.NewFileSet() + methods := map[string]map[string]struct{}{} + for _, entry := range entries { + name := entry.Name() + if entry.IsDir() || !strings.HasSuffix(name, ".go") || strings.HasSuffix(name, "_test.go") { + continue + } + file, err := parser.ParseFile(fset, filepath.Join(dir, name), nil, parser.SkipObjectResolution) + if err != nil { + return "", err + } + for _, decl := range file.Decls { + fn, ok := decl.(*ast.FuncDecl) + if !ok || fn.Recv == nil || len(fn.Recv.List) != 1 { + continue + } + receiver := receiverName(fn.Recv.List[0].Type) + if receiver == "" { + continue + } + if methods[receiver] == nil { + methods[receiver] = map[string]struct{}{} + } + methods[receiver][fn.Name.Name] = struct{}{} + } + } + + var found []string + for receiver, declared := range methods { + implements := true + for _, method := range pluginMetaMethods { + if _, ok := declared[method]; !ok { + implements = false + break + } + } + if implements { + found = append(found, receiver) + } + } + + switch len(found) { + case 1: + return found[0], nil + case 0: + return "", fmt.Errorf("no type implements plugin.PluginMeta") + default: + sort.Strings(found) + return "", fmt.Errorf("several types implement plugin.PluginMeta: %s", strings.Join(found, ", ")) + } +} + +func receiverName(expr ast.Expr) string { + switch t := expr.(type) { + case *ast.Ident: + return t.Name + case *ast.StarExpr: + return receiverName(t.X) + case *ast.IndexExpr: // generic receiver, e.g. Foo[T] + return receiverName(t.X) + case *ast.IndexListExpr: + return receiverName(t.X) + } + return "" +} + +func render(plugins []plugin) ([]byte, error) { + var buf bytes.Buffer + buf.WriteString(licenseHeader) + buf.WriteString("//go:build !devlake_external_plugins\n\n") + buf.WriteString("// Code generated by tools/pluginsgen. DO NOT EDIT.\n\n") + buf.WriteString("// Package all links the official Go plugins into the server binary.\n") + buf.WriteString("//\n") + buf.WriteString("// Importing this package registers every plugin listed below and fixes the\n") + buf.WriteString("// official plugin set at build time. It does not close the door on extensions:\n") + buf.WriteString("// any .so found in PLUGIN_DIR is still opened at startup and registered on\n") + buf.WriteString("// top of these. Build with the devlake_external_plugins tag to leave this\n") + buf.WriteString("// registry empty and ship a core-only binary instead.\n") + buf.WriteString("package all\n\n") + + if len(plugins) == 0 { + src, err := format.Source(buf.Bytes()) + if err != nil { + return nil, fmt.Errorf("formatting generated source: %w", err) + } + return src, nil + } + + buf.WriteString("import (\n") + buf.WriteString("\t\"github.com/apache/devlake/core/plugin\"\n\n") + for _, p := range plugins { + fmt.Fprintf(&buf, "\t%s \"%s/plugins/%s/impl\"\n", p.alias, modulePath, p.name) + } + buf.WriteString(")\n\n") + buf.WriteString("func init() {\n") + for _, p := range plugins { + fmt.Fprintf(&buf, "\tplugin.MustRegister(%s.%s{})\n", p.alias, p.typ) + } + buf.WriteString("}\n") + + src, err := format.Source(buf.Bytes()) + if err != nil { + return nil, fmt.Errorf("formatting generated source: %w", err) + } + return src, nil +} diff --git a/env.example b/env.example index effca30dbaa..68be8d6d249 100755 --- a/env.example +++ b/env.example @@ -46,7 +46,10 @@ # certificates) on the dashboard container to file paths mounted into it. # GRAFANA_DATABASE_SSL_MODE=disable -# Lake plugin dir, absolute path or relative path +# Lake plugin dir, absolute path or relative path. +# The release binary already contains every official plugin. This directory is +# scanned at startup for sideloaded .so plugins, which are registered on top of +# the compiled-in set. PLUGIN_DIR=bin/plugins REMOTE_PLUGIN_DIR=python/plugins