feat(plugins): add plugin telemetry

README.md

@@ -476,6 +476,8 @@ Usage data points are collected during use of this CLI. This includes the CLI ve
 
 Auth0 uses this data to understand how the tool is used, to prioritize the features, enhancements and fixes that matter most to our users, and to attribute usage to a tenant for support and product purposes.
 
+The events that count plugin runs, installs, updates and uninstalls carry only the plugin name and version, plus whether a run succeeded and a short reason code. Plugin arguments, flags, tenant names and output are never part of those events.
+
 To **opt-out** of this collection, set the `AUTH0_CLI_ANALYTICS` environment variable to `false`.
 
 ## Feedback

internal/analytics/analytics.go

@@ -8,6 +8,7 @@ import (
 	"net/http"
 	"os"
 	"runtime"
+	"strconv"
 	"strings"
 	"sync"
 	"time"
@@ -25,10 +26,18 @@ const (
 	versionKey        = "version"
 	osKey             = "os"
 	archKey           = "arch"
+	pluginKey         = "plugin"
+	pluginVersionKey  = "plugin_version"
+	successKey        = "success"
+	errorReasonKey    = "error_reason"
 )
 
 type Tracker struct {
 	wg sync.WaitGroup
+
+	// The client that sends the events. When it is nil, a default client is used. Tests
+	// replace it, so that no test reaches the analytics endpoint.
+	client *http.Client
 }
 
 type event struct {
@@ -43,6 +52,13 @@ func NewTracker() *Tracker {
 	return &Tracker{}
 }
 
+// NewTrackerWithClient returns a tracker that sends its events with the given
+// client. It exists so that tests can see what is sent without any request
+// reaching the analytics endpoint.
+func NewTrackerWithClient(client *http.Client) *Tracker {
+	return &Tracker{client: client}
+}
+
 func (t *Tracker) TrackFirstLogin(id string, loginType string, properties map[string]string) {
 	eventName := fmt.Sprintf("%s - Auth0 - First Login", eventNamePrefix)
 	t.track(eventName, id, properties)
@@ -56,6 +72,32 @@ func (t *Tracker) TrackCommandRun(commandPath string, id string, properties map[
 	t.track(eventName, id, properties)
 }
 
+// TrackPlugin counts one event about a plugin. The event name comes from the
+// closed set of actions, and the properties are built here from the named fields
+// of the event, never from a map that the caller fills, so nothing the user typed
+// or received can be added to it. An action that is not one of the constants
+// sends nothing.
+func (t *Tracker) TrackPlugin(action PluginAction, pluginEvent PluginEvent, id string) {
+	if !action.valid() {
+		return
+	}
+
+	properties := map[string]string{
+		pluginKey:        pluginEvent.Plugin,
+		pluginVersionKey: pluginEvent.Version,
+	}
+
+	if action == PluginActionRun {
+		properties[successKey] = strconv.FormatBool(pluginEvent.Success)
+
+		if !pluginEvent.Success {
+			properties[errorReasonKey] = pluginErrorReason(pluginEvent.ErrorReason)
+		}
+	}
+
+	t.track(fmt.Sprintf("%s - Plugin - %s", eventNamePrefix, action), id, properties)
+}
+
 func (t *Tracker) Wait(ctx context.Context) {
 	ch := make(chan struct{})
 
@@ -96,7 +138,11 @@ func (t *Tracker) sendEvent(event *event) {
 
 	req.Header.Set("Content-Type", "application/json")
 
-	client := &http.Client{}
+	client := t.client
+	if client == nil {
+		client = &http.Client{}
+	}
+
 	resp, err := client.Do(req)
 	if err != nil {
 		println(err.Error())
@@ -166,3 +212,85 @@ func timestamp() int64 {
 	ms := int64(t.Nanosecond()) / 1e6
 	return s + ms
 }
+
+// PluginAction is what happened to a plugin. It is a closed set: the name of a
+// plugin event is made from it, so a name chosen by someone else never becomes
+// part of an event name.
+type PluginAction string
+
+const (
+	// PluginActionRun is a run of a command of an installed plugin.
+	PluginActionRun PluginAction = "Run"
+
+	// PluginActionInstall is an install that put a plugin version on this machine.
+	PluginActionInstall PluginAction = "Install"
+
+	// PluginActionUpdate is an update that moved a plugin to another version.
+	PluginActionUpdate PluginAction = "Update"
+
+	// PluginActionUninstall is the removal of an installed plugin.
+	PluginActionUninstall PluginAction = "Uninstall"
+)
+
+// valid reports whether the action is one of the constants.
+func (a PluginAction) valid() bool {
+	switch a {
+	case PluginActionRun, PluginActionInstall, PluginActionUpdate, PluginActionUninstall:
+		return true
+	default:
+		return false
+	}
+}
+
+// PluginEvent is everything a plugin event can carry. There is no field for
+// arguments, flags, a tenant, an exit code or output, so none can be sent.
+type PluginEvent struct {
+	// Plugin is the name of the plugin without its registry, such as checkmate.
+	Plugin string
+
+	// Version is the version of the plugin the event is about.
+	Version string
+
+	// Success and ErrorReason describe the outcome of a run, and are ignored for
+	// every other action. ErrorReason is the reason code of the CLI for a failed
+	// run, and is sent as it is only when it is one of the known plugin codes.
+	Success     bool
+	ErrorReason string
+}
+
+// otherReason is what a reason that is not on the allowlist is sent as.
+const otherReason = "other"
+
+// pluginReasons are the reason codes of the CLI that a failed plugin run can
+// have. A reason that is not in the set is sent as "other", so free text can
+// never reach an event.
+var pluginReasons = map[string]bool{
+	"plugin_approval_check":       true,
+	"plugin_approvals_damaged":    true,
+	"plugin_changed":              true,
+	"plugin_command_missing":      true,
+	"plugin_exit":                 true,
+	"plugin_interactive":          true,
+	"plugin_interrupted":          true,
+	"plugin_no_tenant":            true,
+	"plugin_not_approved":         true,
+	"plugin_output_unsupported":   true,
+	"plugin_registry_unavailable": true,
+	"plugin_revoked":              true,
+	"plugin_session_unavailable":  true,
+	"plugin_start_failed":         true,
+	"plugin_tenant_mismatch":      true,
+	"plugin_timeout":              true,
+	"plugin_timeout_invalid":      true,
+	"plugin_verification_failed":  true,
+}
+
+// pluginErrorReason returns the reason if it is a known plugin reason code, and
+// "other" otherwise.
+func pluginErrorReason(reason string) string {
+	if pluginReasons[reason] {
+		return reason
+	}
+
+	return otherReason
+}

internal/analytics/analytics_test.go

@@ -1,11 +1,19 @@
 package analytics
 
 import (
+	"context"
+	"encoding/json"
+	"io"
+	"net/http"
 	"runtime"
+	"sync"
 	"testing"
 	"time"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/auth0/auth0-cli/internal/buildinfo"
 )
 
 func TestGenerateEventName(t *testing.T) {
@@ -73,3 +81,193 @@ func TestNewEvent(t *testing.T) {
 		assert.Equal(t, runtime.GOOS, event.Properties[osKey])
 	})
 }
+
+// recordingTransport keeps the events that the tracker sends instead of sending
+// them, so that no test reaches the analytics endpoint.
+type recordingTransport struct {
+	mu     sync.Mutex
+	events []event
+	urls   []string
+}
+
+func (r *recordingTransport) RoundTrip(request *http.Request) (*http.Response, error) {
+	body, err := io.ReadAll(request.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	var sent event
+	if err := json.Unmarshal(body, &sent); err != nil {
+		return nil, err
+	}
+
+	r.mu.Lock()
+	r.events = append(r.events, sent)
+	r.urls = append(r.urls, request.URL.String())
+	r.mu.Unlock()
+
+	return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(http.NoBody), Request: request}, nil
+}
+
+// newRecordedTracker returns a tracker that records what it sends, for a build
+// with a version, so that tracking is on unless the test turns it off.
+func newRecordedTracker(t *testing.T) (*Tracker, *recordingTransport) {
+	t.Helper()
+
+	previous := buildinfo.Version
+	buildinfo.Version = "1.2.3"
+
+	t.Cleanup(func() { buildinfo.Version = previous })
+	t.Setenv("AUTH0_CLI_ANALYTICS", "")
+
+	transport := &recordingTransport{}
+
+	return NewTrackerWithClient(&http.Client{Transport: transport}), transport
+}
+
+// sent waits for the events in flight and returns what was sent.
+func (r *recordingTransport) sent(t *testing.T, tracker *Tracker) []event {
+	t.Helper()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+
+	tracker.Wait(ctx)
+	require.NoError(t, ctx.Err())
+
+	r.mu.Lock()
+	defer r.mu.Unlock()
+
+	return append([]event{}, r.events...)
+}
+
+func TestTrackPlugin(t *testing.T) {
+	base := func(extra map[string]string) map[string]string {
+		properties := map[string]string{
+			versionKey: "1.2.3",
+			osKey:      runtime.GOOS,
+			archKey:    runtime.GOARCH,
+		}
+
+		for k, v := range extra {
+			properties[k] = v
+		}
+
+		return properties
+	}
+
+	tests := []struct {
+		name       string
+		action     PluginAction
+		event      PluginEvent
+		wantName   string
+		wantProps  map[string]string
+		wantNoSend bool
+	}{
+		{
+			name:      "a successful run",
+			action:    PluginActionRun,
+			event:     PluginEvent{Plugin: "checkmate", Version: "1.0.0", Success: true},
+			wantName:  "CLI - Plugin - Run",
+			wantProps: base(map[string]string{"plugin": "checkmate", "plugin_version": "1.0.0", "success": "true"}),
+		},
+		{
+			name:      "a failed run with a known reason",
+			action:    PluginActionRun,
+			event:     PluginEvent{Plugin: "checkmate", Version: "1.0.0", ErrorReason: "plugin_exit"},
+			wantName:  "CLI - Plugin - Run",
+			wantProps: base(map[string]string{"plugin": "checkmate", "plugin_version": "1.0.0", "success": "false", "error_reason": "plugin_exit"}),
+		},
+		{
+			name:      "a failed run with free text as the reason",
+			action:    PluginActionRun,
+			event:     PluginEvent{Plugin: "checkmate", Version: "1.0.0", ErrorReason: "open /Users/jane/acme.us.auth0.com: denied"},
+			wantName:  "CLI - Plugin - Run",
+			wantProps: base(map[string]string{"plugin": "checkmate", "plugin_version": "1.0.0", "success": "false", "error_reason": "other"}),
+		},
+		{
+			name:      "a failed run without a reason",
+			action:    PluginActionRun,
+			event:     PluginEvent{Plugin: "checkmate", Version: "1.0.0"},
+			wantName:  "CLI - Plugin - Run",
+			wantProps: base(map[string]string{"plugin": "checkmate", "plugin_version": "1.0.0", "success": "false", "error_reason": "other"}),
+		},
+		{
+			name:      "an install ignores the outcome fields",
+			action:    PluginActionInstall,
+			event:     PluginEvent{Plugin: "checkmate", Version: "1.0.0", Success: true, ErrorReason: "plugin_exit"},
+			wantName:  "CLI - Plugin - Install",
+			wantProps: base(map[string]string{"plugin": "checkmate", "plugin_version": "1.0.0"}),
+		},
+		{
+			name:      "an update",
+			action:    PluginActionUpdate,
+			event:     PluginEvent{Plugin: "checkmate", Version: "2.0.0"},
+			wantName:  "CLI - Plugin - Update",
+			wantProps: base(map[string]string{"plugin": "checkmate", "plugin_version": "2.0.0"}),
+		},
+		{
+			name:      "an uninstall",
+			action:    PluginActionUninstall,
+			event:     PluginEvent{Plugin: "checkmate", Version: "2.0.0"},
+			wantName:  "CLI - Plugin - Uninstall",
+			wantProps: base(map[string]string{"plugin": "checkmate", "plugin_version": "2.0.0"}),
+		},
+		{
+			name:       "an action that is not one of the constants",
+			action:     PluginAction("Run - Everything"),
+			event:      PluginEvent{Plugin: "checkmate", Version: "1.0.0"},
+			wantNoSend: true,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			tracker, transport := newRecordedTracker(t)
+
+			tracker.TrackPlugin(test.action, test.event, "install-id")
+
+			sent := transport.sent(t, tracker)
+
+			if test.wantNoSend {
+				assert.Empty(t, sent)
+				return
+			}
+
+			require.Len(t, sent, 1)
+			assert.Equal(t, test.wantName, sent[0].Event)
+			assert.Equal(t, "install-id", sent[0].ID)
+			assert.Equal(t, appID, sent[0].App)
+			assert.Equal(t, test.wantProps, sent[0].Properties)
+			assert.Equal(t, []string{analyticsEndpoint}, transport.urls)
+		})
+	}
+}
+
+func TestTrackPlugin_OptOut(t *testing.T) {
+	t.Run("AUTH0_CLI_ANALYTICS=false sends nothing", func(t *testing.T) {
+		tracker, transport := newRecordedTracker(t)
+		t.Setenv("AUTH0_CLI_ANALYTICS", "false")
+
+		for _, action := range []PluginAction{PluginActionRun, PluginActionInstall, PluginActionUpdate, PluginActionUninstall} {
+			tracker.TrackPlugin(action, PluginEvent{Plugin: "checkmate", Version: "1.0.0", Success: true}, "install-id")
+		}
+
+		assert.Empty(t, transport.sent(t, tracker))
+	})
+
+	t.Run("a build without a version sends nothing", func(t *testing.T) {
+		tracker, transport := newRecordedTracker(t)
+		buildinfo.Version = ""
+
+		tracker.TrackPlugin(PluginActionRun, PluginEvent{Plugin: "checkmate", Version: "1.0.0", Success: true}, "install-id")
+
+		assert.Empty(t, transport.sent(t, tracker))
+	})
+}
+
+func TestPluginErrorReason(t *testing.T) {
+	for _, reason := range []string{"", "other", "none", "unclassified", "auth_failed", "PLUGIN_EXIT", "plugin_exit ", "plugin_unknown", "exit status 3"} {
+		assert.Equal(t, "other", pluginErrorReason(reason), reason)
+	}
+}