From 3ff2681aac5e786d422d5663164135f2381caee2 Mon Sep 17 00:00:00 2001 From: Cristian Magherusan-Stanciu Date: Sat, 10 Oct 2026 04:42:32 +0200 Subject: [PATCH 1/3] fix(cmd): cancel the invocation on SIGINT and finish an in-flight purchase The CLI ran on context.Background(), so Ctrl-C only set a flag polled between purchases. SIGINT now cancels the invocation context; further Ctrl-Cs are absorbed until the run returns. The purchase call runs on a context detached from that cancellation, so an in-flight purchase is not aborted with its result lost; its audit record is written (before the rate-limit delay, which previously ran first in the --input-csv loop), the delay returns on cancel, no further purchase starts, and the report and an "interrupted: N of M attempted" summary are still produced. Errored purchases in an interrupted run are flagged "verify in AWS before re-running". Refs #2151 Co-Authored-By: Claude Sonnet 5.5 --- cmd/cancellation_test.go | 307 +++++++++++++++++++++++++++++++++++++++ cmd/interrupt.go | 51 +++++++ cmd/main.go | 16 +- cmd/multi_service.go | 95 ++++++++---- 4 files changed, 439 insertions(+), 30 deletions(-) create mode 100644 cmd/cancellation_test.go create mode 100644 cmd/interrupt.go diff --git a/cmd/cancellation_test.go b/cmd/cancellation_test.go new file mode 100644 index 000000000..e57825f19 --- /dev/null +++ b/cmd/cancellation_test.go @@ -0,0 +1,307 @@ +package main + +import ( + "bytes" + "context" + "encoding/csv" + "io" + "log" + "net/http" + "os" + "path/filepath" + "strings" + "sync" + "syscall" + "testing" + "time" + + "github.com/LeanerCloud/cloud-commitments-go/pkg/common" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" +) + +// hookTransport is an offline AWS RoundTripper whose responses and side +// effects are chosen by the test. The library clones an *http.Client and keeps +// its Transport, so it is wrapped in one. +type hookTransport struct { + mu sync.Mutex + paths []string + handle func(r *http.Request) (status int, body string) + handleMu sync.Mutex +} + +func (h *hookTransport) RoundTrip(r *http.Request) (*http.Response, error) { + h.mu.Lock() + h.paths = append(h.paths, r.URL.Path) + h.mu.Unlock() + h.handleMu.Lock() + status, body := h.handle(r) + h.handleMu.Unlock() + return &http.Response{ + StatusCode: status, + Header: http.Header{"Content-Type": []string{"application/x-amz-json-1.0"}}, + Body: io.NopCloser(strings.NewReader(body)), + Request: r, + }, nil +} + +func (h *hookTransport) count(suffix string) int { + h.mu.Lock() + defer h.mu.Unlock() + n := 0 + for _, p := range h.paths { + if strings.HasSuffix(p, suffix) { + n++ + } + } + return n +} + +func hookedAWSConfig(h *hookTransport) aws.Config { + return aws.Config{ + HTTPClient: &http.Client{Transport: h}, + Credentials: aws.CredentialsProviderFunc(func(context.Context) (aws.Credentials, error) { + return aws.Credentials{AccessKeyID: "AKIDEXAMPLE", SecretAccessKey: "secret"}, nil + }), + Retryer: func() aws.Retryer { return aws.NopRetryer{} }, + } +} + +func captureAppLog(t *testing.T) *bytes.Buffer { + t.Helper() + var buf bytes.Buffer + orig := AppLogger.Writer() + AppLogger.SetOutput(&buf) + t.Cleanup(func() { AppLogger.SetOutput(orig) }) + return &buf +} + +func slowDelay(t *testing.T) { + t.Helper() + t.Setenv("DISABLE_PURCHASE_DELAY", "") + orig := purchaseDelay + purchaseDelay = time.Hour + t.Cleanup(func() { purchaseDelay = orig }) +} + +func offeringSP(id string, commitment float64) common.Recommendation { + rec := parserShapedSP() + rec.Details = &common.SavingsPlanDetails{PlanType: "Compute", HourlyCommitment: commitment, OfferingID: id} + return rec +} + +func auditRows(t *testing.T, path string) []string { + t.Helper() + data, err := os.ReadFile(path) + if os.IsNotExist(err) { + return nil + } + require.NoError(t, err) + var rows []string + for _, l := range strings.Split(string(data), "\n") { + if strings.TrimSpace(l) != "" { + rows = append(rows, l) + } + } + return rows +} + +func TestExecuteAndReport_CancelDuringPurchaseFinishesInFlightAndReportsWhatRan(t *testing.T) { + slowDelay(t) + out := captureAppLog(t) + dir := t.TempDir() + cfg := Config{AuditLog: filepath.Join(dir, "audit.jsonl"), CSVOutput: filepath.Join(dir, "report.csv")} + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + var ctxErrAtPurchase error + h := &hookTransport{} + h.handle = func(r *http.Request) (int, string) { + switch { + case strings.HasSuffix(r.URL.Path, "/CreateSavingsPlan"): + cancel() // Ctrl-C arrives while the purchase request is in flight + ctxErrAtPurchase = r.Context().Err() + return http.StatusOK, `{"savingsPlanId":"sp-111"}` + case strings.HasSuffix(r.URL.Path, "/DescribeSavingsPlans"): + return http.StatusOK, `{"savingsPlans":[]}` + } + return http.StatusBadRequest, `{"__type":"ValidationException","message":"unexpected call"}` + } + recs := []common.Recommendation{offeringSP("o-1", 1), offeringSP("o-2", 2), offeringSP("o-3", 3)} + + executeAndReport(ctx, hookedAWSConfig(h), recs, false, "run-1", cfg, nil) + + require.NoError(t, ctxErrAtPurchase, "the in-flight purchase must not see the parent cancellation") + assert.Equal(t, 1, h.count("/CreateSavingsPlan"), "no purchase may start after the interrupt") + rows := auditRows(t, cfg.AuditLog) + require.Len(t, rows, 1, "the completed purchase is audited") + assert.Contains(t, rows[0], `"status":"success"`) + assert.Contains(t, rows[0], "sp-111") + + f, err := os.Open(cfg.CSVOutput) + require.NoError(t, err, "the report is still written") + defer f.Close() + csvRows, err := csv.NewReader(f).ReadAll() + require.NoError(t, err) + require.Len(t, csvRows, 3, "header + the one purchase + TOTAL") + assert.Contains(t, out.String(), "Run interrupted: 1 of 3 recommendation(s) attempted") +} + +func TestExecuteAndReport_ErroredPurchaseOnInterruptSaysVerifyInAWS(t *testing.T) { + slowDelay(t) + out := captureAppLog(t) + dir := t.TempDir() + cfg := Config{AuditLog: filepath.Join(dir, "audit.jsonl"), CSVOutput: filepath.Join(dir, "report.csv")} + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + h := &hookTransport{} + h.handle = func(r *http.Request) (int, string) { + if strings.HasSuffix(r.URL.Path, "/CreateSavingsPlan") { + cancel() + return http.StatusInternalServerError, `{"__type":"InternalServerException","message":"boom"}` + } + return http.StatusOK, `{"savingsPlans":[]}` + } + + executeAndReport(ctx, hookedAWSConfig(h), []common.Recommendation{offeringSP("o-1", 1), offeringSP("o-2", 2)}, false, "run-1", cfg, nil) + + assert.Contains(t, out.String(), "verify in AWS before re-running") + rows := auditRows(t, cfg.AuditLog) + require.Len(t, rows, 1) + assert.Contains(t, rows[0], `"status":"error"`, "audit status stays as it is today") +} + +func TestProcessPurchaseLoop_AuditRecordWrittenBeforeDelay(t *testing.T) { + slowDelay(t) + captureAppLog(t) + cfg := Config{AuditLog: filepath.Join(t.TempDir(), "audit.jsonl")} + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + recs := []common.Recommendation{validEC2Rec(), validEC2Rec()} + recs[1].ResourceType = "m5.xlarge" + mockClient := &MockServiceClient{} + mockClient.On("PurchaseCommitment", mock.Anything, mock.Anything, mock.Anything). + Return(common.PurchaseResult{Success: true, CommitmentID: "ri-1"}, nil) + + var rowsAtWait []string + seen := false + orig := waitBetweenPurchases + waitBetweenPurchases = func(ctx context.Context) { + if !seen { + seen = true + rowsAtWait = auditRows(t, cfg.AuditLog) + } + cancel() // Ctrl-C during the delay + } + t.Cleanup(func() { waitBetweenPurchases = orig }) + + results := processPurchaseLoop(ctx, recs, "us-east-1", false, mockClient, cfg, "run-1") + + require.Len(t, rowsAtWait, 1, "the purchase that just completed must already be audited when the delay starts") + require.Len(t, results, 1, "the second recommendation is not attempted after the interrupt") + mockClient.AssertNumberOfCalls(t, "PurchaseCommitment", 1) +} + +func TestExecutePurchasePipeline_AuditRecordWrittenBeforeDelay(t *testing.T) { + slowDelay(t) + captureAppLog(t) + cfg := Config{AuditLog: filepath.Join(t.TempDir(), "audit.jsonl")} + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + h := &hookTransport{handle: func(r *http.Request) (int, string) { + if strings.HasSuffix(r.URL.Path, "/CreateSavingsPlan") { + return http.StatusOK, `{"savingsPlanId":"sp-1"}` + } + return http.StatusOK, `{"savingsPlans":[]}` + }} + var rowsAtWait []string + seen := false + orig := waitBetweenPurchases + waitBetweenPurchases = func(context.Context) { + if !seen { + seen = true + rowsAtWait = auditRows(t, cfg.AuditLog) + } + cancel() + } + t.Cleanup(func() { waitBetweenPurchases = orig }) + + executePurchasePipeline(ctx, hookedAWSConfig(h), []common.Recommendation{offeringSP("o-1", 1), offeringSP("o-2", 2)}, false, "run-1", cfg) + + require.Len(t, rowsAtWait, 1) + assert.Equal(t, 1, h.count("/CreateSavingsPlan")) +} + +func TestWaitBetweenPurchases_ReturnsPromptlyOnCancel(t *testing.T) { + slowDelay(t) + ctx, cancel := context.WithCancel(context.Background()) + go func() { time.Sleep(20 * time.Millisecond); cancel() }() + start := time.Now() + waitBetweenPurchases(ctx) + assert.Less(t, time.Since(start), 5*time.Second) +} + +func TestRegisterShutdownSignalHandler_CancelSetsFlagAndPrintsNoticeOnce(t *testing.T) { + var buf bytes.Buffer + orig := log.Writer() + log.SetOutput(&buf) + t.Cleanup(func() { log.SetOutput(orig) }) + ctx, cancel := context.WithCancel(context.Background()) + cleanup := registerShutdownSignalHandler(ctx) + require.False(t, shutdownRequested.Load()) + + cancel() + require.Eventually(t, shutdownRequested.Load, 2*time.Second, 5*time.Millisecond) + cleanup() + shutdownRequested.Store(false) + assert.Equal(t, 1, strings.Count(buf.String(), "stopping after the in-flight purchase")) + assert.Contains(t, buf.String(), `Ctrl-\ force-quits`) +} + +// A second Ctrl-C must be absorbed, not kill the process mid-purchase. If the +// context's stop func were called on the first signal, the default SIGINT +// handler would return and the second signal below would terminate the test +// binary. +func TestInvocationContext_FurtherSigintsAreAbsorbed(t *testing.T) { + ctx, stop := invocationContext() + defer stop() + + require.NoError(t, syscall.Kill(os.Getpid(), syscall.SIGINT)) + select { + case <-ctx.Done(): + case <-time.After(5 * time.Second): + t.Fatal("SIGINT did not cancel the invocation context") + } + time.Sleep(200 * time.Millisecond) // let any early stop() take effect + require.NoError(t, syscall.Kill(os.Getpid(), syscall.SIGINT)) + time.Sleep(200 * time.Millisecond) // the process is still here to run this line + assert.Error(t, ctx.Err()) +} + +func TestProcessPurchaseLoop_CancelDuringPurchaseDoesNotCancelTheInFlightCall(t *testing.T) { + slowDelay(t) + captureAppLog(t) + cfg := Config{AuditLog: filepath.Join(t.TempDir(), "audit.jsonl")} + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + recs := []common.Recommendation{validEC2Rec(), validEC2Rec()} + recs[1].ResourceType = "m5.xlarge" + var ctxErrAtPurchase error + mockClient := &MockServiceClient{} + mockClient.On("PurchaseCommitment", mock.Anything, mock.Anything, mock.Anything). + Run(func(args mock.Arguments) { + cancel() // Ctrl-C while the purchase call is in flight + ctxErrAtPurchase = args.Get(0).(context.Context).Err() + }). + Return(common.PurchaseResult{Success: true, CommitmentID: "ri-1"}, nil) + + results := processPurchaseLoop(ctx, recs, "us-east-1", false, mockClient, cfg, "run-1") + + require.NoError(t, ctxErrAtPurchase) + require.Len(t, results, 1) + assert.True(t, results[0].Success) + assert.Len(t, auditRows(t, cfg.AuditLog), 1) + mockClient.AssertNumberOfCalls(t, "PurchaseCommitment", 1) +} diff --git a/cmd/interrupt.go b/cmd/interrupt.go new file mode 100644 index 000000000..136ee2194 --- /dev/null +++ b/cmd/interrupt.go @@ -0,0 +1,51 @@ +package main + +import ( + "context" + "fmt" + "os" + "time" + + "github.com/LeanerCloud/cloud-commitments-go/pkg/common" +) + +// purchaseDelay is the pause between consecutive real purchases. +var purchaseDelay = PurchaseDelaySeconds * time.Second + +// waitBetweenPurchases pauses between real purchases to avoid rate limiting +// and returns at once when ctx is canceled, so Ctrl-C does not wait out the +// delay. It is a variable so tests can observe the moment the wait starts. +var waitBetweenPurchases = func(ctx context.Context) { + if os.Getenv("DISABLE_PURCHASE_DELAY") == "true" { + return + } + t := time.NewTimer(purchaseDelay) + defer t.Stop() + select { + case <-ctx.Done(): + case <-t.C: + } +} + +// printInterruptedSummary reports how far an interrupted purchase run got. +// Records of failed real purchases are called out because a failure reported +// while the run was being stopped may still have created the commitment at +// AWS; the audit status stays "error" and a re-run relies on the duplicate +// check, so the operator must look before re-running. +func printInterruptedSummary(attempted, total int, results []common.PurchaseResult, isDryRun bool) { + AppLogger.Printf("\nāš ļø Run interrupted: %d of %d recommendation(s) attempted, the rest were not purchased.\n", attempted, total) + if isDryRun { + return + } + for i := range results { + r := results[i] + if r.Success { + continue + } + AppLogger.Printf(" %s\n", unverifiedPurchaseLine(r)) + } +} + +func unverifiedPurchaseLine(r common.PurchaseResult) string { + return fmt.Sprintf("purchase of %s %s errored (%v): verify in AWS before re-running", r.Recommendation.Service, r.Recommendation.ResourceType, r.Error) +} diff --git a/cmd/main.go b/cmd/main.go index d269360ef..8d927b4e7 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -4,6 +4,8 @@ import ( "context" "fmt" "log" + "os" + "os/signal" "regexp" "sort" "strings" @@ -379,8 +381,20 @@ func sanitizeAccountName(accountName string) string { } func runTool(cmd *cobra.Command, args []string) { - ctx := context.Background() + ctx, stop := invocationContext() + defer stop() // Always use the multi-service implementation runToolMultiService(ctx, toolCfg) } + +// invocationContext returns the context for one CLI invocation, canceled by +// SIGINT. stop is for the caller to defer, never to call on the first signal: +// stopping NotifyContext restores the default handler, so a second Ctrl-C +// would kill the process mid-purchase. Keeping SIGINT registered until the +// run returns absorbs further Ctrl-Cs; the wait is bounded by the per-call SDK +// timeouts and the pre-purchase phases exit at once (see +// registerShutdownSignalHandler for the operator notice). +func invocationContext() (ctx context.Context, stop context.CancelFunc) { + return signal.NotifyContext(context.Background(), os.Interrupt) +} diff --git a/cmd/multi_service.go b/cmd/multi_service.go index 8a8091308..911da87a3 100644 --- a/cmd/multi_service.go +++ b/cmd/multi_service.go @@ -4,8 +4,6 @@ import ( "context" "fmt" "log" - "os" - "os/signal" "sync/atomic" "time" @@ -89,17 +87,34 @@ func coverageFetchFailure(cfg Config, err error) error { // shutdownRequested is set to true when SIGINT is received during a purchase run. var shutdownRequested atomic.Bool +// interruptNotice is printed once when the run is interrupted. +const interruptNotice = "interrupt received: stopping after the in-flight purchase, if any; Ctrl-\\ force-quits and may lose its audit record" + // registerShutdownSignalHandler arms shutdownRequested for the duration of a // purchase run and returns the cleanup func the caller must defer (e.g. -// `defer registerShutdownSignalHandler()()`). Shared by both purchase entry -// points -- runToolMultiService and runToolFromCSV -- so an in-flight run on -// either path can be stopped cleanly between purchases with Ctrl-C. -func registerShutdownSignalHandler() func() { +// `defer registerShutdownSignalHandler(ctx)()`). ctx is the invocation context +// that SIGINT cancels (see runTool): when it is canceled the flag is set and +// the notice is printed once. Shared by both purchase entry points -- +// runToolMultiService and runToolFromCSV -- so an in-flight run on either +// path can be stopped cleanly between purchases with Ctrl-C. +func registerShutdownSignalHandler(ctx context.Context) func() { shutdownRequested.Store(false) - sigCh := make(chan os.Signal, 1) - signal.Notify(sigCh, os.Interrupt) - go func() { <-sigCh; shutdownRequested.Store(true) }() - return func() { signal.Stop(sigCh) } + done := make(chan struct{}) + go func() { + select { + case <-ctx.Done(): + log.Print(interruptNotice) + shutdownRequested.Store(true) + case <-done: + } + }() + return func() { close(done) } +} + +// stopRequested reports whether the run was interrupted: SIGINT cancels the +// invocation context and sets shutdownRequested. +func stopRequested(ctx context.Context) bool { + return shutdownRequested.Load() || ctx.Err() != nil } // effectiveDryRun reports whether the run must stay in dry-run mode. A run is @@ -132,7 +147,7 @@ func runToolMultiService(ctx context.Context, cfg Config) { isDryRun := effectiveDryRun(cfg) // Register SIGINT handler so a running purchase loop can be interrupted cleanly. - defer registerShutdownSignalHandler()() + defer registerShutdownSignalHandler(ctx)() // Verify the audit log and its immediate parents before making cloud API calls. if err := CheckAuditLogWritable(cfg.AuditLog); err != nil { @@ -194,10 +209,19 @@ func runPurchaseAndReport(ctx context.Context, awsCfg aws.Config, scoredResult s return } - allResults := executePurchasePipeline(ctx, awsCfg, scoredResult.Passed, isDryRun, runID, cfg) + executeAndReport(ctx, awsCfg, scoredResult.Passed, isDryRun, runID, cfg, drops) +} - // Produce summary outputs. - writeReportAndSummary(scoredResult.Passed, allResults, isDryRun, cfg, drops) +// executeAndReport runs the purchase pipeline and produces the report and +// summary for whatever ran, interrupted or not. Split from +// runPurchaseAndReport so the post-confirmation half is testable without a +// terminal. +func executeAndReport(ctx context.Context, awsCfg aws.Config, passed []common.Recommendation, isDryRun bool, runID string, cfg Config, drops *common.DropSummary) { + allResults := executePurchasePipeline(ctx, awsCfg, passed, isDryRun, runID, cfg) + writeReportAndSummary(passed, allResults, isDryRun, cfg, drops) + if stopRequested(ctx) { + printInterruptedSummary(len(allResults), len(passed), allResults, isDryRun) + } } // confirmPurchaseRun asks for confirmation once against the full @@ -439,15 +463,19 @@ func executePurchasePipeline(ctx context.Context, awsCfg aws.Config, recs []comm results := make([]common.PurchaseResult, 0, len(recs)) for i := range recs { rec := recs[i] - if shutdownRequested.Load() { + if stopRequested(ctx) { log.Printf("Shutdown requested — skipping %d remaining recommendations", len(recs)-i) break } - result, status := purchaseSingleRec(ctx, awsCfg, rec, i+1, isDryRun, cfg) + // The purchase runs on a context that survives Ctrl-C: canceling the + // request mid-flight can leave AWS having created the commitment + // while the CLI records a failure, and a re-run then buys it again. + // The per-call SDK timeouts still bound it. + result, status := purchaseSingleRec(context.WithoutCancel(ctx), awsCfg, rec, i+1, isDryRun, cfg) results = append(results, result) writePurchaseAuditRecord(runID, rec, result, status, isDryRun, cfg.AuditLog) - if !isDryRun && i < len(recs)-1 && os.Getenv("DISABLE_PURCHASE_DELAY") != "true" { - time.Sleep(PurchaseDelaySeconds * time.Second) + if !isDryRun && i < len(recs)-1 { + waitBetweenPurchases(ctx) } } return results @@ -614,7 +642,7 @@ func runToolFromCSV(ctx context.Context, cfg Config) error { // (runToolMultiService). Before #1610 this path had no SIGINT handling // at all: runToolMultiService registers it only on the non-CSV branch, // in code unreachable from CSV mode (the CSV branch returns first). - defer registerShutdownSignalHandler()() + defer registerShutdownSignalHandler(ctx)() csvModeCoverage := determineCSVCoverage(cfg) @@ -644,7 +672,7 @@ func runToolFromCSV(ctx context.Context, cfg Config) error { allAdjustedRecs := make([]common.Recommendation, 0) for service, regionRecs := range recsByServiceRegion { - if shutdownRequested.Load() { + if stopRequested(ctx) { log.Printf("Shutdown requested; stopping before %s", getServiceDisplayName(service)) break } @@ -658,7 +686,7 @@ func runToolFromCSV(ctx context.Context, cfg Config) error { serviceRecs := make([]common.Recommendation, 0) for region, recs := range regionRecs { - if shutdownRequested.Load() { + if stopRequested(ctx) { log.Printf("Shutdown requested; skipping remaining regions for %s", getServiceDisplayName(service)) break } @@ -697,6 +725,9 @@ func runToolFromCSV(ctx context.Context, cfg Config) error { // Print final summary using the post-dedup slice so counts match what was // actually processed, not the pre-dedup input passed into the outer loop. printMultiServiceSummary(allAdjustedRecs, allResults, serviceStats, isDryRun) + if stopRequested(ctx) { + printInterruptedSummary(len(allResults), len(recs), allResults, isDryRun) + } return nil } @@ -862,7 +893,7 @@ func processPurchaseLoop(ctx context.Context, recs []common.Recommendation, regi results := make([]common.PurchaseResult, 0, len(recs)) for j := range recs { - if shutdownRequested.Load() { + if stopRequested(ctx) { log.Printf("Shutdown requested; skipping %d remaining recommendation(s) in %s", len(recs)-j, region) break } @@ -874,6 +905,7 @@ func processPurchaseLoop(ctx context.Context, recs []common.Recommendation, regi var result common.PurchaseResult var status string + attempted := false if err := validatePurchasePreconditions(rec, region); err != nil { result = preconditionFailure(rec, err, isDryRun) status = "error" @@ -881,16 +913,16 @@ func processPurchaseLoop(ctx context.Context, recs []common.Recommendation, regi result = createDryRunResult(rec, label, j+1, cfg) status = "skipped" } else { - // Execute actual purchase - result = executePurchase(ctx, rec, label, j+1, serviceClient, cfg) + // Execute actual purchase. + attempted = true + // Detached from Ctrl-C for the same reason as in + // executePurchasePipeline. + result = executePurchase(context.WithoutCancel(ctx), rec, label, j+1, serviceClient, cfg) status = purchaseAuditStatus(result) - - // Add delay between purchases to avoid rate limiting - if j < len(recs)-1 && os.Getenv("DISABLE_PURCHASE_DELAY") != "true" { - time.Sleep(PurchaseDelaySeconds * time.Second) - } } + // Audit first, then the rate-limit delay: a Ctrl-C during the delay + // must not lose the record of a purchase that already completed. writePurchaseAuditRecord(runID, rec, result, status, isDryRun, cfg.AuditLog) results = append(results, result) @@ -903,6 +935,11 @@ func processPurchaseLoop(ctx context.Context, recs []common.Recommendation, regi } AppLogger.Printf(" āŒ Failed: %s\n", errMsg) } + + // Rate-limit delay between real purchases; returns at once on Ctrl-C. + if attempted && j < len(recs)-1 { + waitBetweenPurchases(ctx) + } } return results From 2397471b82b9e1b9910b42c26a54e7fd7481589c Mon Sep 17 00:00:00 2001 From: Cristian Magherusan-Stanciu Date: Sat, 10 Oct 2026 04:56:38 +0200 Subject: [PATCH 2/3] fix(cmd): treat caller cancellation as terminal before any purchase The lifecycle queries ran on context.Background() and, with a real context, swallowed cancellation into empty maps; region discovery, the recommendation fetch, the coverage fetch and the duplicate checks did the same. A canceled or expired context is now returned as an error at each boundary (nothing is purchased, partial data is discarded, a dry run is not exempt), while provider errors with an active context stay recoverable. Closes #2151 Co-Authored-By: Claude Sonnet 5.5 --- CHANGELOG.md | 12 ++ cmd/cancellation_prepurchase_test.go | 174 ++++++++++++++++++++++++ cmd/multi_service.go | 46 ++++++- cmd/multi_service_coverage_test.go | 8 +- cmd/multi_service_helpers.go | 61 ++++++--- cmd/multi_service_max_instances_test.go | 18 +-- cmd/multi_service_test.go | 2 +- 7 files changed, 284 insertions(+), 37 deletions(-) create mode 100644 cmd/cancellation_prepurchase_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index e94c2e2b3..de0ade427 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,18 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/). ### Fixed +- Ctrl-C now cancels the whole invocation instead of only setting a flag that + was polled between purchases. The engine-version lifecycle queries, region + discovery, recommendation fetch, coverage fetch and duplicate checks treat a + canceled or expired context as terminal (the run stops and buys nothing) + instead of continuing with empty lifecycle data, "no recommendations" or + un-deduplicated counts. A purchase already in flight is not aborted: it + finishes, is written to the audit log (before the delay between purchases, + which the `--input-csv` path used to run first), and the report and an + "interrupted: N of M attempted" summary are still produced. Errored + purchases in an interrupted run are flagged "verify in AWS before + re-running". Further Ctrl-Cs are absorbed; Ctrl-\ force-quits and may lose + the in-flight audit record (#2151). - Reject NaN and infinite values for `--coverage`, `--target-coverage`, `--min-pool-size` and `--min-savings-pct` before any API call. NaN used to pass validation and silently switch `--target-coverage` runs to diff --git a/cmd/cancellation_prepurchase_test.go b/cmd/cancellation_prepurchase_test.go new file mode 100644 index 000000000..21979a121 --- /dev/null +++ b/cmd/cancellation_prepurchase_test.go @@ -0,0 +1,174 @@ +package main + +import ( + "context" + "errors" + "os" + "path/filepath" + "testing" + "time" + + "github.com/LeanerCloud/cloud-commitments-go/pkg/common" + awsprovider "github.com/LeanerCloud/cloud-commitments-go/providers/aws" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" +) + +func canceledCtx() context.Context { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + return ctx +} + +func expiredCtx(t *testing.T) context.Context { + t.Helper() + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(-time.Second)) + t.Cleanup(cancel) + return ctx +} + +func TestFetchEngineVersionData_CancellationAndDeadlineAreTerminal(t *testing.T) { + captureAppLog(t) + for name, ctx := range map[string]context.Context{"canceled": canceledCtx(), "deadline": expiredCtx(t)} { + _, err := fetchEngineVersionData(ctx, Config{}) + require.Error(t, err, name) + assert.ErrorIs(t, err, ctx.Err(), name) + } +} + +func TestFilterAndAdjustRecommendations_CancellationAndDeadlineAreTerminal(t *testing.T) { + for name, ctx := range map[string]context.Context{"canceled": canceledCtx(), "deadline": expiredCtx(t)} { + recs := []common.Recommendation{{Service: common.ServiceRDS, Region: "us-east-1", ResourceType: "db.t3.small", Count: 2}} + got, err := filterAndAdjustRecommendations(ctx, recs, 100, Config{}) + require.Error(t, err, name) + assert.ErrorIs(t, err, ctx.Err(), name) + assert.Empty(t, got, name) + } +} + +func TestPrepareCSVPurchaseRun_CanceledContextStopsBeforeConfirmationAndPurchase(t *testing.T) { + dir := t.TempDir() + csvPath := filepath.Join(dir, "in.csv") + require.NoError(t, os.WriteFile(csvPath, []byte("Service,Region,ResourceType,Count,Engine\nrds,us-east-1,db.t3.small,2,mysql\n"), 0o600)) + cfg := Config{CSVInput: csvPath, AuditLog: filepath.Join(dir, "audit.jsonl")} + + for _, dry := range []bool{true, false} { + recs, _, _, err := prepareCSVPurchaseRun(canceledCtx(), cfg, 100, dry) + require.ErrorIs(t, err, context.Canceled, "dryRun=%v", dry) + assert.Empty(t, recs) + } +} + +func TestFetchExistingCoverage_CanceledContextIsTerminalEvenOnDryRun(t *testing.T) { + realClient := awsprovider.NewRecommendationsClientDirect(aws.Config{Region: "us-east-1"}) + cfg := Config{TargetCoverage: 80, Regions: []string{"us-east-1"}, ActualPurchase: false} + + got, err := fetchExistingCoverage(canceledCtx(), aws.Config{Region: "us-east-1"}, realClient, cfg) + + require.ErrorIs(t, err, context.Canceled, "a dry run used to swallow this and size against an empty map") + assert.Nil(t, got) +} + +func TestHandleRegionDiscoveryError_CanceledContextDoesNotFallBackToDiscovery(t *testing.T) { + captureAppLog(t) + client := &MockRecommendationsClient{} // no expectations: any call fails the test + + regions, err := handleRegionDiscoveryError(canceledCtx(), client, common.ServiceRDS, errors.New("describe regions: context canceled")) + + require.ErrorIs(t, err, context.Canceled) + assert.Nil(t, regions) + client.AssertNotCalled(t, "GetRecommendationsForService", mock.Anything, mock.Anything) +} + +func TestFetchRecommendationsForRegion_InterruptIsNotReportedAsProviderFailure(t *testing.T) { + out := captureAppLog(t) + ctx := canceledCtx() + client := &MockRecommendationsClient{} + client.On("GetRecommendations", mock.Anything, mock.Anything).Return(nil, ctx.Err()) + + got := fetchRecommendationsForRegion(ctx, client, common.ServiceRDS, "us-east-1", Config{TermYears: 1}) + + assert.Nil(t, got) + assert.NotContains(t, out.String(), "Failed to fetch recommendations") +} + +func TestFetchAllRecs_InterruptStopsFanOutAndDiscardsPartialData(t *testing.T) { + captureAppLog(t) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + cfg := Config{Regions: []string{"us-east-1", "us-west-2", "eu-west-1"}, Coverage: 100, TermYears: 1, PaymentOption: "partial-upfront"} + client := &MockRecommendationsClient{} + rec := common.Recommendation{Service: common.ServiceRDS, Region: "us-east-1", ResourceType: "db.t3.small", Count: 3, EstimatedSavings: 10, SavingsPercentage: 10} + client.On("GetRecommendations", mock.Anything, mock.MatchedBy(func(p *common.RecommendationParams) bool { return p.Region == "us-east-1" })). + Run(func(mock.Arguments) { cancel() }). // Ctrl-C after the first region answered + Return([]common.Recommendation{rec}, nil).Once() + + recs, _ := fetchAllRecs(ctx, aws.Config{}, client, NewAccountAliasCache(aws.Config{}), []common.ServiceType{common.ServiceRDS}, engineVersionData{}, cfg, nil) + + assert.Empty(t, recs, "partial data from an interrupted fetch is not a result") + client.AssertNumberOfCalls(t, "GetRecommendations", 1) // later regions are never queried +} + +func TestFetchAllRecs_ActiveContextProviderErrorStaysRecoverable(t *testing.T) { + out := captureAppLog(t) + cfg := Config{Regions: []string{"us-east-1", "us-west-2"}, Coverage: 100, TermYears: 1, PaymentOption: "partial-upfront"} + client := &MockRecommendationsClient{} + client.On("GetRecommendations", mock.Anything, mock.MatchedBy(func(p *common.RecommendationParams) bool { return p.Region == "us-east-1" })). + Return(nil, errors.New("throttling")).Once() + rec := common.Recommendation{Service: common.ServiceRDS, Region: "us-west-2", ResourceType: "db.t3.small", Count: 3, EstimatedSavings: 10, SavingsPercentage: 10} + client.On("GetRecommendations", mock.Anything, mock.MatchedBy(func(p *common.RecommendationParams) bool { return p.Region == "us-west-2" })). + Return([]common.Recommendation{rec}, nil).Once() + + recs, _ := fetchAllRecs(context.Background(), aws.Config{}, client, NewAccountAliasCache(aws.Config{}), []common.ServiceType{common.ServiceRDS}, engineVersionData{}, cfg, nil) + + assert.Len(t, recs, 1, "a provider error with an active context only loses that region") + assert.Contains(t, out.String(), "Failed to fetch recommendations") +} + +func TestCheckDuplicates_InterruptedCheckIsTerminalInDryAndRealRuns(t *testing.T) { + captureAppLog(t) + recs := []common.Recommendation{{ResourceType: "db.t3.small", Count: 5}} + for _, dry := range []bool{true, false} { + ctx, cancel := context.WithCancel(context.Background()) + client := &MockServiceClient{} + client.On("GetExistingCommitments", mock.Anything).Return([]common.Commitment(nil), context.Canceled).Run(func(mock.Arguments) { cancel() }) + + drops := common.NewDropSummary() + got := checkDuplicates(ctx, recs, client, dry, drops) + + assert.Nil(t, got, "dryRun=%v: never continue with un-deduplicated counts after an interrupt", dry) + assert.Equal(t, 0, drops.Total(), "dryRun=%v: an interrupt is not a failed duplicate check", dry) + cancel() + } +} + +func TestCheckDuplicatesForCSVRegion_InterruptedCheckStopsEvenOnDryRun(t *testing.T) { + captureAppLog(t) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + client := &MockServiceClient{} + client.On("GetExistingCommitments", mock.Anything).Return([]common.Commitment(nil), context.Canceled).Run(func(mock.Arguments) { cancel() }) + + got, ok := checkDuplicatesForCSVRegion(ctx, []common.Recommendation{{ResourceType: "db.t3.small", Count: 5}}, client, common.ServiceRDS, "us-east-1", true) + + assert.False(t, ok, "a dry run used to continue with un-deduplicated counts") + assert.Nil(t, got) +} + +func TestFetchAllRecs_InterruptDuringLastRegionDiscardsItsData(t *testing.T) { + captureAppLog(t) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + cfg := Config{Regions: []string{"us-east-1"}, Coverage: 100, TermYears: 1, PaymentOption: "partial-upfront"} + client := &MockRecommendationsClient{} + rec := common.Recommendation{Service: common.ServiceRDS, Region: "us-east-1", ResourceType: "db.t3.small", Count: 3, EstimatedSavings: 10, SavingsPercentage: 10} + client.On("GetRecommendations", mock.Anything, mock.Anything). + Run(func(mock.Arguments) { cancel() }). + Return([]common.Recommendation{rec}, nil).Once() + + recs, _ := fetchAllRecs(ctx, aws.Config{}, client, NewAccountAliasCache(aws.Config{}), []common.ServiceType{common.ServiceRDS}, engineVersionData{}, cfg, nil) + + assert.Empty(t, recs) +} diff --git a/cmd/multi_service.go b/cmd/multi_service.go index 911da87a3..5b6ed11e7 100644 --- a/cmd/multi_service.go +++ b/cmd/multi_service.go @@ -58,6 +58,9 @@ func fetchExistingCoverage(ctx context.Context, awsCfg aws.Config, recClient pro regions := cfg.Regions if len(regions) == 0 { allRegions, err := getAllAWSRegions(ctx, awsCfg) + if ctxErr := ctx.Err(); ctxErr != nil { + return nil, ctxErr + } if err != nil { return nil, coverageFetchFailure(cfg, fmt.Errorf("could not list AWS regions for coverage fetch: %w", err)) } @@ -65,6 +68,9 @@ func fetchExistingCoverage(ctx context.Context, awsCfg aws.Config, recClient pro } AppLogger.Printf("\nšŸ”Ž Fetching existing-RI coverage from Cost Explorer per-account across %d regions (lookback %d days)...\n", len(regions), lookbackDays) cov, err := adapter.GetRICoverageMap(ctx, lookbackDays, regions) + if ctxErr := ctx.Err(); ctxErr != nil { + return nil, ctxErr + } if err != nil { return nil, coverageFetchFailure(cfg, fmt.Errorf("could not fetch existing-RI coverage: %w", err)) } @@ -168,7 +174,7 @@ func runToolMultiService(ctx context.Context, cfg Config) { if adapter, ok := recClient.(*awsprovider.RecommendationsClientAdapter); ok && cfg.RecLookbackPeriod != "" { adapter.SetRecLookbackPeriod(cfg.RecLookbackPeriod) } - engineData := fetchEngineVersionData(ctx, cfg) + engineData := mustFetchEngineVersionData(ctx, cfg) // Fetch existing-RI coverage so --target-coverage can subtract what // the user already owns. On a dry run, a failure logs a warning and @@ -185,6 +191,7 @@ func runToolMultiService(ctx context.Context, cfg Config) { // Phase 1: collect all recommendations without purchasing. AppLogger.Printf("\nšŸ“„ Fetching recommendations from all services...\n") allRecs, drops := fetchAllRecs(ctx, awsCfg, recClient, accountCache, servicesToProcess, engineData, cfg, coverageMap) + exitIfInterrupted(ctx, "fetching recommendations") // Phase 2: score, enforce the run-wide instance cap, and display. scoredResult := scoreLimitAndDisplay(allRecs, cfg, drops) @@ -198,6 +205,25 @@ func runToolMultiService(ctx context.Context, cfg Config) { runPurchaseAndReport(ctx, awsCfg, scoredResult, isDryRun, cfg, drops) } +// mustFetchEngineVersionData is fetchEngineVersionData for the main pipeline: +// an interrupt during the lifecycle queries ends the run, since sizing and +// offering purchases on missing lifecycle data would be wrong. +func mustFetchEngineVersionData(ctx context.Context, cfg Config) engineVersionData { + data, err := fetchEngineVersionData(ctx, cfg) + if err != nil { + log.Fatalf("Interrupted while querying engine versions (%v); nothing was purchased", err) + } + return data +} + +// exitIfInterrupted ends the run when the invocation was canceled during a +// pre-purchase phase. Nothing has been bought at that point. +func exitIfInterrupted(ctx context.Context, phase string) { + if ctx.Err() != nil { + log.Fatalf("Interrupted while %s; nothing was purchased", phase) + } +} + // runPurchaseAndReport handles the confirm, execute, and report phases of // the multi-service pipeline. It is a separate function to keep // runToolMultiService within the cyclomatic-complexity limit. @@ -609,7 +635,7 @@ func prepareCSVPurchaseRun(ctx context.Context, cfg Config, csvModeCoverage floa return nil, aws.Config{}, "", err } - recs, err = filterAndAdjustRecommendations(recs, csvModeCoverage, cfg) + recs, err = filterAndAdjustRecommendations(ctx, recs, csvModeCoverage, cfg) if err != nil { return nil, aws.Config{}, "", err } @@ -787,6 +813,10 @@ func checkDuplicatesForCSVRegion(ctx context.Context, recs []common.Recommendati if err == nil { return adjustedRecs, true } + if ctx.Err() != nil { + // Interrupted: terminal in dry and real runs; runToolFromCSV stops on ctx.Err(). + return nil, false + } if !isDryRun { AppLogger.Printf(" āŒ Refusing to purchase %s/%s: could not check for existing RIs (%v). Skipping this region rather than risking a duplicate purchase.\n", getServiceDisplayName(service), region, err) return nil, false @@ -801,10 +831,13 @@ func checkDuplicatesForCSVRegion(ctx context.Context, recs []common.Recommendati // // It returns an error when the cap cannot be enforced honestly; see // requireRankingSignal. -func filterAndAdjustRecommendations(recs []common.Recommendation, csvModeCoverage float64, cfg Config) ([]common.Recommendation, error) { +func filterAndAdjustRecommendations(ctx context.Context, recs []common.Recommendation, csvModeCoverage float64, cfg Config) ([]common.Recommendation, error) { // Query running instances for engine version validation log.Printf("šŸ” Querying running RDS instances across all regions to validate engine versions...") - instanceVersions, err := queryRunningInstanceEngineVersions(context.Background(), cfg) + instanceVersions, err := queryRunningInstanceEngineVersions(ctx, cfg) + if ctxErr := ctx.Err(); ctxErr != nil { + return nil, ctxErr + } if err != nil { log.Printf("āš ļø Warning: Failed to query running instances for engine version validation: %v", err) log.Printf(" Continuing without engine version filtering") @@ -815,7 +848,10 @@ func filterAndAdjustRecommendations(recs []common.Recommendation, csvModeCoverag // Query major engine versions for extended support detection log.Printf("šŸ” Querying AWS RDS major engine versions for extended support information...") - versionInfo, err := queryMajorEngineVersions(context.Background(), cfg) + versionInfo, err := queryMajorEngineVersions(ctx, cfg) + if ctxErr := ctx.Err(); ctxErr != nil { + return nil, ctxErr + } if err != nil { log.Printf("āš ļø Warning: Failed to query major engine versions: %v", err) log.Printf(" Continuing without extended support detection") diff --git a/cmd/multi_service_coverage_test.go b/cmd/multi_service_coverage_test.go index 30cea2dfa..2db2a421b 100644 --- a/cmd/multi_service_coverage_test.go +++ b/cmd/multi_service_coverage_test.go @@ -730,7 +730,7 @@ func TestFilterAndAdjustRecommendations_ZeroCoverage(t *testing.T) { toolCfg.MaxInstances = 0 toolCfg.OverrideCount = 0 - result, err := filterAndAdjustRecommendations(recommendations, 0.0, toolCfg) + result, err := filterAndAdjustRecommendations(context.Background(), recommendations, 0.0, toolCfg) require.NoError(t, err) // 0% coverage should return empty @@ -757,7 +757,7 @@ func TestFilterAndAdjustRecommendations_WithEngineVersionFiltering(t *testing.T) toolCfg.OverrideCount = 0 toolCfg.IncludeExtendedSupport = false - result, err := filterAndAdjustRecommendations(recommendations, 100.0, toolCfg) + result, err := filterAndAdjustRecommendations(context.Background(), recommendations, 100.0, toolCfg) require.NoError(t, err) // Should return recommendations (engine version filtering is done inside the function) @@ -779,7 +779,7 @@ func TestFilterAndAdjustRecommendations_MaxInstancesApplied(t *testing.T) { toolCfg.MaxInstances = 15 toolCfg.OverrideCount = 0 - result, err := filterAndAdjustRecommendations(recommendations, 100.0, toolCfg) + result, err := filterAndAdjustRecommendations(context.Background(), recommendations, 100.0, toolCfg) require.NoError(t, err) // Total instances should not exceed maxInstances @@ -802,7 +802,7 @@ func TestFilterAndAdjustRecommendations_OverrideCountApplied(t *testing.T) { toolCfg.MaxInstances = 0 toolCfg.OverrideCount = 5 - result, err := filterAndAdjustRecommendations(recommendations, 100.0, toolCfg) + result, err := filterAndAdjustRecommendations(context.Background(), recommendations, 100.0, toolCfg) require.NoError(t, err) // All recommendations should have count = OverrideCount diff --git a/cmd/multi_service_helpers.go b/cmd/multi_service_helpers.go index d490afc2b..a4d170744 100644 --- a/cmd/multi_service_helpers.go +++ b/cmd/multi_service_helpers.go @@ -284,6 +284,9 @@ func determineRegionsForService(ctx context.Context, awsCfg aws.Config, recClien // handleRegionDiscoveryError handles errors during region discovery by falling back to auto-discovery. func handleRegionDiscoveryError(ctx context.Context, recClient provider.RecommendationsClient, service common.ServiceType, originalErr error) ([]string, error) { + if ctxErr := ctx.Err(); ctxErr != nil { + return nil, ctxErr + } AppLogger.Printf("āŒ Failed to get AWS regions: %v\n", originalErr) AppLogger.Printf("šŸ” Falling back to auto-discovery...\n") @@ -302,47 +305,54 @@ type engineVersionData struct { } // fetchEngineVersionData queries running instances and major engine versions for validation. -func fetchEngineVersionData(ctx context.Context, cfg Config) engineVersionData { - data := engineVersionData{ - instanceVersions: make(map[string][]InstanceEngineVersion), - versionInfo: make(map[string]MajorEngineVersionInfo), +// A canceled or expired ctx is terminal and returned as an error; any other +// query failure is a warning and leaves that map empty. +func fetchEngineVersionData(ctx context.Context, cfg Config) (engineVersionData, error) { + instanceVersions, err := queryInstanceVersions(ctx, cfg) + if err != nil { + return engineVersionData{}, err } - - // Query running instances for engine version validation - data.instanceVersions = queryInstanceVersions(ctx, cfg) - - // Query major engine versions for extended support detection - data.versionInfo = queryMajorVersions(ctx, cfg) - - return data + versionInfo, err := queryMajorVersions(ctx, cfg) + if err != nil { + return engineVersionData{}, err + } + return engineVersionData{instanceVersions: instanceVersions, versionInfo: versionInfo}, nil } // queryInstanceVersions queries running instances for engine version validation. -func queryInstanceVersions(ctx context.Context, cfg Config) map[string][]InstanceEngineVersion { +func queryInstanceVersions(ctx context.Context, cfg Config) (map[string][]InstanceEngineVersion, error) { AppLogger.Printf("šŸ” Querying running RDS instances across all regions to validate engine versions...\n") instanceVersions, err := queryRunningInstanceEngineVersions(ctx, cfg) + // Checked on ctx, not on err: a canceled run can return partial data + // with a nil error, and that is not a completed query. + if ctxErr := ctx.Err(); ctxErr != nil { + return nil, ctxErr + } if err != nil { AppLogger.Printf("āš ļø Warning: Failed to query running instances for engine version validation: %v\n", err) AppLogger.Printf(" Continuing without engine version filtering\n") - return make(map[string][]InstanceEngineVersion) + return make(map[string][]InstanceEngineVersion), nil } AppLogger.Printf("āœ… Found %d instance types with version information across all regions\n", len(instanceVersions)) - return instanceVersions + return instanceVersions, nil } // queryMajorVersions queries major engine versions for extended support detection. -func queryMajorVersions(ctx context.Context, cfg Config) map[string]MajorEngineVersionInfo { +func queryMajorVersions(ctx context.Context, cfg Config) (map[string]MajorEngineVersionInfo, error) { AppLogger.Printf("šŸ” Querying AWS RDS major engine versions for extended support information...\n") versionInfo, err := queryMajorEngineVersions(ctx, cfg) + if ctxErr := ctx.Err(); ctxErr != nil { + return nil, ctxErr + } if err != nil { AppLogger.Printf("āš ļø Warning: Failed to query major engine versions: %v\n", err) AppLogger.Printf(" Continuing without extended support detection\n") - return make(map[string]MajorEngineVersionInfo) + return make(map[string]MajorEngineVersionInfo), nil } AppLogger.Printf("āœ… Found support information for %d major engine versions\n", len(versionInfo)) - return versionInfo + return versionInfo, nil } // regionRecommendations holds the processed recommendations for a single region. @@ -490,6 +500,10 @@ func fetchRecommendationsForRegion( } return recs } + if ctx.Err() != nil { + // Interrupted, not a provider failure: the caller stops on ctx.Err(). + return nil + } if err != nil { AppLogger.Printf(" āŒ Failed to fetch recommendations: %v\n", err) return nil @@ -619,6 +633,11 @@ func checkDuplicates( duplicateChecker := NewDuplicateChecker(toolCfg.IdempotencyWindowHours) adjustedRecs, dedupedOut, err := duplicateChecker.AdjustRecommendationsForExistingRIs(ctx, filteredRecs, serviceClient) if err != nil { + if ctx.Err() != nil { + // Interrupted: terminal in dry and real runs alike, never + // "continue with un-deduplicated counts". The caller stops on ctx.Err(). + return nil + } if !isDryRun { AppLogger.Printf(" āŒ Refusing to purchase %d instance(s): could not check for existing RIs (%v). Dropping these recommendations rather than risking a duplicate purchase.\n", CalculateTotalInstances(filteredRecs), err) drops.Add(dropDuplicateCheckFailed, len(filteredRecs)) @@ -730,10 +749,16 @@ func fetchAllRecs( regions, err := determineRegionsForService(ctx, awsCfg, recClient, service, cfg.Regions) if err != nil { + if ctx.Err() != nil { + return nil, drops + } log.Printf("āŒ Failed to determine regions for %s: %v", getServiceDisplayName(service), err) continue } for i, region := range regions { + if ctx.Err() != nil { + return nil, drops // partial data from an interrupted fetch is not a result + } recs := fetchAndFilterRegionRecs(ctx, awsCfg, recClient, accountCache, service, region, i+1, len(regions), engineData, cfg, coverageMap, drops) all = append(all, recs...) } diff --git a/cmd/multi_service_max_instances_test.go b/cmd/multi_service_max_instances_test.go index 0da06c8f1..e6840bb80 100644 --- a/cmd/multi_service_max_instances_test.go +++ b/cmd/multi_service_max_instances_test.go @@ -494,7 +494,7 @@ rds,us-east-1,db.t3.large,postgres,6,100.00,1yr,All Upfront,123456789012 err error ) out := captureAppOutput(t, func() { - got, err = filterAndAdjustRecommendations(recs, 100.0, Config{MaxInstances: maxInstances}) + got, err = filterAndAdjustRecommendations(context.Background(), recs, 100.0, Config{MaxInstances: maxInstances}) }) require.NoError(t, err) @@ -546,7 +546,7 @@ rds,us-east-1,db.t3.medium,postgres,6,500.00,1yr,All Upfront,123456789012 err error ) out := captureAppOutput(t, func() { - got, err = filterAndAdjustRecommendations(recs, 100.0, Config{MaxInstances: maxInstances, MinCount: minCount}) + got, err = filterAndAdjustRecommendations(context.Background(), recs, 100.0, Config{MaxInstances: maxInstances, MinCount: minCount}) }) require.NoError(t, err) @@ -580,7 +580,7 @@ rds,us-east-1,db.t3.medium,postgres,6,100.00,1yr,All Upfront,123456789012 err error ) out := captureAppOutput(t, func() { - got, err = filterAndAdjustRecommendations(recs, 100.0, Config{MinCount: minCount}) + got, err = filterAndAdjustRecommendations(context.Background(), recs, 100.0, Config{MinCount: minCount}) }) require.NoError(t, err) @@ -697,7 +697,7 @@ rds,us-east-1,db.rich.large,postgres,6,500.00,1yr,All Upfront,123456789012 err error ) out := captureAppOutput(t, func() { - got, err = filterAndAdjustRecommendations(recs, 100.0, Config{MaxInstances: maxInstances}) + got, err = filterAndAdjustRecommendations(context.Background(), recs, 100.0, Config{MaxInstances: maxInstances}) }) require.NoError(t, err) @@ -743,7 +743,7 @@ rds,us-east-1,db.aaa.large,postgres,6,1yr,All Upfront,123456789012 var err error captureAppOutput(t, func() { - _, err = filterAndAdjustRecommendations(recs, 100.0, Config{MaxInstances: 6}) + _, err = filterAndAdjustRecommendations(context.Background(), recs, 100.0, Config{MaxInstances: 6}) }) require.Error(t, err, "a binding cap with nothing to rank on must refuse, not pick by name") @@ -762,7 +762,7 @@ rds,us-east-1,db.t3.small,postgres,6,,1yr,All Upfront,123456789012 var err error captureAppOutput(t, func() { - _, err = filterAndAdjustRecommendations(recs, 100.0, Config{MaxInstances: 10}) + _, err = filterAndAdjustRecommendations(context.Background(), recs, 100.0, Config{MaxInstances: 10}) }) require.Error(t, err) @@ -783,7 +783,7 @@ rds,us-east-1,db.aaa.large,postgres,6,1yr,All Upfront,123456789012 err error ) captureAppOutput(t, func() { - got, err = filterAndAdjustRecommendations(recs, 100.0, Config{MaxInstances: 12}) + got, err = filterAndAdjustRecommendations(context.Background(), recs, 100.0, Config{MaxInstances: 12}) }) require.NoError(t, err) @@ -801,7 +801,7 @@ rds,us-east-1,db.t3.medium,postgres,6,500.00,1yr,All Upfront,123456789012 err error ) captureAppOutput(t, func() { - got, err = filterAndAdjustRecommendations(recs, 100.0, Config{MaxInstances: 6}) + got, err = filterAndAdjustRecommendations(context.Background(), recs, 100.0, Config{MaxInstances: 6}) }) require.NoError(t, err) @@ -885,7 +885,7 @@ func TestCSVCapOrderIsIndependentOfFileOrder(t *testing.T) { err error ) captureAppOutput(t, func() { - got, err = filterAndAdjustRecommendations(recs, 100.0, Config{MaxInstances: 6}) + got, err = filterAndAdjustRecommendations(context.Background(), recs, 100.0, Config{MaxInstances: 6}) }) require.NoError(t, err) diff --git a/cmd/multi_service_test.go b/cmd/multi_service_test.go index ce53e8e34..99fb48b46 100644 --- a/cmd/multi_service_test.go +++ b/cmd/multi_service_test.go @@ -1604,7 +1604,7 @@ func TestFilterAndAdjustRecommendations(t *testing.T) { // Suppress logger // Logger output disabled for testing - result, err := filterAndAdjustRecommendations(tt.recommendations, tt.coverage, toolCfg) + result, err := filterAndAdjustRecommendations(context.Background(), tt.recommendations, tt.coverage, toolCfg) require.NoError(t, err) // Verify result is within expected range From 9ebfcaf5a390a6b35eb859252e96030a9e55b8a3 Mon Sep 17 00:00:00 2001 From: Cristian Magherusan-Stanciu Date: Sat, 10 Oct 2026 05:11:47 +0200 Subject: [PATCH 3/3] test(cmd): match PurchaseCommitment mocks on any context Purchase calls now run on a context detached from cancellation, so mocks that matched the exact parent context no longer match. Refs #2151 Co-Authored-By: Claude Sonnet 5.5 --- cmd/multi_service_helpers_test.go | 2 +- cmd/multi_service_test.go | 12 ++++++------ cmd/purchase_safeguards_2121_test.go | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/cmd/multi_service_helpers_test.go b/cmd/multi_service_helpers_test.go index 494e2bf5e..e5088a8c4 100644 --- a/cmd/multi_service_helpers_test.go +++ b/cmd/multi_service_helpers_test.go @@ -340,7 +340,7 @@ func TestExecutePurchase(t *testing.T) { Timestamp: time.Now(), } var capturedOpts common.PurchaseOptions - mockClient.On("PurchaseCommitment", ctx, rec, mock.MatchedBy(func(o common.PurchaseOptions) bool { + mockClient.On("PurchaseCommitment", mock.Anything, rec, mock.MatchedBy(func(o common.PurchaseOptions) bool { capturedOpts = o return o.Source == common.PurchaseSourceCLI })).Return(expectedResult, nil) diff --git a/cmd/multi_service_test.go b/cmd/multi_service_test.go index 99fb48b46..2d54e0873 100644 --- a/cmd/multi_service_test.go +++ b/cmd/multi_service_test.go @@ -1278,7 +1278,7 @@ func TestProcessPurchaseLoopPurchaseFailure(t *testing.T) { Error: fmt.Errorf("API error: quota exceeded"), Timestamp: time.Now(), } - mockClient.On("PurchaseCommitment", ctx, recs[0], mock.MatchedBy(func(o common.PurchaseOptions) bool { return o.Source == common.PurchaseSourceCLI })).Return(failureResult, nil) + mockClient.On("PurchaseCommitment", mock.Anything, recs[0], mock.MatchedBy(func(o common.PurchaseOptions) bool { return o.Source == common.PurchaseSourceCLI })).Return(failureResult, nil) t.Setenv("DISABLE_PURCHASE_DELAY", "true") @@ -1326,7 +1326,7 @@ func TestProcessServicePurchasesUserCancellation(t *testing.T) { CommitmentID: "cache-purchase-123", Timestamp: time.Now(), } - mockClient.On("PurchaseCommitment", ctx, recs[0], mock.MatchedBy(func(o common.PurchaseOptions) bool { return o.Source == common.PurchaseSourceCLI })).Return(result, nil) + mockClient.On("PurchaseCommitment", mock.Anything, recs[0], mock.MatchedBy(func(o common.PurchaseOptions) bool { return o.Source == common.PurchaseSourceCLI })).Return(result, nil) t.Setenv("DISABLE_PURCHASE_DELAY", "true") @@ -1398,7 +1398,7 @@ func TestExecutePurchaseWithEmptyPurchaseID(t *testing.T) { Error: nil, Timestamp: time.Now(), } - mockClient.On("PurchaseCommitment", ctx, rec, mock.MatchedBy(func(o common.PurchaseOptions) bool { return o.Source == common.PurchaseSourceCLI })).Return(expectedResult, nil) + mockClient.On("PurchaseCommitment", mock.Anything, rec, mock.MatchedBy(func(o common.PurchaseOptions) bool { return o.Source == common.PurchaseSourceCLI })).Return(expectedResult, nil) // Logger output disabled for testing @@ -1472,7 +1472,7 @@ func TestProcessPurchaseLoopActualPurchase(t *testing.T) { Error: nil, Timestamp: time.Now(), } - mockClient.On("PurchaseCommitment", ctx, rec, mock.MatchedBy(func(o common.PurchaseOptions) bool { return o.Source == common.PurchaseSourceCLI })).Return(result, nil) + mockClient.On("PurchaseCommitment", mock.Anything, rec, mock.MatchedBy(func(o common.PurchaseOptions) bool { return o.Source == common.PurchaseSourceCLI })).Return(result, nil) } // Logger output disabled for testing @@ -1518,7 +1518,7 @@ func TestProcessPurchaseLoopWithConfirmation(t *testing.T) { Error: nil, Timestamp: time.Now(), } - mockClient.On("PurchaseCommitment", ctx, recs[0], mock.MatchedBy(func(o common.PurchaseOptions) bool { return o.Source == common.PurchaseSourceCLI })).Return(result, nil) + mockClient.On("PurchaseCommitment", mock.Anything, recs[0], mock.MatchedBy(func(o common.PurchaseOptions) bool { return o.Source == common.PurchaseSourceCLI })).Return(result, nil) // Logger output disabled for testing @@ -1984,7 +1984,7 @@ func TestProcessPurchaseLoop_WritesAuditRecordForRealPurchase(t *testing.T) { result.Recommendation = recs[0] mockClient := &MockServiceClient{} - mockClient.On("PurchaseCommitment", ctx, recs[0], mock.MatchedBy(func(o common.PurchaseOptions) bool { return o.Source == common.PurchaseSourceCLI })).Return(result, nil) + mockClient.On("PurchaseCommitment", mock.Anything, recs[0], mock.MatchedBy(func(o common.PurchaseOptions) bool { return o.Source == common.PurchaseSourceCLI })).Return(result, nil) processPurchaseLoop(ctx, recs, "us-east-1", false /* isDryRun */, mockClient, toolCfg, "test-run-id") diff --git a/cmd/purchase_safeguards_2121_test.go b/cmd/purchase_safeguards_2121_test.go index e1d4ed8e1..c44f76eab 100644 --- a/cmd/purchase_safeguards_2121_test.go +++ b/cmd/purchase_safeguards_2121_test.go @@ -173,7 +173,7 @@ func TestExecutePurchase_PreservesExplicitZeroVsAbsentCost_2121(t *testing.T) { t.Run("explicit zero upfront cost stays a non-nil zero", func(t *testing.T) { zero := 0.0 mockClient := &MockServiceClient{} - mockClient.On("PurchaseCommitment", ctx, rec, mock.Anything). + mockClient.On("PurchaseCommitment", mock.Anything, rec, mock.Anything). Return(common.PurchaseResult{Recommendation: rec, Success: true, Cost: &zero}, nil) result := executePurchase(ctx, rec, rec.Region, 1, mockClient, toolCfg) require.True(t, result.Success) @@ -185,7 +185,7 @@ func TestExecutePurchase_PreservesExplicitZeroVsAbsentCost_2121(t *testing.T) { }) t.Run("absent upfront cost stays nil", func(t *testing.T) { mockClient := &MockServiceClient{} - mockClient.On("PurchaseCommitment", ctx, rec, mock.Anything). + mockClient.On("PurchaseCommitment", mock.Anything, rec, mock.Anything). Return(common.PurchaseResult{Recommendation: rec, Success: true, Cost: nil}, nil) result := executePurchase(ctx, rec, rec.Region, 1, mockClient, toolCfg) require.True(t, result.Success)