fix(dora): stream the diff-based deployment fetch in calculateChangeLeadTime

backend/plugins/dora/e2e/change_lead_time/cicd_deployment_commits.csv

@@ -15,4 +15,6 @@ id,result,started_date,duration_sec,cicd_deployment_id,cicd_scope_id,repo_url,en
 14,FAILURE,2023-04-13T07:57:26.000+00:00,60,pipeline8,cicd3,REPO111,PRODUCTION,,,commit14,2023-4-13 7:57:26,2023-04-13T07:58:26.000+00:00
 15,SUCCESS,2023-04-13T07:57:45.000+00:00,60,pipeline9,cicd3,REPO111,PRODUCTION,,,commit15,2023-4-13 7:57:45,2023-04-13T07:58:45.000+00:00
 16,SUCCESS,2023-04-13T07:58:24.000+00:00,60,pipeline10,cicd3,REPO333,,,,commit16,2023-4-13 7:58:24,2023-04-13T07:59:24.000+00:00
-17,SUCCESS,2023-04-13T07:59:00.000+00:00,60,pipeline11,cicd1,REPO111,PRODUCTION,,project1,direct_commit1,2023-4-13 7:59:00,2023-04-13T08:00:00.000+00:00
+17,SUCCESS,2023-04-13T07:59:00.000+00:00,60,pipeline11,cicd1,REPO111,PRODUCTION,,project1,direct_commit1,2023-4-13 7:59:00,2023-04-13T08:00:00.000+00:00
+18,SUCCESS,2023-04-13T08:10:00.000+00:00,60,pipeline12,cicd1,REPO111,PRODUCTION,17,project1,commit18,2023-4-13 8:10:00,2023-04-13T08:11:00.000+00:00
+19,SUCCESS,2023-04-13T08:05:00.000+00:00,60,pipeline13,cicd2,REPO111,PRODUCTION,13,,commit19,2023-4-13 8:05:00,2023-04-13T08:06:00.000+00:00

backend/plugins/dora/e2e/change_lead_time/commits_diffs.csv

@@ -5,3 +5,5 @@ commit2,commit6,pr_merge_commit3
 commit7,commit8,pr_merge_commit4
 ,commit11,pr_merge_commit5
 ,commit1,pr_merge_commit0
+direct_commit1,commit18,pr_merge_commit8
+commit13,commit19,pr_merge_commit8

backend/plugins/dora/e2e/change_lead_time/project_pr_metrics.csv

@@ -5,4 +5,5 @@ pr2,project1,2537845559d8db99e9cda6190f32b50ec979c722,,comment04,1,60,5,1538,159
 pr3,project1,55f445997abbd5918da59d202d28762cd56fbd44,5883,comment07,,5760,6,,10203,2023-04-07T04:51:47.000+00:00,2023-04-10T06:53:51.000+00:00,2023-04-11T06:53:51.000+00:00,2023-04-14T06:53:51.000+00:00,2023-04-13T07:30:34.000+00:00
 pr4,project1,5ad0c09c447c19338f1dfbb65d89a3728962b3b7,11704,comment10,1500,,,,11764,2023-04-05T04:51:47.000+00:00,2023-04-14T08:55:01.000+00:00,2023-04-13T07:55:01.000+00:00,2023-04-13T08:55:01.000+00:00,
 pr5,project1,62535543802631a0d3daf0b0b78c6a7e05e508fb,13144,comment12,,313068,,,13204,2023-04-04T04:51:47.000+00:00,2022-09-07T23:07:13.000+00:00,2023-04-13T07:55:01.000+00:00,2023-04-13T08:55:01.000+00:00,
-pr7,project1,pr7_commit0,1440,comment13,30,30,9,1433,2933,2023-04-11T07:00:00.000+00:00,2023-04-12T07:30:00.000+00:00,2023-04-12T07:00:00.000+00:00,2023-04-12T08:00:00.000+00:00,2023-04-13T07:52:26.000+00:00
+pr7,project1,pr7_commit0,1440,comment13,30,30,9,1433,2933,2023-04-11T07:00:00.000+00:00,2023-04-12T07:30:00.000+00:00,2023-04-12T07:00:00.000+00:00,2023-04-12T08:00:00.000+00:00,2023-04-13T07:52:26.000+00:00
+pr8,project1,pr8_commit0,1440,comment14,30,30,19,66,1566,2023-04-12T06:00:00.000+00:00,2023-04-13T06:30:00.000+00:00,2023-04-13T06:00:00.000+00:00,2023-04-13T07:00:00.000+00:00,2023-04-13T08:06:00.000+00:00

backend/plugins/dora/e2e/change_lead_time/pull_request_comments.csv

@@ -13,4 +13,5 @@ comment09,pr3,2023-4-12 6:53:51,i
 comment10,pr4,2023-4-14 8:55:01,j
 comment11,pr4,2023-4-14 8:55:01,k
 comment12,pr5,2022-09-07 23:07:13,l
-comment13,pr7,2023-4-12 7:30:00,m
+comment13,pr7,2023-4-12 7:30:00,m
+comment14,pr8,2023-4-13 6:30:00,n

backend/plugins/dora/e2e/change_lead_time/pull_request_commits.csv

@@ -12,4 +12,5 @@ pr0_commit0,pr0,2022-1-10 4:51:47,
 56b895f0443730c6d7abfbc51a05ab35abd2971f,pr4,2023-4-06 4:51:47,
 5ad0c09c447c19338f1dfbb65d89a3728962b3b7,pr4,2023-4-05 4:51:47,
 62535543802631a0d3daf0b0b78c6a7e05e508fb,pr5,2023-4-04 4:51:47,
-pr7_commit0,pr7,2023-4-11 7:00:00,
+pr7_commit0,pr7,2023-4-11 7:00:00,
+pr8_commit0,pr8,2023-4-12 6:00:00,

backend/plugins/dora/e2e/change_lead_time/pull_requests.csv

@@ -7,3 +7,4 @@ pr4,repo1,,pr_merge_commit4,2023-4-13 7:55:01,2023-4-13 8:55:01,,,,refs/heads/fe
 pr5,repo1,,pr_merge_commit5,2023-4-13 7:55:01,2023-4-13 8:55:01,,,,
 pr6,repo1,,pr_merge_commit6,2023-4-13 7:55:01,,,,,feat/e
 pr7,repo1,a,commit9,2023-4-12 7:00:00,2023-4-12 8:00:00,,,,hotfix/g
+pr8,repo1,a,pr_merge_commit8,2023-4-13 6:00:00,2023-4-13 7:00:00,,,,chore/h

backend/plugins/dora/tasks/change_lead_time_calculator.go

@@ -192,11 +192,13 @@ func computeTimeSpan(start, end *time.Time) *int64 {
 	return &minutes
 }
 
-// deploymentCommitWithMergeSha is a helper struct to capture both the deployment commit
-// and the associated merge_sha from the commits_diffs join query.
-type deploymentCommitWithMergeSha struct {
-	devops.CicdDeploymentCommit
-	MergeSha string `gorm:"column:merge_sha"`
+// diffDeployment holds the columns of a diff-based deployment match that the
+// lead time conversion reads: the deployment commit id, its finish date, and
+// the merge commit found in its commits_diffs range.
+type diffDeployment struct {
+	Id           string
+	FinishedDate *time.Time
+	MergeSha     string `gorm:"column:merge_sha"`
 }
 
 // batchFetchFirstCommits retrieves the first commit for all pull requests in the given project.
@@ -289,7 +291,8 @@ func batchFetchFirstReviews(projectName string, db dal.Dal) (map[string]*code.Pu
 }
 
 // batchFetchDeployments retrieves deployment commits for all merge commits in the given project.
-// Returns a map indexed by merge commit SHA for O(1) lookup performance.
+// Returns a map indexed by merge commit SHA for O(1) lookup performance. The map values carry
+// only the deployment commit Id and FinishedDate, the two fields the lead time conversion reads.
 //
 // Uses a two-phase strategy to avoid the "first deployment over-mapping" problem:
 //
@@ -303,7 +306,7 @@ func batchFetchDeployments(projectName string, db dal.Dal) (map[string]*devops.C
 	var directResults []*devops.CicdDeploymentCommit
 	err := db.All(
 		&directResults,
-		dal.Select("dc.*"),
+		dal.Select("dc.id, dc.finished_date, dc.commit_sha"),
 		dal.From("cicd_deployment_commits dc"),
 		dal.Join("LEFT JOIN project_mapping pm ON pm.table = 'cicd_scopes' AND pm.row_id = dc.cicd_scope_id"),
 		dal.Where("dc.environment = 'PRODUCTION'"), // TODO: remove when multi-environment is supported
@@ -319,10 +322,8 @@ func batchFetchDeployments(projectName string, db dal.Dal) (map[string]*devops.C
 			deploymentMap[dc.CommitSha] = &deploymentCopy
 		}
 	}
-	var diffResults []*deploymentCommitWithMergeSha
-	err = db.All(
-		&diffResults,
-		dal.Select("dc.*, cd.commit_sha as merge_sha"),
+	cursor, err := db.Cursor(
+		dal.Select("dc.id, dc.finished_date, cd.commit_sha as merge_sha"),
 		dal.From("cicd_deployment_commits dc"),
 		dal.Join("LEFT JOIN cicd_deployment_commits p ON dc.prev_success_deployment_commit_id = p.id"),
 		dal.Join("INNER JOIN commits_diffs cd ON cd.new_commit_sha = dc.commit_sha AND cd.old_commit_sha = COALESCE(p.commit_sha, '')"),
@@ -335,11 +336,24 @@ func batchFetchDeployments(projectName string, db dal.Dal) (map[string]*devops.C
 	if err != nil {
 		return nil, errors.Default.Wrap(err, "failed to batch fetch diff-based deployments")
 	}
-	for _, result := range diffResults {
-		if _, exists := deploymentMap[result.MergeSha]; !exists {
-			deploymentCopy := result.CicdDeploymentCommit
-			deploymentMap[result.MergeSha] = &deploymentCopy
+	defer cursor.Close()
+	// Rows are ordered by merge sha, then by deployment start, so the first row
+	// seen for a merge sha is its earliest deployment. Streaming keeps memory
+	// proportional to the distinct merge shas instead of the join size.
+	for cursor.Next() {
+		row := &diffDeployment{}
+		if err := db.Fetch(cursor, row); err != nil {
+			return nil, errors.Default.Wrap(err, "failed to read diff-based deployment")
+		}
+		if _, exists := deploymentMap[row.MergeSha]; !exists {
+			deployment := &devops.CicdDeploymentCommit{}
+			deployment.Id = row.Id
+			deployment.FinishedDate = row.FinishedDate
+			deploymentMap[row.MergeSha] = deployment
 		}
 	}
+	if err := cursor.Err(); err != nil {
+		return nil, errors.Default.Wrap(err, "failed to read diff-based deployments")
+	}
 	return deploymentMap, nil
 }