diff --git a/backend/plugins/dora/e2e/change_lead_time/cicd_deployment_commits.csv b/backend/plugins/dora/e2e/change_lead_time/cicd_deployment_commits.csv index 9d1e488f072..13bc22dd5da 100644 --- a/backend/plugins/dora/e2e/change_lead_time/cicd_deployment_commits.csv +++ b/backend/plugins/dora/e2e/change_lead_time/cicd_deployment_commits.csv @@ -15,4 +15,6 @@ id,result,started_date,duration_sec,cicd_deployment_id,cicd_scope_id,repo_url,en 14,FAILURE,2023-04-13T07:57:26.000+00:00,60,pipeline8,cicd3,REPO111,PRODUCTION,,,commit14,2023-4-13 7:57:26,2023-04-13T07:58:26.000+00:00 15,SUCCESS,2023-04-13T07:57:45.000+00:00,60,pipeline9,cicd3,REPO111,PRODUCTION,,,commit15,2023-4-13 7:57:45,2023-04-13T07:58:45.000+00:00 16,SUCCESS,2023-04-13T07:58:24.000+00:00,60,pipeline10,cicd3,REPO333,,,,commit16,2023-4-13 7:58:24,2023-04-13T07:59:24.000+00:00 -17,SUCCESS,2023-04-13T07:59:00.000+00:00,60,pipeline11,cicd1,REPO111,PRODUCTION,,project1,direct_commit1,2023-4-13 7:59:00,2023-04-13T08:00:00.000+00:00 \ No newline at end of file +17,SUCCESS,2023-04-13T07:59:00.000+00:00,60,pipeline11,cicd1,REPO111,PRODUCTION,,project1,direct_commit1,2023-4-13 7:59:00,2023-04-13T08:00:00.000+00:00 +18,SUCCESS,2023-04-13T08:10:00.000+00:00,60,pipeline12,cicd1,REPO111,PRODUCTION,17,project1,commit18,2023-4-13 8:10:00,2023-04-13T08:11:00.000+00:00 +19,SUCCESS,2023-04-13T08:05:00.000+00:00,60,pipeline13,cicd2,REPO111,PRODUCTION,13,,commit19,2023-4-13 8:05:00,2023-04-13T08:06:00.000+00:00 \ No newline at end of file diff --git a/backend/plugins/dora/e2e/change_lead_time/commits_diffs.csv b/backend/plugins/dora/e2e/change_lead_time/commits_diffs.csv index 440a68a4a40..f3979dc5b0f 100644 --- a/backend/plugins/dora/e2e/change_lead_time/commits_diffs.csv +++ b/backend/plugins/dora/e2e/change_lead_time/commits_diffs.csv @@ -5,3 +5,5 @@ commit2,commit6,pr_merge_commit3 commit7,commit8,pr_merge_commit4 ,commit11,pr_merge_commit5 ,commit1,pr_merge_commit0 +direct_commit1,commit18,pr_merge_commit8 +commit13,commit19,pr_merge_commit8 diff --git a/backend/plugins/dora/e2e/change_lead_time/project_pr_metrics.csv b/backend/plugins/dora/e2e/change_lead_time/project_pr_metrics.csv index 1d9c736a503..2354cfc6db9 100644 --- a/backend/plugins/dora/e2e/change_lead_time/project_pr_metrics.csv +++ b/backend/plugins/dora/e2e/change_lead_time/project_pr_metrics.csv @@ -5,4 +5,5 @@ pr2,project1,2537845559d8db99e9cda6190f32b50ec979c722,,comment04,1,60,5,1538,159 pr3,project1,55f445997abbd5918da59d202d28762cd56fbd44,5883,comment07,,5760,6,,10203,2023-04-07T04:51:47.000+00:00,2023-04-10T06:53:51.000+00:00,2023-04-11T06:53:51.000+00:00,2023-04-14T06:53:51.000+00:00,2023-04-13T07:30:34.000+00:00 pr4,project1,5ad0c09c447c19338f1dfbb65d89a3728962b3b7,11704,comment10,1500,,,,11764,2023-04-05T04:51:47.000+00:00,2023-04-14T08:55:01.000+00:00,2023-04-13T07:55:01.000+00:00,2023-04-13T08:55:01.000+00:00, pr5,project1,62535543802631a0d3daf0b0b78c6a7e05e508fb,13144,comment12,,313068,,,13204,2023-04-04T04:51:47.000+00:00,2022-09-07T23:07:13.000+00:00,2023-04-13T07:55:01.000+00:00,2023-04-13T08:55:01.000+00:00, -pr7,project1,pr7_commit0,1440,comment13,30,30,9,1433,2933,2023-04-11T07:00:00.000+00:00,2023-04-12T07:30:00.000+00:00,2023-04-12T07:00:00.000+00:00,2023-04-12T08:00:00.000+00:00,2023-04-13T07:52:26.000+00:00 \ No newline at end of file +pr7,project1,pr7_commit0,1440,comment13,30,30,9,1433,2933,2023-04-11T07:00:00.000+00:00,2023-04-12T07:30:00.000+00:00,2023-04-12T07:00:00.000+00:00,2023-04-12T08:00:00.000+00:00,2023-04-13T07:52:26.000+00:00 +pr8,project1,pr8_commit0,1440,comment14,30,30,19,66,1566,2023-04-12T06:00:00.000+00:00,2023-04-13T06:30:00.000+00:00,2023-04-13T06:00:00.000+00:00,2023-04-13T07:00:00.000+00:00,2023-04-13T08:06:00.000+00:00 \ No newline at end of file diff --git a/backend/plugins/dora/e2e/change_lead_time/pull_request_comments.csv b/backend/plugins/dora/e2e/change_lead_time/pull_request_comments.csv index cc8d062c8f4..e0a95b83452 100644 --- a/backend/plugins/dora/e2e/change_lead_time/pull_request_comments.csv +++ b/backend/plugins/dora/e2e/change_lead_time/pull_request_comments.csv @@ -13,4 +13,5 @@ comment09,pr3,2023-4-12 6:53:51,i comment10,pr4,2023-4-14 8:55:01,j comment11,pr4,2023-4-14 8:55:01,k comment12,pr5,2022-09-07 23:07:13,l -comment13,pr7,2023-4-12 7:30:00,m \ No newline at end of file +comment13,pr7,2023-4-12 7:30:00,m +comment14,pr8,2023-4-13 6:30:00,n \ No newline at end of file diff --git a/backend/plugins/dora/e2e/change_lead_time/pull_request_commits.csv b/backend/plugins/dora/e2e/change_lead_time/pull_request_commits.csv index e183d14cce3..544e2e8020c 100644 --- a/backend/plugins/dora/e2e/change_lead_time/pull_request_commits.csv +++ b/backend/plugins/dora/e2e/change_lead_time/pull_request_commits.csv @@ -12,4 +12,5 @@ pr0_commit0,pr0,2022-1-10 4:51:47, 56b895f0443730c6d7abfbc51a05ab35abd2971f,pr4,2023-4-06 4:51:47, 5ad0c09c447c19338f1dfbb65d89a3728962b3b7,pr4,2023-4-05 4:51:47, 62535543802631a0d3daf0b0b78c6a7e05e508fb,pr5,2023-4-04 4:51:47, -pr7_commit0,pr7,2023-4-11 7:00:00, \ No newline at end of file +pr7_commit0,pr7,2023-4-11 7:00:00, +pr8_commit0,pr8,2023-4-12 6:00:00, \ No newline at end of file diff --git a/backend/plugins/dora/e2e/change_lead_time/pull_requests.csv b/backend/plugins/dora/e2e/change_lead_time/pull_requests.csv index a83370ab06f..52c068eb8f5 100644 --- a/backend/plugins/dora/e2e/change_lead_time/pull_requests.csv +++ b/backend/plugins/dora/e2e/change_lead_time/pull_requests.csv @@ -7,3 +7,4 @@ pr4,repo1,,pr_merge_commit4,2023-4-13 7:55:01,2023-4-13 8:55:01,,,,refs/heads/fe pr5,repo1,,pr_merge_commit5,2023-4-13 7:55:01,2023-4-13 8:55:01,,,, pr6,repo1,,pr_merge_commit6,2023-4-13 7:55:01,,,,,feat/e pr7,repo1,a,commit9,2023-4-12 7:00:00,2023-4-12 8:00:00,,,,hotfix/g +pr8,repo1,a,pr_merge_commit8,2023-4-13 6:00:00,2023-4-13 7:00:00,,,,chore/h diff --git a/backend/plugins/dora/tasks/change_lead_time_calculator.go b/backend/plugins/dora/tasks/change_lead_time_calculator.go index fdcf628b59e..e3b0a0cde2f 100644 --- a/backend/plugins/dora/tasks/change_lead_time_calculator.go +++ b/backend/plugins/dora/tasks/change_lead_time_calculator.go @@ -192,11 +192,13 @@ func computeTimeSpan(start, end *time.Time) *int64 { return &minutes } -// deploymentCommitWithMergeSha is a helper struct to capture both the deployment commit -// and the associated merge_sha from the commits_diffs join query. -type deploymentCommitWithMergeSha struct { - devops.CicdDeploymentCommit - MergeSha string `gorm:"column:merge_sha"` +// diffDeployment holds the columns of a diff-based deployment match that the +// lead time conversion reads: the deployment commit id, its finish date, and +// the merge commit found in its commits_diffs range. +type diffDeployment struct { + Id string + FinishedDate *time.Time + MergeSha string `gorm:"column:merge_sha"` } // batchFetchFirstCommits retrieves the first commit for all pull requests in the given project. @@ -289,7 +291,8 @@ func batchFetchFirstReviews(projectName string, db dal.Dal) (map[string]*code.Pu } // batchFetchDeployments retrieves deployment commits for all merge commits in the given project. -// Returns a map indexed by merge commit SHA for O(1) lookup performance. +// Returns a map indexed by merge commit SHA for O(1) lookup performance. The map values carry +// only the deployment commit Id and FinishedDate, the two fields the lead time conversion reads. // // Uses a two-phase strategy to avoid the "first deployment over-mapping" problem: // @@ -303,7 +306,7 @@ func batchFetchDeployments(projectName string, db dal.Dal) (map[string]*devops.C var directResults []*devops.CicdDeploymentCommit err := db.All( &directResults, - dal.Select("dc.*"), + dal.Select("dc.id, dc.finished_date, dc.commit_sha"), dal.From("cicd_deployment_commits dc"), dal.Join("LEFT JOIN project_mapping pm ON pm.table = 'cicd_scopes' AND pm.row_id = dc.cicd_scope_id"), dal.Where("dc.environment = 'PRODUCTION'"), // TODO: remove when multi-environment is supported @@ -319,10 +322,8 @@ func batchFetchDeployments(projectName string, db dal.Dal) (map[string]*devops.C deploymentMap[dc.CommitSha] = &deploymentCopy } } - var diffResults []*deploymentCommitWithMergeSha - err = db.All( - &diffResults, - dal.Select("dc.*, cd.commit_sha as merge_sha"), + cursor, err := db.Cursor( + dal.Select("dc.id, dc.finished_date, cd.commit_sha as merge_sha"), dal.From("cicd_deployment_commits dc"), dal.Join("LEFT JOIN cicd_deployment_commits p ON dc.prev_success_deployment_commit_id = p.id"), dal.Join("INNER JOIN commits_diffs cd ON cd.new_commit_sha = dc.commit_sha AND cd.old_commit_sha = COALESCE(p.commit_sha, '')"), @@ -335,11 +336,24 @@ func batchFetchDeployments(projectName string, db dal.Dal) (map[string]*devops.C if err != nil { return nil, errors.Default.Wrap(err, "failed to batch fetch diff-based deployments") } - for _, result := range diffResults { - if _, exists := deploymentMap[result.MergeSha]; !exists { - deploymentCopy := result.CicdDeploymentCommit - deploymentMap[result.MergeSha] = &deploymentCopy + defer cursor.Close() + // Rows are ordered by merge sha, then by deployment start, so the first row + // seen for a merge sha is its earliest deployment. Streaming keeps memory + // proportional to the distinct merge shas instead of the join size. + for cursor.Next() { + row := &diffDeployment{} + if err := db.Fetch(cursor, row); err != nil { + return nil, errors.Default.Wrap(err, "failed to read diff-based deployment") + } + if _, exists := deploymentMap[row.MergeSha]; !exists { + deployment := &devops.CicdDeploymentCommit{} + deployment.Id = row.Id + deployment.FinishedDate = row.FinishedDate + deploymentMap[row.MergeSha] = deployment } } + if err := cursor.Err(); err != nil { + return nil, errors.Default.Wrap(err, "failed to read diff-based deployments") + } return deploymentMap, nil }