From 8c11d3428f3edc541acc7d7af75e8eeb9fbb829e Mon Sep 17 00:00:00 2001 From: Wei Quan Date: Thu, 27 Aug 2026 14:16:08 +0200 Subject: [PATCH] Remove lifecycle_type_backfill job and related config --- app/jobs/runtime/lifecycle_type_backfill.rb | 98 ---------- config/cloud_controller.yml | 4 - lib/cloud_controller/clock/scheduler.rb | 4 +- .../config_schemas/clock_schema.rb | 4 - lib/cloud_controller/jobs.rb | 1 - .../runtime/lifecycle_type_backfill_spec.rb | 184 ------------------ .../cloud_controller/clock/scheduler_spec.rb | 7 - 7 files changed, 1 insertion(+), 301 deletions(-) delete mode 100644 app/jobs/runtime/lifecycle_type_backfill.rb delete mode 100644 spec/unit/jobs/runtime/lifecycle_type_backfill_spec.rb diff --git a/app/jobs/runtime/lifecycle_type_backfill.rb b/app/jobs/runtime/lifecycle_type_backfill.rb deleted file mode 100644 index 993a83908b0..00000000000 --- a/app/jobs/runtime/lifecycle_type_backfill.rb +++ /dev/null @@ -1,98 +0,0 @@ -# NOTE: This is a one-off backfill job. It populates the `lifecycle_type` -# column on `apps`, `droplets`, and `builds` for rows that pre-date the -# column's introduction. Once all installations have run it long enough to -# drain those rows, this job will be removed. -# -# Operators can also run `rake db:lifecycle_type_backfill` manually. - -module VCAP::CloudController - module Jobs - module Runtime - class LifecycleTypeBackfill < VCAP::CloudController::Jobs::CCJob - BATCH_SIZE = 1000 - BATCHES_PER_RUN = 10 - - TABLES = [ - { table: :apps, guid_column: :app_guid }, - { table: :droplets, guid_column: :droplet_guid }, - { table: :builds, guid_column: :build_guid } - ].freeze - - # Pass -1 for +batches_per_run+ to drain until no rows remain. - def initialize(batch_size: BATCH_SIZE, batches_per_run: BATCHES_PER_RUN) - super() - @batch_size = batch_size - @batches_per_run = batches_per_run - end - - def perform - TABLES.each { |t| backfill(**t) } - end - - def job_name_in_configuration - :lifecycle_type_backfill - end - - def max_attempts - 1 - end - - private - - def backfill(table:, guid_column:) - return unless column_exists?(table, :lifecycle_type) - - total_rows = 0 - remaining_batches = @batches_per_run - while remaining_batches != 0 # -1 means: drain until no rows remain - updated_rows = update_batch(table, guid_column) - total_rows += updated_rows - break if updated_rows < @batch_size - - remaining_batches -= 1 if remaining_batches > 0 - end - logger.info("lifecycle_type_backfill: updated #{total_rows} rows in #{table}") if total_rows > 0 - end - - def update_batch(table, guid_column) - guids = db[table].where(lifecycle_type: nil).limit(@batch_size).select_map(:guid) - return 0 if guids.empty? - - # If a row appears in both *_lifecycle_data tables (which it shouldn't), buildpack wins - # (matches the runtime fallback in {app,build,droplet}_model.rb#lifecycle_type). - guids_with_buildpack_lifecycle_data = db[:buildpack_lifecycle_data].where(guid_column => guids).select_map(guid_column) - guids_with_cnb_lifecycle_data = db[:cnb_lifecycle_data].where(guid_column => guids).select_map(guid_column) - guids_with_buildpack_lifecycle_data - guids_without_lifecycle_data = guids - guids_with_buildpack_lifecycle_data - guids_with_cnb_lifecycle_data - - db.transaction do - update_lifecycle(table, guids_with_buildpack_lifecycle_data, BuildpackLifecycleDataModel::LIFECYCLE_TYPE) - update_lifecycle(table, guids_with_cnb_lifecycle_data, CNBLifecycleDataModel::LIFECYCLE_TYPE) - update_lifecycle(table, guids_without_lifecycle_data, DockerLifecycleDataModel::LIFECYCLE_TYPE) - end - - guids.size - end - - def update_lifecycle(table, guids, value) - return if guids.empty? - - db[table].where(guid: guids, lifecycle_type: nil).update(lifecycle_type: value) - end - - def column_exists?(table, column) - db.schema(table, reload: true).map(&:first).include?(column) - rescue Sequel::Error - false - end - - def db - Sequel::Model.db - end - - def logger - @logger ||= Steno.logger('cc.background.lifecycle-type-backfill') - end - end - end - end -end diff --git a/config/cloud_controller.yml b/config/cloud_controller.yml index 30edab7f28b..63afb38df65 100644 --- a/config/cloud_controller.yml +++ b/config/cloud_controller.yml @@ -58,10 +58,6 @@ service_operations_initial_cleanup: service_operations_create_in_progress_cleanup: frequency_in_seconds: 3600 #1h -# One-off backfill - to be removed in a future version. -lifecycle_type_backfill: - frequency_in_seconds: 3600 #1h - completed_tasks: cutoff_age_in_days: 31 diff --git a/lib/cloud_controller/clock/scheduler.rb b/lib/cloud_controller/clock/scheduler.rb index e128e4db906..06c85526c55 100644 --- a/lib/cloud_controller/clock/scheduler.rb +++ b/lib/cloud_controller/clock/scheduler.rb @@ -25,9 +25,7 @@ class Scheduler { name: 'pending_builds', class: Jobs::Runtime::PendingBuildCleanup }, { name: 'failed_jobs', class: Jobs::Runtime::FailedJobsCleanup }, { name: 'service_operations_initial_cleanup', class: Jobs::Runtime::ServiceOperationsInitialCleanup }, - { name: 'service_operations_create_in_progress_cleanup', class: Jobs::Runtime::ServiceOperationsCreateInProgressCleanup }, - # One-off backfill - to be removed in a future version. - { name: 'lifecycle_type_backfill', class: Jobs::Runtime::LifecycleTypeBackfill } + { name: 'service_operations_create_in_progress_cleanup', class: Jobs::Runtime::ServiceOperationsCreateInProgressCleanup } ].freeze def self.queue_names diff --git a/lib/cloud_controller/config_schemas/clock_schema.rb b/lib/cloud_controller/config_schemas/clock_schema.rb index 79b4461b66c..db6290818fa 100644 --- a/lib/cloud_controller/config_schemas/clock_schema.rb +++ b/lib/cloud_controller/config_schemas/clock_schema.rb @@ -37,10 +37,6 @@ class ClockSchema < VCAP::Config service_operations_create_in_progress_cleanup: { frequency_in_seconds: Integer }, - # One-off backfill - to be removed in a future version. - lifecycle_type_backfill: { - frequency_in_seconds: Integer - }, default_health_check_timeout: Integer, uaa: { diff --git a/lib/cloud_controller/jobs.rb b/lib/cloud_controller/jobs.rb index eaeae0dc0d9..5dfb1f71d30 100644 --- a/lib/cloud_controller/jobs.rb +++ b/lib/cloud_controller/jobs.rb @@ -38,7 +38,6 @@ require 'jobs/runtime/prune_completed_deployments' require 'jobs/runtime/prune_completed_builds' require 'jobs/runtime/prune_excess_app_revisions' -require 'jobs/runtime/lifecycle_type_backfill' require 'jobs/v2/services/service_usage_events_cleanup' diff --git a/spec/unit/jobs/runtime/lifecycle_type_backfill_spec.rb b/spec/unit/jobs/runtime/lifecycle_type_backfill_spec.rb deleted file mode 100644 index ab793f5a96f..00000000000 --- a/spec/unit/jobs/runtime/lifecycle_type_backfill_spec.rb +++ /dev/null @@ -1,184 +0,0 @@ -require 'spec_helper' - -module VCAP::CloudController - module Jobs::Runtime - RSpec.describe LifecycleTypeBackfill, job_context: :worker do - subject(:job) { LifecycleTypeBackfill.new } - - let(:db) { Sequel::Model.db } - - it { is_expected.to be_a_valid_job } - - it 'knows its job name' do - expect(job.job_name_in_configuration).to eq(:lifecycle_type_backfill) - end - - it 'has max_attempts of 1' do - expect(job.max_attempts).to eq(1) - end - - describe '#perform' do - context 'when the lifecycle_type column is missing on every table' do - let!(:app) { create(:app_model) } - let!(:droplet) { create(:droplet_model) } - let!(:build) { create(:build_model) } - - before do - db[:apps].where(guid: app.guid).update(lifecycle_type: nil) - db[:droplets].where(guid: droplet.guid).update(lifecycle_type: nil) - db[:builds].where(guid: build.guid).update(lifecycle_type: nil) - allow(db).to receive(:schema).and_call_original - %i[apps droplets builds].each do |table| - allow(db).to receive(:schema).with(table, reload: true).and_return( - [[:guid, {}], [:name, {}]] - ) - end - end - - it 'does not issue any UPDATE statements' do - expect { job.perform }.to have_queried_db_times(/update .(apps|droplets|builds). set/i, 0) - end - - it 'leaves NULL rows untouched' do - job.perform - expect(db[:apps].where(guid: app.guid).get(:lifecycle_type)).to be_nil - expect(db[:droplets].where(guid: droplet.guid).get(:lifecycle_type)).to be_nil - expect(db[:builds].where(guid: build.guid).get(:lifecycle_type)).to be_nil - end - end - - context 'when no rows have NULL lifecycle_type on any table' do - before do - create(:app_model) - create(:droplet_model) - create(:build_model) - end - - it 'does not issue any UPDATE statements' do - expect { job.perform }.to have_queried_db_times(/update .(apps|droplets|builds). set/i, 0) - end - end - - context 'when there are apps with NULL lifecycle_type' do - let(:buildpack_app) { create(:app_model) } - let(:cnb_app) { create(:app_model, :cnb) } - let(:docker_app) { create(:app_model, :docker) } - - before do - db[:apps].where(guid: [buildpack_app.guid, cnb_app.guid, docker_app.guid]).update(lifecycle_type: nil) - end - - it 'sets lifecycle_type accordingly' do - job.perform - expect(db[:apps].where(guid: buildpack_app.guid).get(:lifecycle_type)).to eq(BuildpackLifecycleDataModel::LIFECYCLE_TYPE) - expect(db[:apps].where(guid: cnb_app.guid).get(:lifecycle_type)).to eq(CNBLifecycleDataModel::LIFECYCLE_TYPE) - expect(db[:apps].where(guid: docker_app.guid).get(:lifecycle_type)).to eq(DockerLifecycleDataModel::LIFECYCLE_TYPE) - end - - it 'does not touch updated_at' do - original_updated_at = db[:apps].where(guid: [buildpack_app.guid, cnb_app.guid, docker_app.guid]).select_map(%i[guid updated_at]).to_h - job.perform - expect(db[:apps].where(guid: [buildpack_app.guid, cnb_app.guid, docker_app.guid]).select_map(%i[guid updated_at]).to_h).to eq(original_updated_at) - end - end - - context 'when there are droplets with NULL lifecycle_type' do - let(:buildpack_droplet) { create(:droplet_model) } - let(:cnb_droplet) { create(:droplet_model, :cnb) } - let(:docker_droplet) { create(:droplet_model, :docker) } - - before do - db[:droplets].where(guid: [buildpack_droplet.guid, cnb_droplet.guid, docker_droplet.guid]).update(lifecycle_type: nil) - end - - it 'sets lifecycle_type accordingly' do - job.perform - expect(db[:droplets].where(guid: buildpack_droplet.guid).get(:lifecycle_type)).to eq(BuildpackLifecycleDataModel::LIFECYCLE_TYPE) - expect(db[:droplets].where(guid: cnb_droplet.guid).get(:lifecycle_type)).to eq(CNBLifecycleDataModel::LIFECYCLE_TYPE) - expect(db[:droplets].where(guid: docker_droplet.guid).get(:lifecycle_type)).to eq(DockerLifecycleDataModel::LIFECYCLE_TYPE) - end - - it 'does not touch updated_at' do - original_updated_at = db[:droplets].where(guid: [buildpack_droplet.guid, cnb_droplet.guid, docker_droplet.guid]).select_map(%i[guid updated_at]).to_h - job.perform - expect(db[:droplets].where(guid: [buildpack_droplet.guid, cnb_droplet.guid, docker_droplet.guid]).select_map(%i[guid updated_at]).to_h).to eq(original_updated_at) - end - end - - context 'when there are builds with NULL lifecycle_type' do - let(:buildpack_build) { create(:build_model) } - let(:cnb_build) { create(:build_model, :cnb) } - let(:docker_build) { create(:build_model, :docker) } - - before do - db[:builds].where(guid: [buildpack_build.guid, cnb_build.guid, docker_build.guid]).update(lifecycle_type: nil) - end - - it 'sets lifecycle_type accordingly' do - job.perform - expect(db[:builds].where(guid: buildpack_build.guid).get(:lifecycle_type)).to eq(BuildpackLifecycleDataModel::LIFECYCLE_TYPE) - expect(db[:builds].where(guid: cnb_build.guid).get(:lifecycle_type)).to eq(CNBLifecycleDataModel::LIFECYCLE_TYPE) - expect(db[:builds].where(guid: docker_build.guid).get(:lifecycle_type)).to eq(DockerLifecycleDataModel::LIFECYCLE_TYPE) - end - - it 'does not touch updated_at' do - original_updated_at = db[:builds].where(guid: [buildpack_build.guid, cnb_build.guid, docker_build.guid]).select_map(%i[guid updated_at]).to_h - job.perform - expect(db[:builds].where(guid: [buildpack_build.guid, cnb_build.guid, docker_build.guid]).select_map(%i[guid updated_at]).to_h).to eq(original_updated_at) - end - end - - context 'with more rows than batch_size * batches_per_run' do - subject(:job) { LifecycleTypeBackfill.new(batch_size: 2, batches_per_run: 2) } - - before do - create_list(:app_model, 5) - db[:apps].update(lifecycle_type: nil) - end - - it 'updates at most batch_size * batches_per_run rows in a single perform' do - expect { job.perform }.to change { db[:apps].where(lifecycle_type: nil).count }.from(5).to(1) - end - - it 'processes the remainder on the next perform' do - job.perform - job.perform - expect(db[:apps].where(lifecycle_type: nil).count).to eq(0) - end - end - - context 'with fewer rows than batch_size' do - subject(:job) { LifecycleTypeBackfill.new(batch_size: 2, batches_per_run: 2) } - - before do - create(:app_model) - db[:apps].update(lifecycle_type: nil) - end - - it 'updates every NULL row in a single perform' do - job.perform - expect(db[:apps].where(lifecycle_type: nil).count).to eq(0) - end - - it 'issues exactly one SELECT for guids, subsequent batch is skipped' do - expect { job.perform }.to have_queried_db_times(/select .guid. from .apps. where \(.lifecycle_type. is null\)/i, 1) - end - end - - context 'when batches_per_run is -1 (drain mode)' do - subject(:job) { LifecycleTypeBackfill.new(batch_size: 2, batches_per_run: -1) } - - before do - create_list(:app_model, 5) - db[:apps].update(lifecycle_type: nil) - end - - it 'keeps batching until no NULL rows remain' do - job.perform - expect(db[:apps].where(lifecycle_type: nil).count).to eq(0) - end - end - end - end - end -end diff --git a/spec/unit/lib/cloud_controller/clock/scheduler_spec.rb b/spec/unit/lib/cloud_controller/clock/scheduler_spec.rb index 21c5b94d459..932abcc98b6 100644 --- a/spec/unit/lib/cloud_controller/clock/scheduler_spec.rb +++ b/spec/unit/lib/cloud_controller/clock/scheduler_spec.rb @@ -22,7 +22,6 @@ module VCAP::CloudController pollable_jobs: { cutoff_age_in_days: 2 }, service_operations_initial_cleanup: { frequency_in_seconds: 600 }, service_operations_create_in_progress_cleanup: { frequency_in_seconds: 600 }, - lifecycle_type_backfill: { frequency_in_seconds: 500 }, service_usage_events: { cutoff_age_in_days: 5 }, completed_tasks: { cutoff_age_in_days: 6 }, pending_droplets: { frequency_in_seconds: 300, expiration_in_seconds: 600 }, @@ -169,12 +168,6 @@ module VCAP::CloudController expect(block.call).to be_instance_of(Jobs::Runtime::ServiceOperationsCreateInProgressCleanup) end - expect(clock).to receive(:schedule_frequent_worker_job) do |args, &block| - expect(args).to eql(name: 'lifecycle_type_backfill', interval: 500) - expect(Jobs::Runtime::LifecycleTypeBackfill).to receive(:new).and_call_original - expect(block.call).to be_instance_of(Jobs::Runtime::LifecycleTypeBackfill) - end - schedule.start end