Skip to content

Commit

Permalink
Improve DestroyStackJob
Browse files Browse the repository at this point in the history
Do all the delete in batch of 1k all in the DB.

Rather than fetch a list of IDs and generate a big `IN` delete
query, we can just do it with a sub query and a LIMIT.

On MySQL specifically we could directly do a `DELETE ... LIMIT`,
but SQLite and PG don't support that syntax.
  • Loading branch information
byroot committed Nov 14, 2024
1 parent 1283edc commit 2fa442c
Showing 1 changed file with 24 additions and 20 deletions.
44 changes: 24 additions & 20 deletions app/jobs/shipit/destroy_stack_job.rb
Original file line number Diff line number Diff line change
Expand Up @@ -16,29 +16,33 @@ class DestroyStackJob < BackgroundJob
# +-- chunks

def perform(stack)
Shipit::ApiClient.where(stack_id: stack.id).delete_all
commits_ids = Shipit::Commit.where(stack_id: stack.id).pluck(:id)
tasks_ids = Shipit::Task.where(stack_id: stack.id).pluck(:id)
commit_deployments_ids = Shipit::CommitDeployment.where(task_id: tasks_ids).pluck(:id)
Shipit::CommitDeploymentStatus.where(commit_deployment_id: commit_deployments_ids).in_batches(&:delete_all)
Shipit::CommitDeployment.where(id: commit_deployments_ids).in_batches(&:delete_all)

Shipit::Status.where(commit_id: commits_ids).find_in_batches do |batch|
Shipit::Status.where(id: batch.map(&:id)).delete_all
end
delete(Shipit::ApiClient.where(stack_id: stack.id))

commits_ids.each_slice(1000) do |batch|
Shipit::Commit.where(id: batch).delete_all
end
delete(
Shipit::CommitDeploymentStatus.
joins(commit_deployment: [:task]).
where(commit_deployment: { tasks: { stack_id: stack.id }})
)

Shipit::GithubHook.where(stack_id: stack.id).destroy_all
Shipit::Hook.where(stack_id: stack.id).in_batches(&:delete_all)
Shipit::MergeRequest.where(stack_id: stack.id).in_batches(&:delete_all)
tasks_ids.each_slice(100) do |ids|
Shipit::OutputChunk.where(task_id: ids).in_batches(&:delete_all)
Shipit::Task.where(id: ids).in_batches(&:delete_all)
end
delete(Shipit::CommitDeployment.joins(:task).where(task: {stack_id: stack.id }))
delete(Shipit::Status.joins(:commit).where(commit: { stack_id: stack.id }))
delete(Shipit::GithubHook.where(stack_id: stack.id))
delete(Shipit::Hook.where(stack_id: stack.id))
delete(Shipit::MergeRequest.where(stack_id: stack.id))

delete(Shipit::OutputChunk.joins(:task).where(task: {stack_id: stack.id }))
delete(Shipit::Task.where(stack_id: stack.id))
stack.destroy!
end

private

BATCH_SIZE = 1000

def delete(relation)
while relation.model.where(id: relation.select(:id).limit(BATCH_SIZE)).delete_all == BATCH_SIZE
true # loop
end
end
end
end

0 comments on commit 2fa442c

Please sign in to comment.