feat: delete pending canceled prebuilds (coder#20499) (coder#20554)

Related to PR: coder#20499

(cherry picked from commit c3e3bb5)

Author: ssncferreira

coderd/database/dbauthz/dbauthz.go

@@ -4934,10 +4934,10 @@ func (q *querier) UpdateOrganizationDeletedByID(ctx context.Context, arg databas
 	return deleteQ(q.log, q.auth, q.db.GetOrganizationByID, deleteF)(ctx, arg.ID)
 }
 
-func (q *querier) UpdatePrebuildProvisionerJobWithCancel(ctx context.Context, arg database.UpdatePrebuildProvisionerJobWithCancelParams) ([]uuid.UUID, error) {
+func (q *querier) UpdatePrebuildProvisionerJobWithCancel(ctx context.Context, arg database.UpdatePrebuildProvisionerJobWithCancelParams) ([]database.UpdatePrebuildProvisionerJobWithCancelRow, error) {
 	// Prebuild operation for canceling pending prebuild jobs from non-active template versions
 	if err := q.authorizeContext(ctx, policy.ActionUpdate, rbac.ResourcePrebuiltWorkspace); err != nil {
-		return []uuid.UUID{}, err
+		return []database.UpdatePrebuildProvisionerJobWithCancelRow{}, err
 	}
 	return q.db.UpdatePrebuildProvisionerJobWithCancel(ctx, arg)
 }

coderd/database/dbauthz/dbauthz_test.go

@@ -646,10 +646,13 @@ func (s *MethodTestSuite) TestProvisionerJob() {
 			PresetID: uuid.NullUUID{UUID: uuid.New(), Valid: true},
 			Now:      dbtime.Now(),
 		}
-		jobIDs := []uuid.UUID{uuid.New(), uuid.New()}
+		canceledJobs := []database.UpdatePrebuildProvisionerJobWithCancelRow{
+			{ID: uuid.New(), WorkspaceID: uuid.New(), TemplateID: uuid.New(), TemplateVersionPresetID: uuid.NullUUID{UUID: uuid.New(), Valid: true}},
+			{ID: uuid.New(), WorkspaceID: uuid.New(), TemplateID: uuid.New(), TemplateVersionPresetID: uuid.NullUUID{UUID: uuid.New(), Valid: true}},
+		}
 
-		dbm.EXPECT().UpdatePrebuildProvisionerJobWithCancel(gomock.Any(), arg).Return(jobIDs, nil).AnyTimes()
-		check.Args(arg).Asserts(rbac.ResourcePrebuiltWorkspace, policy.ActionUpdate).Returns(jobIDs)
+		dbm.EXPECT().UpdatePrebuildProvisionerJobWithCancel(gomock.Any(), arg).Return(canceledJobs, nil).AnyTimes()
+		check.Args(arg).Asserts(rbac.ResourcePrebuiltWorkspace, policy.ActionUpdate).Returns(canceledJobs)
 	}))
 	s.Run("GetProvisionerJobsByIDs", s.Mocked(func(dbm *dbmock.MockStore, faker *gofakeit.Faker, check *expects) {
 		org := testutil.Fake(s.T(), faker, database.Organization{})

coderd/database/dbmetrics/querymetrics.go

@@ -300,12 +300,8 @@ GROUP BY wpb.template_version_preset_id;
 -- Cancels all pending provisioner jobs for prebuilt workspaces on a specific preset from an
 -- inactive template version.
 -- This is an optimization to clean up stale pending jobs.
-UPDATE provisioner_jobs
-SET
-	canceled_at = @now::timestamptz,
-	completed_at = @now::timestamptz
-WHERE id IN (
-	SELECT pj.id
+WITH jobs_to_cancel AS (
+	SELECT pj.id, w.id AS workspace_id, w.template_id, wpb.template_version_preset_id
 	FROM provisioner_jobs pj
 	INNER JOIN workspace_prebuild_builds wpb ON wpb.job_id = pj.id
 	INNER JOIN workspaces w ON w.id = wpb.workspace_id
@@ -324,4 +320,10 @@ WHERE id IN (
 		AND pj.canceled_at IS NULL
 		AND pj.completed_at IS NULL
 )
-RETURNING id;
+UPDATE provisioner_jobs
+SET
+	canceled_at = @now::timestamptz,
+	completed_at = @now::timestamptz
+FROM jobs_to_cancel
+WHERE provisioner_jobs.id = jobs_to_cancel.id
+RETURNING jobs_to_cancel.id, jobs_to_cancel.workspace_id, jobs_to_cancel.template_id, jobs_to_cancel.template_version_preset_id;

coderd/database/dbmock/dbmock.go

@@ -57,6 +57,24 @@ type StoreReconciler struct {
 
 var _ prebuilds.ReconciliationOrchestrator = &StoreReconciler{}
 
+type DeprovisionMode int
+
+const (
+	DeprovisionModeNormal DeprovisionMode = iota
+	DeprovisionModeOrphan
+)
+
+func (d DeprovisionMode) String() string {
+	switch d {
+	case DeprovisionModeOrphan:
+		return "orphan"
+	case DeprovisionModeNormal:
+		return "normal"
+	default:
+		return "unknown"
+	}
+}
+
 func NewStoreReconciler(store database.Store,
 	ps pubsub.Pubsub,
 	fileCache *files.Cache,
@@ -642,34 +660,7 @@ func (c *StoreReconciler) executeReconciliationAction(ctx context.Context, logge
 		return multiErr.ErrorOrNil()
 
 	case prebuilds.ActionTypeCancelPending:
-		// Cancel pending prebuild jobs from non-active template versions to avoid
-		// provisioning obsolete workspaces that would immediately be deprovisioned.
-		// This uses a criteria-based update to ensure only jobs that are still pending
-		// at execution time are canceled, avoiding race conditions where jobs may have
-		// transitioned to running status between query and update.
-		canceledJobs, err := c.store.UpdatePrebuildProvisionerJobWithCancel(
-			ctx,
-			database.UpdatePrebuildProvisionerJobWithCancelParams{
-				Now: c.clock.Now(),
-				PresetID: uuid.NullUUID{
-					UUID:  ps.Preset.ID,
-					Valid: true,
-				},
-			})
-		if err != nil {
-			logger.Error(ctx, "failed to cancel pending prebuild jobs",
-				slog.F("template_version_id", ps.Preset.TemplateVersionID.String()),
-				slog.F("preset_id", ps.Preset.ID),
-				slog.Error(err))
-			return err
-		}
-		if len(canceledJobs) > 0 {
-			logger.Info(ctx, "canceled pending prebuild jobs for inactive version",
-				slog.F("template_version_id", ps.Preset.TemplateVersionID.String()),
-				slog.F("preset_id", ps.Preset.ID),
-				slog.F("count", len(canceledJobs)))
-		}
-		return nil
+		return c.cancelAndOrphanDeletePendingPrebuilds(ctx, ps.Preset.TemplateID, ps.Preset.TemplateVersionID, ps.Preset.ID)
 
 	default:
 		return xerrors.Errorf("unknown action type: %v", action.ActionType)
@@ -717,33 +708,100 @@ func (c *StoreReconciler) createPrebuiltWorkspace(ctx context.Context, prebuiltW
 		c.logger.Info(ctx, "attempting to create prebuild", slog.F("name", name),
 			slog.F("workspace_id", prebuiltWorkspaceID.String()), slog.F("preset_id", presetID.String()))
 
-		return c.provision(ctx, db, prebuiltWorkspaceID, template, presetID, database.WorkspaceTransitionStart, workspace)
+		return c.provision(ctx, db, prebuiltWorkspaceID, template, presetID, database.WorkspaceTransitionStart, workspace, DeprovisionModeNormal)
 	}, &database.TxOptions{
 		Isolation: sql.LevelRepeatableRead,
 		ReadOnly:  false,
 	})
 }
 
-func (c *StoreReconciler) deletePrebuiltWorkspace(ctx context.Context, prebuiltWorkspaceID uuid.UUID, templateID uuid.UUID, presetID uuid.UUID) error {
+// provisionDelete provisions a delete transition for a prebuilt workspace.
+//
+// If mode is DeprovisionModeOrphan, the builder will not send Terraform state to the provisioner.
+// This allows the workspace to be deleted even when no provisioners are available, and is safe
+// when no Terraform resources were actually created (e.g., for pending prebuilds that were canceled
+// before provisioning started).
+//
+// IMPORTANT: This function must be called within a database transaction. It does not create its own transaction.
+// The caller is responsible for managing the transaction boundary via db.InTx().
+func (c *StoreReconciler) provisionDelete(ctx context.Context, db database.Store, workspaceID uuid.UUID, templateID uuid.UUID, presetID uuid.UUID, mode DeprovisionMode) error {
+	workspace, err := db.GetWorkspaceByID(ctx, workspaceID)
+	if err != nil {
+		return xerrors.Errorf("get workspace by ID: %w", err)
+	}
+
+	template, err := db.GetTemplateByID(ctx, templateID)
+	if err != nil {
+		return xerrors.Errorf("failed to get template: %w", err)
+	}
+
+	if workspace.OwnerID != database.PrebuildsSystemUserID {
+		return xerrors.Errorf("prebuilt workspace is not owned by prebuild user anymore, probably it was claimed")
+	}
+
+	c.logger.Info(ctx, "attempting to delete prebuild", slog.F("orphan", mode.String()),
+		slog.F("name", workspace.Name), slog.F("workspace_id", workspaceID.String()), slog.F("preset_id", presetID.String()))
+
+	return c.provision(ctx, db, workspaceID, template, presetID,
+		database.WorkspaceTransitionDelete, workspace, mode)
+}
+
+// cancelAndOrphanDeletePendingPrebuilds cancels pending prebuild jobs from inactive template versions
+// and orphan-deletes their associated workspaces.
+//
+// The cancel operation uses a criteria-based update to ensure only jobs that are still pending at
+// execution time are canceled, avoiding race conditions where jobs may have transitioned to running.
+//
+// Since these jobs were never processed by a provisioner, no Terraform resources were created,
+// making it safe to orphan-delete the workspaces (skipping Terraform destroy).
+func (c *StoreReconciler) cancelAndOrphanDeletePendingPrebuilds(ctx context.Context, templateID uuid.UUID, templateVersionID uuid.UUID, presetID uuid.UUID) error {
 	return c.store.InTx(func(db database.Store) error {
-		workspace, err := db.GetWorkspaceByID(ctx, prebuiltWorkspaceID)
+		canceledJobs, err := db.UpdatePrebuildProvisionerJobWithCancel(
+			ctx,
+			database.UpdatePrebuildProvisionerJobWithCancelParams{
+				Now: c.clock.Now(),
+				PresetID: uuid.NullUUID{
+					UUID:  presetID,
+					Valid: true,
+				},
+			})
 		if err != nil {
-			return xerrors.Errorf("get workspace by ID: %w", err)
+			c.logger.Error(ctx, "failed to cancel pending prebuild jobs",
+				slog.F("template_id", templateID.String()),
+				slog.F("template_version_id", templateVersionID.String()),
+				slog.F("preset_id", presetID.String()),
+				slog.Error(err))
+			return err
 		}
 
-		template, err := db.GetTemplateByID(ctx, templateID)
-		if err != nil {
-			return xerrors.Errorf("failed to get template: %w", err)
+		if len(canceledJobs) > 0 {
+			c.logger.Info(ctx, "canceled pending prebuild jobs for inactive version",
+				slog.F("template_id", templateID.String()),
+				slog.F("template_version_id", templateVersionID.String()),
+				slog.F("preset_id", presetID.String()),
+				slog.F("count", len(canceledJobs)))
 		}
 
-		if workspace.OwnerID != database.PrebuildsSystemUserID {
-			return xerrors.Errorf("prebuilt workspace is not owned by prebuild user anymore, probably it was claimed")
+		var multiErr multierror.Error
+		for _, job := range canceledJobs {
+			err = c.provisionDelete(ctx, db, job.WorkspaceID, job.TemplateID, presetID, DeprovisionModeOrphan)
+			if err != nil {
+				c.logger.Error(ctx, "failed to orphan delete canceled prebuild",
+					slog.F("workspace_id", job.WorkspaceID.String()), slog.Error(err))
+				multiErr.Errors = append(multiErr.Errors, err)
+			}
 		}
 
-		c.logger.Info(ctx, "attempting to delete prebuild",
-			slog.F("workspace_id", prebuiltWorkspaceID.String()), slog.F("preset_id", presetID.String()))
+		return multiErr.ErrorOrNil()
+	}, &database.TxOptions{
+		Isolation: sql.LevelRepeatableRead,
+		ReadOnly:  false,
+	})
+}
 
-		return c.provision(ctx, db, prebuiltWorkspaceID, template, presetID, database.WorkspaceTransitionDelete, workspace)
+func (c *StoreReconciler) deletePrebuiltWorkspace(ctx context.Context, prebuiltWorkspaceID uuid.UUID, templateID uuid.UUID, presetID uuid.UUID) error {
+	return c.store.InTx(func(db database.Store) error {
+		return c.provisionDelete(ctx, db, prebuiltWorkspaceID, templateID, presetID, DeprovisionModeNormal)
 	}, &database.TxOptions{
 		Isolation: sql.LevelRepeatableRead,
 		ReadOnly:  false,
@@ -758,6 +816,7 @@ func (c *StoreReconciler) provision(
 	presetID uuid.UUID,
 	transition database.WorkspaceTransition,
 	workspace database.Workspace,
+	mode DeprovisionMode,
 ) error {
 	tvp, err := db.GetPresetParametersByTemplateVersionID(ctx, template.ActiveVersionID)
 	if err != nil {
@@ -795,6 +854,11 @@ func (c *StoreReconciler) provision(
 		builder = builder.RichParameterValues(params)
 	}
 
+	// Use orphan mode for deletes when no Terraform resources exist
+	if transition == database.WorkspaceTransitionDelete && mode == DeprovisionModeOrphan {
+		builder = builder.Orphan()
+	}
+
 	_, provisionerJob, _, err := builder.Build(
 		ctx,
 		db,