feat: delete pending canceled prebuilds (coder#20499)

## Description

PR coder#20387 introduced canceling
pending prebuild jobs from inactive template versions to avoid
provisioning obsolete workspaces. However, the associated prebuilds
remained in the database with "Canceled" status, visible in the UI.

This PR now orphan-deletes these canceled prebuilt workspaces. Since the
canceled jobs were never processed by a provisioner, no Terraform
resources were created, making orphan deletion safe.

Orphan deletion always creates a provisioner job, but behaves
differently based on provisioner availability:
- If no provisioner daemon is available, the job is immediately marked
as completed and the workspace is marked as deleted without any
provisioner processing
- If a provisioner daemon is available, it processes the delete job with
empty Terraform state (no actual resources to destroy)

The job cancellation and workspace deletion occur atomically in the same
transaction. We don't split this into two separate reconciliation runs
because there's no way to distinguish between system-canceled prebuilds
and user-canceled workspaces. If we deleted canceled workspaces in a
later run, we'd delete user-canceled workspaces that users may want to
keep for troubleshooting.

Note: This only applies to system-generated prebuilds from inactive
template versions.

## Changes

* Update `UpdatePrebuildProvisionerJobWithCancel` query to return job
ID, workspace ID, template ID, and template version preset ID
* Add `DeprovisionMode` enum to support orphan deletion in the provision
flow
* Update `ActionTypeCancelPending` handler to cancel jobs and
orphan-delete associated workspaces atomically

Author: ssncferreira

coderd/database/dbauthz/dbauthz.go

@@ -4933,10 +4933,10 @@ func (q *querier) UpdateOrganizationDeletedByID(ctx context.Context, arg databas
 	return deleteQ(q.log, q.auth, q.db.GetOrganizationByID, deleteF)(ctx, arg.ID)
 }
 
-func (q *querier) UpdatePrebuildProvisionerJobWithCancel(ctx context.Context, arg database.UpdatePrebuildProvisionerJobWithCancelParams) ([]uuid.UUID, error) {
+func (q *querier) UpdatePrebuildProvisionerJobWithCancel(ctx context.Context, arg database.UpdatePrebuildProvisionerJobWithCancelParams) ([]database.UpdatePrebuildProvisionerJobWithCancelRow, error) {
 	// Prebuild operation for canceling pending prebuild jobs from non-active template versions
 	if err := q.authorizeContext(ctx, policy.ActionUpdate, rbac.ResourcePrebuiltWorkspace); err != nil {
-		return []uuid.UUID{}, err
+		return []database.UpdatePrebuildProvisionerJobWithCancelRow{}, err
 	}
 	return q.db.UpdatePrebuildProvisionerJobWithCancel(ctx, arg)
 }

coderd/database/dbauthz/dbauthz_test.go

@@ -646,10 +646,13 @@ func (s *MethodTestSuite) TestProvisionerJob() {
 			PresetID: uuid.NullUUID{UUID: uuid.New(), Valid: true},
 			Now:      dbtime.Now(),
 		}
-		jobIDs := []uuid.UUID{uuid.New(), uuid.New()}
+		canceledJobs := []database.UpdatePrebuildProvisionerJobWithCancelRow{
+			{ID: uuid.New(), WorkspaceID: uuid.New(), TemplateID: uuid.New(), TemplateVersionPresetID: uuid.NullUUID{UUID: uuid.New(), Valid: true}},
+			{ID: uuid.New(), WorkspaceID: uuid.New(), TemplateID: uuid.New(), TemplateVersionPresetID: uuid.NullUUID{UUID: uuid.New(), Valid: true}},
+		}
 
-		dbm.EXPECT().UpdatePrebuildProvisionerJobWithCancel(gomock.Any(), arg).Return(jobIDs, nil).AnyTimes()
-		check.Args(arg).Asserts(rbac.ResourcePrebuiltWorkspace, policy.ActionUpdate).Returns(jobIDs)
+		dbm.EXPECT().UpdatePrebuildProvisionerJobWithCancel(gomock.Any(), arg).Return(canceledJobs, nil).AnyTimes()
+		check.Args(arg).Asserts(rbac.ResourcePrebuiltWorkspace, policy.ActionUpdate).Returns(canceledJobs)
 	}))
 	s.Run("GetProvisionerJobsByIDs", s.Mocked(func(dbm *dbmock.MockStore, faker *gofakeit.Faker, check *expects) {
 		org := testutil.Fake(s.T(), faker, database.Organization{})

coderd/database/dbmetrics/querymetrics.go

@@ -300,12 +300,8 @@ GROUP BY wpb.template_version_preset_id;
 -- Cancels all pending provisioner jobs for prebuilt workspaces on a specific preset from an
 -- inactive template version.
 -- This is an optimization to clean up stale pending jobs.
-UPDATE provisioner_jobs
-SET
-	canceled_at = @now::timestamptz,
-	completed_at = @now::timestamptz
-WHERE id IN (
-	SELECT pj.id
+WITH jobs_to_cancel AS (
+	SELECT pj.id, w.id AS workspace_id, w.template_id, wpb.template_version_preset_id
 	FROM provisioner_jobs pj
 	INNER JOIN workspace_prebuild_builds wpb ON wpb.job_id = pj.id
 	INNER JOIN workspaces w ON w.id = wpb.workspace_id
@@ -324,4 +320,10 @@ WHERE id IN (
 		AND pj.canceled_at IS NULL
 		AND pj.completed_at IS NULL
 )
-RETURNING id;
+UPDATE provisioner_jobs
+SET
+	canceled_at = @now::timestamptz,
+	completed_at = @now::timestamptz
+FROM jobs_to_cancel
+WHERE provisioner_jobs.id = jobs_to_cancel.id
+RETURNING jobs_to_cancel.id, jobs_to_cancel.workspace_id, jobs_to_cancel.template_id, jobs_to_cancel.template_version_preset_id;

coderd/database/dbmock/dbmock.go

@@ -57,6 +57,24 @@ type StoreReconciler struct {
 
 var _ prebuilds.ReconciliationOrchestrator = &StoreReconciler{}
 
+type DeprovisionMode int
+
+const (
+	DeprovisionModeNormal DeprovisionMode = iota
+	DeprovisionModeOrphan
+)
+
+func (d DeprovisionMode) String() string {
+	switch d {
+	case DeprovisionModeOrphan:
+		return "orphan"
+	case DeprovisionModeNormal:
+		return "normal"
+	default:
+		return "unknown"
+	}
+}
+
 func NewStoreReconciler(store database.Store,
 	ps pubsub.Pubsub,
 	fileCache *files.Cache,
@@ -642,34 +660,7 @@ func (c *StoreReconciler) executeReconciliationAction(ctx context.Context, logge
 		return multiErr.ErrorOrNil()
 
 	case prebuilds.ActionTypeCancelPending:
-		// Cancel pending prebuild jobs from non-active template versions to avoid
-		// provisioning obsolete workspaces that would immediately be deprovisioned.
-		// This uses a criteria-based update to ensure only jobs that are still pending
-		// at execution time are canceled, avoiding race conditions where jobs may have
-		// transitioned to running status between query and update.
-		canceledJobs, err := c.store.UpdatePrebuildProvisionerJobWithCancel(
-			ctx,
-			database.UpdatePrebuildProvisionerJobWithCancelParams{
-				Now: c.clock.Now(),
-				PresetID: uuid.NullUUID{
-					UUID:  ps.Preset.ID,
-					Valid: true,
-				},
-			})
-		if err != nil {
-			logger.Error(ctx, "failed to cancel pending prebuild jobs",
-				slog.F("template_version_id", ps.Preset.TemplateVersionID.String()),
-				slog.F("preset_id", ps.Preset.ID),
-				slog.Error(err))
-			return err
-		}
-		if len(canceledJobs) > 0 {
-			logger.Info(ctx, "canceled pending prebuild jobs for inactive version",
-				slog.F("template_version_id", ps.Preset.TemplateVersionID.String()),
-				slog.F("preset_id", ps.Preset.ID),
-				slog.F("count", len(canceledJobs)))
-		}
-		return nil
+		return c.cancelAndOrphanDeletePendingPrebuilds(ctx, ps.Preset.TemplateID, ps.Preset.TemplateVersionID, ps.Preset.ID)
 
 	default:
 		return xerrors.Errorf("unknown action type: %v", action.ActionType)
@@ -717,33 +708,100 @@ func (c *StoreReconciler) createPrebuiltWorkspace(ctx context.Context, prebuiltW
 		c.logger.Info(ctx, "attempting to create prebuild", slog.F("name", name),
 			slog.F("workspace_id", prebuiltWorkspaceID.String()), slog.F("preset_id", presetID.String()))
 
-		return c.provision(ctx, db, prebuiltWorkspaceID, template, presetID, database.WorkspaceTransitionStart, workspace)
+		return c.provision(ctx, db, prebuiltWorkspaceID, template, presetID, database.WorkspaceTransitionStart, workspace, DeprovisionModeNormal)
 	}, &database.TxOptions{
 		Isolation: sql.LevelRepeatableRead,
 		ReadOnly:  false,
 	})
 }
 
-func (c *StoreReconciler) deletePrebuiltWorkspace(ctx context.Context, prebuiltWorkspaceID uuid.UUID, templateID uuid.UUID, presetID uuid.UUID) error {
+// provisionDelete provisions a delete transition for a prebuilt workspace.
+//
+// If mode is DeprovisionModeOrphan, the builder will not send Terraform state to the provisioner.
+// This allows the workspace to be deleted even when no provisioners are available, and is safe
+// when no Terraform resources were actually created (e.g., for pending prebuilds that were canceled
+// before provisioning started).
+//
+// IMPORTANT: This function must be called within a database transaction. It does not create its own transaction.
+// The caller is responsible for managing the transaction boundary via db.InTx().
+func (c *StoreReconciler) provisionDelete(ctx context.Context, db database.Store, workspaceID uuid.UUID, templateID uuid.UUID, presetID uuid.UUID, mode DeprovisionMode) error {
+	workspace, err := db.GetWorkspaceByID(ctx, workspaceID)
+	if err != nil {
+		return xerrors.Errorf("get workspace by ID: %w", err)
+	}
+
+	template, err := db.GetTemplateByID(ctx, templateID)
+	if err != nil {
+		return xerrors.Errorf("failed to get template: %w", err)
+	}
+
+	if workspace.OwnerID != database.PrebuildsSystemUserID {
+		return xerrors.Errorf("prebuilt workspace is not owned by prebuild user anymore, probably it was claimed")
+	}
+
+	c.logger.Info(ctx, "attempting to delete prebuild", slog.F("orphan", mode.String()),
+		slog.F("name", workspace.Name), slog.F("workspace_id", workspaceID.String()), slog.F("preset_id", presetID.String()))
+
+	return c.provision(ctx, db, workspaceID, template, presetID,
+		database.WorkspaceTransitionDelete, workspace, mode)
+}
+
+// cancelAndOrphanDeletePendingPrebuilds cancels pending prebuild jobs from inactive template versions
+// and orphan-deletes their associated workspaces.
+//
+// The cancel operation uses a criteria-based update to ensure only jobs that are still pending at
+// execution time are canceled, avoiding race conditions where jobs may have transitioned to running.
+//
+// Since these jobs were never processed by a provisioner, no Terraform resources were created,
+// making it safe to orphan-delete the workspaces (skipping Terraform destroy).
+func (c *StoreReconciler) cancelAndOrphanDeletePendingPrebuilds(ctx context.Context, templateID uuid.UUID, templateVersionID uuid.UUID, presetID uuid.UUID) error {
 	return c.store.InTx(func(db database.Store) error {
-		workspace, err := db.GetWorkspaceByID(ctx, prebuiltWorkspaceID)
+		canceledJobs, err := db.UpdatePrebuildProvisionerJobWithCancel(
+			ctx,
+			database.UpdatePrebuildProvisionerJobWithCancelParams{
+				Now: c.clock.Now(),
+				PresetID: uuid.NullUUID{
+					UUID:  presetID,
+					Valid: true,
+				},
+			})
 		if err != nil {
-			return xerrors.Errorf("get workspace by ID: %w", err)
+			c.logger.Error(ctx, "failed to cancel pending prebuild jobs",
+				slog.F("template_id", templateID.String()),
+				slog.F("template_version_id", templateVersionID.String()),
+				slog.F("preset_id", presetID.String()),
+				slog.Error(err))
+			return err
 		}
 
-		template, err := db.GetTemplateByID(ctx, templateID)
-		if err != nil {
-			return xerrors.Errorf("failed to get template: %w", err)
+		if len(canceledJobs) > 0 {
+			c.logger.Info(ctx, "canceled pending prebuild jobs for inactive version",
+				slog.F("template_id", templateID.String()),
+				slog.F("template_version_id", templateVersionID.String()),
+				slog.F("preset_id", presetID.String()),
+				slog.F("count", len(canceledJobs)))
 		}
 
-		if workspace.OwnerID != database.PrebuildsSystemUserID {
-			return xerrors.Errorf("prebuilt workspace is not owned by prebuild user anymore, probably it was claimed")
+		var multiErr multierror.Error
+		for _, job := range canceledJobs {
+			err = c.provisionDelete(ctx, db, job.WorkspaceID, job.TemplateID, presetID, DeprovisionModeOrphan)
+			if err != nil {
+				c.logger.Error(ctx, "failed to orphan delete canceled prebuild",
+					slog.F("workspace_id", job.WorkspaceID.String()), slog.Error(err))
+				multiErr.Errors = append(multiErr.Errors, err)
+			}
 		}
 
-		c.logger.Info(ctx, "attempting to delete prebuild",
-			slog.F("workspace_id", prebuiltWorkspaceID.String()), slog.F("preset_id", presetID.String()))
+		return multiErr.ErrorOrNil()
+	}, &database.TxOptions{
+		Isolation: sql.LevelRepeatableRead,
+		ReadOnly:  false,
+	})
+}
 
-		return c.provision(ctx, db, prebuiltWorkspaceID, template, presetID, database.WorkspaceTransitionDelete, workspace)
+func (c *StoreReconciler) deletePrebuiltWorkspace(ctx context.Context, prebuiltWorkspaceID uuid.UUID, templateID uuid.UUID, presetID uuid.UUID) error {
+	return c.store.InTx(func(db database.Store) error {
+		return c.provisionDelete(ctx, db, prebuiltWorkspaceID, templateID, presetID, DeprovisionModeNormal)
 	}, &database.TxOptions{
 		Isolation: sql.LevelRepeatableRead,
 		ReadOnly:  false,
@@ -758,6 +816,7 @@ func (c *StoreReconciler) provision(
 	presetID uuid.UUID,
 	transition database.WorkspaceTransition,
 	workspace database.Workspace,
+	mode DeprovisionMode,
 ) error {
 	tvp, err := db.GetPresetParametersByTemplateVersionID(ctx, template.ActiveVersionID)
 	if err != nil {
@@ -795,6 +854,11 @@ func (c *StoreReconciler) provision(
 		builder = builder.RichParameterValues(params)
 	}
 
+	// Use orphan mode for deletes when no Terraform resources exist
+	if transition == database.WorkspaceTransitionDelete && mode == DeprovisionModeOrphan {
+		builder = builder.Orphan()
+	}
+
 	_, provisionerJob, _, err := builder.Build(
 		ctx,
 		db,