templates: reconcile fork refcounts and sweep orphans on boot

lib/forkvm/copy.go

@@ -11,10 +11,28 @@ import (
 
 var ErrSparseCopyUnsupported = errors.New("sparse copy unsupported")
 
+// CopyOptions tunes CopyGuestDirectory behavior. The zero value reproduces
+// the original full-copy semantics; callers can opt into skipping specific
+// paths when the consumer arranges its own substitute (e.g. a symlink to a
+// template-shared mem-file).
+type CopyOptions struct {
+	// SkipRelPaths lists relative paths under srcDir that should not be
+	// materialized in dstDir. Comparison is exact and uses forward-slash
+	// separators on all platforms.
+	SkipRelPaths []string
+}
+
 // CopyGuestDirectory recursively copies a guest directory to a new destination.
 // Regular files are copied using sparse extent copy only (SEEK_DATA/SEEK_HOLE).
 // Runtime sockets and logs are skipped because they are host-runtime artifacts.
 func CopyGuestDirectory(srcDir, dstDir string) error {
+	return CopyGuestDirectoryWithOptions(srcDir, dstDir, CopyOptions{})
+}
+
+// CopyGuestDirectoryWithOptions is the option-taking variant of
+// CopyGuestDirectory. Use this when forking with template-shared assets, so
+// the caller can install a symlink in place of a heavy copied file.
+func CopyGuestDirectoryWithOptions(srcDir, dstDir string, opts CopyOptions) error {
 	srcInfo, err := os.Stat(srcDir)
 	if err != nil {
 		return fmt.Errorf("stat source directory: %w", err)
@@ -27,6 +45,11 @@ func CopyGuestDirectory(srcDir, dstDir string) error {
 		return fmt.Errorf("create destination directory: %w", err)
 	}
 
+	skipSet := make(map[string]struct{}, len(opts.SkipRelPaths))
+	for _, p := range opts.SkipRelPaths {
+		skipSet[filepath.ToSlash(p)] = struct{}{}
+	}
+
 	return filepath.WalkDir(srcDir, func(path string, d fs.DirEntry, walkErr error) error {
 		if walkErr != nil {
 			return walkErr
@@ -39,6 +62,12 @@ func CopyGuestDirectory(srcDir, dstDir string) error {
 		if relPath == "." {
 			return nil
 		}
+		if _, skip := skipSet[filepath.ToSlash(relPath)]; skip {
+			if d.IsDir() {
+				return filepath.SkipDir
+			}
+			return nil
+		}
 		if d.IsDir() && shouldSkipDirectory(relPath) {
 			return filepath.SkipDir
 		}

lib/forkvm/copy_test.go

@@ -44,6 +44,25 @@ func TestCopyGuestDirectory(t *testing.T) {
 	assert.Equal(t, "metadata.json", linkTarget)
 }
 
+func TestCopyGuestDirectory_SkipRelPaths(t *testing.T) {
+	src := filepath.Join(t.TempDir(), "src")
+	dst := filepath.Join(t.TempDir(), "dst")
+
+	require.NoError(t, os.MkdirAll(filepath.Join(src, "snapshots", "snapshot-latest"), 0755))
+	require.NoError(t, os.WriteFile(filepath.Join(src, "snapshots", "snapshot-latest", "config.json"), []byte(`{}`), 0644))
+	require.NoError(t, os.WriteFile(filepath.Join(src, "snapshots", "snapshot-latest", "memory"), []byte("the heavy mem-file"), 0644))
+	require.NoError(t, os.WriteFile(filepath.Join(src, "snapshots", "snapshot-latest", "state"), []byte("device state"), 0644))
+
+	err := CopyGuestDirectoryWithOptions(src, dst, CopyOptions{
+		SkipRelPaths: []string{"snapshots/snapshot-latest/memory"},
+	})
+	require.NoError(t, err)
+
+	assert.NoFileExists(t, filepath.Join(dst, "snapshots", "snapshot-latest", "memory"))
+	assert.FileExists(t, filepath.Join(dst, "snapshots", "snapshot-latest", "config.json"))
+	assert.FileExists(t, filepath.Join(dst, "snapshots", "snapshot-latest", "state"))
+}
+
 func TestCopyGuestDirectory_DoesNotSkipTmpSuffixedDirectories(t *testing.T) {
 	src := filepath.Join(t.TempDir(), "src")
 	dst := filepath.Join(t.TempDir(), "dst")

lib/instances/delete.go

@@ -35,6 +35,18 @@ func (m *manager) deleteInstance(
 	stored := &meta.StoredMetadata
 	log.DebugContext(ctx, "loaded instance", "instance_id", id, "state", inst.State)
 
+	// If this instance was promoted to a template parent, refuse to delete
+	// it while live forks reference it. Removing the registry entry now
+	// (instead of after the data wipe) gives us a single transactional
+	// "in-use" check via templates.ErrInUse.
+	if stored.IsTemplate && stored.TemplateID != "" && m.templateRegistry != nil {
+		if err := m.templateRegistry.Delete(ctx, stored.TemplateID); err != nil {
+			return fmt.Errorf("delete template registry entry for instance %s: %w", id, err)
+		}
+		stored.IsTemplate = false
+		stored.TemplateID = ""
+	}
+
 	target, err := m.cancelAndWaitCompressionJob(ctx, m.snapshotJobKeyForInstance(id))
 	if err != nil {
 		return fmt.Errorf("wait for instance compression to stop: %w", err)
@@ -136,6 +148,12 @@ func (m *manager) deleteInstance(
 		return fmt.Errorf("delete instance data: %w", err)
 	}
 
+	// 9. If this instance was a fork of a template, drop the template's
+	// fork refcount so the template can eventually be deleted.
+	if stored.ForkOfTemplate != "" {
+		m.dropTemplateForkRefcount(ctx, stored.ForkOfTemplate)
+	}
+
 	log.InfoContext(ctx, "instance deleted successfully", "instance_id", id)
 	return nil
 }

lib/instances/fork.go

@@ -15,6 +15,7 @@ import (
 	"github.com/kernel/hypeman/lib/hypervisor"
 	"github.com/kernel/hypeman/lib/logger"
 	"github.com/kernel/hypeman/lib/network"
+	"github.com/kernel/hypeman/lib/templates"
 	"github.com/nrednav/cuid2"
 	"go.opentelemetry.io/otel/attribute"
 	"gvisor.dev/gvisor/pkg/cleanup"
@@ -36,11 +37,22 @@ func (m *manager) forkInstance(ctx context.Context, id string, req ForkInstanceR
 		return nil, "", err
 	}
 
+	resolvedID, tpl, err := m.resolveForkFromTemplateRequest(ctx, id, req)
+	if err != nil {
+		return nil, "", err
+	}
+	id = resolvedID
+
 	meta, err := m.loadMetadata(id)
 	if err != nil {
 		return nil, "", err
 	}
 	source := m.toInstance(ctx, meta)
+	if tpl != nil {
+		if err := validateForkResolvedFromTemplate(tpl, source.HypervisorType); err != nil {
+			return nil, "", err
+		}
+	}
 	targetState, err := resolveForkTargetState(req.TargetState, source.State)
 	if err != nil {
 		return nil, "", err
@@ -65,7 +77,7 @@ func (m *manager) forkInstance(ctx context.Context, id string, req ForkInstanceR
 			return nil, "", fmt.Errorf("standby source instance: %w", err)
 		}
 
-		forked, forkErr := m.forkInstanceFromStoppedOrStandby(ctx, id, req, true)
+		forked, forkErr := m.forkInstanceFromStoppedOrStandby(ctx, id, req, true, tpl)
 		if forkErr == nil {
 			if err := m.rotateSourceVsockForRestore(ctx, id, forked.Id); err != nil {
 				forkErr = fmt.Errorf("prepare source snapshot for restore: %w", err)
@@ -104,7 +116,7 @@ func (m *manager) forkInstance(ctx context.Context, id string, req ForkInstanceR
 		}
 		return forked, targetState, nil
 	case StateStopped, StateStandby:
-		forked, err := m.forkInstanceFromStoppedOrStandby(ctx, id, req, false)
+		forked, err := m.forkInstanceFromStoppedOrStandby(ctx, id, req, false, tpl)
 		if err != nil {
 			return nil, "", err
 		}
@@ -192,7 +204,7 @@ func generateForkSourceVsockCID(sourceID, forkID string, current int64) int64 {
 	return cid
 }
 
-func (m *manager) forkInstanceFromStoppedOrStandby(ctx context.Context, id string, req ForkInstanceRequest, supportValidated bool) (*Instance, error) {
+func (m *manager) forkInstanceFromStoppedOrStandby(ctx context.Context, id string, req ForkInstanceRequest, supportValidated bool, tpl *templates.Template) (*Instance, error) {
 	log := logger.FromContext(ctx)
 
 	meta, err := m.loadMetadata(id)
@@ -202,6 +214,9 @@ func (m *manager) forkInstanceFromStoppedOrStandby(ctx context.Context, id strin
 
 	source := m.toInstance(ctx, meta)
 	stored := &meta.StoredMetadata
+	if tpl != nil && !stored.IsTemplate {
+		return nil, fmt.Errorf("%w: template %s source instance %s is not flagged as a template parent", ErrInvalidState, tpl.ID, id)
+	}
 
 	switch source.State {
 	case StateStopped, StateStandby:
@@ -255,12 +270,21 @@ func (m *manager) forkInstanceFromStoppedOrStandby(ctx context.Context, id strin
 		}
 	}
 
-	if err := forkvm.CopyGuestDirectory(srcDir, dstDir); err != nil {
+	copyOpts := forkvm.CopyOptions{}
+	if tpl != nil {
+		copyOpts.SkipRelPaths = []string{templateSharedMemFileRelPath}
+	}
+	if err := forkvm.CopyGuestDirectoryWithOptions(srcDir, dstDir, copyOpts); err != nil {
 		if errors.Is(err, forkvm.ErrSparseCopyUnsupported) {
 			return nil, fmt.Errorf("fork requires sparse-capable filesystem (SEEK_DATA/SEEK_HOLE unsupported): %w", err)
 		}
 		return nil, fmt.Errorf("clone guest directory: %w", err)
 	}
+	if tpl != nil {
+		if err := m.installForkSharedMemFile(dstDir, tpl); err != nil {
+			return nil, fmt.Errorf("install shared mem-file: %w", err)
+		}
+	}
 
 	starter, err := m.getVMStarter(stored.HypervisorType)
 	if err != nil {
@@ -280,6 +304,15 @@ func (m *manager) forkInstanceFromStoppedOrStandby(ctx context.Context, id strin
 	forkMeta.VsockSocket = m.paths.InstanceSocket(forkID, hypervisor.VsockSocketNameForType(forkMeta.HypervisorType))
 	forkMeta.ExitCode = nil
 	forkMeta.ExitMessage = ""
+	// Forks of a template carry the template id but never inherit the
+	// IsTemplate flag — they are working copies.
+	forkMeta.IsTemplate = false
+	forkMeta.TemplateID = ""
+	if tpl != nil {
+		forkMeta.ForkOfTemplate = tpl.ID
+	} else {
+		forkMeta.ForkOfTemplate = stored.ForkOfTemplate
+	}
 
 	// Keep the original CID for snapshot-based forks.
 	// Rewriting CID in restored memory snapshots is not reliable across
@@ -324,6 +357,14 @@ func (m *manager) forkInstanceFromStoppedOrStandby(ctx context.Context, id strin
 		return nil, fmt.Errorf("save fork metadata: %w", err)
 	}
 
+	if tpl != nil {
+		// Bumped before cu.Release so a refcount failure leaves no orphan
+		// fork directory (deferred cu.Clean removes the data dir + metadata).
+		if err := m.bumpTemplateForkRefcount(ctx, tpl); err != nil {
+			return nil, fmt.Errorf("record template fork refcount: %w", err)
+		}
+	}
+
 	cu.Release()
 	forked := m.toInstance(ctx, newMeta)
 	log.InfoContext(ctx, "instance forked successfully",

lib/instances/fork_test.go

@@ -265,7 +265,7 @@ func TestForkInstanceFromStandbyCancelsCompressionJobAndCopiesRawMemory(t *testi
 	forked, err := manager.forkInstanceFromStoppedOrStandby(ctx, sourceID, ForkInstanceRequest{
 		Name:        "fork-standby-compressed-copy",
 		TargetState: StateStopped,
-	}, true)
+	}, true, nil)
 	require.NoError(t, err)
 	require.NotNil(t, forked)
 

lib/instances/manager.go

@@ -17,6 +17,7 @@ import (
 	"github.com/kernel/hypeman/lib/paths"
 	"github.com/kernel/hypeman/lib/resources"
 	"github.com/kernel/hypeman/lib/system"
+	"github.com/kernel/hypeman/lib/templates"
 	"github.com/kernel/hypeman/lib/volumes"
 	"go.opentelemetry.io/otel/attribute"
 	"go.opentelemetry.io/otel/metric"
@@ -147,6 +148,12 @@ type manager struct {
 	vmStarters        map[hypervisor.Type]hypervisor.VMStarter
 	defaultHypervisor hypervisor.Type // Default hypervisor type when not specified in request
 	guestMemoryPolicy guestmemory.Policy
+
+	// Template registry. Owned by the manager because template lifecycle
+	// is coupled to instance lifecycle (promotion + refcount on
+	// fork/delete). Constructed lazily so existing managers without
+	// template support keep working unchanged.
+	templateRegistry templates.Registry
 }
 
 // platformStarters is populated by platform-specific init functions.
@@ -201,6 +208,7 @@ func NewManagerWithConfig(p *paths.Paths, imageManager images.Manager, systemMan
 		compressionJobs:   make(map[string]*compressionJob),
 		nativeCodecPaths:  make(map[string]string),
 		lifecycleEvents:   newLifecycleSubscribersWithBufferSize(managerConfig.LifecycleEventBufferSize),
+		templateRegistry:  templates.NewFileRegistry(p.TemplatesDir()),
 	}
 	m.deleteSnapshotFn = m.deleteSnapshot
 
@@ -218,9 +226,21 @@ func NewManagerWithConfig(p *paths.Paths, imageManager images.Manager, systemMan
 		logger.FromContext(context.Background()).WarnContext(context.Background(), "failed to recover pending standby compression jobs", "error", err)
 	}
 
+	// Heal any drift between the templates registry and on-disk
+	// instances after a crash or out-of-band fork delete.
+	if err := m.reconcileTemplateState(context.Background()); err != nil {
+		logger.FromContext(context.Background()).WarnContext(context.Background(), "failed to reconcile template state at boot", "error", err)
+	}
+
 	return m
 }
 
+// ReconcileTemplateState heals registry/instances drift on demand. Useful
+// for tests and for periodic GC tickers driven by the host process.
+func (m *manager) ReconcileTemplateState(ctx context.Context) error {
+	return m.reconcileTemplateState(ctx)
+}
+
 // SetResourceValidator sets the resource validator for aggregate limit checking.
 // This is called after initialization to avoid circular dependencies.
 func (m *manager) SetResourceValidator(v ResourceValidator) {
@@ -373,7 +393,13 @@ func (m *manager) DeleteSnapshot(ctx context.Context, snapshotID string) error {
 
 // ForkInstance creates a forked copy of an instance.
 func (m *manager) ForkInstance(ctx context.Context, id string, req ForkInstanceRequest) (*Instance, error) {
-	lock := m.getInstanceLock(id)
+	// Resolve TemplateID outside the lock so we hold the source instance
+	// lock — not an empty string lock — when forking from a template.
+	resolvedID, _, err := m.resolveForkFromTemplateRequest(ctx, id, req)
+	if err != nil {
+		return nil, err
+	}
+	lock := m.getInstanceLock(resolvedID)
 	lock.Lock()
 	forked, targetState, err := m.forkInstance(ctx, id, req)
 	lock.Unlock()

lib/instances/restore.go

@@ -57,6 +57,10 @@ func (m *manager) restoreInstance(
 		log.ErrorContext(ctx, "no snapshot available", "instance_id", id)
 		return nil, fmt.Errorf("no snapshot available for instance %s", id)
 	}
+	if err := m.templateGuard(stored, "restore"); err != nil {
+		log.ErrorContext(ctx, "refusing to restore template instance", "instance_id", id, "template_id", stored.TemplateID)
+		return nil, err
+	}
 
 	// 2b. Validate aggregate resource limits before allocating resources (if configured)
 	reservedResources := false

lib/instances/start.go

@@ -47,6 +47,10 @@ func (m *manager) startInstance(
 		log.ErrorContext(ctx, "invalid state for start", "instance_id", id, "state", inst.State)
 		return nil, fmt.Errorf("%w: cannot start from state %s, must be Stopped", ErrInvalidState, inst.State)
 	}
+	if err := m.templateGuard(stored, "start"); err != nil {
+		log.ErrorContext(ctx, "refusing to start template instance", "instance_id", id, "template_id", stored.TemplateID)
+		return nil, err
+	}
 
 	// 2a. Clear stale exit info from previous run and apply command overrides
 	stored.ExitCode = nil

lib/instances/storage.go

@@ -77,6 +77,21 @@ func (m *manager) loadMetadata(id string) (*metadata, error) {
 	return &meta, nil
 }
 
+// loadMetadataFromFile reads a metadata file by path. Used by sweepers
+// that already have the path from listMetadataFiles and don't want to
+// reverse-derive an instance id.
+func loadMetadataFromFile(path string) (*metadata, error) {
+	data, err := os.ReadFile(path)
+	if err != nil {
+		return nil, fmt.Errorf("read metadata: %w", err)
+	}
+	var meta metadata
+	if err := json.Unmarshal(data, &meta); err != nil {
+		return nil, fmt.Errorf("unmarshal metadata: %w", err)
+	}
+	return &meta, nil
+}
+
 // saveMetadata saves instance metadata to disk
 func (m *manager) saveMetadata(meta *metadata) error {
 	metaPath := m.paths.InstanceMetadata(meta.Id)