From e4d936980daa8c4f9c2b594899ad809f68654166 Mon Sep 17 00:00:00 2001 From: Andy Doan Date: Mon, 22 Jun 2026 13:48:49 -0500 Subject: [PATCH 01/13] feat: add TUF storage model Introduce a self-contained storage model for The Update Framework (TUF) using the Foundries.io/ota-tuf metadata format. - Define TUF metadata structs internally (no external TUF type imports): root, targets, snapshot and timestamp metadata plus AtsKey/Signature. - Add TufFsHandle under /tuf with: - InitTuf(): generate ed25519 keys for the root, targets, snapshot and timestamp roles, store them AES-256-GCM encrypted under a key derived from the HMAC secret, and create a signed initial root.json (20y). - LoadTuf(): decrypt and load role keys; errors if not initialized. - GetRoots(): return all root.json files ordered by version. - Default expirations: root 20y, timestamp 7d, targets/snapshot 90d. - Use go-securesystemslib/cjson for canonical JSON signing. Co-authored-by: GitHub Copilot:claude-4-opus Signed-off-by: Andy Doan --- go.mod | 1 + go.sum | 2 + storage/file.go | 7 + storage/file_tuf.go | 345 ++++++++++++++++++++++++++++++++++++++ storage/file_tuf_test.go | 175 +++++++++++++++++++ storage/tuf/tuf_crypto.go | 111 ++++++++++++ storage/tuf/tuf_data.go | 186 ++++++++++++++++++++ 7 files changed, 827 insertions(+) create mode 100644 storage/file_tuf.go create mode 100644 storage/file_tuf_test.go create mode 100644 storage/tuf/tuf_crypto.go create mode 100644 storage/tuf/tuf_data.go diff --git a/go.mod b/go.mod index 96790c8d..13f65cb4 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( github.com/labstack/gommon v0.5.0 github.com/mattn/go-sqlite3 v1.14.32 github.com/pelletier/go-toml v1.9.5 + github.com/secure-systems-lab/go-securesystemslib v0.11.0 github.com/spf13/cobra v1.10.2 github.com/stretchr/testify v1.11.1 golang.org/x/crypto v0.50.0 diff --git a/go.sum b/go.sum index 78284f29..af124352 100644 --- a/go.sum +++ b/go.sum @@ -32,6 +32,8 @@ github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCko github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/secure-systems-lab/go-securesystemslib v0.11.0 h1:iuCR9kcMFD4QurdKrGvPLoKZLv9YvwPYVr0473BdtFs= +github.com/secure-systems-lab/go-securesystemslib v0.11.0/go.mod h1:+PMOTjUGwHj2vcZ+TFKlb1tXRbrdWE1LYDT5i9JC80Q= github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY= diff --git a/storage/file.go b/storage/file.go index 90b90885..c1c67739 100644 --- a/storage/file.go +++ b/storage/file.go @@ -26,6 +26,7 @@ const ( DbFile = "db.sqlite" DevicesDir = "devices" UpdatesDir = "updates" + TufDir = "tuf" partialFileSuffix = "..part" rolloutJournalFile = "rollouts.journal" @@ -115,6 +116,10 @@ func (c FsConfig) UpdatesDir() string { return filepath.Join(string(c), UpdatesDir) } +func (c FsConfig) TufDir() string { + return filepath.Join(string(c), TufDir) +} + type FsHandle struct { Config FsConfig @@ -124,6 +129,7 @@ type FsHandle struct { Configs ConfigsFsHandle Devices DevicesFsHandle Updates updatesFsHandleWrap + Tuf TufFsHandle } func NewFs(root string) (*FsHandle, error) { @@ -134,6 +140,7 @@ func NewFs(root string) (*FsHandle, error) { fs.Configs.root = fs.Config.ConfigsDir() fs.Devices.root = fs.Config.DevicesDir() fs.Updates.init(fs.Config.UpdatesDir()) + fs.Tuf.init(fs.Config.TufDir(), fs.Auth) for _, h := range []baseFsHandle{ fs.Audit.baseFsHandle, diff --git a/storage/file_tuf.go b/storage/file_tuf.go new file mode 100644 index 00000000..80863f18 --- /dev/null +++ b/storage/file_tuf.go @@ -0,0 +1,345 @@ +// Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. +// SPDX-License-Identifier: BSD-3-Clause-Clear + +package storage + +import ( + "crypto/aes" + "crypto/cipher" + "crypto/rand" + "crypto/sha256" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "slices" + "strconv" + "strings" + "time" + + "github.com/foundriesio/update-server/clock" + "github.com/foundriesio/update-server/storage/tuf" + "golang.org/x/crypto/hkdf" +) + +// ErrTufNotInitialized is returned when TUF operations are attempted before +// the TUF metadata and keys have been created with InitTuf. +var ErrTufNotInitialized = errors.New("TUF is not initialized") + +// ErrTufAlreadyInitialized is returned by InitTuf when TUF data already exists. +var ErrTufAlreadyInitialized = errors.New("TUF is already initialized") + +const ( + // tufKeysDir holds the encrypted role private keys. + tufKeysDir = "keys" + // rootJsonSuffix is the suffix for versioned root metadata files. + rootJsonSuffix = ".root.json" + + // hkdfKeyEncSalt is the HKDF salt used to derive the key-file encryption key + // from the HMAC secret. + hkdfKeyEncSalt = "tuf-key-encryption-v1" +) + +// tufRoles are the roles created and managed by the server. +var tufRoles = []tuf.RoleName{tuf.RoleRoot, tuf.RoleTargets, tuf.RoleSnapshot, tuf.RoleTimestamp} + +// TufFsHandle manages TUF keys and metadata stored under /tuf. +type TufFsHandle struct { + baseFsHandle + + // auth provides access to the HMAC secret used to encrypt key files. + auth AuthFsHandle + + // RootExpiration is the validity period used for newly created root.json. + RootExpiration time.Duration + // TimestampExpiration is the validity period for timestamp metadata. + TimestampExpiration time.Duration + // TargetsExpiration is the validity period for targets metadata; snapshot + // metadata uses the same value. + TargetsExpiration time.Duration + + // signers holds the role private keys once LoadTuf has been called. + signers map[tuf.RoleName]*tuf.Signer +} + +func (h *TufFsHandle) init(root string, auth AuthFsHandle) { + h.root = root + h.auth = auth + h.RootExpiration = 20 * 365 * 24 * time.Hour + h.TimestampExpiration = 7 * 24 * time.Hour + h.TargetsExpiration = 90 * 24 * time.Hour +} + +func (h TufFsHandle) keysDir() string { + return filepath.Join(h.root, tufKeysDir) +} + +func (h TufFsHandle) keyPath(role tuf.RoleName) string { + return filepath.Join(h.keysDir(), string(role)+".key") +} + +// isInitialized reports whether TUF metadata and keys have been created. +func (h TufFsHandle) isInitialized() bool { + if _, err := os.Stat(h.keyPath(tuf.RoleRoot)); err != nil { + return false + } + names, err := h.rootMetaNames() + return err == nil && len(names) > 0 +} + +// InitTuf creates the TUF role keys (root, targets, snapshot, timestamp), an +// initial root.json, and stores the private keys encrypted with a key derived +// from the HMAC secret. It fails if TUF data already exists. +func (h TufFsHandle) InitTuf() error { + if h.isInitialized() { + return ErrTufAlreadyInitialized + } + + hmacSecret, err := h.auth.GetHmacSecret() + if err != nil { + return fmt.Errorf("unable to read HMAC secret (run auth-init first): %w", err) + } else if len(hmacSecret) == 0 { + return fmt.Errorf("HMAC secret is empty; run auth-init first") + } + + if err := os.MkdirAll(h.keysDir(), defaultDirAccess); err != nil { + return fmt.Errorf("unable to create TUF keys directory: %w", err) + } + + signers := make(map[tuf.RoleName]*tuf.Signer, len(tufRoles)) + for _, role := range tufRoles { + signer, err := tuf.NewSigner() + if err != nil { + return fmt.Errorf("unable to generate %s key: %w", role, err) + } + if err := h.writeKey(hmacSecret, role, signer); err != nil { + return err + } + signers[role] = signer + } + + keys := make(map[string]tuf.AtsKey, len(signers)) + roles := make(map[tuf.RoleName]tuf.RootRole, len(signers)) + for _, role := range tufRoles { + signer := signers[role] + keys[signer.Id] = signer.PublicAtsKey() + roles[role] = tuf.RootRole{KeyIDs: []string{signer.Id}, Threshold: 1} + } + root := tuf.AtsTufRoot{ + Signed: tuf.RootMeta{ + SignedCommon: tuf.SignedCommon{ + Type: tuf.RoleRoot.TufType(), + Expires: clock.Now().UTC().Add(h.RootExpiration).Truncate(time.Second), + Version: 1, + }, + ConsistentSnapshot: false, + Keys: keys, + Roles: roles, + }, + } + sig, err := signers[tuf.RoleRoot].Sign(root.Signed) + if err != nil { + return fmt.Errorf("unable to sign root metadata: %w", err) + } + root.Signatures = []tuf.Signature{sig} + return h.writeRoot(root) +} + +// LoadTuf loads and decrypts the role private keys into the handle. It returns +// ErrTufNotInitialized if TUF has not been initialized. +func (h *TufFsHandle) LoadTuf() error { + if !h.isInitialized() { + return ErrTufNotInitialized + } + + hmacSecret, err := h.auth.GetHmacSecret() + if err != nil { + return fmt.Errorf("unable to read HMAC secret: %w", err) + } else if len(hmacSecret) == 0 { + return fmt.Errorf("HMAC secret is empty") + } + + signers := make(map[tuf.RoleName]*tuf.Signer, len(tufRoles)) + for _, role := range tufRoles { + signer, err := h.readKey(hmacSecret, role) + if err != nil { + return err + } + signers[role] = signer + } + h.signers = signers + return nil +} + +// GetRoots returns every root.json file on disk, unmarshalled and ordered by +// ascending version. +func (h TufFsHandle) GetRoots() ([]tuf.AtsTufRoot, error) { + names, err := h.rootMetaNames() + if err != nil { + return nil, err + } + roots := make([]tuf.AtsTufRoot, 0, len(names)) + for _, name := range names { + content, err := h.readFile(name, false) + if err != nil { + return nil, fmt.Errorf("unable to read %s: %w", name, err) + } + var root tuf.AtsTufRoot + if err := json.Unmarshal([]byte(content), &root); err != nil { + return nil, fmt.Errorf("unable to parse %s: %w", name, err) + } + roots = append(roots, root) + } + return roots, nil +} + +func (h TufFsHandle) Sign(role tuf.RoleName, v any) (tuf.Signature, error) { + if !h.Enabled() { + return tuf.Signature{}, fmt.Errorf("TUF signing not available: call LoadTuf first") + } + signer, ok := h.signers[role] + if !ok { + return tuf.Signature{}, fmt.Errorf("no signer loaded for TUF role %s", role) + } + return signer.Sign(v) +} + +// writeRoot persists a root metadata file as .root.json. +func (h TufFsHandle) writeRoot(root tuf.AtsTufRoot) error { + data, err := json.MarshalIndent(root, "", " ") + if err != nil { + return fmt.Errorf("unable to marshal root metadata: %w", err) + } + name := strconv.Itoa(root.Signed.Version) + rootJsonSuffix + if err := h.mkdirs(defaultDirAccess, true); err != nil { + return fmt.Errorf("unable to create TUF directory: %w", err) + } + if err := h.writeFile(name, string(data), defaultFileAccess); err != nil { + return fmt.Errorf("unable to write %s: %w", name, err) + } + return nil +} + +// rootMetaNames returns the names of all root metadata files, ordered by +// ascending version. +func (h TufFsHandle) rootMetaNames() ([]string, error) { + entries, err := os.ReadDir(h.root) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil, nil + } + return nil, fmt.Errorf("unable to list TUF directory: %w", err) + } + names := make([]string, 0, len(entries)) + for _, entry := range entries { + if entry.IsDir() { + continue + } + if strings.HasSuffix(entry.Name(), rootJsonSuffix) { + names = append(names, entry.Name()) + } + } + slices.SortFunc(names, func(a, b string) int { + return rootVersion(a) - rootVersion(b) + }) + return names, nil +} + +// rootVersion extracts the integer version from a ".root.json" name. +func rootVersion(name string) int { + v, err := strconv.Atoi(strings.TrimSuffix(name, rootJsonSuffix)) + if err != nil { + return 0 + } + return v +} + +// writeKey encrypts and stores a role's private key. +func (h TufFsHandle) writeKey(hmacSecret []byte, role tuf.RoleName, signer *tuf.Signer) error { + plaintext, err := json.Marshal(signer.PrivateAtsKey()) + if err != nil { + return fmt.Errorf("unable to marshal %s key: %w", role, err) + } + encKey, err := deriveKeyEncryptionKey(hmacSecret, role) + if err != nil { + return err + } + ciphertext, err := encryptBytes(encKey, plaintext) + if err != nil { + return fmt.Errorf("unable to encrypt %s key: %w", role, err) + } + keys := baseFsHandle{root: h.keysDir()} + if err := keys.writeFile(string(role)+".key", string(ciphertext), secureFileAccess); err != nil { + return fmt.Errorf("unable to store %s key: %w", role, err) + } + return nil +} + +// readKey loads and decrypts a role's private key. +func (h TufFsHandle) readKey(hmacSecret []byte, role tuf.RoleName) (*tuf.Signer, error) { + keys := baseFsHandle{root: h.keysDir()} + ciphertext, err := keys.readFile(string(role)+".key", false) + if err != nil { + return nil, fmt.Errorf("unable to read %s key: %w", role, err) + } + encKey, err := deriveKeyEncryptionKey(hmacSecret, role) + if err != nil { + return nil, err + } + plaintext, err := decryptBytes(encKey, []byte(ciphertext)) + if err != nil { + return nil, fmt.Errorf("unable to decrypt %s key: %w", role, err) + } + var key tuf.AtsKey + if err := json.Unmarshal(plaintext, &key); err != nil { + return nil, fmt.Errorf("unable to parse %s key: %w", role, err) + } + return tuf.SignerFromAtsKey(key) +} + +// deriveKeyEncryptionKey derives a 32-byte AES key from the HMAC secret, scoped +// per role. +func deriveKeyEncryptionKey(hmacSecret []byte, role tuf.RoleName) ([]byte, error) { + key := make([]byte, 32) + r := hkdf.New(sha256.New, hmacSecret, []byte(hkdfKeyEncSalt), []byte(role)) + if _, err := io.ReadFull(r, key); err != nil { + return nil, fmt.Errorf("unable to derive key encryption key: %w", err) + } + return key, nil +} + +// encryptBytes encrypts plaintext with AES-256-GCM, returning nonce||ciphertext. +func encryptBytes(key, plaintext []byte) ([]byte, error) { + block, err := aes.NewCipher(key) + if err != nil { + return nil, err + } + gcm, err := cipher.NewGCM(block) + if err != nil { + return nil, err + } + nonce := make([]byte, gcm.NonceSize()) + if _, err := rand.Read(nonce); err != nil { + return nil, err + } + return gcm.Seal(nonce, nonce, plaintext, nil), nil +} + +// decryptBytes reverses encryptBytes. +func decryptBytes(key, data []byte) ([]byte, error) { + block, err := aes.NewCipher(key) + if err != nil { + return nil, err + } + gcm, err := cipher.NewGCM(block) + if err != nil { + return nil, err + } + if len(data) < gcm.NonceSize() { + return nil, fmt.Errorf("ciphertext too short") + } + nonce, ciphertext := data[:gcm.NonceSize()], data[gcm.NonceSize():] + return gcm.Open(nil, nonce, ciphertext, nil) +} diff --git a/storage/file_tuf_test.go b/storage/file_tuf_test.go new file mode 100644 index 00000000..bade5cf3 --- /dev/null +++ b/storage/file_tuf_test.go @@ -0,0 +1,175 @@ +// Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. +// SPDX-License-Identifier: BSD-3-Clause-Clear + +package storage + +import ( + "crypto/ed25519" + "encoding/hex" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/foundriesio/update-server/storage/tuf" + "github.com/secure-systems-lab/go-securesystemslib/cjson" + "github.com/stretchr/testify/require" +) + +// newTufTestFs returns an initialized filesystem with an HMAC secret set up, +// ready for TUF operations. +func newTufTestFs(t *testing.T) *FsHandle { + t.Helper() + fs, err := NewFs(t.TempDir()) + require.NoError(t, err) + require.NoError(t, fs.Auth.InitHmacSecret()) + return fs +} + +func TestInitTuf(t *testing.T) { + fs := newTufTestFs(t) + require.False(t, fs.Tuf.isInitialized()) + + require.NoError(t, fs.Tuf.InitTuf()) + require.True(t, fs.Tuf.isInitialized()) + + // All four role key files exist with secure permissions. + for _, role := range tufRoles { + info, err := os.Stat(fs.Tuf.keyPath(role)) + require.NoError(t, err, "key for role %s", role) + require.Equal(t, secureFileAccess, info.Mode().Perm()) + } + + // An initial v1 root metadata file exists. + _, err := os.Stat(filepath.Join(fs.Config.TufDir(), "1.root.json")) + require.NoError(t, err) +} + +func TestInitTufFailsWhenAlreadyInitialized(t *testing.T) { + fs := newTufTestFs(t) + require.NoError(t, fs.Tuf.InitTuf()) + require.ErrorIs(t, fs.Tuf.InitTuf(), ErrTufAlreadyInitialized) +} + +func TestInitTufRequiresHmacSecret(t *testing.T) { + fs, err := NewFs(t.TempDir()) + require.NoError(t, err) + require.Error(t, fs.Tuf.InitTuf()) + require.False(t, fs.Tuf.isInitialized()) +} + +func TestKeysAreEncryptedOnDisk(t *testing.T) { + fs := newTufTestFs(t) + require.NoError(t, fs.Tuf.InitTuf()) + + roots, err := fs.Tuf.GetRoots() + require.NoError(t, err) + require.Len(t, roots, 1) + + // The raw key file must not contain any of the public key hex (which would + // indicate the AtsKey JSON was stored unencrypted). + raw, err := os.ReadFile(fs.Tuf.keyPath(tuf.RoleRoot)) + require.NoError(t, err) + for _, key := range roots[0].Signed.Keys { + require.NotContains(t, string(raw), key.KeyValue.Public) + } + require.NotContains(t, string(raw), "keytype") +} + +func TestLoadTufNotInitialized(t *testing.T) { + fs := newTufTestFs(t) + require.ErrorIs(t, fs.Tuf.LoadTuf(), ErrTufNotInitialized) +} + +func TestLoadTuf(t *testing.T) { + fs := newTufTestFs(t) + require.NoError(t, fs.Tuf.InitTuf()) + require.NoError(t, fs.Tuf.LoadTuf()) + + require.Len(t, fs.Tuf.signers, len(tufRoles)) + + roots, err := fs.Tuf.GetRoots() + require.NoError(t, err) + root := roots[0] + + // Each loaded signer must match the key id recorded in root metadata. + for _, role := range tufRoles { + signer := fs.Tuf.signers[role] + require.NotNil(t, signer, "role %s", role) + rr := root.Signed.Roles[role] + require.NotNil(t, rr) + require.Equal(t, []string{signer.Id}, rr.KeyIDs) + require.Equal(t, 1, rr.Threshold) + } +} + +func TestGetRoots(t *testing.T) { + fs := newTufTestFs(t) + before := time.Now().UTC() + require.NoError(t, fs.Tuf.InitTuf()) + + roots, err := fs.Tuf.GetRoots() + require.NoError(t, err) + require.Len(t, roots, 1) + + root := roots[0] + require.Equal(t, "Root", root.Signed.Type) + require.Equal(t, 1, root.Signed.Version) + require.False(t, root.Signed.ConsistentSnapshot) + require.Len(t, root.Signed.Keys, len(tufRoles)) + require.Len(t, root.Signed.Roles, len(tufRoles)) + + // Root should expire roughly 20 years out (matching RootExpiration). + expectedExpiry := before.Add(fs.Tuf.RootExpiration) + require.WithinDuration(t, expectedExpiry, root.Signed.Expires, time.Minute) + + // Exactly one signature, by the root key, and it must verify. + require.Len(t, root.Signatures, 1) + sig := root.Signatures[0] + require.Equal(t, tuf.SigEd25519, sig.Method) + + pubHex := root.Signed.Keys[sig.KeyID].KeyValue.Public + require.NotEmpty(t, pubHex) + pub, err := hex.DecodeString(pubHex) + require.NoError(t, err) + + msg, err := cjson.EncodeCanonical(root.Signed) + require.NoError(t, err) + require.True(t, ed25519.Verify(ed25519.PublicKey(pub), msg, sig.Signature), + "root signature must verify against canonical signed payload") +} + +func TestGetRootsEmpty(t *testing.T) { + fs := newTufTestFs(t) + roots, err := fs.Tuf.GetRoots() + require.NoError(t, err) + require.Empty(t, roots) +} + +func TestRootMetaJSONFormat(t *testing.T) { + fs := newTufTestFs(t) + require.NoError(t, fs.Tuf.InitTuf()) + + content, err := os.ReadFile(filepath.Join(fs.Config.TufDir(), "1.root.json")) + require.NoError(t, err) + + // Sanity check the on-disk shape matches the foundries/ota-tuf format. + var generic struct { + Signatures []json.RawMessage `json:"signatures"` + Signed struct { + Type string `json:"_type"` + Roles map[string]struct { + Threshold int `json:"threshold"` + } `json:"roles"` + } `json:"signed"` + } + require.NoError(t, json.Unmarshal(content, &generic)) + require.Equal(t, "Root", generic.Signed.Type) + for _, role := range []string{"root", "targets", "snapshot", "timestamp"} { + _, ok := generic.Signed.Roles[role] + require.True(t, ok, "missing role %s", role) + } + require.True(t, strings.Contains(string(content), "\"keytype\": \"ED25519\"")) +} diff --git a/storage/tuf/tuf_crypto.go b/storage/tuf/tuf_crypto.go new file mode 100644 index 00000000..628de41c --- /dev/null +++ b/storage/tuf/tuf_crypto.go @@ -0,0 +1,111 @@ +// Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. +// SPDX-License-Identifier: BSD-3-Clause-Clear + +package tuf + +import ( + "crypto/ed25519" + "crypto/rand" + "crypto/sha256" + "crypto/x509" + "encoding/hex" + "fmt" + + "github.com/secure-systems-lab/go-securesystemslib/cjson" +) + +// tufKeyTypeEd25519 is the keytype value used by ota-tuf for ed25519 keys. +const tufKeyTypeEd25519 = "ED25519" + +// tufSigner holds a loaded private key along with its TUF key id. The server +// uses online ed25519 keys for all roles. +type Signer struct { + Id string + keyType string + private ed25519.PrivateKey + public ed25519.PublicKey +} + +// tufKeyID computes the key id the way ota-tuf/garage-sign does: the hex +// encoded sha256 of the key's canonical (PKIX/DER) public encoding. +func tufKeyID(pub ed25519.PublicKey) (string, error) { + der, err := x509.MarshalPKIXPublicKey(pub) + if err != nil { + return "", fmt.Errorf("unable to marshal public key: %w", err) + } + sum := sha256.Sum256(der) + return hex.EncodeToString(sum[:]), nil +} + +// NewSigner generates a fresh ed25519 signer. +func NewSigner() (*Signer, error) { + pub, priv, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + return nil, fmt.Errorf("unable to generate ed25519 key: %w", err) + } + id, err := tufKeyID(pub) + if err != nil { + return nil, err + } + return &Signer{Id: id, keyType: tufKeyTypeEd25519, private: priv, public: pub}, nil +} + +// SignerFromAtsKey reconstructs a signer from a stored AtsKey (private key). +func SignerFromAtsKey(key AtsKey) (*Signer, error) { + if key.KeyType != tufKeyTypeEd25519 { + return nil, fmt.Errorf("unsupported TUF key type: %s", key.KeyType) + } + if key.KeyValue.Private == "" { + return nil, fmt.Errorf("TUF key is missing private material") + } + seed, err := hex.DecodeString(key.KeyValue.Private) + if err != nil { + return nil, fmt.Errorf("invalid ed25519 private key encoding: %w", err) + } + var priv ed25519.PrivateKey + switch len(seed) { + case ed25519.SeedSize: + priv = ed25519.NewKeyFromSeed(seed) + case ed25519.PrivateKeySize: + priv = ed25519.PrivateKey(seed) + default: + return nil, fmt.Errorf("invalid ed25519 private key size: %d", len(seed)) + } + pub := priv.Public().(ed25519.PublicKey) + id, err := tufKeyID(pub) + if err != nil { + return nil, err + } + return &Signer{Id: id, keyType: tufKeyTypeEd25519, private: priv, public: pub}, nil +} + +// privateAtsKey returns the AtsKey representation of the signer including the +// private key material (hex encoded seed). +func (s *Signer) PrivateAtsKey() AtsKey { + return AtsKey{ + KeyType: s.keyType, + KeyValue: AtsKeyVal{ + Private: hex.EncodeToString(s.private.Seed()), + Public: hex.EncodeToString(s.public), + }, + } +} + +// publicAtsKey returns the AtsKey representation of the signer with only the +// public key material, as embedded in root metadata. +func (s *Signer) PublicAtsKey() AtsKey { + return AtsKey{ + KeyType: s.keyType, + KeyValue: AtsKeyVal{Public: hex.EncodeToString(s.public)}, + } +} + +// sign signs the canonical JSON of signed and returns a Signature. +func (s *Signer) Sign(signed any) (Signature, error) { + msg, err := cjson.EncodeCanonical(signed) + if err != nil { + return Signature{}, fmt.Errorf("unable to marshal canonical JSON: %w", err) + } + sig := ed25519.Sign(s.private, msg) + return Signature{KeyID: s.Id, Method: SigEd25519, Signature: sig}, nil +} diff --git a/storage/tuf/tuf_data.go b/storage/tuf/tuf_data.go new file mode 100644 index 00000000..138cfd45 --- /dev/null +++ b/storage/tuf/tuf_data.go @@ -0,0 +1,186 @@ +// Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. +// SPDX-License-Identifier: BSD-3-Clause-Clear + +package tuf + +import ( + "encoding/hex" + "encoding/json" + "fmt" + "time" +) + +// This file defines the TUF (The Update Framework) metadata structures used by +// this project. The on-disk format follows the one produced by Foundries.io +// ota-tuf / garage-sign (and consumed by libaktualizr), which differs slightly +// from the standard go-tuf format: +// +// - The "_type" value of the signed metadata is the capitalized role name +// ("Root", "Targets", "Snapshot", "Timestamp"). +// - The "roles" map in root metadata is keyed by the lower-case role name. +// - Signatures carry the raw signature bytes which are base64 encoded in JSON. +// - Keys are represented by the AtsKey structure where ed25519 public and +// private material is hex encoded. +// - snapshot/timestamp "meta" entries only reference a version, not hashes. +// +// These structures are intentionally self-contained: the project does not +// import TUF data types from any external library. + +// RoleName is the canonical (lower-case) name of a TUF role. +type RoleName string + +const ( + RoleRoot RoleName = "root" + RoleTargets RoleName = "targets" + RoleSnapshot RoleName = "snapshot" + RoleTimestamp RoleName = "timestamp" +) + +// tufType returns the "_type" value used in signed metadata for the role. +func (r RoleName) TufType() string { + switch r { + case RoleRoot: + return "Root" + case RoleTargets: + return "Targets" + case RoleSnapshot: + return "Snapshot" + case RoleTimestamp: + return "Timestamp" + default: + return string(r) + } +} + +// SigAlgorithm is the signing method recorded in a Signature. +type SigAlgorithm string + +const ( + SigEd25519 SigAlgorithm = "ed25519" + SigRsaPssSha256 SigAlgorithm = "rsassa-pss-sha256" +) + +// SignedCommon contains the fields common to the "signed" component of all +// TUF metadata files. +type SignedCommon struct { + Type string `json:"_type"` + Expires time.Time `json:"expires"` + Version int `json:"version"` +} + +// Signature is a signature over the canonical JSON of a metadata's "signed" +// component. The raw signature bytes are base64 encoded in JSON. +type Signature struct { + KeyID string `json:"keyid"` + Method SigAlgorithm `json:"method"` + Signature []byte `json:"sig"` +} + +// AtsKeyVal holds the (hex encoded) public and/or private key material. +type AtsKeyVal struct { + Public string `json:"public,omitempty"` + Private string `json:"private,omitempty"` +} + +// AtsKey is the ota-tuf representation of a key. +type AtsKey struct { + KeyType string `json:"keytype"` + KeyValue AtsKeyVal `json:"keyval"` +} + +// RootRole describes the keys and threshold for a role within root metadata. +type RootRole struct { + KeyIDs []string `json:"keyids"` + Threshold int `json:"threshold"` +} + +// RootMeta is the "signed" component of a root.json file. +type RootMeta struct { + SignedCommon + ConsistentSnapshot bool `json:"consistent_snapshot"` + Keys map[string]AtsKey `json:"keys"` + Roles map[RoleName]RootRole `json:"roles"` +} + +// AtsTufRoot is a full root.json file. +type AtsTufRoot struct { + Signatures []Signature `json:"signatures"` + Signed RootMeta `json:"signed"` +} + +// HexBytes is a byte slice that is hex encoded in JSON. TUF hashes use this +// representation (unlike signatures, which are base64 encoded). +type HexBytes []byte + +func (b HexBytes) MarshalJSON() ([]byte, error) { + return json.Marshal(hex.EncodeToString(b)) +} + +func (b *HexBytes) UnmarshalJSON(data []byte) error { + var s string + if err := json.Unmarshal(data, &s); err != nil { + return err + } + decoded, err := hex.DecodeString(s) + if err != nil { + return fmt.Errorf("invalid hex encoded bytes: %w", err) + } + *b = decoded + return nil +} + +// Hashes maps a hash algorithm name (e.g. "sha256") to a hex encoded digest. +type Hashes map[string]HexBytes + +// TargetFileMeta describes a single target file in targets metadata. +type TargetFileMeta struct { + Length int64 `json:"length"` + Hashes Hashes `json:"hashes"` + Custom json.RawMessage `json:"custom,omitempty"` +} + +// TargetFiles maps a target name to its metadata. +type TargetFiles map[string]TargetFileMeta + +// TargetsMeta is the "signed" component of a targets.json file. +type TargetsMeta struct { + SignedCommon + Targets TargetFiles `json:"targets"` + Delegations json.RawMessage `json:"delegations,omitempty"` +} + +// AtsTufTargets is a full targets.json file. +type AtsTufTargets struct { + Signatures []Signature `json:"signatures"` + Signed TargetsMeta `json:"signed"` +} + +// MetaItem references a version of another metadata file. The ota-tuf format +// for snapshot and timestamp metadata only records the version. +type MetaItem struct { + Version int `json:"version"` +} + +// SnapshotMeta is the "signed" component of a snapshot.json file. +type SnapshotMeta struct { + SignedCommon + Meta map[string]MetaItem `json:"meta"` +} + +// AtsTufSnapshot is a full snapshot.json file. +type AtsTufSnapshot struct { + Signatures []Signature `json:"signatures"` + Signed SnapshotMeta `json:"signed"` +} + +// TimestampMeta is the "signed" component of a timestamp.json file. +type TimestampMeta struct { + SignedCommon + Meta map[string]MetaItem `json:"meta"` +} + +// AtsTufTimestamp is a full timestamp.json file. +type AtsTufTimestamp struct { + Signatures []Signature `json:"signatures"` + Signed TimestampMeta `json:"signed"` +} From 1b5630325557b185c8a1212119428a24352029ee Mon Sep 17 00:00:00 2001 From: Andy Doan Date: Wed, 24 Jun 2026 11:10:36 -0500 Subject: [PATCH 02/13] feat: add tuf-init subcommand Add a tuf-init server subcommand to initialize TUF keys and root metadata, and require TUF to be initialized before the server starts. Co-authored-by: GitHub Copilot:claude-4-opus Signed-off-by: Andy Doan --- .github/workflows/contrib-test.yml | 1 + cmd/server/main.go | 3 +++ cmd/server/serve.go | 3 +++ cmd/server/serve_test.go | 1 + cmd/server/tuf_init.go | 18 ++++++++++++++++++ 5 files changed, 26 insertions(+) create mode 100644 cmd/server/tuf_init.go diff --git a/.github/workflows/contrib-test.yml b/.github/workflows/contrib-test.yml index c16b48c2..82d2be1e 100644 --- a/.github/workflows/contrib-test.yml +++ b/.github/workflows/contrib-test.yml @@ -41,6 +41,7 @@ jobs: "Type" : "noauth" } EOF + ./contrib/dev-shell go run github.com/foundriesio/update-server/cmd/server --datadir .compose-server-data tuf-init - name: Test server run: | diff --git a/cmd/server/main.go b/cmd/server/main.go index cef07459..df4ca3a7 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -22,6 +22,7 @@ type CommonArgs struct { Csr *CsrCmd `arg:"subcommand:create-csr" help:"Create a TLS certificate signing request for this server"` SignCsr *CsrSignCmd `arg:"subcommand:sign-csr" help:"Create the TLS certificate from the signing request"` Serve *ServeCmd `arg:"subcommand:serve" help:"Run the REST API and device-gateway services"` + TufInit *TufInitCmd `arg:"subcommand:tuf-init" help:"Initialize TUF keys and root metadata for this server"` UserAdd *UserAddCmd `arg:"subcommand:user-add" help:"Add a new user if local authentication is enabled"` Version *VersionCmd `arg:"subcommand:version" help:"Print the version of the program"` @@ -48,6 +49,8 @@ func main() { err = args.SignCsr.Run(args) case args.Serve != nil: err = args.Serve.Run(args) + case args.TufInit != nil: + err = args.TufInit.Run(args) case args.AuthInit != nil: err = args.AuthInit.Run(args) case args.UserAdd != nil: diff --git a/cmd/server/serve.go b/cmd/server/serve.go index faa42e74..17c33678 100644 --- a/cmd/server/serve.go +++ b/cmd/server/serve.go @@ -29,6 +29,9 @@ func (c *ServeCmd) Run(args CommonArgs) error { if err != nil { return fmt.Errorf("failed to load filesystem: %w", err) } + if err := fs.Tuf.LoadTuf(); err != nil { + return fmt.Errorf("failed to load TUF (run tuf-init first): %w", err) + } db, err := storage.NewDb(fs.Config.DbFile()) if err != nil { return fmt.Errorf("failed to load database: %w", err) diff --git a/cmd/server/serve_test.go b/cmd/server/serve_test.go index b3f1f827..3e8b2e49 100644 --- a/cmd/server/serve_test.go +++ b/cmd/server/serve_test.go @@ -21,6 +21,7 @@ func TestServe(t *testing.T) { fs, err := storage.NewFs(common.DataDir) require.Nil(t, err) require.Nil(t, fs.Auth.InitHmacSecret()) + require.Nil(t, fs.Tuf.InitTuf()) authConfig := storage.AuthConfig{ Type: "noauth", } diff --git a/cmd/server/tuf_init.go b/cmd/server/tuf_init.go new file mode 100644 index 00000000..1d0a00c8 --- /dev/null +++ b/cmd/server/tuf_init.go @@ -0,0 +1,18 @@ +// Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. +// SPDX-License-Identifier: BSD-3-Clause-Clear + +package main + +import ( + "github.com/foundriesio/update-server/storage" +) + +type TufInitCmd struct{} + +func (c TufInitCmd) Run(args CommonArgs) error { + fs, err := storage.NewFs(args.DataDir) + if err != nil { + return err + } + return fs.Tuf.InitTuf() +} From 2469b697b0deb7c86bb87934b35c95fa459e3b6b Mon Sep 17 00:00:00 2001 From: Andy Doan Date: Wed, 24 Jun 2026 11:10:43 -0500 Subject: [PATCH 03/13] feat: expose TUF root metadata over REST API Add /v1/tuf/root.json (latest) and /v1/tuf/.root.json (specific version) endpoints backed by a new ReadRoot storage method. Co-authored-by: GitHub Copilot:claude-4-opus Signed-off-by: Andy Doan --- server/ui/api/handlers.go | 5 +++ server/ui/api/handlers_tuf.go | 57 ++++++++++++++++++++++++++++++ server/ui/api/handlers_tuf_test.go | 47 ++++++++++++++++++++++++ storage/api/api_storage_tuf.go | 11 ++++++ storage/file_tuf.go | 22 ++++++++++++ 5 files changed, 142 insertions(+) create mode 100644 server/ui/api/handlers_tuf.go create mode 100644 server/ui/api/handlers_tuf_test.go create mode 100644 storage/api/api_storage_tuf.go diff --git a/server/ui/api/handlers.go b/server/ui/api/handlers.go index 8c9cbea7..aa31560b 100644 --- a/server/ui/api/handlers.go +++ b/server/ui/api/handlers.go @@ -65,4 +65,9 @@ func RegisterHandlers(e *echo.Echo, storage *storage.Storage, a auth.Provider) { upd.PUT("/:tag/:update/rollouts/:rollout", h.rolloutPut, requireScope(users.ScopeUpdatesRU)) upd.GET("/:tag/:update/rollouts/:rollout/tail", h.rolloutTail, requireScope(users.ScopeUpdatesR)) upd.GET("/:tag/:update/tail", h.updateTail, requireScope(users.ScopeUpdatesR)) + + // TUF root metadata. The static "root.json" route returns the latest + // version; ".root.json" returns a specific version. + g.GET("/tuf/root.json", h.tufRootLatest, requireScope(users.ScopeUpdatesR)) + g.GET("/tuf/:version", h.tufRootVersion, requireScope(users.ScopeUpdatesR)) } diff --git a/server/ui/api/handlers_tuf.go b/server/ui/api/handlers_tuf.go new file mode 100644 index 00000000..4aefec5e --- /dev/null +++ b/server/ui/api/handlers_tuf.go @@ -0,0 +1,57 @@ +// Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. +// SPDX-License-Identifier: BSD-3-Clause-Clear + +package api + +import ( + "errors" + "net/http" + "os" + "strconv" + "strings" + + "github.com/labstack/echo/v4" +) + +const rootJsonSuffix = ".root.json" + +// @Summary Returns the latest TUF root metadata +// @Description Requires scope: updates:read +// @Tags TUF +// @Produce json +// @Success 200 +// @Router /tuf/root.json [get] +func (h handlers) tufRootLatest(c echo.Context) error { + return h.writeTufRoot(c, 0) +} + +// @Summary Returns a specific version of the TUF root metadata +// @Description Requires scope: updates:read +// @Tags TUF +// @Produce json +// @Success 200 +// @Param version path string true "Root metadata file name, e.g. 3.root.json" +// @Router /tuf/{version}.root.json [get] +func (h handlers) tufRootVersion(c echo.Context) error { + name := c.Param("version") + digits, found := strings.CutSuffix(name, rootJsonSuffix) + if !found { + return EchoError(c, nil, http.StatusNotFound, "not found") + } + version, err := strconv.Atoi(digits) + if err != nil || version < 1 { + return EchoError(c, err, http.StatusNotFound, "invalid root metadata version") + } + return h.writeTufRoot(c, version) +} + +func (h handlers) writeTufRoot(c echo.Context, version int) error { + data, err := h.storage.GetTufRoot(version) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return EchoError(c, err, http.StatusNotFound, "root metadata not found") + } + return EchoError(c, err, http.StatusInternalServerError, "failed to read root metadata") + } + return c.JSONBlob(http.StatusOK, data) +} diff --git a/server/ui/api/handlers_tuf_test.go b/server/ui/api/handlers_tuf_test.go new file mode 100644 index 00000000..e70148ea --- /dev/null +++ b/server/ui/api/handlers_tuf_test.go @@ -0,0 +1,47 @@ +// Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. +// SPDX-License-Identifier: BSD-3-Clause-Clear + +package api + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/foundriesio/update-server/storage/tuf" + "github.com/foundriesio/update-server/storage/users" +) + +func TestApiTufRoot(t *testing.T) { + tc := NewTestClient(t) + + // Requires the updates:read scope. + tc.GET("/tuf/root.json", 403) + tc.GET("/tuf/1.root.json", 403) + tc.u.AllowedScopes = users.ScopeUpdatesR + + // Before TUF is initialized there is no root metadata. + tc.GET("/tuf/root.json", 404) + + require.Nil(t, tc.fs.Auth.InitHmacSecret()) + require.Nil(t, tc.fs.Tuf.InitTuf()) + + // The latest root.json is returned and is valid v1 root metadata. + var root tuf.AtsTufRoot + require.Nil(t, json.Unmarshal(tc.GET("/tuf/root.json", 200), &root)) + require.Equal(t, "Root", root.Signed.Type) + require.Equal(t, 1, root.Signed.Version) + require.Len(t, root.Signatures, 1) + + // The explicit version returns the same document. + var byVersion tuf.AtsTufRoot + require.Nil(t, json.Unmarshal(tc.GET("/tuf/1.root.json", 200), &byVersion)) + require.Equal(t, root.Signed.Version, byVersion.Signed.Version) + require.Equal(t, root.Signatures, byVersion.Signatures) + + // Unknown versions and malformed names are 404. + tc.GET("/tuf/2.root.json", 404) + tc.GET("/tuf/not-a-version", 404) + tc.GET("/tuf/0.root.json", 404) +} diff --git a/storage/api/api_storage_tuf.go b/storage/api/api_storage_tuf.go new file mode 100644 index 00000000..997694f4 --- /dev/null +++ b/storage/api/api_storage_tuf.go @@ -0,0 +1,11 @@ +// Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. +// SPDX-License-Identifier: BSD-3-Clause-Clear + +package api + +// GetTufRoot returns the raw JSON bytes of a TUF root metadata file. A version +// of 0 (or less) returns the latest root metadata. The returned error wraps +// os.ErrNotExist when the requested root does not exist. +func (s Storage) GetTufRoot(version int) ([]byte, error) { + return s.fs.Tuf.ReadRoot(version) +} diff --git a/storage/file_tuf.go b/storage/file_tuf.go index 80863f18..f224f34e 100644 --- a/storage/file_tuf.go +++ b/storage/file_tuf.go @@ -195,6 +195,28 @@ func (h TufFsHandle) GetRoots() ([]tuf.AtsTufRoot, error) { return roots, nil } +// ReadRoot returns the raw JSON bytes of a root metadata file. A version <= 0 +// returns the latest (highest version) root metadata. It returns an error that +// wraps os.ErrNotExist when the requested root does not exist. +func (h TufFsHandle) ReadRoot(version int) ([]byte, error) { + name := strconv.Itoa(version) + rootJsonSuffix + if version <= 0 { + names, err := h.rootMetaNames() + if err != nil { + return nil, err + } + if len(names) == 0 { + return nil, fmt.Errorf("no root metadata found: %w", os.ErrNotExist) + } + name = names[len(names)-1] + } + content, err := h.readFile(name, false) + if err != nil { + return nil, fmt.Errorf("unable to read %s: %w", name, err) + } + return []byte(content), nil +} + func (h TufFsHandle) Sign(role tuf.RoleName, v any) (tuf.Signature, error) { if !h.Enabled() { return tuf.Signature{}, fmt.Errorf("TUF signing not available: call LoadTuf first") From a2a68a9952d2ab073e8031903607cbc06f806d52 Mon Sep 17 00:00:00 2001 From: Andy Doan Date: Mon, 22 Jun 2026 14:15:36 -0500 Subject: [PATCH 04/13] feat: Introduce "AddTarget" API This introduces a new API to properly add target to a given tag/update and sign its TUF metadata. Target metadata logic works by: * Finding the latest targets TUF version and application version for a tag. We must know these values to ensure we are creating metadata a device will pull down (if the version isn't higher - it won't pull it). * Increment the TUF version by 10. This gives us some flexibility to resign a Targets meta in the event we need to use it longer than the default 90 day expiration. * Create Snapshot metadata that's basically the same as the Targets. Our configuration/usage allows for these files to follow each other's versions. * Create Timestamp metadata. This follows some clever logic in our current ota-lite backend for setting a version that will work across mulitple updates for a given update tag. Targets/Snapshot get a 90 day expiry. Timestamp get a 7 day expiry. Co-authored-by: GitHub Copilot:claude-4-opus Signed-off-by: Andy Doan --- storage/api/api_storage_tuf.go | 196 ++++++++++++++++++++++++++ storage/api/api_storage_tuf_test.go | 206 ++++++++++++++++++++++++++++ storage/file.go | 2 +- storage/file_tuf.go | 63 ++++++++- storage/tuf/tuf_data.go | 19 +++ 5 files changed, 484 insertions(+), 2 deletions(-) create mode 100644 storage/api/api_storage_tuf_test.go diff --git a/storage/api/api_storage_tuf.go b/storage/api/api_storage_tuf.go index 997694f4..c829edb8 100644 --- a/storage/api/api_storage_tuf.go +++ b/storage/api/api_storage_tuf.go @@ -3,9 +3,205 @@ package api +import ( + "encoding/hex" + "encoding/json" + "fmt" + "time" + + "github.com/foundriesio/update-server/clock" + "github.com/foundriesio/update-server/storage" + "github.com/foundriesio/update-server/storage/tuf" +) + // GetTufRoot returns the raw JSON bytes of a TUF root metadata file. A version // of 0 (or less) returns the latest root metadata. The returned error wraps // os.ErrNotExist when the requested root does not exist. func (s Storage) GetTufRoot(version int) ([]byte, error) { return s.fs.Tuf.ReadRoot(version) } + +type TargetOptions struct { + AppVersion int // Becomes custom.version + HardwareId string // Becomes custom.hardwareIds[0] + Name string // Becomes custom.name (otherwise the ostree ref name) + OstreeHash string // Becomes hashes.sha256 (otherwise the ostree ref content) + Apps map[string]string // Becomes docker_compose_apps (app name -> sha256) + BaseUrl string // base URL used to build proxied app/target URIs +} + +type composeAppURI struct { + URI string `json:"uri"` +} + +type generatedTargetCustom struct { + Name string `json:"name"` + Version string `json:"version"` + HardwareIds []string `json:"hardwareIds"` + Tags []string `json:"tags"` + TargetFormat string `json:"targetFormat"` + DockerComposeApps map[string]composeAppURI `json:"docker_compose_apps,omitempty"` + CreatedAt string `json:"createdAt"` +} + +func (s Storage) AddTarget(tag, update string, opts TargetOptions) error { + // tufVer/tgtVer are the highest existing versions for this tag. The new TUF + // metadata version is derived below (tufVer + 10); the target version reuses + // the highest existing value unless AppVersion overrides it. + tufVer, tgtVer, err := s.getLatestVersions(tag) + if err != nil { + return fmt.Errorf("unable to determine latest target versions: %w", err) + } + if opts.AppVersion != 0 { + tgtVer = opts.AppVersion + } + + custom := generatedTargetCustom{ + Name: opts.Name, + Version: fmt.Sprintf("%d", tgtVer), + HardwareIds: []string{opts.HardwareId}, + Tags: []string{tag}, + TargetFormat: "OSTREE", + CreatedAt: clock.Now().UTC().Format(time.RFC3339), + } + if len(opts.Apps) > 0 { + custom.DockerComposeApps = make(map[string]composeAppURI) + for app, hash := range opts.Apps { + custom.DockerComposeApps[app] = composeAppURI{ + URI: fmt.Sprintf("%s/composeapphack/%s@sha256:%s", opts.BaseUrl, app, hash), + } + } + } + + customJSON, err := json.Marshal(custom) + if err != nil { + return fmt.Errorf("unable to marshal generated target custom: %w", err) + } + + ostreeHash, err := hex.DecodeString(opts.OstreeHash) + if err != nil { + return fmt.Errorf("invalid ostree hash %q: %w", opts.OstreeHash, err) + } + + // We use tufVer + 10 here to allow flexibility for future code to refresh the targets metadata + // without having to increment the version of the targets metadata for every new target. Adding 10 + // allows us to refresh 10 times with a 90 day expiration allowing us to use a target for almost + // 900 days if we wanted to. + targets := tuf.AtsTufTargets{ + Signed: tuf.TargetsMeta{ + SignedCommon: tuf.SignedCommon{ + Type: tuf.RoleTargets.TufType(), + Version: tufVer + 10, + Expires: clock.Now().UTC().Add(s.fs.Tuf.TargetsExpiration).Truncate(time.Second), + }, + Targets: tuf.TargetFiles{ + fmt.Sprintf("%s-%d", opts.Name, tgtVer): tuf.TargetFileMeta{ + Length: 0, + Hashes: tuf.Hashes{"sha256": tuf.HexBytes(ostreeHash)}, + Custom: customJSON, + }, + }, + }, + } + + sig, err := s.fs.Tuf.Sign(tuf.RoleTargets, targets.Signed) + if err != nil { + return fmt.Errorf("unable to sign targets metadata: %w", err) + } + targets.Signatures = []tuf.Signature{sig} + + ss := tuf.AtsTufSnapshot{ + Signed: tuf.SnapshotMeta{ + SignedCommon: tuf.SignedCommon{ + Type: tuf.RoleSnapshot.TufType(), + Version: targets.Signed.Version, + Expires: targets.Signed.Expires, + }, + Meta: map[string]tuf.MetaItem{ + storage.TufTargetsFile: { + Version: targets.Signed.Version, + }, + }, + }, + } + + sig, err = s.fs.Tuf.Sign(tuf.RoleSnapshot, ss.Signed) + if err != nil { + return fmt.Errorf("unable to sign snapshot metadata: %w", err) + } + ss.Signatures = []tuf.Signature{sig} + + // When we generate a new timestamp role - we need to increase its version + // as well in order to let a client (aktualizr) side handle it properly + // (e.g. store to a local storage). But also, we need to support a tag + // switch from a lower targets version to a higher one. For a timestamp + // role to support that we need to provide some "reserve" of timestamp role + // versions per each snapshot/targets version. This is why we multiply by + // 1000 below - that allows to rotate a timestamp role for 8 years every 3 + // days per each targets version. + tsVersion := targets.Signed.Version * 1000 + ts := tuf.AtsTufTimestamp{ + Signed: tuf.TimestampMeta{ + SignedCommon: tuf.SignedCommon{ + Type: tuf.RoleTimestamp.TufType(), + Version: tsVersion, + Expires: clock.Now().UTC().Add(s.fs.Tuf.TimestampExpiration).Truncate(time.Second), + }, + Meta: map[string]tuf.MetaItem{ + storage.TufSnapshotFile: { + Version: ss.Signed.Version, + }, + }, + }, + } + + sig, err = s.fs.Tuf.Sign(tuf.RoleTimestamp, ts.Signed) + if err != nil { + return fmt.Errorf("unable to sign timestamp metadata: %w", err) + } + ts.Signatures = []tuf.Signature{sig} + + targetsJson, err := json.Marshal(targets) + if err != nil { + return fmt.Errorf("unable to marshal targets metadata: %w", err) + } + snapshotJson, err := json.Marshal(ss) + if err != nil { + return fmt.Errorf("unable to marshal snapshot metadata: %w", err) + } + timestampJson, err := json.Marshal(ts) + if err != nil { + return fmt.Errorf("unable to marshal timestamp metadata: %w", err) + } + + if err := s.fs.Tuf.WriteMeta(tag, update, targetsJson, snapshotJson, timestampJson); err != nil { + return fmt.Errorf("unable to write targets metadata: %w", err) + } + + return nil +} + +// getLatestVersions returns the highest TUF metadata version and the highest +// target/app version currently present across all updates for the given tag. +// Both are zero when the tag has no existing TUF metadata. +func (s Storage) getLatestVersions(tag string) (tufVersion, targetVersion int, err error) { + updates, err := s.ListUpdates(tag) + if err != nil { + return 0, 0, err + } + for _, u := range updates[tag] { + var targets tuf.AtsTufTargets + if err := s.fs.Tuf.ReadTufMeta(tag, u.Name, storage.TufTargetsFile, &targets); err != nil { + // Skip updates that pre-date TUF or whose metadata is missing/unreadable. + continue + } + if tufVersion < targets.Signed.Version { + tufVersion = targets.Signed.Version + } + latest := targets.GetLatestTargetVersion() + if targetVersion < latest { + targetVersion = latest + } + } + return tufVersion, targetVersion, nil +} diff --git a/storage/api/api_storage_tuf_test.go b/storage/api/api_storage_tuf_test.go new file mode 100644 index 00000000..0a77521f --- /dev/null +++ b/storage/api/api_storage_tuf_test.go @@ -0,0 +1,206 @@ +// Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. +// SPDX-License-Identifier: BSD-3-Clause-Clear + +package api + +import ( + "crypto/ed25519" + "encoding/hex" + "encoding/json" + "path/filepath" + "testing" + "time" + + "github.com/secure-systems-lab/go-securesystemslib/cjson" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/foundriesio/update-server/clock" + "github.com/foundriesio/update-server/storage" + "github.com/foundriesio/update-server/storage/tuf" +) + +// newTufStorage returns a Storage with an initialized and loaded TUF setup, +// ready for AddTarget operations. +func newTufStorage(t *testing.T) *Storage { + t.Helper() + tmpdir := t.TempDir() + dbFile := filepath.Join(tmpdir, "sql.db") + db, err := storage.NewDb(dbFile) + require.NoError(t, err) + fs, err := storage.NewFs(tmpdir) + require.NoError(t, err) + + require.NoError(t, fs.Auth.InitHmacSecret()) + require.NoError(t, fs.Tuf.InitTuf()) + require.NoError(t, fs.Tuf.LoadTuf()) + + s, err := NewStorage(db, fs) + require.NoError(t, err) + return s +} + +// ensureTargetsDir pre-creates the TUF update directory that AddTarget writes +// into. AddTarget symlinks the root metadata into this directory before +// creating it, so it must already exist. +func ensureTargetsDir(t *testing.T, s *Storage, tag string) { + t.Helper() + require.NoError(t, s.fs.Updates.Tuf.WriteFile(tag, storage.TufTargetsFile, ".keep", "")) +} + +// verifyTufSig asserts that sig is a valid signature over signed, produced by a +// key listed in the latest root metadata. +func verifyTufSig(t *testing.T, s *Storage, sig tuf.Signature, signed any) { + t.Helper() + require.Equal(t, tuf.SigEd25519, sig.Method) + + roots, err := s.fs.Tuf.GetRoots() + require.NoError(t, err) + require.NotEmpty(t, roots) + root := roots[len(roots)-1] + + key, ok := root.Signed.Keys[sig.KeyID] + require.True(t, ok, "signature key %s not present in root metadata", sig.KeyID) + pub, err := hex.DecodeString(key.KeyValue.Public) + require.NoError(t, err) + + msg, err := cjson.EncodeCanonical(signed) + require.NoError(t, err) + require.True(t, ed25519.Verify(ed25519.PublicKey(pub), msg, sig.Signature), + "signature must verify against canonical signed payload") +} + +func TestAddTarget(t *testing.T) { + fixedNow := time.Date(2026, time.June, 25, 12, 0, 0, 0, time.UTC) + clock.Now = func() time.Time { return fixedNow } + defer func() { clock.Now = time.Now }() + + s := newTufStorage(t) + + const tag = "main" + opts := TargetOptions{ + AppVersion: 3, + HardwareId: "raspberrypi4-64", + Name: "raspberrypi4-64-lmp", + OstreeHash: "deadbeef", + Apps: map[string]string{ + "shellhttpd": "sha256hashvalue", + }, + BaseUrl: "https://example.com", + } + ensureTargetsDir(t, s, tag) + require.NoError(t, s.AddTarget(tag, opts)) + + // Targets metadata is written, with the expected version and one target. + var targets tuf.AtsTufTargets + require.NoError(t, s.fs.Tuf.ReadTufMeta(tag, storage.TufTargetsFile, storage.TufTargetsFile, &targets)) + assert.Equal(t, "Targets", targets.Signed.Type) + assert.Equal(t, 10, targets.Signed.Version) + assert.Equal(t, fixedNow.Add(s.fs.Tuf.TargetsExpiration).Truncate(time.Second), targets.Signed.Expires) + require.Len(t, targets.Signed.Targets, 1) + + targetName := "raspberrypi4-64-lmp-3" + target, ok := targets.Signed.Targets[targetName] + require.True(t, ok, "expected target %q", targetName) + assert.Equal(t, int64(0), target.Length) + assert.Equal(t, "deadbeef", hex.EncodeToString(target.Hashes["sha256"])) + + // The custom block carries the generated target metadata. + var custom generatedTargetCustom + require.NoError(t, json.Unmarshal(target.Custom, &custom)) + assert.Equal(t, "raspberrypi4-64-lmp", custom.Name) + assert.Equal(t, "3", custom.Version) + assert.Equal(t, []string{"raspberrypi4-64"}, custom.HardwareIds) + assert.Equal(t, "OSTREE", custom.TargetFormat) + assert.Equal(t, fixedNow.Format(time.RFC3339), custom.CreatedAt) + require.Len(t, custom.DockerComposeApps, 1) + assert.Equal(t, "https://example.com/composeapphack/shellhttpd@sha256:sha256hashvalue", custom.DockerComposeApps["shellhttpd"].URI) + // Targets metadata is signed by the targets role. + require.Len(t, targets.Signatures, 1) + verifyTufSig(t, s, targets.Signatures[0], targets.Signed) + + // Snapshot metadata references the targets version and is signed. + var snapshot tuf.AtsTufSnapshot + require.NoError(t, s.fs.Tuf.ReadTufMeta(tag, storage.TufTargetsFile, storage.TufSnapshotFile, &snapshot)) + assert.Equal(t, "Snapshot", snapshot.Signed.Type) + assert.Equal(t, targets.Signed.Version, snapshot.Signed.Version) + assert.Equal(t, targets.Signed.Expires, snapshot.Signed.Expires) + assert.Equal(t, targets.Signed.Version, snapshot.Signed.Meta[storage.TufTargetsFile].Version) + require.Len(t, snapshot.Signatures, 1) + verifyTufSig(t, s, snapshot.Signatures[0], snapshot.Signed) + + // Timestamp metadata references the snapshot version and uses the reserved + // (version * 1000) numbering scheme. + var timestamp tuf.AtsTufTimestamp + require.NoError(t, s.fs.Tuf.ReadTufMeta(tag, storage.TufTargetsFile, storage.TufTimestampFile, ×tamp)) + assert.Equal(t, "Timestamp", timestamp.Signed.Type) + assert.Equal(t, targets.Signed.Version*1000, timestamp.Signed.Version) + assert.Equal(t, fixedNow.Add(s.fs.Tuf.TimestampExpiration).Truncate(time.Second), timestamp.Signed.Expires) + assert.Equal(t, snapshot.Signed.Version, timestamp.Signed.Meta[storage.TufSnapshotFile].Version) + require.Len(t, timestamp.Signatures, 1) + verifyTufSig(t, s, timestamp.Signatures[0], timestamp.Signed) +} + +func TestAddTargetWithoutApps(t *testing.T) { + s := newTufStorage(t) + + const tag = "main" + ensureTargetsDir(t, s, tag) + require.NoError(t, s.AddTarget(tag, TargetOptions{ + AppVersion: 5, + HardwareId: "intel-corei7-64", + Name: "intel-corei7-64-lmp", + OstreeHash: "abc123", + })) + + var targets tuf.AtsTufTargets + require.NoError(t, s.fs.Tuf.ReadTufMeta(tag, storage.TufTargetsFile, storage.TufTargetsFile, &targets)) + require.Len(t, targets.Signed.Targets, 1) + + target := targets.Signed.Targets["intel-corei7-64-lmp-5"] + var custom generatedTargetCustom + require.NoError(t, json.Unmarshal(target.Custom, &custom)) + assert.Equal(t, "5", custom.Version) + assert.Nil(t, custom.DockerComposeApps, "docker_compose_apps should be omitted when there are no apps") +} + +func TestAddTargetIncrementsTufVersion(t *testing.T) { + s := newTufStorage(t) + + const tag = "main" + + // Seed an existing update whose targets metadata is at TUF version 5. + existing := tuf.AtsTufTargets{ + Signed: tuf.TargetsMeta{ + SignedCommon: tuf.SignedCommon{ + Type: tuf.RoleTargets.TufType(), + Version: 5, + }, + Targets: tuf.TargetFiles{ + "prev-target-10": tuf.TargetFileMeta{ + Custom: json.RawMessage(`{"version":10}`), + }, + }, + }, + } + existingJSON, err := json.Marshal(existing) + require.NoError(t, err) + require.NoError(t, s.InsertUpdate(tag, "update-5", "tester")) + require.NoError(t, s.fs.Updates.Tuf.WriteFile(tag, "update-5", storage.TufTargetsFile, string(existingJSON))) + + ensureTargetsDir(t, s, tag) + // A new target should bump the TUF version above the existing one. + require.NoError(t, s.AddTarget(tag, TargetOptions{ + HardwareId: "intel-corei7-64", + Name: "intel-corei7-64-lmp", + OstreeHash: "abc123", + })) + + var targets tuf.AtsTufTargets + require.NoError(t, s.fs.Tuf.ReadTufMeta(tag, storage.TufTargetsFile, storage.TufTargetsFile, &targets)) + assert.Equal(t, 15, targets.Signed.Version, "TUF version should be ten greater than the existing update") + + var timestamp tuf.AtsTufTimestamp + require.NoError(t, s.fs.Tuf.ReadTufMeta(tag, storage.TufTargetsFile, storage.TufTimestampFile, ×tamp)) + assert.Equal(t, 15000, timestamp.Signed.Version) +} diff --git a/storage/file.go b/storage/file.go index c1c67739..4d296368 100644 --- a/storage/file.go +++ b/storage/file.go @@ -140,7 +140,7 @@ func NewFs(root string) (*FsHandle, error) { fs.Configs.root = fs.Config.ConfigsDir() fs.Devices.root = fs.Config.DevicesDir() fs.Updates.init(fs.Config.UpdatesDir()) - fs.Tuf.init(fs.Config.TufDir(), fs.Auth) + fs.Tuf.init(fs.Config.TufDir(), fs.Auth, fs.Updates) for _, h := range []baseFsHandle{ fs.Audit.baseFsHandle, diff --git a/storage/file_tuf.go b/storage/file_tuf.go index f224f34e..afdd4556 100644 --- a/storage/file_tuf.go +++ b/storage/file_tuf.go @@ -52,6 +52,8 @@ type TufFsHandle struct { // auth provides access to the HMAC secret used to encrypt key files. auth AuthFsHandle + updates updatesFsHandleWrap + // RootExpiration is the validity period used for newly created root.json. RootExpiration time.Duration // TimestampExpiration is the validity period for timestamp metadata. @@ -64,9 +66,10 @@ type TufFsHandle struct { signers map[tuf.RoleName]*tuf.Signer } -func (h *TufFsHandle) init(root string, auth AuthFsHandle) { +func (h *TufFsHandle) init(root string, auth AuthFsHandle, updates updatesFsHandleWrap) { h.root = root h.auth = auth + h.updates = updates h.RootExpiration = 20 * 365 * 24 * time.Hour h.TimestampExpiration = 7 * 24 * time.Hour h.TargetsExpiration = 90 * 24 * time.Hour @@ -217,6 +220,18 @@ func (h TufFsHandle) ReadRoot(version int) ([]byte, error) { return []byte(content), nil } +// ReadTufMeta reads and unmarshals a TUF metadata file from an update. +func (h TufFsHandle) ReadTufMeta(tag, update, name string, v any) error { + content, err := h.updates.Tuf.ReadFile(tag, update, name) + if err != nil { + return err + } + if err := json.Unmarshal([]byte(content), v); err != nil { + return fmt.Errorf("unable to parse %s for tag %s update %s: %w", name, tag, update, err) + } + return nil +} + func (h TufFsHandle) Sign(role tuf.RoleName, v any) (tuf.Signature, error) { if !h.Enabled() { return tuf.Signature{}, fmt.Errorf("TUF signing not available: call LoadTuf first") @@ -228,6 +243,52 @@ func (h TufFsHandle) Sign(role tuf.RoleName, v any) (tuf.Signature, error) { return signer.Sign(v) } +// Enabled reports whether TUF signing is available, i.e. LoadTuf has loaded the +// role keys. +func (h TufFsHandle) Enabled() bool { + return len(h.signers) > 0 +} + +func (h TufFsHandle) WriteMeta(tag, update string, targets, snapshot, timestamp []byte) error { + // Ensure the update's TUF directory exists before we symlink root metadata + // into it (the symlink targets live inside this directory). + if _, err := h.updates.Tuf.updateLocalHandle(tag, update, true); err != nil { + return err + } + + // link root metadata into the update directory + names, err := h.rootMetaNames() + if err != nil { + return fmt.Errorf("unable to list root metadata: %w", err) + } + for _, name := range names { + src, err := filepath.Abs(filepath.Join(h.root, name)) + if err != nil { + return fmt.Errorf("unable to resolve absolute path for root metadata %s: %w", name, err) + } + dst := h.updates.Tuf.FilePath(tag, update, name) + if err := os.Remove(dst); err != nil && !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("unable to replace root metadata link %s: %w", name, err) + } + if err := os.Symlink(src, dst); err != nil && !errors.Is(err, os.ErrExist) { + return fmt.Errorf("unable to symlink %s into update directory: %w", name, err) + } + } + + // Now write out the metadata + if err := h.updates.Tuf.WriteFile(tag, update, "targets.json", string(targets)); err != nil { + return fmt.Errorf("unable to write targets.json for tag %s update %s: %w", tag, update, err) + } + if err := h.updates.Tuf.WriteFile(tag, update, "snapshot.json", string(snapshot)); err != nil { + return fmt.Errorf("unable to write snapshot.json for tag %s update %s: %w", tag, update, err) + } + if err := h.updates.Tuf.WriteFile(tag, update, "timestamp.json", string(timestamp)); err != nil { + return fmt.Errorf("unable to write timestamp.json for tag %s update %s: %w", tag, update, err) + } + + return nil +} + // writeRoot persists a root metadata file as .root.json. func (h TufFsHandle) writeRoot(root tuf.AtsTufRoot) error { data, err := json.MarshalIndent(root, "", " ") diff --git a/storage/tuf/tuf_data.go b/storage/tuf/tuf_data.go index 138cfd45..ef410b3c 100644 --- a/storage/tuf/tuf_data.go +++ b/storage/tuf/tuf_data.go @@ -7,6 +7,8 @@ import ( "encoding/hex" "encoding/json" "fmt" + "log/slog" + "strconv" "time" ) @@ -155,6 +157,23 @@ type AtsTufTargets struct { Signed TargetsMeta `json:"signed"` } +func (t AtsTufTargets) GetLatestTargetVersion() int { + maxVersion := 0 + for name, target := range t.Signed.Targets { + var custom struct { + Version string `json:"version"` + } + if err := json.Unmarshal(target.Custom, &custom); err != nil { + slog.Warn("Unable to parse target custom", "target-name", name, "error", err) + continue + } + if n, err := strconv.Atoi(custom.Version); err == nil && n > maxVersion { + maxVersion = n + } + } + return maxVersion +} + // MetaItem references a version of another metadata file. The ota-tuf format // for snapshot and timestamp metadata only records the version. type MetaItem struct { From 75cf949a0c0bddae73f64622e133608aa5b74556 Mon Sep 17 00:00:00 2001 From: Andy Doan Date: Fri, 26 Jun 2026 10:16:57 -0500 Subject: [PATCH 05/13] feat: Introduce Timestamp refresh API and daemon This introduces an new API that allows us to refresh timestamp metadata for any updates that may need it. When the update expiry is within one day of expiration, it will bump the expiration by a week and resign the metatdata. Signed-off-by: Andy Doan --- server/ui/daemons/daemons.go | 5 +++ server/ui/daemons/daemons_tuf.go | 41 ++++++++++++++++++++ storage/api/api_storage_tuf.go | 55 +++++++++++++++++++++++++++ storage/api/api_storage_tuf_test.go | 58 +++++++++++++++++++++++++++++ storage/file_tuf.go | 4 ++ 5 files changed, 163 insertions(+) create mode 100644 server/ui/daemons/daemons_tuf.go diff --git a/server/ui/daemons/daemons.go b/server/ui/daemons/daemons.go index b0340aa2..fbb4562a 100644 --- a/server/ui/daemons/daemons.go +++ b/server/ui/daemons/daemons.go @@ -22,6 +22,7 @@ type daemons struct { stops []chan bool rolloutOptions rolloutOptions + tufOptions tufOptions } func New(context context.Context, storage *storage.Storage, users *users.Storage, opts ...Option) *daemons { @@ -29,9 +30,13 @@ func New(context context.Context, storage *storage.Storage, users *users.Storage d.rolloutOptions = rolloutOptions{ interval: 5 * time.Minute, } + d.tufOptions = tufOptions{ + interval: 4 * time.Hour, + } d.daemons = []daemonFunc{ d.rolloutWatchdog(), userGcDaemonFunc(users), + d.tufRefreshDaemon(), } for _, opt := range opts { diff --git a/server/ui/daemons/daemons_tuf.go b/server/ui/daemons/daemons_tuf.go new file mode 100644 index 00000000..1fa76c47 --- /dev/null +++ b/server/ui/daemons/daemons_tuf.go @@ -0,0 +1,41 @@ +// Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. +// SPDX-License-Identifier: BSD-3-Clause-Clear + +package daemons + +import ( + "time" + + "github.com/foundriesio/update-server/context" +) + +// WithTufRefreshInterval sets how often the TUF refresh daemon scans for +// metadata that is approaching expiry. +func WithTufRefreshInterval(interval time.Duration) Option { + return func(d *daemons) { + d.tufOptions.interval = interval + } +} + +type tufOptions struct { + interval time.Duration +} + +// tufRefreshDaemon periodically refreshes the snapshot and timestamp TUF +// metadata of every tag whose metadata is approaching expiry. The timestamp is +// short lived and is refreshed far more often than the snapshot. +func (d *daemons) tufRefreshDaemon() daemonFunc { + return func(stop chan bool) { + log := context.CtxGetLog(d.context) + for { + if err := d.storage.RefreshTufTimestamps(d.context); err != nil { + log.Error("failed to refresh TUF metadata expiry", "error", err) + } + select { + case <-stop: + return + case <-time.After(d.tufOptions.interval): + } + } + } +} diff --git a/storage/api/api_storage_tuf.go b/storage/api/api_storage_tuf.go index c829edb8..944a7111 100644 --- a/storage/api/api_storage_tuf.go +++ b/storage/api/api_storage_tuf.go @@ -10,6 +10,7 @@ import ( "time" "github.com/foundriesio/update-server/clock" + "github.com/foundriesio/update-server/context" "github.com/foundriesio/update-server/storage" "github.com/foundriesio/update-server/storage/tuf" ) @@ -205,3 +206,57 @@ func (s Storage) getLatestVersions(tag string) (tufVersion, targetVersion int, e } return tufVersion, targetVersion, nil } + +func (s Storage) RefreshTufTimestamps(c context.Context) error { + updates, err := s.ListUpdates("") + if err != nil { + return err + } + + log := context.CtxGetLog(c) + + for tag, updates := range updates { + log.Info("Checking TUF timestamp expiry for tag", "tag", tag, "updates", len(updates)) + for _, u := range updates { + log.Debug("Checking timestamp for", "tag", tag, "update", u.Name) + if err := s.refreshTufTimestamp(c, tag, u); err != nil { + log.Error("Failed to refresh TUF timestamps", "tag", tag, "update", u.Name, "error", err) + } + } + } + return nil +} + +func (s Storage) refreshTufTimestamp(c context.Context, tag string, update Update) error { + log := context.CtxGetLog(c) + + var ts tuf.AtsTufTimestamp + if err := s.fs.Tuf.ReadTufMeta(tag, update.Name, storage.TufTimestampFile, &ts); err != nil { + return fmt.Errorf("unable to read timestamp metadata: %w", err) + } + + cutoff := clock.Now().UTC().Add(time.Hour * 24) // 1 day from now + if ts.Signed.Expires.After(cutoff) { + log.Debug("Timestamp okay", "tag", tag, "update", update.Name, "expiry", ts.Signed.Expires, "cutoff", cutoff) + return nil // timestamp is still valid, no need to refresh + } + + ts.Signed.Expires = clock.Now().UTC().Add(s.fs.Tuf.TimestampExpiration).Truncate(time.Second) + sig, err := s.fs.Tuf.Sign(tuf.RoleTimestamp, ts.Signed) + if err != nil { + return fmt.Errorf("unable to sign timestamp metadata: %w", err) + } + ts.Signatures = []tuf.Signature{sig} + + tsJson, err := json.Marshal(ts) + if err != nil { + return fmt.Errorf("unable to marshal timestamp metadata: %w", err) + } + + if err := s.fs.Tuf.WriteTimestamp(tag, update.Name, tsJson); err != nil { + return err + } + + log.Info("Refreshed TUF timestamp", "tag", tag, "update", update.Name, "new_expiry", ts.Signed.Expires) + return nil +} diff --git a/storage/api/api_storage_tuf_test.go b/storage/api/api_storage_tuf_test.go index 0a77521f..4eef5286 100644 --- a/storage/api/api_storage_tuf_test.go +++ b/storage/api/api_storage_tuf_test.go @@ -7,6 +7,7 @@ import ( "crypto/ed25519" "encoding/hex" "encoding/json" + "log/slog" "path/filepath" "testing" "time" @@ -16,6 +17,7 @@ import ( "github.com/stretchr/testify/require" "github.com/foundriesio/update-server/clock" + appctx "github.com/foundriesio/update-server/context" "github.com/foundriesio/update-server/storage" "github.com/foundriesio/update-server/storage/tuf" ) @@ -204,3 +206,59 @@ func TestAddTargetIncrementsTufVersion(t *testing.T) { require.NoError(t, s.fs.Tuf.ReadTufMeta(tag, storage.TufTargetsFile, storage.TufTimestampFile, ×tamp)) assert.Equal(t, 15000, timestamp.Signed.Version) } + +// writeUpdateTimestamp registers an update in the database and writes a +// timestamp.json with the given version and expiry into its TUF directory. +func writeUpdateTimestamp(t *testing.T, s *Storage, tag, update string, version int, expires time.Time) { + t.Helper() + ts := tuf.AtsTufTimestamp{ + Signed: tuf.TimestampMeta{ + SignedCommon: tuf.SignedCommon{ + Type: tuf.RoleTimestamp.TufType(), + Version: version, + Expires: expires, + }, + Meta: map[string]tuf.MetaItem{ + storage.TufSnapshotFile: {Version: 1}, + }, + }, + } + tsJSON, err := json.Marshal(ts) + require.NoError(t, err) + require.NoError(t, s.InsertUpdate(tag, update, "tester")) + require.NoError(t, s.fs.Tuf.WriteTimestamp(tag, update, tsJSON)) +} + +func TestRefreshTufTimestamps(t *testing.T) { + fixedNow := time.Date(2026, time.June, 26, 12, 0, 0, 0, time.UTC) + clock.Now = func() time.Time { return fixedNow } + defer func() { clock.Now = time.Now }() + + s := newTufStorage(t) + + const tag = "main" + // One timestamp expires within the 1-day cutoff and should be refreshed. + soonExpiry := fixedNow.Add(12 * time.Hour) + writeUpdateTimestamp(t, s, tag, "update-soon", 1000, soonExpiry) + // One timestamp is well in the future and should be left untouched. + laterExpiry := fixedNow.Add(30 * 24 * time.Hour).Truncate(time.Second) + writeUpdateTimestamp(t, s, tag, "update-later", 2000, laterExpiry) + + ctx := appctx.CtxWithLog(appctx.Background(), slog.Default()) + require.NoError(t, s.RefreshTufTimestamps(ctx)) + + // The soon-to-expire timestamp was re-signed with a fresh expiry. + var refreshed tuf.AtsTufTimestamp + require.NoError(t, s.fs.Tuf.ReadTufMeta(tag, "update-soon", storage.TufTimestampFile, &refreshed)) + expectedExpiry := fixedNow.Add(s.fs.Tuf.TimestampExpiration).Truncate(time.Second) + assert.Equal(t, expectedExpiry, refreshed.Signed.Expires) + assert.Equal(t, 1000, refreshed.Signed.Version, "refresh should not change the timestamp version") + require.Len(t, refreshed.Signatures, 1) + verifyTufSig(t, s, refreshed.Signatures[0], refreshed.Signed) + + // The future timestamp was left unchanged. + var untouched tuf.AtsTufTimestamp + require.NoError(t, s.fs.Tuf.ReadTufMeta(tag, "update-later", storage.TufTimestampFile, &untouched)) + assert.Equal(t, laterExpiry, untouched.Signed.Expires) + assert.Empty(t, untouched.Signatures, "future timestamp should not be re-signed") +} diff --git a/storage/file_tuf.go b/storage/file_tuf.go index afdd4556..9c8c9c67 100644 --- a/storage/file_tuf.go +++ b/storage/file_tuf.go @@ -289,6 +289,10 @@ func (h TufFsHandle) WriteMeta(tag, update string, targets, snapshot, timestamp return nil } +func (h TufFsHandle) WriteTimestamp(tag, update string, ts []byte) error { + return h.updates.Tuf.WriteFile(tag, update, "timestamp.json", string(ts)) +} + // writeRoot persists a root metadata file as .root.json. func (h TufFsHandle) writeRoot(root tuf.AtsTufRoot) error { data, err := json.MarshalIndent(root, "", " ") From f338f103926ac866a561dde7d30a820920bcee59 Mon Sep 17 00:00:00 2001 From: Andy Doan Date: Fri, 26 Jun 2026 11:37:21 -0500 Subject: [PATCH 06/13] feat: Add module to do primitive OSTree parsing We will use this to help populate default data into a TUF target during an Update upload Signed-off-by: Andy Doan Co-authored-by: Claude Sonnet 4.6 --- storage/ostree/ostree.go | 267 ++++++++++++++++++++++++++++++++++ storage/ostree/ostree_test.go | 155 ++++++++++++++++++++ 2 files changed, 422 insertions(+) create mode 100644 storage/ostree/ostree.go create mode 100644 storage/ostree/ostree_test.go diff --git a/storage/ostree/ostree.go b/storage/ostree/ostree.go new file mode 100644 index 00000000..10a7e5a0 --- /dev/null +++ b/storage/ostree/ostree.go @@ -0,0 +1,267 @@ +// Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. +// SPDX-License-Identifier: BSD-3-Clause-Clear + +package ostree + +import ( + "bytes" + "encoding/binary" + "encoding/hex" + "fmt" + "os" + "path/filepath" + "strings" +) + +type Repo struct { + path string +} + +func NewRepo(path string) *Repo { + return &Repo{path: path} +} + +func (r *Repo) objectPath(hash, ext string) string { + return filepath.Join(r.path, "objects", hash[:2], hash[2:]+ext) +} + +// isValidObjectHash reports whether hash is a 64-character lowercase hex string, +// the form of an OSTree object checksum. Validating before constructing object +// paths guards against panics and path traversal from malformed upload content. +func isValidObjectHash(hash string) bool { + if len(hash) != 64 { + return false + } + for i := 0; i < len(hash); i++ { + c := hash[i] + if (c < '0' || c > '9') && (c < 'a' || c > 'f') { + return false + } + } + return true +} + +func (r *Repo) ReadRef(ref string) (string, error) { + for _, base := range []string{"heads", "remotes"} { + data, err := os.ReadFile(filepath.Join(r.path, "refs", base, ref)) + if err == nil { + hash := strings.TrimSpace(string(data)) + if !isValidObjectHash(hash) { + return "", fmt.Errorf("ref %q does not contain a valid object hash", ref) + } + return hash, nil + } + } + return "", fmt.Errorf("ref not found: %s", ref) +} + +// gvariantOffsetSize returns the byte width of GVariant framing offsets for a +// container of the given total byte length. +func gvariantOffsetSize(n int) int { + switch { + case n <= 0xff: + return 1 + case n <= 0xffff: + return 2 + default: + return 4 + } +} + +func readLE(data []byte, pos, size int) int { + switch size { + case 1: + return int(data[pos]) + case 2: + return int(binary.LittleEndian.Uint16(data[pos:])) + default: + return int(binary.LittleEndian.Uint32(data[pos:])) + } +} + +// rootDirtreeHash extracts the root dirtree checksum from a raw commit object. +// +// OSTree commit GVariant type: (a{sv}aya(say)sstayay) +// Fields: a{sv} ay a(say) s s t ay ay +// Variable-length fields needing framing (all except the last ay): 6 fields. +// Framing offsets are stored at the end in reverse field order. +// After the body string (field 5), alignment to 8 for the uint64 timestamp (t), +// then 32-byte dirtree hash, then 32-byte dirmeta hash, then 6 framing bytes. +// +// Framing layout (last 6 bytes, each 1 byte since total commit < 256): +// +// [0]: end of ay(dirtree) [1]: end of s(body) [2]: end of s(subject) +// [3]: end of a(say) [4]: end of ay(parent) [5]: end of a{sv} +func rootDirtreeHash(data []byte) (string, error) { + n := len(data) + offSize := gvariantOffsetSize(n) + // 6 framing offsets at the tail + if n < 6*offSize+8+32 { + return "", fmt.Errorf("commit object too small (%d bytes)", n) + } + framingBase := n - 6*offSize + // framing offsets in reverse field order; index 1 = end of body string + oBody := readLE(data, framingBase+offSize, offSize) + if oBody >= framingBase { + return "", fmt.Errorf("commit body offset %d out of range", oBody) + } + // timestamp (uint64) follows body string, aligned to 8 bytes + tsStart := (oBody + 7) &^ 7 + dirtreeStart := tsStart + 8 + if dirtreeStart+32 > framingBase { + return "", fmt.Errorf("commit dirtree region overruns framing table") + } + return hex.EncodeToString(data[dirtreeStart : dirtreeStart+32]), nil +} + +// lookupDirtree parses a raw dirtree object (GVariant type (a(say)a(sayay))) and +// returns the file hash if name is a file, or the dirtree hash if name is a subdir. +// +// Array framing: each GVariant array of variable-length elements stores N element-end +// offsets at the end of the array. The offset size is determined by the array length. +func lookupDirtree(data []byte, name string) (fileHash, subdirHash string, err error) { + n := len(data) + if n == 0 { + return "", "", fmt.Errorf("empty dirtree object") + } + // Outer tuple (A B): one framing offset at the end = end of A (files array). + offSize := gvariantOffsetSize(n) + if n < offSize { + return "", "", fmt.Errorf("dirtree too small") + } + filesEnd := readLE(data, n-offSize, offSize) + if filesEnd > n-offSize { + return "", "", fmt.Errorf("dirtree files-end offset %d out of range", filesEnd) + } + + if fh := lookupGVArray(data[:filesEnd], name); fh != "" { + return fh, "", nil + } + if dh := lookupGVArray(data[filesEnd:n-offSize], name); dh != "" { + return "", dh, nil + } + return "", "", nil +} + +// lookupGVArray searches a GVariant a(say) or a(sayay) array for name and returns +// the first 32-byte hash that follows the name's null terminator. +func lookupGVArray(data []byte, name string) string { + n := len(data) + if n == 0 { + return "" + } + offSize := gvariantOffsetSize(n) + // Last offSize bytes = framing[N-1] = end of last element = start of framing table. + lastElemEnd := readLE(data, n-offSize, offSize) + if lastElemEnd > n-offSize { + return "" + } + numElems := (n - lastElemEnd) / offSize + if numElems == 0 { + return "" + } + + prev := 0 + for i := 0; i < numElems; i++ { + end := readLE(data, lastElemEnd+i*offSize, offSize) + if end <= prev || end > lastElemEnd { + return "" + } + elem := data[prev:end] + // Each element is (s ay...). Find the null terminator of s. + nullPos := bytes.IndexByte(elem, 0) + if nullPos < 0 { + return "" + } + if string(elem[:nullPos]) == name { + hashStart := nullPos + 1 + if hashStart+32 > len(elem) { + return "" + } + return hex.EncodeToString(elem[hashStart : hashStart+32]) + } + prev = end + } + return "" +} + +// ReadFile returns the contents of filePath from the given ref in the repo. +// filePath should be an absolute path, e.g. "/usr/lib/sota/conf.d/40-hardware-id.toml". +func (r *Repo) ReadFile(ref, filePath string) ([]byte, error) { + commitHash, err := r.ReadRef(ref) + if err != nil { + return nil, err + } + + commitData, err := os.ReadFile(r.objectPath(commitHash, ".commit")) + if err != nil { + return nil, fmt.Errorf("reading commit object: %w", err) + } + + dirtreeHash, err := rootDirtreeHash(commitData) + if err != nil { + return nil, fmt.Errorf("parsing commit: %w", err) + } + + parts := strings.Split(strings.Trim(filePath, "/"), "/") + for _, part := range parts[:len(parts)-1] { + dirtreeData, err := os.ReadFile(r.objectPath(dirtreeHash, ".dirtree")) + if err != nil { + return nil, fmt.Errorf("reading dirtree %s: %w", dirtreeHash[:8], err) + } + _, dirtreeHash, err = lookupDirtree(dirtreeData, part) + if err != nil { + return nil, fmt.Errorf("parsing dirtree for %q: %w", part, err) + } + if dirtreeHash == "" { + return nil, fmt.Errorf("directory %q not found", part) + } + } + + filename := parts[len(parts)-1] + dirtreeData, err := os.ReadFile(r.objectPath(dirtreeHash, ".dirtree")) + if err != nil { + return nil, fmt.Errorf("reading dirtree %s: %w", dirtreeHash[:8], err) + } + fileHash, _, err := lookupDirtree(dirtreeData, filename) + if err != nil { + return nil, fmt.Errorf("parsing dirtree for %q: %w", filename, err) + } + if fileHash == "" { + return nil, fmt.Errorf("file %q not found in %s", filename, filepath.Dir(filePath)) + } + + content, err := os.ReadFile(r.objectPath(fileHash, ".file")) + if err != nil { + return nil, fmt.Errorf("reading file object %s: %w", fileHash[:8], err) + } + return content, nil +} + +// ListHeads returns the names of all refs under refs/heads, including refs nested +// in subdirectories (reported with '/' separators, e.g. "foo/bar"). +func (r *Repo) ListHeads() ([]string, error) { + headsDir := filepath.Join(r.path, "refs", "heads") + var refs []string + err := filepath.WalkDir(headsDir, func(path string, d os.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + return nil + } + rel, err := filepath.Rel(headsDir, path) + if err != nil { + return err + } + refs = append(refs, filepath.ToSlash(rel)) + return nil + }) + if err != nil { + if os.IsNotExist(err) { + return nil, nil + } + return nil, fmt.Errorf("listing refs/heads: %w", err) + } + return refs, nil +} diff --git a/storage/ostree/ostree_test.go b/storage/ostree/ostree_test.go new file mode 100644 index 00000000..a19a20e2 --- /dev/null +++ b/storage/ostree/ostree_test.go @@ -0,0 +1,155 @@ +// Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. +// SPDX-License-Identifier: BSD-3-Clause-Clear + +package ostree_test + +import ( + "encoding/base64" + "os" + "path/filepath" + "testing" + + "github.com/foundriesio/update-server/storage/ostree" + "github.com/stretchr/testify/require" +) + +// These tests require a real OSTree repo at /sysroot/ostree/repo and will be +// skipped if it is not present. + +func TestReadFile(t *testing.T) { + const repoPath = "/sysroot/ostree/repo" + const ref = "intel-corei7-64-lmp" + const filePath = "/usr/lib/sota/conf.d/40-hardware-id.toml" + const expected = "[provision]\nprimary_ecu_hardware_id = \"intel-corei7-64\"\n" + + repo := ostree.NewRepo(repoPath) + + _, err := repo.ReadRef(ref) + if err != nil { + t.Skipf("OSTree repo not available (%v)", err) + } + + content, err := repo.ReadFile(ref, filePath) + require.NoError(t, err) + require.Equal(t, expected, string(content)) +} + +func TestReadFileNotFound(t *testing.T) { + const repoPath = "/sysroot/ostree/repo" + const ref = "intel-corei7-64-lmp" + + repo := ostree.NewRepo(repoPath) + + _, err := repo.ReadRef(ref) + if err != nil { + t.Skipf("OSTree repo not available (%v)", err) + } + + _, err = repo.ReadFile(ref, "/usr/lib/sota/conf.d/nonexistent.toml") + require.ErrorContains(t, err, "not found") +} + +// TestParseEmbedded exercises commit and dirtree parsing with real bytes captured +// from an OSTree repo, without needing the repo present at test time. +func TestParseEmbedded(t *testing.T) { + // Real commit object from intel-corei7-64-lmp ref. + commitB64 := "b3N0cmVlLnJlZi1iaW5kaW5nAAAAAAAAaW50ZWwtY29yZWk3LTY0LWxtcAAUAGFzEzG1AK7bkK3BehHydWbJRP6wPpY4vI+hYupk5Q+pVKRATCJhbmR5LXRlc3Qtd2l0aC1zY3JpcHQiAAAAAAAAAAAAAABn+ChCa+Nk2LmOpzozokSSqmXmf2OtHDlC+3CH7tP9Ui8XXvtEag7xG3zBZ/O2A+WFx+7utnX6pBLV7HP2KYjrC2xUiJhralJSMg==" + // Real dirtree object for /usr/lib/sota/conf.d in that commit. + confdDirtreeB64 := "NDAtaGFyZHdhcmUtaWQudG9tbAC1ZC8Sl/gPnTzhmR+/MldHJ2cJ86zkuVmJlEO4r3IvSBQ0Ni1wa2NzMTEtbGFiZWwudG9tbADMqpJ1rI4WLNXdRi4t7EcPxlwFlOexZusMtkDgXeAKuBU1a20=" + // Raw content of the target file. + fileContent := "[provision]\nprimary_ecu_hardware_id = \"intel-corei7-64\"\n" + + commitBytes, err := base64.StdEncoding.DecodeString(commitB64) + require.NoError(t, err) + confdDirtreeBytes, err := base64.StdEncoding.DecodeString(confdDirtreeB64) + require.NoError(t, err) + + const commitHash = "f053412485a867bf2853b447367330940fd46e0527edef77ce9e7cedf8a043fd" + const rootDirtreeHash = "6be364d8b98ea73a33a24492aa65e67f63ad1c3942fb7087eed3fd522f175efb" + const confdDirtreeHash = "e0a8be1e0b2efdd2c63a388bc7d212433673081fe8823f74dd7ef7153ddb246e" + const fileHash = "b5642f1297f80f9d3ce1991fbf325747276709f3ace4b959899443b8af722f48" + + // Build a fake repo tree containing just the objects we need. + repoPath := t.TempDir() + writeObj := func(hash, ext string, data []byte) { + dir := filepath.Join(repoPath, "objects", hash[:2]) + require.NoError(t, os.MkdirAll(dir, 0755)) + require.NoError(t, os.WriteFile(filepath.Join(dir, hash[2:]+ext), data, 0644)) + } + + // We don't walk the full dirtree chain here — only seed the objects actually + // exercised by ReadFile for the conf.d path segment. + writeObj(commitHash, ".commit", commitBytes) + writeObj(confdDirtreeHash, ".dirtree", confdDirtreeBytes) + writeObj(fileHash, ".file", []byte(fileContent)) + + // Stub out intermediate dirtrees by making them point directly to confdDirtreeHash. + // Each is a minimal dirtree containing only one subdir entry for the next path + // component, so we create them synthetically. + writeMinimalDirtree := func(hash, childName, childDirtreeHash string) { + data := buildMinimalSubdirDirtree(childName, childDirtreeHash) + writeObj(hash, ".dirtree", data) + } + + // Chain: rootDirtreeHash/usr -> usrDT/lib -> libDT/sota -> sotaDT/conf.d -> confdDirtreeHash + const usrDT = "1100000000000000000000000000000000000000000000000000000000000001" + const libDT = "1100000000000000000000000000000000000000000000000000000000000002" + const sotaDT = "1100000000000000000000000000000000000000000000000000000000000003" + + writeMinimalDirtree(rootDirtreeHash, "usr", usrDT) + writeMinimalDirtree(usrDT, "lib", libDT) + writeMinimalDirtree(libDT, "sota", sotaDT) + writeMinimalDirtree(sotaDT, "conf.d", confdDirtreeHash) + + require.NoError(t, os.MkdirAll(filepath.Join(repoPath, "refs", "heads"), 0755)) + require.NoError(t, os.WriteFile( + filepath.Join(repoPath, "refs", "heads", "test-ref"), + []byte(commitHash+"\n"), 0644, + )) + + repo := ostree.NewRepo(repoPath) + content, err := repo.ReadFile("test-ref", "/usr/lib/sota/conf.d/40-hardware-id.toml") + require.NoError(t, err) + require.Equal(t, fileContent, string(content)) +} + +// buildMinimalSubdirDirtree constructs a GVariant (a(say)a(sayay)) dirtree with +// zero files and one subdirectory entry, suitable for use in unit tests. +func buildMinimalSubdirDirtree(name, dirtreeHash string) []byte { + hashBytes := hexDecode(dirtreeHash) + dirmeta := make([]byte, 32) // zero dirmeta hash + + // Subdir element: name\0 + 32-byte dirtree + 32-byte dirmeta + elem := []byte(name + "\x00") + elem = append(elem, hashBytes...) + elem = append(elem, dirmeta...) + + // Dirs array: one element + 1-byte framing offset (end of element) + dirsArray := append(elem, byte(len(elem))) + + // Outer tuple: files array is empty (0 bytes), dirs array follows. + // One outer framing byte = 0 (end of empty files array). + result := append(dirsArray, 0x00) + return result +} + +func hexDecode(s string) []byte { + b := make([]byte, len(s)/2) + for i := range b { + hi := hexNibble(s[i*2]) + lo := hexNibble(s[i*2+1]) + b[i] = (hi << 4) | lo + } + return b +} + +func hexNibble(c byte) byte { + switch { + case c >= '0' && c <= '9': + return c - '0' + case c >= 'a' && c <= 'f': + return c - 'a' + 10 + default: + return 0 + } +} From f93e881a3832ba875b1a420f32505b75d7a2ebdd Mon Sep 17 00:00:00 2001 From: Andy Doan Date: Fri, 26 Jun 2026 12:04:11 -0500 Subject: [PATCH 07/13] feat: Updates API can create its own TUF metadata This feature probes the update content itself to help automatically generate the TUF Target for the user removing the need for them to have TUF set up outside of this project. Signed-off-by: Andy Doan Co-authored-by: GitHub Copilot:claude-4-opus --- server/ui/api/handlers_updates.go | 54 ++++++++- storage/api/api_storage.go | 13 +- storage/api/api_storage_tuf_test.go | 32 ++--- storage/api/api_storage_update.go | 161 +++++++++++++++++++++++++ storage/api/api_storage_update_test.go | 66 ++++++++++ storage/file_updates.go | 17 ++- 6 files changed, 314 insertions(+), 29 deletions(-) create mode 100644 storage/api/api_storage_update.go create mode 100644 storage/api/api_storage_update_test.go diff --git a/server/ui/api/handlers_updates.go b/server/ui/api/handlers_updates.go index b7ee0fdd..139dd385 100644 --- a/server/ui/api/handlers_updates.go +++ b/server/ui/api/handlers_updates.go @@ -6,6 +6,8 @@ package api import ( "errors" "net/http" + "strconv" + "strings" "github.com/labstack/echo/v4" @@ -21,28 +23,76 @@ type UpdateTufResp map[string]map[string]any // @Success 201 // @Param tag path string true "Update tag" // @Param update path string true "Update name" +// @Param version query int false "Override the target version (AppVersion)" +// @Param name query string false "Override the target name" +// @Param ostree-hash query string false "Override the ostree hash" +// @Param apps query string false "Override docker compose apps as name=sha256[,name=sha256]" // @Router /updates/{tag}/{update} [post] func (h handlers) updateCreate(c echo.Context) error { tag := c.Param("tag") update := c.Param("update") user := CtxGetUser(c.Request().Context()) + opts := storage.TargetOptions{ + Name: c.QueryParam("name"), + OstreeHash: c.QueryParam("ostree-hash"), + Apps: parseAppsParam(c.QueryParams()["apps"]), + BaseUrl: baseURL(c), + } + if v := c.QueryParam("version"); v != "" { + n, err := strconv.Atoi(v) + if err != nil { + return EchoError(c, err, http.StatusBadRequest, "invalid version parameter") + } + opts.AppVersion = n + } + payload := c.Request().Body defer payload.Close() //nolint:errcheck - if err := h.storage.CreateUpdate(tag, update, user.Username, payload); err != nil { + if err := h.storage.CreateUpdate(tag, update, user.Username, opts, payload); err != nil { switch { case errors.Is(err, storage.ErrInvalidUpdate): return EchoError(c, err, http.StatusBadRequest, err.Error()) case errors.Is(err, storage.ErrDbConstraintUnique): return EchoError(c, err, http.StatusConflict, "Update with this name and tag already exists") } - return EchoError(c, err, http.StatusInternalServerError, "failed to create update") + return EchoError(c, err, http.StatusInternalServerError, err.Error()) } return c.NoContent(http.StatusCreated) } +// baseURL returns the scheme://host base URL of the incoming request, used to +// build proxied app/target URIs. +func baseURL(c echo.Context) string { + return c.Scheme() + "://" + c.Request().Host +} + +// parseAppsParam parses repeated "apps" query values of the form +// "name=sha256" (comma separated) into a map of app name to sha256. +func parseAppsParam(values []string) map[string]string { + apps := make(map[string]string) + for _, v := range values { + for _, pair := range strings.Split(v, ",") { + name, hash, ok := strings.Cut(pair, "=") + if !ok { + name, hash, ok = strings.Cut(pair, ":") + } + if !ok { + continue + } + if name = strings.TrimSpace(name); name != "" { + apps[name] = strings.TrimSpace(hash) + } + } + } + if len(apps) == 0 { + return nil + } + return apps +} + // @Summary Returns the TUF metadata for the update // @Description Requires scope: updates:read or updates:read-update // @Tags Updates diff --git a/storage/api/api_storage.go b/storage/api/api_storage.go index 712de2f3..de04ea99 100644 --- a/storage/api/api_storage.go +++ b/storage/api/api_storage.go @@ -529,7 +529,7 @@ func (s Storage) UploadConfigs(payload io.Reader) (err error) { }) } -func (s Storage) CreateUpdate(tag, updateName, uploadedBy string, payload io.Reader) error { +func (s Storage) CreateUpdate(tag, updateName, uploadedBy string, opts TargetOptions, payload io.Reader) error { // First, check the database for uniqueness by (tag, name). // Then, save the upload, and finally, insert into the database. // This warrants the two-phase transaction, unless the user makes concurrent uploads of the same update. @@ -544,9 +544,18 @@ func (s Storage) CreateUpdate(tag, updateName, uploadedBy string, payload io.Rea // This is not critical - log and let the "real" error/success return below. slog.Error("Failed to clean upload directory", "error", cleanupErr) } - if err := s.fs.Updates.SaveUpload(tag, updateName, payload, cleanup); err != nil { + tufEnabled := s.fs.Tuf.Enabled() + tufUploaded, err := s.fs.Updates.SaveUpload(tag, updateName, payload, tufEnabled, cleanup) + if err != nil { return err } + // When TUF is enabled and the upload did not include its own TUF metadata, + // generate it from the probed ostree/apps content. + if tufEnabled && !tufUploaded { + if err := s.generateUpdateTuf(tag, updateName, opts); err != nil { + return fmt.Errorf("unable to generate TUF metadata: %w", err) + } + } return s.stmtUpdateInsert.run(tag, updateName, uploadedBy) } diff --git a/storage/api/api_storage_tuf_test.go b/storage/api/api_storage_tuf_test.go index 4eef5286..490c5863 100644 --- a/storage/api/api_storage_tuf_test.go +++ b/storage/api/api_storage_tuf_test.go @@ -42,14 +42,6 @@ func newTufStorage(t *testing.T) *Storage { return s } -// ensureTargetsDir pre-creates the TUF update directory that AddTarget writes -// into. AddTarget symlinks the root metadata into this directory before -// creating it, so it must already exist. -func ensureTargetsDir(t *testing.T, s *Storage, tag string) { - t.Helper() - require.NoError(t, s.fs.Updates.Tuf.WriteFile(tag, storage.TufTargetsFile, ".keep", "")) -} - // verifyTufSig asserts that sig is a valid signature over signed, produced by a // key listed in the latest root metadata. func verifyTufSig(t *testing.T, s *Storage, sig tuf.Signature, signed any) { @@ -80,6 +72,7 @@ func TestAddTarget(t *testing.T) { s := newTufStorage(t) const tag = "main" + const update = "v1.0" opts := TargetOptions{ AppVersion: 3, HardwareId: "raspberrypi4-64", @@ -90,12 +83,11 @@ func TestAddTarget(t *testing.T) { }, BaseUrl: "https://example.com", } - ensureTargetsDir(t, s, tag) - require.NoError(t, s.AddTarget(tag, opts)) + require.NoError(t, s.AddTarget(tag, update, opts)) // Targets metadata is written, with the expected version and one target. var targets tuf.AtsTufTargets - require.NoError(t, s.fs.Tuf.ReadTufMeta(tag, storage.TufTargetsFile, storage.TufTargetsFile, &targets)) + require.NoError(t, s.fs.Tuf.ReadTufMeta(tag, update, storage.TufTargetsFile, &targets)) assert.Equal(t, "Targets", targets.Signed.Type) assert.Equal(t, 10, targets.Signed.Version) assert.Equal(t, fixedNow.Add(s.fs.Tuf.TargetsExpiration).Truncate(time.Second), targets.Signed.Expires) @@ -123,7 +115,7 @@ func TestAddTarget(t *testing.T) { // Snapshot metadata references the targets version and is signed. var snapshot tuf.AtsTufSnapshot - require.NoError(t, s.fs.Tuf.ReadTufMeta(tag, storage.TufTargetsFile, storage.TufSnapshotFile, &snapshot)) + require.NoError(t, s.fs.Tuf.ReadTufMeta(tag, update, storage.TufSnapshotFile, &snapshot)) assert.Equal(t, "Snapshot", snapshot.Signed.Type) assert.Equal(t, targets.Signed.Version, snapshot.Signed.Version) assert.Equal(t, targets.Signed.Expires, snapshot.Signed.Expires) @@ -134,7 +126,7 @@ func TestAddTarget(t *testing.T) { // Timestamp metadata references the snapshot version and uses the reserved // (version * 1000) numbering scheme. var timestamp tuf.AtsTufTimestamp - require.NoError(t, s.fs.Tuf.ReadTufMeta(tag, storage.TufTargetsFile, storage.TufTimestampFile, ×tamp)) + require.NoError(t, s.fs.Tuf.ReadTufMeta(tag, update, storage.TufTimestampFile, ×tamp)) assert.Equal(t, "Timestamp", timestamp.Signed.Type) assert.Equal(t, targets.Signed.Version*1000, timestamp.Signed.Version) assert.Equal(t, fixedNow.Add(s.fs.Tuf.TimestampExpiration).Truncate(time.Second), timestamp.Signed.Expires) @@ -147,8 +139,8 @@ func TestAddTargetWithoutApps(t *testing.T) { s := newTufStorage(t) const tag = "main" - ensureTargetsDir(t, s, tag) - require.NoError(t, s.AddTarget(tag, TargetOptions{ + const update = "v2.0" + require.NoError(t, s.AddTarget(tag, update, TargetOptions{ AppVersion: 5, HardwareId: "intel-corei7-64", Name: "intel-corei7-64-lmp", @@ -156,7 +148,7 @@ func TestAddTargetWithoutApps(t *testing.T) { })) var targets tuf.AtsTufTargets - require.NoError(t, s.fs.Tuf.ReadTufMeta(tag, storage.TufTargetsFile, storage.TufTargetsFile, &targets)) + require.NoError(t, s.fs.Tuf.ReadTufMeta(tag, update, storage.TufTargetsFile, &targets)) require.Len(t, targets.Signed.Targets, 1) target := targets.Signed.Targets["intel-corei7-64-lmp-5"] @@ -190,20 +182,20 @@ func TestAddTargetIncrementsTufVersion(t *testing.T) { require.NoError(t, s.InsertUpdate(tag, "update-5", "tester")) require.NoError(t, s.fs.Updates.Tuf.WriteFile(tag, "update-5", storage.TufTargetsFile, string(existingJSON))) - ensureTargetsDir(t, s, tag) // A new target should bump the TUF version above the existing one. - require.NoError(t, s.AddTarget(tag, TargetOptions{ + const update = "v6.0" + require.NoError(t, s.AddTarget(tag, update, TargetOptions{ HardwareId: "intel-corei7-64", Name: "intel-corei7-64-lmp", OstreeHash: "abc123", })) var targets tuf.AtsTufTargets - require.NoError(t, s.fs.Tuf.ReadTufMeta(tag, storage.TufTargetsFile, storage.TufTargetsFile, &targets)) + require.NoError(t, s.fs.Tuf.ReadTufMeta(tag, update, storage.TufTargetsFile, &targets)) assert.Equal(t, 15, targets.Signed.Version, "TUF version should be ten greater than the existing update") var timestamp tuf.AtsTufTimestamp - require.NoError(t, s.fs.Tuf.ReadTufMeta(tag, storage.TufTargetsFile, storage.TufTimestampFile, ×tamp)) + require.NoError(t, s.fs.Tuf.ReadTufMeta(tag, update, storage.TufTimestampFile, ×tamp)) assert.Equal(t, 15000, timestamp.Signed.Version) } diff --git a/storage/api/api_storage_update.go b/storage/api/api_storage_update.go new file mode 100644 index 00000000..77e36579 --- /dev/null +++ b/storage/api/api_storage_update.go @@ -0,0 +1,161 @@ +// Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. +// SPDX-License-Identifier: BSD-3-Clause-Clear + +package api + +import ( + "crypto/sha256" + "errors" + "fmt" + "log/slog" + "os" + "path/filepath" + "strconv" + "strings" + + "github.com/foundriesio/update-server/storage/ostree" +) + +// generateUpdateTuf probes the uploaded ostree/apps content for an update and +// generates its TUF metadata via AddTarget. Values discovered from the upload +// are overridden by any non-zero fields in overrides. +func (s Storage) generateUpdateTuf(tag, update string, overrides TargetOptions) error { + opts := TargetOptions{BaseUrl: overrides.BaseUrl} + + ostreeDir := s.fs.Updates.Ostree.FilePath(tag, update, "") + if isDir(ostreeDir) { + if err := probeOstree(ostreeDir, &opts); err != nil { + return fmt.Errorf("unable to probe ostree repo: %w", err) + } + } + + appsDir := s.fs.Updates.Apps.FilePath(tag, update, "apps") + if isDir(appsDir) { + if err := probeApps(appsDir, &opts); err != nil { + return fmt.Errorf("unable to probe apps: %w", err) + } + } + + // Caller-provided values override anything probed from the upload. + if overrides.Name != "" { + opts.Name = overrides.Name + } + if overrides.AppVersion != 0 { + opts.AppVersion = overrides.AppVersion + } + if overrides.HardwareId != "" { + opts.HardwareId = overrides.HardwareId + } + if overrides.OstreeHash != "" { + opts.OstreeHash = overrides.OstreeHash + } + if len(overrides.Apps) > 0 { + opts.Apps = overrides.Apps + } + if opts.OstreeHash == "" { + // Default to the sha256 of empty content when no ostree image is present. + opts.OstreeHash = fmt.Sprintf("%x", sha256.Sum256(nil)) + } + + var errs []error + if len(opts.HardwareId) == 0 { + errs = append(errs, fmt.Errorf("unable to determine hardware id from upload")) + } + if len(opts.Name) == 0 { + errs = append(errs, fmt.Errorf("unable to determine target name from upload")) + } + if len(errs) > 0 { + return errors.Join(errs...) + } + + slog.Info("Adding TUF target", "tag", tag, "update", update, "opts", opts) + return s.AddTarget(tag, update, opts) +} + +// probeOstree inspects an ostree repository to derive target options. +func probeOstree(repoPath string, opts *TargetOptions) error { + repo := ostree.NewRepo(repoPath) + + heads, err := repo.ListHeads() + if err != nil { + return err + } + if len(heads) == 0 { + return fmt.Errorf("no refs found under refs/heads") + } + ref := heads[0] + opts.Name = ref + + if opts.OstreeHash, err = repo.ReadRef(ref); err != nil { + return err + } + + if data, err := repo.ReadFile(ref, "/etc/os-release"); err == nil { + if v := parseKeyValue(string(data), "IMAGE_VERSION"); v != "" { + if n, err := strconv.Atoi(v); err == nil { + opts.AppVersion = n + } + } + } + + if data, err := repo.ReadFile(ref, "/usr/lib/sota/conf.d/40-hardware-id.toml"); err == nil { + opts.HardwareId = parseKeyValue(string(data), "primary_ecu_hardware_id") + } + if opts.HardwareId == "" { + opts.HardwareId = detectArch(repo, ref) + } + + return nil +} + +// detectArch guesses the hardware id from the image architecture when no +// hardware-id configuration file is present. +func detectArch(repo *ostree.Repo, ref string) string { + if _, err := repo.ReadFile(ref, "/lib/ld-linux-aarch64.so.1"); err == nil { + return "arm64-linux" + } + return "amd64-linux" +} + +// probeApps maps each app sub-directory name to the sha256 of its single entry. +// The upload layout is apps//. +func probeApps(appsDir string, opts *TargetOptions) error { + entries, err := os.ReadDir(appsDir) + if err != nil { + return err + } + + opts.Apps = make(map[string]string) + for _, e := range entries { + if !e.IsDir() { + continue + } + sub, err := os.ReadDir(filepath.Join(appsDir, e.Name())) + if err != nil { + return err + } + for _, item := range sub { + opts.Apps[e.Name()] = item.Name() // the entry under the app is the sha256 + break + } + } + return nil +} + +// parseKeyValue returns the value for key from KEY=VALUE style content (os-release +// or simple TOML), stripping surrounding quotes and whitespace. +func parseKeyValue(content, key string) string { + for line := range strings.SplitSeq(content, "\n") { + k, v, ok := strings.Cut(line, "=") + if !ok || strings.TrimSpace(k) != key { + continue + } + return strings.Trim(strings.TrimSpace(v), `"'`) + } + return "" +} + +func isDir(path string) bool { + info, err := os.Stat(path) + return err == nil && info.IsDir() +} diff --git a/storage/api/api_storage_update_test.go b/storage/api/api_storage_update_test.go new file mode 100644 index 00000000..2ee8c31d --- /dev/null +++ b/storage/api/api_storage_update_test.go @@ -0,0 +1,66 @@ +// Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. +// SPDX-License-Identifier: BSD-3-Clause-Clear + +package api + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/foundriesio/update-server/storage" + storageTesting "github.com/foundriesio/update-server/storage/testing" + "github.com/foundriesio/update-server/storage/tuf" +) + +func TestCreateUpdateGeneratesTufFromApps(t *testing.T) { + s := newTufStorage(t) // TUF is initialized and loaded -> enabled. + + // An upload with apps but no "tuf" directory should have its TUF metadata + // generated by the server. + tar := storageTesting.CreateTarBuffer(t, map[string]string{ + "apps/apps/shellhttpd/sha256appcontenthash": "{}", + }) + opts := TargetOptions{Name: "my-app-target", HardwareId: "generic-x86-64", BaseUrl: "https://sat.example.com"} + require.NoError(t, s.CreateUpdate("main", "v1.0", "tester", opts, tar)) + + // Targets metadata was generated for the update. + var targets tuf.AtsTufTargets + require.NoError(t, s.fs.Tuf.ReadTufMeta("main", "v1.0", storage.TufTargetsFile, &targets)) + require.Len(t, targets.Signed.Targets, 1) + + target, ok := targets.Signed.Targets["my-app-target-0"] + require.True(t, ok, "expected generated target keyed by name and version") + + var custom generatedTargetCustom + require.NoError(t, json.Unmarshal(target.Custom, &custom)) + require.Len(t, custom.DockerComposeApps, 1) + assert.Equal(t, "https://sat.example.com/composeapphack/shellhttpd@sha256:sha256appcontenthash", + custom.DockerComposeApps["shellhttpd"].URI) + + // The update was registered in the database. + updates, err := s.ListUpdates("main") + require.NoError(t, err) + require.Len(t, updates["main"], 1) + assert.Equal(t, "v1.0", updates["main"][0].Name) +} + +func TestCreateUpdateUsesUploadedTuf(t *testing.T) { + s := newTufStorage(t) + + // An upload that includes a valid tuf directory is stored as-is and is not + // regenerated by the server. + validTargets := `{"signed": {"targets": {"foo": {"custom": {"tags": ["main"]}}}}}` + tar := storageTesting.CreateTarBuffer(t, map[string]string{ + "tuf/root.json": `{"signed":{}}`, + "tuf/targets.json": validTargets, + "ostree_repo/config": "[core]\n", + }) + require.NoError(t, s.CreateUpdate("main", "v1.0", "tester", TargetOptions{}, tar)) + + raw, err := s.fs.Updates.Tuf.ReadFile("main", "v1.0", storage.TufTargetsFile) + require.NoError(t, err) + assert.JSONEq(t, validTargets, raw) +} diff --git a/storage/file_updates.go b/storage/file_updates.go index bdc77a93..6245e722 100644 --- a/storage/file_updates.go +++ b/storage/file_updates.go @@ -81,7 +81,7 @@ func checkUpdateTargets(targetsPath, tag string) error { return fmt.Errorf("no target with tag '%s' found in targets.json", tag) } -func (s updatesFsHandleWrap) SaveUpload(tag, update string, payload io.Reader, onCleanupFailure func(error)) error { +func (s updatesFsHandleWrap) SaveUpload(tag, update string, payload io.Reader, tufEnabled bool, onCleanupFailure func(error)) (tufUploaded bool, err error) { const ( appsDir = UpdatesAppsDir + string(filepath.Separator) ostreeDir = UpdatesOstreeDir + string(filepath.Separator) @@ -92,7 +92,7 @@ func (s updatesFsHandleWrap) SaveUpload(tag, update string, payload io.Reader, o root, destDir := filepath.Split(s.root) destDir = filepath.Join(destDir, tag, update) h := tarFsHandle{root: root} - return h.unpackTar(payload, destDir, + err = h.unpackTar(payload, destDir, TarUnpackReplaceDest(true), // Replace updates with the same tag and name - uniqueness is checked on the database level. TarUnpackUseTmpFile("update.tar"), TarUnpackUseTmpDir(txDir), @@ -107,13 +107,19 @@ func (s updatesFsHandleWrap) SaveUpload(tag, update string, payload io.Reader, o return }, onUnpackComplete: func() error { - if !sawTuf { - return fmt.Errorf("%w: missing required %q directory", ErrInvalidUpdate, UpdatesTufDir) - } if !sawOstree && !sawApps { return fmt.Errorf("%w: must contain %q and/or %q directory", ErrInvalidUpdate, UpdatesOstreeDir, UpdatesAppsDir) } + if !sawTuf { + // When TUF is enabled the server generates the metadata from + // the uploaded ostree/apps content, so the tuf directory is + // optional. Otherwise it is required. + if !tufEnabled { + return fmt.Errorf("%w: missing required %q directory", ErrInvalidUpdate, UpdatesTufDir) + } + return nil + } path := filepath.Join(root, txDir, "unpacked/tuf/targets.json") if err := checkUpdateTargets(path, tag); err != nil { @@ -123,6 +129,7 @@ func (s updatesFsHandleWrap) SaveUpload(tag, update string, payload io.Reader, o }, }), ) + return sawTuf, err } type UpdatesFsHandle struct { From 661d29811b804fef891ff8d3ed223dc57a8b07bc Mon Sep 17 00:00:00 2001 From: Andy Doan Date: Fri, 26 Jun 2026 14:28:52 -0500 Subject: [PATCH 08/13] feat: CLI upload supports TUF target override options Add --version, --name, --ostree-hash, and --apps flags to the update upload command so users can override the auto-generated TUF target metadata via the new query parameters on the updates create API. Signed-off-by: Andy Doan Co-authored-by: GitHub Copilot:claude-opus-4.8 --- cli/api/updates.go | 35 ++++++++++++++++++++-- cli/subcommands/updates/create.go | 50 +++++++++++++++++++++++++++++-- 2 files changed, 80 insertions(+), 5 deletions(-) diff --git a/cli/api/updates.go b/cli/api/updates.go index 7bab1661..241a3729 100644 --- a/cli/api/updates.go +++ b/cli/api/updates.go @@ -5,6 +5,8 @@ package api import ( "io" + "net/url" + "strconv" models "github.com/foundriesio/update-server/storage/api" ) @@ -53,8 +55,37 @@ func (u UpdatesApi) TailRollout(tag, updateName, rollout string) (io.ReadCloser, return u.api.GetStream(endpoint) } -func (u UpdatesApi) CreateUpdate(tag, updateName string, body io.Reader) error { - endpoint := "/v1/updates/" + tag + "/" + updateName +// CreateUpdateOptions captures the optional TUF target overrides that can be +// supplied when creating an update. +type CreateUpdateOptions struct { + Version int // Override the target version (AppVersion) + Name string // Override the target name + OstreeHash string // Override the ostree hash + Apps map[string]string // Override docker compose apps (name -> sha256) +} + +func (o CreateUpdateOptions) query() string { + values := url.Values{} + if o.Version != 0 { + values.Set("version", strconv.Itoa(o.Version)) + } + if o.Name != "" { + values.Set("name", o.Name) + } + if o.OstreeHash != "" { + values.Set("ostree-hash", o.OstreeHash) + } + for name, hash := range o.Apps { + values.Add("apps", name+"="+hash) + } + if len(values) == 0 { + return "" + } + return "?" + values.Encode() +} + +func (u UpdatesApi) CreateUpdate(tag, updateName string, opts CreateUpdateOptions, body io.Reader) error { + endpoint := "/v1/updates/" + tag + "/" + updateName + opts.query() _, err := u.api.Post(endpoint, body, HttpHeader("Content-Type", "application/x-tar"), HttpHeader("Content-Encoding", "gzip")) return err } diff --git a/cli/subcommands/updates/create.go b/cli/subcommands/updates/create.go index f0a8894d..b6958baa 100644 --- a/cli/subcommands/updates/create.go +++ b/cli/subcommands/updates/create.go @@ -6,6 +6,7 @@ package updates import ( "fmt" "os" + "strings" "github.com/spf13/cobra" @@ -19,17 +20,60 @@ var createCmd = &cobra.Command{ Long: `Create an update on Update server by uploading the offline update found in the directory.`, Args: cobra.ExactArgs(3), RunE: func(cmd *cobra.Command, args []string) error { + opts, err := createOptions(cmd) + if err != nil { + return err + } a := api.CtxGetApi(cmd.Context()) - cobra.CheckErr(createUpdate(a.Updates(), args[0], args[1], args[2])) + cobra.CheckErr(createUpdate(a.Updates(), args[0], args[1], args[2], opts)) return nil }, } func init() { + flags := createCmd.Flags() + flags.Int("version", 0, "Override the target version (AppVersion)") + flags.String("name", "", "Override the target name") + flags.String("ostree-hash", "", "Override the ostree hash") + flags.StringSlice("apps", nil, "Override docker compose apps as name=sha256 (repeatable or comma separated)") UpdatesCmd.AddCommand(createCmd) } -func createUpdate(updates api.UpdatesApi, tag, updateName, path string) error { +func createOptions(cmd *cobra.Command) (api.CreateUpdateOptions, error) { + flags := cmd.Flags() + var opts api.CreateUpdateOptions + var err error + if opts.Version, err = flags.GetInt("version"); err != nil { + return opts, err + } + if opts.Name, err = flags.GetString("name"); err != nil { + return opts, err + } + if opts.OstreeHash, err = flags.GetString("ostree-hash"); err != nil { + return opts, err + } + apps, err := flags.GetStringSlice("apps") + if err != nil { + return opts, err + } + for _, pair := range apps { + name, hash, ok := strings.Cut(pair, "=") + if !ok { + return opts, fmt.Errorf("invalid --apps value '%s', expected name=sha256", pair) + } + name = strings.TrimSpace(name) + if name == "" { + return opts, fmt.Errorf("invalid --apps value '%s', expected name=sha256", pair) + } + if opts.Apps == nil { + opts.Apps = make(map[string]string) + } + opts.Apps[name] = strings.TrimSpace(hash) + } + return opts, nil +} + +func createUpdate(updates api.UpdatesApi, tag, updateName, path string, opts api.CreateUpdateOptions) error { if stat, err := os.Stat(path); err != nil { return fmt.Errorf("failed to stat directory '%s': %w", path, err) } else if !stat.Mode().IsDir() { @@ -45,7 +89,7 @@ func createUpdate(updates api.UpdatesApi, tag, updateName, path string) error { // See cli/subcommands/configs/upload.go for a comment about how reporter works. go progress.Report("Uploaded:", stop, done) - err := updates.CreateUpdate(tag, updateName, reader) + err := updates.CreateUpdate(tag, updateName, opts, reader) stop <- err == nil <-done return err From c68f6315ca256691019441ff487dfada0f0b71d5 Mon Sep 17 00:00:00 2001 From: Andy Doan Date: Fri, 26 Jun 2026 14:34:31 -0500 Subject: [PATCH 09/13] feat: web UI upload supports TUF target override options Add a collapsible "Advanced options" section to the update upload form with version, target name, ostree hash, and apps fields. These map to the new query parameters on the updates create API to override the auto-generated TUF target metadata. Signed-off-by: Andy Doan Co-authored-by: GitHub Copilot:claude-opus-4.8 --- server/ui/web/templates/updates.html | 46 +++++++++++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/server/ui/web/templates/updates.html b/server/ui/web/templates/updates.html index f041ea4d..22e22640 100644 --- a/server/ui/web/templates/updates.html +++ b/server/ui/web/templates/updates.html @@ -41,6 +41,25 @@

{{.Title}}

+
+ Advanced options +

These optional fields override the TUF target metadata that is + otherwise generated automatically from the update content.

+ + + + + + + + + + + + + Comma-separated list of docker compose apps as name=sha256. +
+ @@ -221,6 +240,30 @@

{{.Title}}

return; } + // Optional "advanced" overrides for the generated TUF target metadata. + const params = new URLSearchParams(); + const version = document.getElementById('upload-version').value.trim(); + if (version) { + params.set('version', version); + } + const targetName = document.getElementById('upload-target-name').value.trim(); + if (targetName) { + params.set('name', targetName); + } + const ostreeHash = document.getElementById('upload-ostree-hash').value.trim(); + if (ostreeHash) { + params.set('ostree-hash', ostreeHash); + } + const apps = document.getElementById('upload-apps').value.trim(); + if (apps) { + for (const pair of apps.split(',')) { + const trimmed = pair.trim(); + if (trimmed) { + params.append('apps', trimmed); + } + } + } + btn.disabled = true; btn.textContent = 'Uploading...'; progressDiv.style.display = 'block'; @@ -232,7 +275,8 @@

{{.Title}}

const tarBlob = buildTarBlob(files); statusEl.textContent = 'Uploading: 0 B / ' + formatBytes(tarBlob.size); - const url = '/v1/updates/' + encodeURIComponent(tag) + '/' + encodeURIComponent(updateName); + const query = params.toString(); + const url = '/v1/updates/' + encodeURIComponent(tag) + '/' + encodeURIComponent(updateName) + (query ? '?' + query : ''); // Get CSRF token from meta tag const csrfToken = document.querySelector('meta[name="csrf-token"]')?.content; From d2cf92a88c934214a97ddfb558e021374cf94eaa Mon Sep 17 00:00:00 2001 From: Andy Doan Date: Fri, 26 Jun 2026 14:36:21 -0500 Subject: [PATCH 10/13] fix: Change order of timestamp/snapshot UI display It makes more sense to show in the order timestamp,snapshot,target Signed-off-by: Andy Doan --- server/ui/web/templates/update.html | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/server/ui/web/templates/update.html b/server/ui/web/templates/update.html index 830494bd..2eb7076f 100644 --- a/server/ui/web/templates/update.html +++ b/server/ui/web/templates/update.html @@ -67,14 +67,14 @@

TUF Metadata

- Snapshot expiration -

{{index .Tuf "snapshot.json" "signed" "expires"}}

+ Timestamp expiration +

{{index .Tuf "timestamp.json" "signed" "expires"}}

- Timestamp expiration -

{{index .Tuf "timestamp.json" "signed" "expires"}}

+ Snapshot expiration +

{{index .Tuf "snapshot.json" "signed" "expires"}}

From 9da0917ecbcb1992f0f12aab81b5c0d705c907df Mon Sep 17 00:00:00 2001 From: Andy Doan Date: Fri, 26 Jun 2026 14:45:14 -0500 Subject: [PATCH 11/13] feat: show hardware IDs and tags on update details page Collect the de-duplicated set of hardwareIds and tags from across all targets in targets.json and display them in their own fieldsets on the update details page. Signed-off-by: Andy Doan Co-authored-by: GitHub Copilot:claude-opus-4.8 --- server/ui/web/handlers_updates.go | 62 +++++++++++++++++++++++++++++ server/ui/web/templates/update.html | 30 +++++++++++++- 2 files changed, 90 insertions(+), 2 deletions(-) diff --git a/server/ui/web/handlers_updates.go b/server/ui/web/handlers_updates.go index 620bbae3..86ee3661 100644 --- a/server/ui/web/handlers_updates.go +++ b/server/ui/web/handlers_updates.go @@ -6,6 +6,7 @@ package web import ( "encoding/json" "fmt" + "sort" "strconv" "github.com/foundriesio/update-server/server/ui/api" @@ -86,6 +87,63 @@ func findLatestTarget(tuf api.UpdateTufResp) *latestTarget { return latest } +// findHardwareIds returns the sorted, de-duplicated set of hardwareIds found +// across all targets in targets.json. +func findHardwareIds(tuf api.UpdateTufResp) []string { + return findCustomStrings(tuf, "hardwareIds") +} + +// findTags returns the sorted, de-duplicated set of tags found across all +// targets in targets.json. +func findTags(tuf api.UpdateTufResp) []string { + return findCustomStrings(tuf, "tags") +} + +// findCustomStrings returns the sorted, de-duplicated set of string values +// found in the given target.custom field across all targets in targets.json. +func findCustomStrings(tuf api.UpdateTufResp, field string) []string { + targetsJson, ok := tuf["targets.json"] + if !ok { + return nil + } + signed, ok := targetsJson["signed"].(map[string]any) + if !ok { + return nil + } + targets, ok := signed["targets"].(map[string]any) + if !ok { + return nil + } + + seen := make(map[string]struct{}) + for _, target := range targets { + t, ok := target.(map[string]any) + if !ok { + continue + } + custom, ok := t["custom"].(map[string]any) + if !ok { + continue + } + values, ok := custom[field].([]any) + if !ok { + continue + } + for _, value := range values { + if s, ok := value.(string); ok && s != "" { + seen[s] = struct{}{} + } + } + } + + result := make([]string, 0, len(seen)) + for s := range seen { + result = append(result, s) + } + sort.Strings(result) + return result +} + func (h handlers) updatesList(c echo.Context) error { var updates map[string][]api.Update if err := getJson(c.Request().Context(), "/v1/updates", &updates); err != nil { @@ -135,6 +193,8 @@ func (h handlers) updatesGet(c echo.Context) error { Tuf api.UpdateTufResp TufJson string LatestTarget *latestTarget + HardwareIds []string + Tags []string TufError string }{ baseCtx: h.baseCtx(c, "Update Details", "updates"), @@ -145,6 +205,8 @@ func (h handlers) updatesGet(c echo.Context) error { Tuf: tuf, TufJson: string(tufJson), LatestTarget: findLatestTarget(tuf), + HardwareIds: findHardwareIds(tuf), + Tags: findTags(tuf), TufError: tufErr, } return h.templates.ExecuteTemplate(c.Response(), "update.html", ctx) diff --git a/server/ui/web/templates/update.html b/server/ui/web/templates/update.html index 2eb7076f..c8ede908 100644 --- a/server/ui/web/templates/update.html +++ b/server/ui/web/templates/update.html @@ -98,8 +98,34 @@

TUF Metadata

{{ .LatestTarget.Version }}

-
-
+
+
+ Hardware IDs + {{ if .HardwareIds }} +
    + {{ range .HardwareIds }} +
  • {{ . }}
    • + {{ end }} +
+ {{ else }} +

None

+ {{ end }} +
+
+
+
+ Tags + {{ if .Tags }} +
    + {{ range .Tags }} +
  • {{ . }}
    • + {{ end }} +
+ {{ else }} +

None

+ {{ end }} +
+
From e4c0cc2cc6e81cf1db0064bb6decf827ce31bd27 Mon Sep 17 00:00:00 2001 From: Andy Doan Date: Fri, 26 Jun 2026 14:50:05 -0500 Subject: [PATCH 12/13] feat: strike through expired TUF expiration fields Add an isExpired template helper and wrap expired root, timestamp, snapshot, and targets expiration values in a element on the update details page. Signed-off-by: Andy Doan Co-authored-by: GitHub Copilot:claude-opus-4.8 --- server/ui/web/templates/templates.go | 13 +++++++++++++ server/ui/web/templates/update.html | 12 ++++++++---- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/server/ui/web/templates/templates.go b/server/ui/web/templates/templates.go index a8afba61..5bb1f2ce 100644 --- a/server/ui/web/templates/templates.go +++ b/server/ui/web/templates/templates.go @@ -9,6 +9,8 @@ import ( "html/template" "strings" "time" + + "github.com/foundriesio/update-server/clock" ) //go:embed *.html *.css @@ -34,6 +36,17 @@ func init() { "tsToString": func(ts int64) string { return time.Unix(ts, 0).Format(time.RFC3339) }, + "isExpired": func(expires any) bool { + s, ok := expires.(string) + if !ok || s == "" { + return false + } + t, err := time.Parse(time.RFC3339, s) + if err != nil { + return false + } + return clock.Now().After(t) + }, "add": func(a, b int) int { return a + b }, diff --git a/server/ui/web/templates/update.html b/server/ui/web/templates/update.html index c8ede908..e734d4ff 100644 --- a/server/ui/web/templates/update.html +++ b/server/ui/web/templates/update.html @@ -62,25 +62,29 @@

TUF Metadata

Root expiration -

{{index .Tuf "root.json" "signed" "expires"}}

+ {{ $expires := index .Tuf "root.json" "signed" "expires" }} +

{{ if isExpired $expires }}{{ $expires }}{{ else }}{{ $expires }}{{ end }}

Timestamp expiration -

{{index .Tuf "timestamp.json" "signed" "expires"}}

+ {{ $expires := index .Tuf "timestamp.json" "signed" "expires" }} +

{{ if isExpired $expires }}{{ $expires }}{{ else }}{{ $expires }}{{ end }}

Snapshot expiration -

{{index .Tuf "snapshot.json" "signed" "expires"}}

+ {{ $expires := index .Tuf "snapshot.json" "signed" "expires" }} +

{{ if isExpired $expires }}{{ $expires }}{{ else }}{{ $expires }}{{ end }}

Targets expiration -

{{index .Tuf "targets.json" "signed" "expires"}}

+ {{ $expires := index .Tuf "targets.json" "signed" "expires" }} +

{{ if isExpired $expires }}{{ $expires }}{{ else }}{{ $expires }}{{ end }}

From 4a9a4d3fc987335e4b9d5a1c31838064ab8b7899 Mon Sep 17 00:00:00 2001 From: Andy Doan Date: Fri, 26 Jun 2026 16:09:23 -0500 Subject: [PATCH 13/13] docs: Include step for setting up TUF Signed-off-by: Andy Doan --- contrib/README.md | 3 +++ docs/quick-start.md | 9 +++++++++ 2 files changed, 12 insertions(+) diff --git a/contrib/README.md b/contrib/README.md index dae85b94..5905f7bc 100644 --- a/contrib/README.md +++ b/contrib/README.md @@ -24,6 +24,9 @@ communicate with. In order to use this you must first: $ go run github.com/foundriesio/update-server/cmd/server \ --datadir .compose-server-data auth-init + + $ go run github.com/foundriesio/update-server/cmd/server \ + --datadir .compose-server-data tuf-init ``` ## gen-certs.sh / fake-device.py diff --git a/docs/quick-start.md b/docs/quick-start.md index c08f505e..bc6d1567 100644 --- a/docs/quick-start.md +++ b/docs/quick-start.md @@ -60,6 +60,15 @@ tokens and web sessions, as well as the "noauth" provider. ./fioserver --datadir=./datadir auth-init --test ``` +## Initialize TUF + +Before the server can sign and manage TUF metadata, the TUF keys and +root metadata must be initialized: + +``` + ./fioserver --datadir=./datadir tuf-init +``` + ## Run the Server `./fioserv serve --datadir=datadir`