Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions .github/workflows/docker.yml
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,7 @@ jobs:
- name: Prepare build args
id: build_args
run: |
set -euo pipefail
if [[ "$GITHUB_REF" == refs/tags/v* ]]; then
VERSION="${GITHUB_REF#refs/tags/}"
else
Expand All @@ -112,6 +113,17 @@ jobs:
echo "build_date=$(date -u +%Y-%m-%dT%H:%M:%SZ)"
} >> "$GITHUB_OUTPUT"

- name: Validate OAuth build args
env:
MSGVAULT_OAUTH_CLIENT_ID: ${{ secrets.MSGVAULT_OAUTH_CLIENT_ID }}
MSGVAULT_OAUTH_CLIENT_SECRET: ${{ secrets.MSGVAULT_OAUTH_CLIENT_SECRET }}
run: |
set -euo pipefail
if [[ -z "$MSGVAULT_OAUTH_CLIENT_ID" || -z "$MSGVAULT_OAUTH_CLIENT_SECRET" ]]; then
echo "MSGVAULT OAuth secrets are required for Docker publish" >&2
exit 1
fi

- name: Build and push
uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf # v7.2.0
with:
Expand All @@ -124,5 +136,7 @@ jobs:
VERSION=${{ steps.build_args.outputs.version }}
COMMIT=${{ steps.build_args.outputs.commit }}
BUILD_DATE=${{ steps.build_args.outputs.build_date }}
MSGVAULT_OAUTH_CLIENT_ID=${{ secrets.MSGVAULT_OAUTH_CLIENT_ID }}
MSGVAULT_OAUTH_CLIENT_SECRET=${{ secrets.MSGVAULT_OAUTH_CLIENT_SECRET }}
cache-from: type=gha
cache-to: type=gha,mode=max
24 changes: 21 additions & 3 deletions .github/workflows/release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -52,12 +52,18 @@ jobs:
GOOS: linux
GOARCH: ${{ matrix.goarch }}
CGO_ENABLED: '1'
MSGVAULT_OAUTH_CLIENT_ID: ${{ secrets.MSGVAULT_OAUTH_CLIENT_ID }}
MSGVAULT_OAUTH_CLIENT_SECRET: ${{ secrets.MSGVAULT_OAUTH_CLIENT_SECRET }}
run: |
if [ -z "$MSGVAULT_OAUTH_CLIENT_ID" ] || [ -z "$MSGVAULT_OAUTH_CLIENT_SECRET" ]; then
echo "FATAL: MSGVAULT_OAUTH_CLIENT_ID and MSGVAULT_OAUTH_CLIENT_SECRET repository secrets must be set" >&2
exit 1
fi
export PATH="/usr/local/go/bin:$HOME/go/bin:$PATH"
VERSION=${GITHUB_REF#refs/tags/v}

mkdir -p dist
LDFLAGS="-s -w -X go.kenn.io/msgvault/cmd/msgvault/cmd.Version=v${VERSION} -X go.kenn.io/msgvault/cmd/msgvault/cmd.Commit=$(printf '%s' "$GITHUB_SHA" | cut -c1-8) -X go.kenn.io/msgvault/cmd/msgvault/cmd.BuildDate=$(date -u +%Y-%m-%dT%H:%M:%SZ) -extldflags '-lstdc++ -lm'"
LDFLAGS="-s -w -X go.kenn.io/msgvault/cmd/msgvault/cmd.Version=v${VERSION} -X go.kenn.io/msgvault/cmd/msgvault/cmd.Commit=$(printf '%s' "$GITHUB_SHA" | cut -c1-8) -X go.kenn.io/msgvault/cmd/msgvault/cmd.BuildDate=$(date -u +%Y-%m-%dT%H:%M:%SZ) -X go.kenn.io/msgvault/internal/oauth.oauthClientID=${MSGVAULT_OAUTH_CLIENT_ID} -X go.kenn.io/msgvault/internal/oauth.oauthClientSecret=${MSGVAULT_OAUTH_CLIENT_SECRET} -extldflags '-lstdc++ -lm'"
go build -tags "fts5 sqlite_vec" -trimpath -buildvcs=false -ldflags="$LDFLAGS" -o dist/msgvault ./cmd/msgvault

echo "--- Binary info ---"
Expand Down Expand Up @@ -106,11 +112,17 @@ jobs:
GOOS: darwin
GOARCH: ${{ matrix.goarch }}
CGO_ENABLED: 1
MSGVAULT_OAUTH_CLIENT_ID: ${{ secrets.MSGVAULT_OAUTH_CLIENT_ID }}
MSGVAULT_OAUTH_CLIENT_SECRET: ${{ secrets.MSGVAULT_OAUTH_CLIENT_SECRET }}
run: |
if [ -z "$MSGVAULT_OAUTH_CLIENT_ID" ] || [ -z "$MSGVAULT_OAUTH_CLIENT_SECRET" ]; then
echo "FATAL: MSGVAULT_OAUTH_CLIENT_ID and MSGVAULT_OAUTH_CLIENT_SECRET repository secrets must be set" >&2
exit 1
fi
VERSION=${GITHUB_REF#refs/tags/v}

mkdir -p dist
LDFLAGS="-s -w -X go.kenn.io/msgvault/cmd/msgvault/cmd.Version=v${VERSION} -X go.kenn.io/msgvault/cmd/msgvault/cmd.Commit=$(printf '%s' "$GITHUB_SHA" | cut -c1-8) -X go.kenn.io/msgvault/cmd/msgvault/cmd.BuildDate=$(date -u +%Y-%m-%dT%H:%M:%SZ)"
LDFLAGS="-s -w -X go.kenn.io/msgvault/cmd/msgvault/cmd.Version=v${VERSION} -X go.kenn.io/msgvault/cmd/msgvault/cmd.Commit=$(printf '%s' "$GITHUB_SHA" | cut -c1-8) -X go.kenn.io/msgvault/cmd/msgvault/cmd.BuildDate=$(date -u +%Y-%m-%dT%H:%M:%SZ) -X go.kenn.io/msgvault/internal/oauth.oauthClientID=${MSGVAULT_OAUTH_CLIENT_ID} -X go.kenn.io/msgvault/internal/oauth.oauthClientSecret=${MSGVAULT_OAUTH_CLIENT_SECRET}"
go build -tags "fts5 sqlite_vec" -trimpath -ldflags="$LDFLAGS" -o dist/msgvault ./cmd/msgvault

echo "--- Binary info ---"
Expand Down Expand Up @@ -172,11 +184,17 @@ jobs:
CGO_ENABLED: '1'
CGO_CFLAGS: "-IC:/msys64/mingw64/include -fgnu89-inline"
CGO_LDFLAGS: "-Wl,--allow-multiple-definition"
MSGVAULT_OAUTH_CLIENT_ID: ${{ secrets.MSGVAULT_OAUTH_CLIENT_ID }}
MSGVAULT_OAUTH_CLIENT_SECRET: ${{ secrets.MSGVAULT_OAUTH_CLIENT_SECRET }}
run: |
if [ -z "$MSGVAULT_OAUTH_CLIENT_ID" ] || [ -z "$MSGVAULT_OAUTH_CLIENT_SECRET" ]; then
echo "FATAL: MSGVAULT_OAUTH_CLIENT_ID and MSGVAULT_OAUTH_CLIENT_SECRET repository secrets must be set" >&2
exit 1
fi
VERSION="${GITHUB_REF#refs/tags/v}"

mkdir -p dist
LDFLAGS="-s -w -X go.kenn.io/msgvault/cmd/msgvault/cmd.Version=v${VERSION} -X go.kenn.io/msgvault/cmd/msgvault/cmd.Commit=$(printf '%s' "$GITHUB_SHA" | cut -c1-8) -X go.kenn.io/msgvault/cmd/msgvault/cmd.BuildDate=$(date -u +%Y-%m-%dT%H:%M:%SZ)"
LDFLAGS="-s -w -X go.kenn.io/msgvault/cmd/msgvault/cmd.Version=v${VERSION} -X go.kenn.io/msgvault/cmd/msgvault/cmd.Commit=$(printf '%s' "$GITHUB_SHA" | cut -c1-8) -X go.kenn.io/msgvault/cmd/msgvault/cmd.BuildDate=$(date -u +%Y-%m-%dT%H:%M:%SZ) -X go.kenn.io/msgvault/internal/oauth.oauthClientID=${MSGVAULT_OAUTH_CLIENT_ID} -X go.kenn.io/msgvault/internal/oauth.oauthClientSecret=${MSGVAULT_OAUTH_CLIENT_SECRET}"
go build -tags "fts5 sqlite_vec" -trimpath -ldflags="$LDFLAGS" -o dist/msgvault.exe ./cmd/msgvault

# Smoke test
Expand Down
9 changes: 8 additions & 1 deletion Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -22,15 +22,22 @@
ARG VERSION=dev
ARG COMMIT=unknown
ARG BUILD_DATE=unknown
ARG MSGVAULT_OAUTH_CLIENT_ID=
ARG MSGVAULT_OAUTH_CLIENT_SECRET=

Check warning on line 26 in Dockerfile

View workflow job for this annotation

GitHub Actions / validate

Sensitive data should not be used in the ARG or ENV commands

SecretsUsedInArgOrEnv: Do not use ARG or ENV instructions for sensitive data (ARG "MSGVAULT_OAUTH_CLIENT_SECRET") More info: https://docs.docker.com/go/dockerfile/rule/secrets-used-in-arg-or-env/

# Note: Module path must match go.mod (go.kenn.io/msgvault)
# Docker builds receive production OAuth values from the publish workflow.
# Local/PR Docker builds default these to empty so they do not accidentally
# embed the source development client in container images.
RUN CGO_ENABLED=1 go build \
-tags fts5 \
-trimpath \
-ldflags="-s -w \
-X go.kenn.io/msgvault/cmd/msgvault/cmd.Version=${VERSION} \
-X go.kenn.io/msgvault/cmd/msgvault/cmd.Commit=${COMMIT} \
-X go.kenn.io/msgvault/cmd/msgvault/cmd.BuildDate=${BUILD_DATE}" \
-X go.kenn.io/msgvault/cmd/msgvault/cmd.BuildDate=${BUILD_DATE} \
-X go.kenn.io/msgvault/internal/oauth.oauthClientID=${MSGVAULT_OAUTH_CLIENT_ID} \
-X go.kenn.io/msgvault/internal/oauth.oauthClientSecret=${MSGVAULT_OAUTH_CLIENT_SECRET}" \
-o /msgvault \
./cmd/msgvault

Expand Down
7 changes: 7 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,13 @@ LDFLAGS := -X go.kenn.io/msgvault/cmd/msgvault/cmd.Version=$(VERSION) \
-X go.kenn.io/msgvault/cmd/msgvault/cmd.Commit=$(COMMIT) \
-X go.kenn.io/msgvault/cmd/msgvault/cmd.BuildDate=$(BUILD_DATE)

# Only inject embedded OAuth credentials when both env vars are set;
# otherwise leave the compiled-in defaults from internal/oauth/embedded.go.
ifneq ($(and $(MSGVAULT_OAUTH_CLIENT_ID),$(MSGVAULT_OAUTH_CLIENT_SECRET)),)
LDFLAGS += -X go.kenn.io/msgvault/internal/oauth.oauthClientID=$(MSGVAULT_OAUTH_CLIENT_ID) \
-X go.kenn.io/msgvault/internal/oauth.oauthClientSecret=$(MSGVAULT_OAUTH_CLIENT_SECRET)
endif

LDFLAGS_RELEASE := $(LDFLAGS) -s -w

# Default build tags applied to every go build/test/bench invocation.
Expand Down
41 changes: 31 additions & 10 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -64,16 +64,22 @@ conda install -c conda-forge msgvault

## Quick Start

> **Prerequisites:** You need a Google Cloud OAuth credential before adding an account.
> Follow the **[OAuth Setup Guide](https://msgvault.io/guides/oauth-setup/)** to create one (~5 minutes).

```bash
```sh
# Initialize the database
msgvault init-db
msgvault add-account you@gmail.com # opens browser for OAuth
msgvault sync-full you@gmail.com --limit 100

# Add a Gmail account — opens your browser for consent
msgvault add-account you@gmail.com

# Sync mail
msgvault sync-full you@gmail.com

# Browse the archive
msgvault tui
```

No Google Cloud Console setup required: msgvault ships with a verified OAuth client.

## Commands

| Command | Description |
Expand Down Expand Up @@ -174,14 +180,29 @@ All data lives in `~/.msgvault/` by default (override with `MSGVAULT_HOME`).

```toml
# ~/.msgvault/config.toml
[oauth]
client_secrets = "/path/to/client_secret.json"

[sync]
rate_limit_qps = 5
```

See the [Configuration Guide](https://msgvault.io/configuration/) for all options.
See the [Configuration Guide](https://msgvault.io/configuration/) for all options. To override the embedded OAuth client, see [Advanced: bring your own OAuth client](#advanced-bring-your-own-oauth-client) below.

### Advanced: bring your own OAuth client

The default flow uses msgvault's centralized verified OAuth client. You only need your own Cloud project if:

- Your Workspace organization prohibits authorizing third-party OAuth apps
- You prefer your own Cloud project's third-party-access listing to show
- You need your own Gmail API quota for very large mailboxes
- You want a fallback before msgvault's centralized client finishes Google verification

Follow the [OAuth setup guide](https://msgvault.io/guides/oauth-setup/) to create a Desktop OAuth client, then add it to `~/.msgvault/config.toml`:

```toml
[oauth]
client_secrets = "/path/to/client_secret.json"
```

Use `--oauth-app NAME` for per-account named-app routing — see the OAuth setup guide for details.

### Multiple OAuth Apps (Google Workspace)

Expand Down
21 changes: 6 additions & 15 deletions cmd/msgvault/cmd/addaccount.go
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,6 @@ Examples:
// Resolve which client secrets to use
resolvedApp := oauthAppName
oauthAppExplicit := cmd.Flags().Changed("oauth-app")
var clientSecretsPath string

// Initialize database (in case it's new)
dbPath := cfg.DatabaseDSN()
Expand Down Expand Up @@ -174,21 +173,13 @@ Examples:
return nil
}

// Resolve client secrets path (standard OAuth flow)
clientSecretsPath, err = cfg.OAuth.ClientSecretsFor(resolvedApp)
// Build the OAuth manager. resolveOAuthManager handles named BYO,
// global BYO, and the embedded fallback automatically.
oauthMgr, err := resolveOAuthManager(cfg, resolvedApp, oauth.Scopes, logger)
if err != nil {
if !cfg.OAuth.HasAnyConfig() {
return errOAuthNotConfigured()
}
return err
}

// Create OAuth manager
oauthMgr, err := oauth.NewManager(clientSecretsPath, cfg.TokensDir(), logger)
if err != nil {
return wrapOAuthError(fmt.Errorf("create oauth manager: %w", err))
}

// If --force, delete existing token so we re-authorize
if forceReauth {
if oauthMgr.HasToken(email) {
Expand All @@ -204,10 +195,10 @@ Examples:
// If a valid token exists, check if we can reuse it.
// Validate the token's client identity when any named app is
// involved — whether from an explicit flag, a binding change,
// or inherited from the DB. A mismatched token would fail on
// next refresh.
// inherited from the DB — or when falling back to the embedded
// client. A mismatched token would fail on next refresh.
needsClientCheck := bindingChanged || oauthAppExplicit ||
resolvedApp != ""
resolvedApp != "" || oauthMgr.UsesEmbeddedClient()
tokenReusable := !forceReauth && oauthMgr.HasToken(email) &&
(!needsClientCheck || oauthMgr.TokenMatchesClient(email))
if tokenReusable {
Expand Down
118 changes: 118 additions & 0 deletions cmd/msgvault/cmd/addaccount_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import (
assertpkg "github.com/stretchr/testify/assert"
requirepkg "github.com/stretchr/testify/require"
"go.kenn.io/msgvault/internal/config"
"go.kenn.io/msgvault/internal/oauth"
"go.kenn.io/msgvault/internal/store"
)

Expand Down Expand Up @@ -369,6 +370,64 @@ func TestAddAccount_ExplicitDefaultRejectsMismatchedToken(t *testing.T) {
require.Error(err, "mismatched token should be rejected with explicit --oauth-app \"\"")
}

// TestAddAccount_EmbeddedDefaultRejectsMismatchedToken verifies that the
// no-config embedded fallback does not silently reuse a token minted by a
// different OAuth client.
func TestAddAccount_EmbeddedDefaultRejectsMismatchedToken(t *testing.T) {
require := requirepkg.New(t)
tmpDir := t.TempDir()

tokensDir := filepath.Join(tmpDir, "tokens")
require.NoError(os.MkdirAll(tokensDir, 0700), "mkdir tokens")
tokenData, err := json.Marshal(map[string]string{
"access_token": "fake-access",
"refresh_token": "fake-refresh",
"token_type": "Bearer",
"client_id": "wrong-client.apps.googleusercontent.com",
})
require.NoError(err, "marshal token")
require.NoError(os.WriteFile(
filepath.Join(tokensDir, "user@example.com.json"),
tokenData, 0600,
), "write token")

savedCfg := cfg
savedLogger := logger
savedOAuthApp := oauthAppName
defer func() {
cfg = savedCfg
logger = savedLogger
oauthAppName = savedOAuthApp
}()

cfg = &config.Config{
HomeDir: tmpDir,
Data: config.DataConfig{DataDir: tmpDir},
}
logger = slog.New(slog.NewTextHandler(os.Stderr, nil))

ctx, cancel := context.WithCancel(context.Background())
cancel()

testCmd := &cobra.Command{
Use: "add-account <email>",
Args: cobra.ExactArgs(1),
RunE: addAccountCmd.RunE,
}
testCmd.Flags().StringVar(&oauthAppName, "oauth-app", "", "")
testCmd.Flags().BoolVar(&headless, "headless", false, "")
testCmd.Flags().BoolVar(&forceReauth, "force", false, "")
testCmd.Flags().StringVar(&accountDisplayName, "display-name", "", "")
testCmd.Flags().BoolVar(&noDefaultIdentityAddAccount, "no-default-identity", false, "")

root := newTestRootCmd()
root.AddCommand(testCmd)
root.SetArgs([]string{"add-account", "user@example.com"})

err = root.ExecuteContext(ctx)
require.Error(err, "embedded client should reject a token minted by another OAuth client")
}

// TestAddAccount_ExplicitDefaultAcceptsMatchingToken verifies that
// --oauth-app "" accepts a token minted by the default client.
func TestAddAccount_ExplicitDefaultAcceptsMatchingToken(t *testing.T) {
Expand Down Expand Up @@ -1027,3 +1086,62 @@ func TestAddAccount_ForceServiceAccountReturnsActionableError(t *testing.T) {
requirepkg.Error(t, err, "expected --force service account error")
requirepkg.ErrorContains(t, err, "service accounts do not use --force")
}

func TestAddAccount_ResolverBranches(t *testing.T) {
tests := []struct {
name string
appName string
setup func(t *testing.T, cfg *config.Config)
wantErr bool
errContains string
}{
{
name: "named BYO with client_secrets",
appName: "acme",
setup: func(t *testing.T, cfg *config.Config) {
t.Helper()
path := writeStubClientSecrets(t, cfg.Data.DataDir, "acme.json")
cfg.OAuth.Apps = map[string]config.OAuthApp{"acme": {ClientSecrets: path}}
},
wantErr: false,
},
{
name: "named app without client_secrets",
appName: "missing",
setup: func(t *testing.T, cfg *config.Config) { t.Helper() },
wantErr: true,
errContains: "missing",
},
{
name: "global BYO",
appName: "",
setup: func(t *testing.T, cfg *config.Config) {
t.Helper()
cfg.OAuth.ClientSecrets = writeStubClientSecrets(t, cfg.Data.DataDir, "default.json")
},
wantErr: false,
},
{
name: "no config falls through to embedded",
appName: "",
setup: func(t *testing.T, cfg *config.Config) { t.Helper() },
wantErr: false,
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
require := requirepkg.New(t)
cfg := newTestConfig(t)
tc.setup(t, cfg)
_, err := resolveOAuthManager(cfg, tc.appName, oauth.Scopes, slog.Default())
if tc.wantErr {
require.Error(err, "expected error")
if tc.errContains != "" {
require.ErrorContains(err, tc.errContains)
}
return
}
require.NoError(err, "resolveOAuthManager")
})
}
}
Loading