Skip to content

Commit 8dfbe47

Browse files
diratheadirathea
authored
feat: template provider (#35)
* feat: template provider for composing secrets * fix: end2end test for template provider * fix: unused attributes * fix: provider refactoring * fix: end2end test * fix: use environment to protect end2end test * feat: split test between short and full test * fix: gotestsum install * fix: ci test reporting * feat: docs
1 parent 932059d commit 8dfbe47

34 files changed

Lines changed: 1138 additions & 171 deletions

‎.github/workflows/ci.yml‎

Lines changed: 6 additions & 134 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ permissions:
1313

1414
jobs:
1515
test:
16-
name: Run End-to-End Tests
16+
name: Run short test
1717
runs-on: ubuntu-latest
1818
steps:
1919
- name: Checkout code
@@ -24,150 +24,22 @@ jobs:
2424
with:
2525
go-version-file: go.mod
2626

27-
- name: Setup env via sstart
28-
uses: dirathea/setup-sstart-env@main
29-
env:
30-
INFISICAL_UNIVERSAL_AUTH_CLIENT_SECRET: ${{ secrets.INFISICAL_UNIVERSAL_AUTH_CLIENT_SECRET }}
31-
INFISICAL_UNIVERSAL_AUTH_CLIENT_ID: ${{ secrets.INFISICAL_UNIVERSAL_AUTH_CLIENT_ID }}
32-
INFISICAL_SITE_URL: ${{ secrets.INFISICAL_SITE_URL }}
33-
with:
34-
config: |
35-
providers:
36-
- kind: infisical
37-
project_id: 8aded323-e110-4f48-9c7f-24c275358609
38-
environment: prod
39-
path: /github
40-
41-
- name: Authenticate to Google Cloud
42-
uses: google-github-actions/auth@v2
43-
with:
44-
credentials_json: ${{ env.GOOGLE_APPLICATION_CREDENTIALS }}
45-
46-
- name: Set up Cloud SDK
47-
uses: google-github-actions/setup-gcloud@v2
48-
49-
- name: Install Bitwarden CLI
50-
run: |
51-
BW_DOWNLOAD_URL=$(curl -s https://api.github.com/repos/bitwarden/cli/releases/latest | grep '"browser_download_url".*linux.*zip' | head -1 | sed -E 's/.*"([^"]+)".*/\1/')
52-
curl -L "${BW_DOWNLOAD_URL}" -o bw.zip
53-
unzip -q bw.zip
54-
chmod +x bw
55-
sudo mv bw /usr/local/bin/
56-
rm bw.zip
57-
bw --version
58-
59-
- name: Determine which tests to run
60-
id: test_filter
61-
uses: actions/github-script@v7
62-
with:
63-
script: |
64-
try {
65-
const { data: files } = await github.rest.pulls.listFiles({
66-
owner: context.repo.owner,
67-
repo: context.repo.repo,
68-
pull_number: context.issue.number,
69-
});
70-
71-
const changedFiles = files.map(f => f.filename);
72-
console.log('Changed files:', changedFiles);
73-
74-
// Map provider directories to test name prefixes
75-
const providerTestMap = {
76-
'internal/provider/aws/': ['TestE2E_AWSSecretsManager'],
77-
'internal/provider/azurekeyvault/': ['TestE2E_AzureKeyVault'],
78-
'internal/provider/bitwarden/': ['TestE2E_Bitwarden', 'TestE2E_BitwardenSM'],
79-
'internal/provider/doppler/': ['TestE2E_Doppler'],
80-
'internal/provider/gcsm/': ['TestE2E_GCSM'],
81-
'internal/provider/infisical/': ['TestE2E_Infisical'],
82-
'internal/provider/onepassword/': ['TestE2E_OnePassword'],
83-
'internal/provider/vault/': ['TestE2E_Vault', 'TestE2E_OpenBao'],
84-
'internal/oidc/': ['TestE2E_SSO'],
85-
};
86-
87-
// Core files that require all tests
88-
const corePaths = [
89-
'internal/secrets/',
90-
'internal/config/',
91-
'internal/app/',
92-
'internal/cli/',
93-
'tests/end2end/',
94-
'cmd/',
95-
];
96-
97-
// Check if any core files changed
98-
const hasCoreChanges = changedFiles.some(file =>
99-
corePaths.some(path => file.startsWith(path))
100-
);
101-
102-
if (hasCoreChanges) {
103-
console.log('Core files changed, running all tests');
104-
core.setOutput('test_filter', '');
105-
core.setOutput('run_all_tests', 'true');
106-
core.setOutput('skip_tests', 'false');
107-
return;
108-
}
109-
110-
// Find which providers changed
111-
const affectedTests = new Set();
112-
113-
for (const file of changedFiles) {
114-
for (const [providerPath, testNames] of Object.entries(providerTestMap)) {
115-
if (file.startsWith(providerPath)) {
116-
testNames.forEach(test => affectedTests.add(test));
117-
}
118-
}
119-
}
120-
121-
if (affectedTests.size === 0) {
122-
console.log('No provider changes detected, skipping end2end tests');
123-
core.setOutput('test_filter', '');
124-
core.setOutput('run_all_tests', 'false');
125-
core.setOutput('skip_tests', 'true');
126-
return;
127-
}
128-
129-
// Construct test filter regex (matches any of the affected tests)
130-
// Format: TestE2E_(Provider1|Provider2) for go test -run flag
131-
const testFilter = Array.from(affectedTests).join('|');
132-
console.log('Running selective tests:', testFilter);
133-
core.setOutput('test_filter', testFilter);
134-
core.setOutput('run_all_tests', 'false');
135-
core.setOutput('skip_tests', 'false');
136-
} catch (error) {
137-
console.log('Error determining test filter, running all tests:', error.message);
138-
core.setOutput('test_filter', '');
139-
core.setOutput('run_all_tests', 'true');
140-
core.setOutput('skip_tests', 'false');
141-
}
142-
143-
- name: Run end-to-end tests
144-
if: steps.test_filter.outputs.skip_tests != 'true'
27+
- name: Run tests in short mode
14528
env:
14629
CGO_LDFLAGS: -lm
147-
# the rest of env supplied by setup-sstart-env
14830
run: |
14931
go install gotest.tools/gotestsum
150-
if [ "${{ steps.test_filter.outputs.run_all_tests }}" = "true" ]; then
151-
echo "Running all end-to-end tests"
152-
gotestsum --junitfile test-results.xml --format testname -- ./tests/end2end/...
153-
else
154-
echo "Running selective tests: ${{ steps.test_filter.outputs.test_filter }}"
155-
gotestsum --junitfile test-results.xml --format testname -- -run "${{ steps.test_filter.outputs.test_filter }}" ./tests/end2end/...
156-
fi
32+
echo "Running tests in short mode"
33+
gotestsum --junitfile test-results.xml --format testname -- -short ./tests/end2end/...
15734
15835
- name: Publish test results
159-
if: always() && steps.test_filter.outputs.skip_tests != 'true'
36+
if: always()
16037
uses: EnricoMi/publish-unit-test-result-action@v2
16138
with:
16239
files: test-results.xml
163-
check_name: End-to-End Test Results
40+
check_name: CI Test Results
16441
fail_on: 'nothing'
16542
comment_mode: off
166-
167-
- name: Skip end-to-end tests
168-
if: steps.test_filter.outputs.skip_tests == 'true'
169-
run: |
170-
echo "No provider changes detected. Skipping end-to-end tests."
17143

17244
build:
17345
name: Build

‎.github/workflows/end2end.yml‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
name: End-to-End Tests
2+
3+
on:
4+
pull_request:
5+
types: [opened, synchronize]
6+
branches:
7+
- main
8+
9+
permissions:
10+
contents: read
11+
checks: write
12+
pull-requests: write
13+
14+
jobs:
15+
test:
16+
name: Run End-to-End Tests
17+
runs-on: ubuntu-latest
18+
environment: end2end
19+
steps:
20+
- name: Checkout code
21+
uses: actions/checkout@v4
22+
23+
- name: Set up Go
24+
uses: actions/setup-go@v5
25+
with:
26+
go-version-file: go.mod
27+
28+
- name: Setup env via sstart
29+
uses: dirathea/setup-sstart-env@main
30+
env:
31+
INFISICAL_UNIVERSAL_AUTH_CLIENT_SECRET: ${{ secrets.INFISICAL_UNIVERSAL_AUTH_CLIENT_SECRET }}
32+
INFISICAL_UNIVERSAL_AUTH_CLIENT_ID: ${{ secrets.INFISICAL_UNIVERSAL_AUTH_CLIENT_ID }}
33+
INFISICAL_SITE_URL: https://eu.infisical.com
34+
with:
35+
config: |
36+
providers:
37+
- kind: infisical
38+
project_id: 8aded323-e110-4f48-9c7f-24c275358609
39+
environment: prod
40+
path: /github
41+
42+
- name: Authenticate to Google Cloud
43+
uses: google-github-actions/auth@v2
44+
with:
45+
credentials_json: ${{ env.GOOGLE_APPLICATION_CREDENTIALS }}
46+
47+
- name: Set up Cloud SDK
48+
uses: google-github-actions/setup-gcloud@v2
49+
50+
- name: Install Bitwarden CLI
51+
run: |
52+
BW_DOWNLOAD_URL=$(curl -s https://api.github.com/repos/bitwarden/cli/releases/latest | grep '"browser_download_url".*linux.*zip' | head -1 | sed -E 's/.*"([^"]+)".*/\1/')
53+
curl -L "${BW_DOWNLOAD_URL}" -o bw.zip
54+
unzip -q bw.zip
55+
chmod +x bw
56+
sudo mv bw /usr/local/bin/
57+
rm bw.zip
58+
bw --version
59+
60+
- name: Run all end-to-end tests
61+
env:
62+
CGO_LDFLAGS: -lm
63+
# the rest of env supplied by setup-sstart-env
64+
run: |
65+
go install gotest.tools/gotestsum
66+
echo "Running all end-to-end tests (including tests that require real services)"
67+
gotestsum --junitfile test-results.xml --format testname -- ./tests/end2end/...
68+
69+
- name: Publish test results
70+
if: always()
71+
uses: EnricoMi/publish-unit-test-result-action@v2
72+
with:
73+
files: test-results.xml
74+
check_name: End-to-End Test Results
75+
fail_on: 'nothing'
76+
comment_mode: off
77+

‎CONFIGURATION.md‎

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ providers:
3434
| `dotenv` | Stable |
3535
| `gcloud_secretmanager` | Stable |
3636
| `infisical` | Stable |
37+
| `template` | Stable |
3738
| `vault` | Stable |
3839

3940
## Provider Configuration
@@ -590,6 +591,100 @@ You can also use simple environment variable expansion with `${VAR}` or `$VAR` s
590591
path: ${HOME}/.config/myapp/.env
591592
```
592593

594+
## Template Providers
595+
596+
The template provider allows you to construct new secrets by combining values from other providers using Go template syntax. This is useful when your application needs secrets in a different format than how they're stored (e.g., building connection URIs from separate credentials).
597+
598+
**Configuration:**
599+
- `uses` (required): List of provider IDs that this template provider depends on. The template provider can only access secrets from providers explicitly listed here (principle of least privilege).
600+
- `templates` (required): Map of output secret keys to template expressions. Each template expression is evaluated using Go's `text/template` package.
601+
602+
**Template Syntax:**
603+
- Use `{{.<provider_id>.<secret_key>}}` to reference secrets from other providers
604+
- The syntax is similar to Helm templates and uses Go's text/template package
605+
- You can use all Go template functions (e.g., `{{if}}`, `{{range}}`, `{{index}}`, etc.)
606+
- Provider IDs and secret keys are case-sensitive
607+
608+
**Security Model:**
609+
The template provider follows the principle of least privilege:
610+
- Only providers listed in the `uses` field are accessible
611+
- If a provider is not in `uses`, references to it will resolve to empty values
612+
- This ensures templates can only access secrets they explicitly declare as dependencies
613+
614+
**Provider Order:**
615+
Template providers must be defined after the providers they depend on. Providers are processed in the order they appear in the configuration file, so ensure all source providers are listed before the template provider.
616+
617+
**Example - Building a Database URI:**
618+
```yaml
619+
providers:
620+
# Fetch database host configuration
621+
- kind: aws_secretsmanager
622+
id: db_config
623+
secret_id: rds/credentials
624+
# Returns: DB_HOST, DB_PORT, DB_NAME
625+
626+
# Fetch database credentials
627+
- kind: aws_secretsmanager
628+
id: db_creds
629+
secret_id: rds/prod/credentials
630+
# Returns: DB_USER, DB_PASSWORD
631+
632+
# Build database URI using template provider
633+
- kind: template
634+
uses:
635+
- db_config
636+
- db_creds
637+
templates:
638+
DATABASE_URI: postgresql://{{.db_creds.DB_USER}}:{{.db_creds.DB_PASSWORD}}@{{.db_config.DB_HOST}}:{{.db_config.DB_PORT}}/{{.db_config.DB_NAME}}
639+
```
640+
641+
**Example - Multiple Templates:**
642+
```yaml
643+
providers:
644+
- kind: aws_secretsmanager
645+
id: api_config
646+
secret_id: api/config
647+
# Returns: API_HOST, API_PORT
648+
649+
- kind: aws_secretsmanager
650+
id: api_creds
651+
secret_id: api/credentials
652+
# Returns: API_KEY, API_SECRET
653+
654+
- kind: template
655+
uses:
656+
- api_config
657+
- api_creds
658+
templates:
659+
API_BASE_URL: https://{{.api_config.API_HOST}}:{{.api_config.API_PORT}}
660+
API_AUTH_HEADER: Bearer {{.api_creds.API_KEY}}
661+
API_FULL_URL: https://{{.api_config.API_HOST}}:{{.api_config.API_PORT}}/v1?key={{.api_creds.API_KEY}}
662+
```
663+
664+
**Example - Using Template Functions:**
665+
```yaml
666+
providers:
667+
- kind: aws_secretsmanager
668+
id: config
669+
secret_id: app/config
670+
# Returns: ENV (e.g., "production")
671+
672+
- kind: template
673+
uses:
674+
- config
675+
templates:
676+
# Use conditional logic based on secret values
677+
LOG_LEVEL: {{if eq .config.ENV "production"}}error{{else}}debug{{end}}
678+
# Combine multiple template expressions
679+
APP_ENV: {{.config.ENV}}
680+
```
681+
682+
**Error Handling:**
683+
- If a referenced provider ID doesn't exist, the template will fail with an error
684+
- If a referenced secret key doesn't exist in a provider, it will resolve to an empty value
685+
- If `uses` is not specified or empty, all provider references will resolve to empty values
686+
- Template parsing errors will be reported with the specific template expression that failed
687+
593688
## Multiple Providers
594689

595690
Each provider loads from a single source. To load multiple secrets from the same provider type, create multiple provider instances:

0 commit comments

Comments
 (0)