From 32b1128f2c71db32f11d812a599c7e8ad3910505 Mon Sep 17 00:00:00 2001 From: bhagyapathak Date: Wed, 1 Jul 2026 17:19:45 +0530 Subject: [PATCH 1/5] feat(projectconfig): accept [tests] and [test-groups] schema in config files --- docs/user/reference/config/components.md | 20 ++ docs/user/reference/config/config-file.md | 2 + docs/user/reference/config/images.md | 4 +- docs/user/reference/config/tests.md | 92 +++++++ go.mod | 2 +- internal/app/azldev/cmds/image/list.go | 2 +- internal/app/azldev/cmds/image/list_test.go | 12 +- internal/projectconfig/component.go | 9 + internal/projectconfig/configfile.go | 13 + internal/projectconfig/configfile_test.go | 32 +++ internal/projectconfig/fingerprint_test.go | 3 + internal/projectconfig/image.go | 11 +- internal/projectconfig/loader_test.go | 1 + internal/projectconfig/tests.go | 196 ++++++++++++++ internal/projectconfig/testsuite_test.go | 6 +- ...ainer_config_generate-schema_stdout_1.snap | 250 ++++++++++++++++++ ...shots_config_generate-schema_stdout_1.snap | 250 ++++++++++++++++++ schemas/azldev.schema.json | 250 ++++++++++++++++++ 18 files changed, 1143 insertions(+), 12 deletions(-) create mode 100644 docs/user/reference/config/tests.md create mode 100644 internal/projectconfig/tests.go diff --git a/docs/user/reference/config/components.md b/docs/user/reference/config/components.md index 8a25d9ef..2b67a42a 100644 --- a/docs/user/reference/config/components.md +++ b/docs/user/reference/config/components.md @@ -16,6 +16,7 @@ A component definition tells azldev where to find the spec file, how to customiz | Render config | `render` | [RenderConfig](#render-configuration) | No | Options controlling spec rendering behavior | | Source files | `source-files` | array of [SourceFileReference](#source-file-references) | No | Additional source files to download for this component | | Package overrides | `packages` | map of string → [PackageConfig](package-groups.md#package-config) | No | Exact per-package configuration overrides; highest priority in the resolution order | +| Tests | `tests` | [ComponentTests](#component-tests) | No | Test references that apply to this component (see [Tests and Test Groups](tests.md)) | ### Bare Components @@ -338,6 +339,24 @@ rpm-channel = "rpm-devel" rpm-channel = "none" ``` +## Component Tests + +The `[components..tests]` subtable lists test or test-group +references that apply to the component. Each entry is a [TestRef](tests.md#test-reference) +with exactly one of `name` or `group`. + +| Field | TOML Key | Type | Required | Description | +|-------|----------|------|----------|-------------| +| Tests | `tests` | array of [TestRef](tests.md#test-reference) | No | References to `[tests.]` entries or `[test-groups.]` entries | + +```toml +[components.kernel.tests] +tests = [ + { group = "kernel-bvt" }, + { name = "kdump-smoke" }, +] +``` + ## Source File References The `[[components..source-files]]` array defines additional source files to fetch or generate before building — binaries, pre-built artifacts, or archives generated on-the-fly by a script. @@ -537,5 +556,6 @@ lines = ["cp -vf %{shimdirx64}/$(basename %{shimefix64}) %{shimefix64} ||:"] - [Distros](distros.md) — distro definitions and `default-component-config` inheritance - [Component Groups](component-groups.md) — grouping components with shared defaults - [Package Groups](package-groups.md) — project-level package groups and full resolution order +- [Tests and Test Groups](tests.md) — definitions referenced by `[components..tests]` - [Configuration System](../../explanation/config-system.md) — inheritance and merge behavior - [JSON Schema](../../../../schemas/azldev.schema.json) — machine-readable schema diff --git a/docs/user/reference/config/config-file.md b/docs/user/reference/config/config-file.md index e1ae30bb..19abd972 100644 --- a/docs/user/reference/config/config-file.md +++ b/docs/user/reference/config/config-file.md @@ -15,6 +15,8 @@ All config files share the same schema — there is no distinction between a "ro | `component-groups` | map of objects | Named groups of components with shared defaults | [Component Groups](component-groups.md) | | `images` | map of objects | Image definitions (VMs, containers) | [Images](images.md) | | `test-suites` | map of objects | Named test suite definitions referenced by images | [Test Suites](test-suites.md) | +| `tests` | map of objects | Named test definitions (new-shape, parse-only) | [Tests and Test Groups](tests.md) | +| `test-groups` | map of objects | Named bundles of test references (new-shape, parse-only) | [Tests and Test Groups](tests.md) | | `tools` | object | Configuration for external tools used by azldev | [Tools](tools.md) | | `default-package-config` | object | Project-wide default applied to all binary packages | [Package Groups — Resolution Order](package-groups.md#resolution-order) | | `package-groups` | map of objects | Named groups of binary packages with shared config | [Package Groups](package-groups.md) | diff --git a/docs/user/reference/config/images.md b/docs/user/reference/config/images.md index ebbd5e53..34dd9cb9 100644 --- a/docs/user/reference/config/images.md +++ b/docs/user/reference/config/images.md @@ -35,11 +35,12 @@ The `capabilities` subtable describes what the image supports. All fields are op ## Image Tests -The `tests` subtable links an image to one or more test suites defined in the top-level [`[test-suites]`](test-suites.md) section. +The `tests` subtable links an image to one or more test suites defined in the top-level [`[test-suites]`](test-suites.md) section, and/or to entries from the new-shape [`[tests]` / `[test-groups]`](tests.md) sections. | Field | TOML Key | Type | Required | Description | |-------|----------|------|----------|-------------| | Test Suites | `test-suites` | array of inline tables | No | List of test suite references. Each entry must have a `name` field matching a key in `[test-suites]`. | +| Tests | `tests` | array of [TestRef](tests.md#test-reference) | No | References to `[tests.]` entries or `[test-groups.]` entries (parse-only; see [Tests and Test Groups](tests.md)). | ## Image Publish @@ -118,4 +119,5 @@ channels = ["registry-prod", "registry-staging"] - [Config File Structure](config-file.md) — top-level config file layout - [Test Suites](test-suites.md) — test suite definitions +- [Tests and Test Groups](tests.md) — new-shape test/group definitions referenced by `[images..tests]` - [Tools](tools.md) — Image Customizer tool configuration diff --git a/docs/user/reference/config/tests.md b/docs/user/reference/config/tests.md new file mode 100644 index 00000000..990e34aa --- /dev/null +++ b/docs/user/reference/config/tests.md @@ -0,0 +1,92 @@ +# Tests and Test Groups + +The `[tests]` and `[test-groups]` sections declare framework-agnostic test +metadata that components and images can target by name. Each test entry +binds a single test (a pytest run, a LISA case, or a TMT plan) +to a named identifier; each group entry bundles tests (and +nested groups) under one name so callers can reference a curated set +without enumerating every member. + +## Test Definition + +Each entry under `[tests.]` describes one configuration of one +runner. Framework-specific options live in a typed subtable +(`pytest`, `lisa`, `tmt`) whose contents are passed through +to the runner; their internal schemas are intentionally not validated +by azldev so frameworks can evolve independently. + +| Field | TOML Key | Type | Required | Description | +|-------|----------|------|----------|-------------| +| Type | `type` | string | Yes | Test framework: `pytest`, `lisa`, or `tmt` | +| Description | `description` | string | No | Human-readable description | +| Kind | `kind` | string array | No | Free-form hints (e.g. `functional`, `performance`, `bvt`) | +| Long running | `long-running` | boolean | No | Hints that this test may run for hours | +| Required capabilities | `required-capabilities` | string array | No | Capability tokens the image must declare for this test to be applicable | +| Lisa | `lisa` | table | No | LISA-specific configuration (opaque to azldev) | +| Tmt | `tmt` | table | No | TMT-specific configuration (opaque to azldev) | +| Pytest | `pytest` | table | No | pytest-specific configuration (opaque to azldev) | + +## Test Group + +Each entry under `[test-groups.]` names an ordered list of test or +nested-group references that callers can target as a single unit. + +| Field | TOML Key | Type | Required | Description | +|-------|----------|------|----------|-------------| +| Description | `description` | string | No | Human-readable description | +| Tests | `tests` | array of [TestRef](#test-reference) | No | Ordered members of the group | + +## Test Reference + +`TestRef` is an inline table with exactly one of `name` or `group`: + +| Field | TOML Key | Type | Description | +|-------|----------|------|-------------| +| Name | `name` | string | References a `[tests.]` entry | +| Group | `group` | string | References a `[test-groups.]` entry | + +## Referencing from Components and Images + +Components and images both expose a `tests` subtable that holds a list +of `TestRef`s: + +```toml +[components.kernel.tests] +tests = [{ group = "kernel-bvt" }, { name = "kdump-smoke" }] + +[images.vm-base.tests] +tests = [{ group = "bvt" }] +``` + +## Example + +```toml +[tests.bvt-ssh] +type = "pytest" +description = "Basic SSH boot verification" +kind = ["functional", "bvt"] +required-capabilities = ["ssh"] +pytest = { working-dir = "tests/bvt", test-paths = ["test_ssh.py"] } + +[tests.kdump-smoke] +type = "lisa" +description = "Smoke test for kdump" +lisa = { case = "kdump.smoke" } + +[test-groups.bvt] +description = "Build verification tests" +tests = [ + { name = "bvt-ssh" }, + { group = "bvt-extras" }, +] + +[test-groups.bvt-extras] +tests = [{ name = "kdump-smoke" }] +``` + +## Related Resources + +- [Test Suites](test-suites.md) - legacy test suite definitions +- [Components](components.md#component-tests) — per-component `tests` field +- [Images](images.md#image-tests) — per-image `tests` field +- [Config File Structure](config-file.md) — top-level config layout diff --git a/go.mod b/go.mod index b8efaf3e..510d3271 100644 --- a/go.mod +++ b/go.mod @@ -40,6 +40,7 @@ require ( github.com/muesli/termenv v0.16.0 github.com/nxadm/tail v1.4.11 github.com/opencontainers/selinux v1.15.1 + github.com/pb33f/ordered-map/v2 v2.3.1 github.com/pelletier/go-toml/v2 v2.4.3 github.com/pmezard/go-difflib v1.0.0 github.com/samber/lo v1.53.0 @@ -131,7 +132,6 @@ require ( github.com/muesli/cancelreader v0.2.2 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.1 // indirect - github.com/pb33f/ordered-map/v2 v2.3.1 // indirect github.com/pjbgf/sha1cd v0.6.0 // indirect github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect github.com/rivo/uniseg v0.4.7 // indirect diff --git a/internal/app/azldev/cmds/image/list.go b/internal/app/azldev/cmds/image/list.go index 0f7e5166..52ff3dd4 100644 --- a/internal/app/azldev/cmds/image/list.go +++ b/internal/app/azldev/cmds/image/list.go @@ -38,7 +38,7 @@ type ImageListResult struct { // Tests holds the test configuration for this image, matching the original config // structure. - Tests projectconfig.ImageTestsConfig `json:"tests" table:"-"` + Tests *projectconfig.ImageTestsConfig `json:"tests,omitempty" table:"-"` // TestsSummary is a comma-separated summary of test suite names for table display. TestsSummary string `json:"-" table:"Tests"` diff --git a/internal/app/azldev/cmds/image/list_test.go b/internal/app/azldev/cmds/image/list_test.go index 6c572ac3..6289c08f 100644 --- a/internal/app/azldev/cmds/image/list_test.go +++ b/internal/app/azldev/cmds/image/list_test.go @@ -89,7 +89,7 @@ func TestListImages_WithCapabilitiesAndTests(t *testing.T) { MachineBootable: lo.ToPtr(true), Systemd: lo.ToPtr(true), }, - Tests: projectconfig.ImageTestsConfig{ + Tests: &projectconfig.ImageTestsConfig{ TestSuites: []projectconfig.TestSuiteRef{ {Name: "smoke"}, {Name: "integration"}, @@ -105,7 +105,7 @@ func TestListImages_WithCapabilitiesAndTests(t *testing.T) { Capabilities: projectconfig.ImageCapabilities{ Container: lo.ToPtr(true), }, - Tests: projectconfig.ImageTestsConfig{ + Tests: &projectconfig.ImageTestsConfig{ TestSuites: []projectconfig.TestSuiteRef{ {Name: "smoke"}, }, @@ -131,9 +131,10 @@ func TestListImages_WithCapabilitiesAndTests(t *testing.T) { assert.Equal(t, lo.ToPtr(true), results[0].Capabilities.Container) assert.Nil(t, results[0].Capabilities.MachineBootable) assert.Equal(t, "container", results[0].CapabilitiesSummary) + require.NotNil(t, results[0].Tests) assert.Equal(t, projectconfig.ImageTestsConfig{ TestSuites: []projectconfig.TestSuiteRef{{Name: "smoke"}}, - }, results[0].Tests) + }, *results[0].Tests) assert.Equal(t, "smoke", results[0].TestsSummary) assert.Equal(t, projectconfig.ImagePublishConfig{ Channels: []string{"registry-prod"}, @@ -144,7 +145,7 @@ func TestListImages_WithCapabilitiesAndTests(t *testing.T) { assert.Nil(t, results[1].Capabilities.MachineBootable) assert.Nil(t, results[1].Capabilities.Container) assert.Empty(t, results[1].CapabilitiesSummary) - assert.Empty(t, results[1].Tests.TestSuites) + assert.Nil(t, results[1].Tests) assert.Empty(t, results[1].TestsSummary) assert.Empty(t, results[1].Publish.Channels) assert.Empty(t, results[1].PublishSummary) @@ -154,9 +155,10 @@ func TestListImages_WithCapabilitiesAndTests(t *testing.T) { assert.Equal(t, lo.ToPtr(true), results[2].Capabilities.Systemd) assert.Nil(t, results[2].Capabilities.Container) assert.Equal(t, "machine-bootable, systemd", results[2].CapabilitiesSummary) + require.NotNil(t, results[2].Tests) assert.Equal(t, projectconfig.ImageTestsConfig{ TestSuites: []projectconfig.TestSuiteRef{{Name: "smoke"}, {Name: "integration"}}, - }, results[2].Tests) + }, *results[2].Tests) assert.Equal(t, "smoke, integration", results[2].TestsSummary) assert.Equal(t, projectconfig.ImagePublishConfig{ Channels: []string{"registry-prod", "registry-staging"}, diff --git a/internal/projectconfig/component.go b/internal/projectconfig/component.go index 02c53c9d..763ac0b8 100644 --- a/internal/projectconfig/component.go +++ b/internal/projectconfig/component.go @@ -422,6 +422,14 @@ type ComponentConfig struct { // all packages produced by this component. Overridden by package-group and per-package settings // for binary and debuginfo channels. Publish ComponentPublishConfig `toml:"publish,omitempty" json:"publish,omitempty" table:"-" jsonschema:"title=Publish settings,description=Component-level publish channel settings" fingerprint:"-"` + + // Tests holds the new-shape per-component tests block: + // + // tests.tests = [{ name = "..." }, { group = "..." }] + // + // References must resolve to entries in the project-level [tests] or + // [test-groups] maps; resolution is the responsibility of the test layer. + Tests *ComponentTestsConfig `toml:"tests,omitempty" json:"tests,omitempty" table:"-" jsonschema:"title=Tests,description=Per-component test or test-group references" fingerprint:"-"` } // AllowedSourceFilesHashTypes defines the set of hash types that are supported @@ -525,6 +533,7 @@ func (c *ComponentConfig) WithAbsolutePaths(referenceDir string) *ComponentConfi // here so inherited patterns can be interpreted relative to the concrete component // config file. OverlayFiles: slices.Clone(c.OverlayFiles), + Tests: deep.MustCopy(c.Tests), } // Fix up paths. diff --git a/internal/projectconfig/configfile.go b/internal/projectconfig/configfile.go index a96829b3..d3b0949a 100644 --- a/internal/projectconfig/configfile.go +++ b/internal/projectconfig/configfile.go @@ -66,6 +66,12 @@ type ConfigFile struct { // Definitions of test suites. TestSuites map[string]TestSuiteConfig `toml:"test-suites,omitempty" validate:"dive" jsonschema:"title=Test Suites,description=Definitions of test suites for this project"` + // Definitions of individual tests (new schema, [tests.X]). + Tests map[string]TestDefinition `toml:"tests,omitempty" validate:"dive" jsonschema:"title=Tests,description=Definitions of individual tests"` + + // Definitions of test groups (new schema, [test-groups.X]). + TestGroups map[string]TestGroup `toml:"test-groups,omitempty" validate:"dive" jsonschema:"title=Test Groups,description=Definitions of named bundles of tests"` + // Internal fields used to track the origin of the config file; `dir` is the directory // that the config file's relative paths are based from. sourcePath string `toml:"-"` @@ -152,6 +158,13 @@ func (f ConfigFile) Validate() error { } } + // Validate individual test definitions and ensure framework subtable/type match. + for testName, testDef := range f.Tests { + if err := testDef.Validate(testName); err != nil { + return fmt.Errorf("invalid test %#q:\n%w", testName, err) + } + } + return nil } diff --git a/internal/projectconfig/configfile_test.go b/internal/projectconfig/configfile_test.go index 054498a1..b0e22bdf 100644 --- a/internal/projectconfig/configfile_test.go +++ b/internal/projectconfig/configfile_test.go @@ -17,6 +17,38 @@ func TestProjectConfigFileValidation_EmptyFile(t *testing.T) { assert.NoError(t, file.Validate()) } +func TestProjectConfigFileValidation_TestDefinitionMismatchedSubtable(t *testing.T) { + file := projectconfig.ConfigFile{ + Tests: map[string]projectconfig.TestDefinition{ + "smoke": { + Type: "pytest", + Pytest: map[string]any{"working-dir": "tests"}, + Lisa: map[string]any{"suite": "vm"}, + }, + }, + } + + err := file.Validate() + require.Error(t, err) + require.ErrorIs(t, err, projectconfig.ErrMismatchedTestSubtable) + assert.Contains(t, err.Error(), "invalid test") + assert.Contains(t, err.Error(), "smoke") + assert.Contains(t, err.Error(), "lisa") +} + +func TestProjectConfigFileValidation_TestDefinitionMatchingSubtable(t *testing.T) { + file := projectconfig.ConfigFile{ + Tests: map[string]projectconfig.TestDefinition{ + "smoke": { + Type: "pytest", + Pytest: map[string]any{"working-dir": "tests"}, + }, + }, + } + + assert.NoError(t, file.Validate()) +} + func TestProjectConfigFileValidation_DefaultProjectInfo(t *testing.T) { file := projectconfig.ConfigFile{ Project: &projectconfig.ProjectInfo{}, diff --git a/internal/projectconfig/fingerprint_test.go b/internal/projectconfig/fingerprint_test.go index 95d860d7..836eee6f 100644 --- a/internal/projectconfig/fingerprint_test.go +++ b/internal/projectconfig/fingerprint_test.go @@ -67,6 +67,9 @@ func TestAllFingerprintedFieldsHaveDecision(t *testing.T) { // ComponentConfig.Publish — post-build routing (where to publish), not a build input. "ComponentConfig.Publish": true, + // ComponentConfig.Tests — test selection metadata (new schema), not a build input. + "ComponentConfig.Tests": true, + // ComponentOverlay.Description — human-readable documentation for the overlay. "ComponentOverlay.Description": true, // ComponentOverlay.Source — absolute path that varies by checkout location. diff --git a/internal/projectconfig/image.go b/internal/projectconfig/image.go index f2000851..b8033dc6 100644 --- a/internal/projectconfig/image.go +++ b/internal/projectconfig/image.go @@ -30,7 +30,7 @@ type ImageConfig struct { // Tests holds the test configuration for this image, including which test suites // apply to it. - Tests ImageTestsConfig `toml:"tests,omitempty" json:"tests,omitempty" jsonschema:"title=Tests,description=Test configuration for this image"` + Tests *ImageTestsConfig `toml:"tests,omitempty" json:"tests,omitempty" jsonschema:"title=Tests,description=Test configuration for this image"` // Publish holds the publish settings for this image. Publish ImagePublishConfig `toml:"publish,omitempty" json:"publish,omitempty" jsonschema:"title=Publish settings,description=Publishing settings for this image"` @@ -115,6 +115,11 @@ type ImageTestsConfig struct { // reference identifies a test suite defined in the top-level [test-suites] section // and may carry per-test metadata in the future (e.g., required vs optional). TestSuites []TestSuiteRef `toml:"test-suites,omitempty" json:"testSuites,omitempty" jsonschema:"title=Test Suites,description=List of test suite references that apply to this image"` + + // Tests is the new-shape list of test or test-group references that apply to this + // image. References must resolve to entries in the project-level [tests] or + // [test-groups] maps; resolution is the responsibility of the test layer. + Tests []TestRef `toml:"tests,omitempty" json:"tests,omitempty" jsonschema:"title=Tests,description=List of test or test-group references that apply to this image"` } // TestSuiteRef is a reference to a named test suite. Using a structured type (rather than @@ -126,6 +131,10 @@ type TestSuiteRef struct { // TestNames returns the test suite names referenced by this image. func (i *ImageConfig) TestNames() []string { + if i.Tests == nil { + return nil + } + names := make([]string, len(i.Tests.TestSuites)) for idx, ref := range i.Tests.TestSuites { names[idx] = ref.Name diff --git a/internal/projectconfig/loader_test.go b/internal/projectconfig/loader_test.go index d362040d..8083e89b 100644 --- a/internal/projectconfig/loader_test.go +++ b/internal/projectconfig/loader_test.go @@ -1055,6 +1055,7 @@ test-suites = [{ name = "smoke" }] require.NoError(t, err) if assert.Contains(t, config.Images, "myimage") { + require.NotNil(t, config.Images["myimage"].Tests) assert.Equal(t, []TestSuiteRef{{Name: "smoke"}}, config.Images["myimage"].Tests.TestSuites) } } diff --git a/internal/projectconfig/tests.go b/internal/projectconfig/tests.go new file mode 100644 index 00000000..601a92a0 --- /dev/null +++ b/internal/projectconfig/tests.go @@ -0,0 +1,196 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package projectconfig + +import ( + "fmt" + + "github.com/invopop/jsonschema" + orderedmap "github.com/pb33f/ordered-map/v2" +) + +func orderedMapWithConst(key string, value any) *orderedmap.OrderedMap[string, *jsonschema.Schema] { + m := orderedmap.New[string, *jsonschema.Schema]() + m.Set(key, &jsonschema.Schema{Const: value}) + + return m +} + +// TestDefinition is the new-shape [tests.X] declaration: one configuration of one +// runner/harness with framework-specific options. Framework subtables are kept as +// loosely-typed maps so the resolver can evolve their schemas without requiring +// matching struct changes here. +type TestDefinition struct { + // Type identifies the framework/runner. Required, and constrained to the + // closed enum in the schema tag at the schema layer. The loader still + // accepts unknown values permissively; the resolver is the source of truth. + Type string `toml:"type" json:"type" jsonschema:"required,title=Type,description=Test framework type,enum=pytest,enum=lisa,enum=tmt"` + + // Human-readable description. + Description string `toml:"description,omitempty" json:"description,omitempty" jsonschema:"title=Description,description=Description of this test"` + + // Kind hints at what the test exercises (e.g. functional, performance). + Kind []string `toml:"kind,omitempty" json:"kind,omitempty" jsonschema:"title=Kind,description=Test kind hints (e.g. functional or performance)"` + + // LongRunning hints to schedulers/policy that this test may take a long time. + LongRunning bool `toml:"long-running,omitempty" json:"longRunning,omitempty" jsonschema:"title=Long running,description=Hints that this test may run for hours"` + + // RequiredCapabilities lists capability tokens an image must declare to be a + // valid target for this test. Tokens are matched against [ImageCapabilities]. + RequiredCapabilities []string `toml:"required-capabilities,omitempty" json:"requiredCapabilities,omitempty" jsonschema:"title=Required capabilities,description=Capability tokens the image must declare"` + + // Framework-specific subtables. Kept untyped so framework schema can evolve + // independently of the dev-tools type definitions. + Lisa map[string]any `toml:"lisa,omitempty" json:"lisa,omitempty" jsonschema:"title=LISA config,description=LISA-specific configuration"` + Tmt map[string]any `toml:"tmt,omitempty" json:"tmt,omitempty" jsonschema:"title=TMT config,description=TMT-specific configuration"` + Pytest map[string]any `toml:"pytest,omitempty" json:"pytest,omitempty" jsonschema:"title=Pytest config,description=pytest-specific configuration"` +} + +// TestGroup is a [test-groups.X] declaration: a named bundle of test references that +// images or components can target via a single name. +type TestGroup struct { + // Human-readable description. + Description string `toml:"description,omitempty" json:"description,omitempty" jsonschema:"title=Description,description=Description of this test group"` + + // Tests is the ordered list of test or nested-group references that make up + // the group's membership. + Tests []TestRef `toml:"tests,omitempty" json:"tests,omitempty" jsonschema:"title=Tests,description=Member references (each is either {name=...} or {group=...})"` +} + +// TestRef is a reference to either a test (by name) or another group (by name). +// Exactly one of Name or Group should be set; semantic validation is the resolver's +// responsibility. +type TestRef struct { + // Name references a [tests.X] entry. + Name string `toml:"name,omitempty" json:"name,omitempty" jsonschema:"title=Name,description=Name of a test (mutually exclusive with group)"` + + // Group references a [test-groups.X] entry. + Group string `toml:"group,omitempty" json:"group,omitempty" jsonschema:"title=Group,description=Name of a test group (mutually exclusive with name)"` +} + +// ComponentTestsConfig holds the new-shape per-component tests block: +// +// tests.tests = [{ name = "..." }, { group = "..." }] +type ComponentTestsConfig struct { + // Tests is the list of test or test-group references that apply to the component. + Tests []TestRef `toml:"tests,omitempty" json:"tests,omitempty" jsonschema:"title=Tests,description=Per-component test or test-group references"` +} + +// Validate checks that exactly one framework subtable is set and it matches Type. +func (t TestDefinition) Validate(testName string) error { + if t.Type == "" { + return fmt.Errorf("%w: test %#q is missing required field 'type'", ErrMissingTestField, testName) + } + + type testTypeRule struct { + required string + disallowed []string + } + + typeRules := map[string]testTypeRule{ + "pytest": {required: "pytest", disallowed: []string{"lisa", "tmt"}}, + "lisa": {required: "lisa", disallowed: []string{"pytest", "tmt"}}, + "tmt": {required: "tmt", disallowed: []string{"pytest", "lisa"}}, + } + + rule, ok := typeRules[t.Type] + if !ok { + return fmt.Errorf("%w: %#q (test: %#q)", ErrUnknownTestType, t.Type, testName) + } + + subtableLengths := map[string]int{ + "pytest": len(t.Pytest), + "lisa": len(t.Lisa), + "tmt": len(t.Tmt), + } + + if subtableLengths[rule.required] == 0 { + return fmt.Errorf( + "%w: test %#q of type %#q requires a [%s] subtable", + ErrMissingTestField, + testName, + t.Type, + rule.required, + ) + } + + for _, subtable := range rule.disallowed { + if subtableLengths[subtable] > 0 { + return fmt.Errorf( + "%w: test %#q of type %#q cannot include subtable '%s'", + ErrMismatchedTestSubtable, + testName, + t.Type, + subtable, + ) + } + } + + return nil +} + +// JSONSchemaExtend tightens [TestDefinition] so the framework-specific subtable +// must match the declared type. +func (TestDefinition) JSONSchemaExtend(schema *jsonschema.Schema) { + if schema == nil { + return + } + + onlyPytest := &jsonschema.Schema{ + Required: []string{"pytest"}, + Not: &jsonschema.Schema{AnyOf: []*jsonschema.Schema{ + {Required: []string{"lisa"}}, + {Required: []string{"tmt"}}, + }}, + } + + onlyLisa := &jsonschema.Schema{ + Required: []string{"lisa"}, + Not: &jsonschema.Schema{AnyOf: []*jsonschema.Schema{ + {Required: []string{"pytest"}}, + {Required: []string{"tmt"}}, + }}, + } + + onlyTmt := &jsonschema.Schema{ + Required: []string{"tmt"}, + Not: &jsonschema.Schema{AnyOf: []*jsonschema.Schema{ + {Required: []string{"pytest"}}, + {Required: []string{"lisa"}}, + }}, + } + + schema.AllOf = append(schema.AllOf, + &jsonschema.Schema{If: &jsonschema.Schema{Properties: orderedMapWithConst("type", "pytest")}, Then: onlyPytest}, + &jsonschema.Schema{If: &jsonschema.Schema{Properties: orderedMapWithConst("type", "lisa")}, Then: onlyLisa}, + &jsonschema.Schema{If: &jsonschema.Schema{Properties: orderedMapWithConst("type", "tmt")}, Then: onlyTmt}, + ) +} + +// JSONSchemaExtend tightens the generated schema for [TestRef] so editors can +// flag refs that set neither or both of name/group, or empty/whitespace-only values. +// The runtime resolver ([ErrInvalidTestRef]) is the source of truth; this keeps the schema in sync. +func (TestRef) JSONSchemaExtend(schema *jsonschema.Schema) { + // Exactly one of name|group: encoded as oneOf with `required` on each and + // `not` excluding the other, which forbids both `{}` and `{name, group}`. + schema.OneOf = []*jsonschema.Schema{ + {Required: []string{"name"}, Not: &jsonschema.Schema{Required: []string{"group"}}}, + {Required: []string{"group"}, Not: &jsonschema.Schema{Required: []string{"name"}}}, + } + + // Prevent empty or leading-whitespace identifiers in both name and group fields. + minLen := uint64(1) + + if schema.Properties != nil { + if nameProp, ok := schema.Properties.Get("name"); ok && nameProp != nil { + nameProp.MinLength = &minLen + nameProp.Pattern = "^\\S" + } + + if groupProp, ok := schema.Properties.Get("group"); ok && groupProp != nil { + groupProp.MinLength = &minLen + groupProp.Pattern = "^\\S" + } + } +} diff --git a/internal/projectconfig/testsuite_test.go b/internal/projectconfig/testsuite_test.go index de95adb6..dcf2cdc1 100644 --- a/internal/projectconfig/testsuite_test.go +++ b/internal/projectconfig/testsuite_test.go @@ -64,7 +64,7 @@ func TestImageCapabilities_EnabledNames(t *testing.T) { func TestImageConfig_TestNames(t *testing.T) { t.Run("with tests", func(t *testing.T) { img := projectconfig.ImageConfig{ - Tests: projectconfig.ImageTestsConfig{ + Tests: &projectconfig.ImageTestsConfig{ TestSuites: []projectconfig.TestSuiteRef{ {Name: "smoke"}, {Name: "integration"}, @@ -357,7 +357,7 @@ func TestValidateTestSuiteReferences(t *testing.T) { Images: map[string]projectconfig.ImageConfig{ "myimage": { Name: "myimage", - Tests: projectconfig.ImageTestsConfig{TestSuites: []projectconfig.TestSuiteRef{{Name: "smoke"}}}, + Tests: &projectconfig.ImageTestsConfig{TestSuites: []projectconfig.TestSuiteRef{{Name: "smoke"}}}, }, }, TestSuites: map[string]projectconfig.TestSuiteConfig{ @@ -383,7 +383,7 @@ func TestValidateTestSuiteReferences(t *testing.T) { Images: map[string]projectconfig.ImageConfig{ "myimage": { Name: "myimage", - Tests: projectconfig.ImageTestsConfig{TestSuites: []projectconfig.TestSuiteRef{{Name: "nonexistent"}}}, + Tests: &projectconfig.ImageTestsConfig{TestSuites: []projectconfig.TestSuiteRef{{Name: "nonexistent"}}}, }, }, TestSuites: make(map[string]projectconfig.TestSuiteConfig), diff --git a/scenario/__snapshots__/TestSnapshotsContainer_config_generate-schema_stdout_1.snap b/scenario/__snapshots__/TestSnapshotsContainer_config_generate-schema_stdout_1.snap index 5f5542a2..f17a8cf0 100755 --- a/scenario/__snapshots__/TestSnapshotsContainer_config_generate-schema_stdout_1.snap +++ b/scenario/__snapshots__/TestSnapshotsContainer_config_generate-schema_stdout_1.snap @@ -162,6 +162,11 @@ "$ref": "#/$defs/ComponentPublishConfig", "title": "Publish settings", "description": "Component-level publish channel settings" + }, + "tests": { + "$ref": "#/$defs/ComponentTestsConfig", + "title": "Tests", + "description": "Per-component test or test-group references" } }, "additionalProperties": false, @@ -346,6 +351,20 @@ "additionalProperties": false, "type": "object" }, + "ComponentTestsConfig": { + "properties": { + "tests": { + "items": { + "$ref": "#/$defs/TestRef" + }, + "type": "array", + "title": "Tests", + "description": "Per-component test or test-group references" + } + }, + "additionalProperties": false, + "type": "object" + }, "ConfigFile": { "properties": { "$schema": { @@ -431,6 +450,22 @@ "type": "object", "title": "Test Suites", "description": "Definitions of test suites for this project" + }, + "tests": { + "additionalProperties": { + "$ref": "#/$defs/TestDefinition" + }, + "type": "object", + "title": "Tests", + "description": "Definitions of individual tests" + }, + "test-groups": { + "additionalProperties": { + "$ref": "#/$defs/TestGroup" + }, + "type": "object", + "title": "Test Groups", + "description": "Definitions of named bundles of tests" } }, "additionalProperties": false, @@ -729,6 +764,14 @@ "type": "array", "title": "Test Suites", "description": "List of test suite references that apply to this image" + }, + "tests": { + "items": { + "$ref": "#/$defs/TestRef" + }, + "type": "array", + "title": "Tests", + "description": "List of test or test-group references that apply to this image" } }, "additionalProperties": false, @@ -1385,6 +1428,213 @@ "subpath" ] }, + "TestDefinition": { + "allOf": [ + { + "if": { + "properties": { + "type": { + "const": "pytest" + } + } + }, + "then": { + "not": { + "anyOf": [ + { + "required": [ + "lisa" + ] + }, + { + "required": [ + "tmt" + ] + } + ] + }, + "required": [ + "pytest" + ] + } + }, + { + "if": { + "properties": { + "type": { + "const": "lisa" + } + } + }, + "then": { + "not": { + "anyOf": [ + { + "required": [ + "pytest" + ] + }, + { + "required": [ + "tmt" + ] + } + ] + }, + "required": [ + "lisa" + ] + } + }, + { + "if": { + "properties": { + "type": { + "const": "tmt" + } + } + }, + "then": { + "not": { + "anyOf": [ + { + "required": [ + "pytest" + ] + }, + { + "required": [ + "lisa" + ] + } + ] + }, + "required": [ + "tmt" + ] + } + } + ], + "properties": { + "type": { + "type": "string", + "enum": [ + "pytest", + "lisa", + "tmt" + ], + "title": "Type", + "description": "Test framework type" + }, + "description": { + "type": "string", + "title": "Description", + "description": "Description of this test" + }, + "kind": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Kind", + "description": "Test kind hints (e.g. functional or performance)" + }, + "long-running": { + "type": "boolean", + "title": "Long running", + "description": "Hints that this test may run for hours" + }, + "required-capabilities": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Required capabilities", + "description": "Capability tokens the image must declare" + }, + "lisa": { + "type": "object", + "title": "LISA config", + "description": "LISA-specific configuration" + }, + "tmt": { + "type": "object", + "title": "TMT config", + "description": "TMT-specific configuration" + }, + "pytest": { + "type": "object", + "title": "Pytest config", + "description": "pytest-specific configuration" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "type" + ] + }, + "TestGroup": { + "properties": { + "description": { + "type": "string", + "title": "Description", + "description": "Description of this test group" + }, + "tests": { + "items": { + "$ref": "#/$defs/TestRef" + }, + "type": "array", + "title": "Tests", + "description": "Member references (each is either {name=...} or {group=...})" + } + }, + "additionalProperties": false, + "type": "object" + }, + "TestRef": { + "oneOf": [ + { + "not": { + "required": [ + "group" + ] + }, + "required": [ + "name" + ] + }, + { + "not": { + "required": [ + "name" + ] + }, + "required": [ + "group" + ] + } + ], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "pattern": "^\\S", + "title": "Name", + "description": "Name of a test (mutually exclusive with group)" + }, + "group": { + "type": "string", + "minLength": 1, + "pattern": "^\\S", + "title": "Group", + "description": "Name of a test group (mutually exclusive with name)" + } + }, + "additionalProperties": false, + "type": "object" + }, "TestSuiteConfig": { "properties": { "description": { diff --git a/scenario/__snapshots__/TestSnapshots_config_generate-schema_stdout_1.snap b/scenario/__snapshots__/TestSnapshots_config_generate-schema_stdout_1.snap index 5f5542a2..f17a8cf0 100755 --- a/scenario/__snapshots__/TestSnapshots_config_generate-schema_stdout_1.snap +++ b/scenario/__snapshots__/TestSnapshots_config_generate-schema_stdout_1.snap @@ -162,6 +162,11 @@ "$ref": "#/$defs/ComponentPublishConfig", "title": "Publish settings", "description": "Component-level publish channel settings" + }, + "tests": { + "$ref": "#/$defs/ComponentTestsConfig", + "title": "Tests", + "description": "Per-component test or test-group references" } }, "additionalProperties": false, @@ -346,6 +351,20 @@ "additionalProperties": false, "type": "object" }, + "ComponentTestsConfig": { + "properties": { + "tests": { + "items": { + "$ref": "#/$defs/TestRef" + }, + "type": "array", + "title": "Tests", + "description": "Per-component test or test-group references" + } + }, + "additionalProperties": false, + "type": "object" + }, "ConfigFile": { "properties": { "$schema": { @@ -431,6 +450,22 @@ "type": "object", "title": "Test Suites", "description": "Definitions of test suites for this project" + }, + "tests": { + "additionalProperties": { + "$ref": "#/$defs/TestDefinition" + }, + "type": "object", + "title": "Tests", + "description": "Definitions of individual tests" + }, + "test-groups": { + "additionalProperties": { + "$ref": "#/$defs/TestGroup" + }, + "type": "object", + "title": "Test Groups", + "description": "Definitions of named bundles of tests" } }, "additionalProperties": false, @@ -729,6 +764,14 @@ "type": "array", "title": "Test Suites", "description": "List of test suite references that apply to this image" + }, + "tests": { + "items": { + "$ref": "#/$defs/TestRef" + }, + "type": "array", + "title": "Tests", + "description": "List of test or test-group references that apply to this image" } }, "additionalProperties": false, @@ -1385,6 +1428,213 @@ "subpath" ] }, + "TestDefinition": { + "allOf": [ + { + "if": { + "properties": { + "type": { + "const": "pytest" + } + } + }, + "then": { + "not": { + "anyOf": [ + { + "required": [ + "lisa" + ] + }, + { + "required": [ + "tmt" + ] + } + ] + }, + "required": [ + "pytest" + ] + } + }, + { + "if": { + "properties": { + "type": { + "const": "lisa" + } + } + }, + "then": { + "not": { + "anyOf": [ + { + "required": [ + "pytest" + ] + }, + { + "required": [ + "tmt" + ] + } + ] + }, + "required": [ + "lisa" + ] + } + }, + { + "if": { + "properties": { + "type": { + "const": "tmt" + } + } + }, + "then": { + "not": { + "anyOf": [ + { + "required": [ + "pytest" + ] + }, + { + "required": [ + "lisa" + ] + } + ] + }, + "required": [ + "tmt" + ] + } + } + ], + "properties": { + "type": { + "type": "string", + "enum": [ + "pytest", + "lisa", + "tmt" + ], + "title": "Type", + "description": "Test framework type" + }, + "description": { + "type": "string", + "title": "Description", + "description": "Description of this test" + }, + "kind": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Kind", + "description": "Test kind hints (e.g. functional or performance)" + }, + "long-running": { + "type": "boolean", + "title": "Long running", + "description": "Hints that this test may run for hours" + }, + "required-capabilities": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Required capabilities", + "description": "Capability tokens the image must declare" + }, + "lisa": { + "type": "object", + "title": "LISA config", + "description": "LISA-specific configuration" + }, + "tmt": { + "type": "object", + "title": "TMT config", + "description": "TMT-specific configuration" + }, + "pytest": { + "type": "object", + "title": "Pytest config", + "description": "pytest-specific configuration" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "type" + ] + }, + "TestGroup": { + "properties": { + "description": { + "type": "string", + "title": "Description", + "description": "Description of this test group" + }, + "tests": { + "items": { + "$ref": "#/$defs/TestRef" + }, + "type": "array", + "title": "Tests", + "description": "Member references (each is either {name=...} or {group=...})" + } + }, + "additionalProperties": false, + "type": "object" + }, + "TestRef": { + "oneOf": [ + { + "not": { + "required": [ + "group" + ] + }, + "required": [ + "name" + ] + }, + { + "not": { + "required": [ + "name" + ] + }, + "required": [ + "group" + ] + } + ], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "pattern": "^\\S", + "title": "Name", + "description": "Name of a test (mutually exclusive with group)" + }, + "group": { + "type": "string", + "minLength": 1, + "pattern": "^\\S", + "title": "Group", + "description": "Name of a test group (mutually exclusive with name)" + } + }, + "additionalProperties": false, + "type": "object" + }, "TestSuiteConfig": { "properties": { "description": { diff --git a/schemas/azldev.schema.json b/schemas/azldev.schema.json index 5f5542a2..f17a8cf0 100644 --- a/schemas/azldev.schema.json +++ b/schemas/azldev.schema.json @@ -162,6 +162,11 @@ "$ref": "#/$defs/ComponentPublishConfig", "title": "Publish settings", "description": "Component-level publish channel settings" + }, + "tests": { + "$ref": "#/$defs/ComponentTestsConfig", + "title": "Tests", + "description": "Per-component test or test-group references" } }, "additionalProperties": false, @@ -346,6 +351,20 @@ "additionalProperties": false, "type": "object" }, + "ComponentTestsConfig": { + "properties": { + "tests": { + "items": { + "$ref": "#/$defs/TestRef" + }, + "type": "array", + "title": "Tests", + "description": "Per-component test or test-group references" + } + }, + "additionalProperties": false, + "type": "object" + }, "ConfigFile": { "properties": { "$schema": { @@ -431,6 +450,22 @@ "type": "object", "title": "Test Suites", "description": "Definitions of test suites for this project" + }, + "tests": { + "additionalProperties": { + "$ref": "#/$defs/TestDefinition" + }, + "type": "object", + "title": "Tests", + "description": "Definitions of individual tests" + }, + "test-groups": { + "additionalProperties": { + "$ref": "#/$defs/TestGroup" + }, + "type": "object", + "title": "Test Groups", + "description": "Definitions of named bundles of tests" } }, "additionalProperties": false, @@ -729,6 +764,14 @@ "type": "array", "title": "Test Suites", "description": "List of test suite references that apply to this image" + }, + "tests": { + "items": { + "$ref": "#/$defs/TestRef" + }, + "type": "array", + "title": "Tests", + "description": "List of test or test-group references that apply to this image" } }, "additionalProperties": false, @@ -1385,6 +1428,213 @@ "subpath" ] }, + "TestDefinition": { + "allOf": [ + { + "if": { + "properties": { + "type": { + "const": "pytest" + } + } + }, + "then": { + "not": { + "anyOf": [ + { + "required": [ + "lisa" + ] + }, + { + "required": [ + "tmt" + ] + } + ] + }, + "required": [ + "pytest" + ] + } + }, + { + "if": { + "properties": { + "type": { + "const": "lisa" + } + } + }, + "then": { + "not": { + "anyOf": [ + { + "required": [ + "pytest" + ] + }, + { + "required": [ + "tmt" + ] + } + ] + }, + "required": [ + "lisa" + ] + } + }, + { + "if": { + "properties": { + "type": { + "const": "tmt" + } + } + }, + "then": { + "not": { + "anyOf": [ + { + "required": [ + "pytest" + ] + }, + { + "required": [ + "lisa" + ] + } + ] + }, + "required": [ + "tmt" + ] + } + } + ], + "properties": { + "type": { + "type": "string", + "enum": [ + "pytest", + "lisa", + "tmt" + ], + "title": "Type", + "description": "Test framework type" + }, + "description": { + "type": "string", + "title": "Description", + "description": "Description of this test" + }, + "kind": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Kind", + "description": "Test kind hints (e.g. functional or performance)" + }, + "long-running": { + "type": "boolean", + "title": "Long running", + "description": "Hints that this test may run for hours" + }, + "required-capabilities": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Required capabilities", + "description": "Capability tokens the image must declare" + }, + "lisa": { + "type": "object", + "title": "LISA config", + "description": "LISA-specific configuration" + }, + "tmt": { + "type": "object", + "title": "TMT config", + "description": "TMT-specific configuration" + }, + "pytest": { + "type": "object", + "title": "Pytest config", + "description": "pytest-specific configuration" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "type" + ] + }, + "TestGroup": { + "properties": { + "description": { + "type": "string", + "title": "Description", + "description": "Description of this test group" + }, + "tests": { + "items": { + "$ref": "#/$defs/TestRef" + }, + "type": "array", + "title": "Tests", + "description": "Member references (each is either {name=...} or {group=...})" + } + }, + "additionalProperties": false, + "type": "object" + }, + "TestRef": { + "oneOf": [ + { + "not": { + "required": [ + "group" + ] + }, + "required": [ + "name" + ] + }, + { + "not": { + "required": [ + "name" + ] + }, + "required": [ + "group" + ] + } + ], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "pattern": "^\\S", + "title": "Name", + "description": "Name of a test (mutually exclusive with group)" + }, + "group": { + "type": "string", + "minLength": 1, + "pattern": "^\\S", + "title": "Group", + "description": "Name of a test group (mutually exclusive with name)" + } + }, + "additionalProperties": false, + "type": "object" + }, "TestSuiteConfig": { "properties": { "description": { From 1866401283de6d161dcc2d4801d03f55b9b5472e Mon Sep 17 00:00:00 2001 From: bhagyapathak Date: Thu, 2 Jul 2026 20:26:03 +0530 Subject: [PATCH 2/5] Strengthen test metadata validation and reference integrity checks --- docs/user/reference/config/tests.md | 17 +- internal/projectconfig/configfile.go | 196 +++++++++- internal/projectconfig/configfile_test.go | 243 +++++++++++++ internal/projectconfig/tests.go | 340 +++++++++++++++++- internal/projectconfig/testsuite.go | 14 + ...ainer_config_generate-schema_stdout_1.snap | 31 +- ...shots_config_generate-schema_stdout_1.snap | 31 +- schemas/azldev.schema.json | 31 +- 8 files changed, 857 insertions(+), 46 deletions(-) diff --git a/docs/user/reference/config/tests.md b/docs/user/reference/config/tests.md index 990e34aa..a6442412 100644 --- a/docs/user/reference/config/tests.md +++ b/docs/user/reference/config/tests.md @@ -4,7 +4,7 @@ The `[tests]` and `[test-groups]` sections declare framework-agnostic test metadata that components and images can target by name. Each test entry binds a single test (a pytest run, a LISA case, or a TMT plan) to a named identifier; each group entry bundles tests (and -nested groups) under one name so callers can reference a curated set +named references) under one name so callers can reference a curated set without enumerating every member. ## Test Definition @@ -19,7 +19,7 @@ by azldev so frameworks can evolve independently. |-------|----------|------|----------|-------------| | Type | `type` | string | Yes | Test framework: `pytest`, `lisa`, or `tmt` | | Description | `description` | string | No | Human-readable description | -| Kind | `kind` | string array | No | Free-form hints (e.g. `functional`, `performance`, `bvt`) | +| Kind | `kind` | string | No | Test kind hint: `functional` or `performance` | | Long running | `long-running` | boolean | No | Hints that this test may run for hours | | Required capabilities | `required-capabilities` | string array | No | Capability tokens the image must declare for this test to be applicable | | Lisa | `lisa` | table | No | LISA-specific configuration (opaque to azldev) | @@ -28,13 +28,13 @@ by azldev so frameworks can evolve independently. ## Test Group -Each entry under `[test-groups.]` names an ordered list of test or -nested-group references that callers can target as a single unit. +Each entry under `[test-groups.]` names an ordered list of test +references that callers can target as a single unit. | Field | TOML Key | Type | Required | Description | |-------|----------|------|----------|-------------| | Description | `description` | string | No | Human-readable description | -| Tests | `tests` | array of [TestRef](#test-reference) | No | Ordered members of the group | +| Tests | `tests` | array of [TestRef](#test-reference) | No | Ordered members of the group (name refs only) | ## Test Reference @@ -64,7 +64,7 @@ tests = [{ group = "bvt" }] [tests.bvt-ssh] type = "pytest" description = "Basic SSH boot verification" -kind = ["functional", "bvt"] +kind = "functional" required-capabilities = ["ssh"] pytest = { working-dir = "tests/bvt", test-paths = ["test_ssh.py"] } @@ -77,11 +77,8 @@ lisa = { case = "kdump.smoke" } description = "Build verification tests" tests = [ { name = "bvt-ssh" }, - { group = "bvt-extras" }, + { name = "kdump-smoke" }, ] - -[test-groups.bvt-extras] -tests = [{ name = "kdump-smoke" }] ``` ## Related Resources diff --git a/internal/projectconfig/configfile.go b/internal/projectconfig/configfile.go index d3b0949a..45b7a7a0 100644 --- a/internal/projectconfig/configfile.go +++ b/internal/projectconfig/configfile.go @@ -142,8 +142,23 @@ func (f ConfigFile) Validate() error { } } - // Validate test suite configurations. - for suiteName, suite := range f.TestSuites { + if err := validateTestSuites(f.TestSuites); err != nil { + return err + } + + if err := validateTestDefinitions(f.Tests); err != nil { + return err + } + + if err := validateNewTestReferences(f); err != nil { + return err + } + + return nil +} + +func validateTestSuites(testSuites map[string]TestSuiteConfig) error { + for suiteName, suite := range testSuites { // Suite names are used as path components (e.g., for the per-suite venv directory), // so reject anything that could escape the intended directory or otherwise be unsafe // across platforms. @@ -158,8 +173,11 @@ func (f ConfigFile) Validate() error { } } - // Validate individual test definitions and ensure framework subtable/type match. - for testName, testDef := range f.Tests { + return nil +} + +func validateTestDefinitions(tests map[string]TestDefinition) error { + for testName, testDef := range tests { if err := testDef.Validate(testName); err != nil { return fmt.Errorf("invalid test %#q:\n%w", testName, err) } @@ -184,6 +202,176 @@ func validateComponentGroupMetadata(groups map[string]ComponentGroupConfig) erro return nil } +func validateNewTestReferences(cfgFile ConfigFile) error { + for groupName, group := range cfgFile.TestGroups { + scope := fmt.Sprintf("test-group %#q tests", groupName) + if err := validateTestGroupMembers(scope, group.Tests, cfgFile.Tests); err != nil { + return err + } + } + + for componentName, component := range cfgFile.Components { + if component.Tests == nil { + continue + } + + scope := fmt.Sprintf("component %#q tests.tests", componentName) + if err := validateTestRefList(scope, component.Tests.Tests, cfgFile.Tests, cfgFile.TestGroups); err != nil { + return err + } + } + + for imageName, image := range cfgFile.Images { + if image.Tests == nil { + continue + } + + scope := fmt.Sprintf("image %#q tests.tests", imageName) + if err := validateTestRefList(scope, image.Tests.Tests, cfgFile.Tests, cfgFile.TestGroups); err != nil { + return err + } + } + + return nil +} + +func validateTestGroupMembers( + scope string, + refs []TestRef, + tests map[string]TestDefinition, +) error { + seenRefs := make(map[string]int, len(refs)) + + for idx, ref := range refs { + hasName := ref.Name != "" + hasGroup := ref.Group != "" + + if hasName == hasGroup { + return fmt.Errorf( + "%w: %s[%d] must set exactly one of 'name' or 'group'", + ErrInvalidTestRef, + scope, + idx, + ) + } + + if hasGroup { + return fmt.Errorf( + "%w: %s[%d].group is not allowed in [test-groups]; use .name to reference a [tests] entry", + ErrNestedTestGroupReference, + scope, + idx, + ) + } + + if _, ok := tests[ref.Name]; !ok { + return fmt.Errorf( + "%w: %s[%d].name references undefined test %#q", + ErrUndefinedTest, + scope, + idx, + ref.Name, + ) + } + + refKey := "name:" + ref.Name + if firstIdx, exists := seenRefs[refKey]; exists { + return fmt.Errorf( + "%w: %s[%d] duplicates %s[%d] (%#q)", + ErrDuplicateTestRef, + scope, + idx, + scope, + firstIdx, + ref.Name, + ) + } + + seenRefs[refKey] = idx + } + + return nil +} + +func validateTestRefList( + scope string, + refs []TestRef, + tests map[string]TestDefinition, + groups map[string]TestGroup, +) error { + seenRefs := make(map[string]int, len(refs)) + + for idx, ref := range refs { + hasName := ref.Name != "" + hasGroup := ref.Group != "" + + if hasName == hasGroup { + return fmt.Errorf( + "%w: %s[%d] must set exactly one of 'name' or 'group'", + ErrInvalidTestRef, + scope, + idx, + ) + } + + if hasName { + if _, ok := tests[ref.Name]; !ok { + return fmt.Errorf( + "%w: %s[%d].name references undefined test %#q", + ErrUndefinedTest, + scope, + idx, + ref.Name, + ) + } + + refKey := "name:" + ref.Name + if firstIdx, exists := seenRefs[refKey]; exists { + return fmt.Errorf( + "%w: %s[%d] duplicates %s[%d] (%#q)", + ErrDuplicateTestRef, + scope, + idx, + scope, + firstIdx, + ref.Name, + ) + } + + seenRefs[refKey] = idx + + continue + } + + if _, ok := groups[ref.Group]; !ok { + return fmt.Errorf( + "%w: %s[%d].group references undefined test-group %#q", + ErrUndefinedTestGroup, + scope, + idx, + ref.Group, + ) + } + + refKey := "group:" + ref.Group + if firstIdx, exists := seenRefs[refKey]; exists { + return fmt.Errorf( + "%w: %s[%d] duplicates %s[%d] (%#q)", + ErrDuplicateTestRef, + scope, + idx, + scope, + firstIdx, + ref.Group, + ) + } + + seenRefs[refKey] = idx + } + + return nil +} + // validateSourceFiles checks 'source-files' configuration for a component: // - All filenames must be unique. // - Hash type must be a supported algorithm when specified. diff --git a/internal/projectconfig/configfile_test.go b/internal/projectconfig/configfile_test.go index b0e22bdf..d1229a72 100644 --- a/internal/projectconfig/configfile_test.go +++ b/internal/projectconfig/configfile_test.go @@ -49,6 +49,249 @@ func TestProjectConfigFileValidation_TestDefinitionMatchingSubtable(t *testing.T assert.NoError(t, file.Validate()) } +func TestProjectConfigFileValidation_TestDefinitionRequiredSubtablePresentButEmpty(t *testing.T) { + file := projectconfig.ConfigFile{ + Tests: map[string]projectconfig.TestDefinition{ + "smoke": { + Type: "pytest", + Pytest: map[string]any{}, + }, + }, + } + + assert.NoError(t, file.Validate()) +} + +func TestProjectConfigFileValidation_TestDefinitionDisallowedSubtablePresentButEmpty(t *testing.T) { + file := projectconfig.ConfigFile{ + Tests: map[string]projectconfig.TestDefinition{ + "smoke": { + Type: "pytest", + Pytest: map[string]any{"working-dir": "tests"}, + Lisa: map[string]any{}, + }, + }, + } + + err := file.Validate() + require.Error(t, err) + require.ErrorIs(t, err, projectconfig.ErrMismatchedTestSubtable) + assert.Contains(t, err.Error(), "smoke") + assert.Contains(t, err.Error(), "lisa") +} + +func TestProjectConfigFileValidation_TestDefinitionInvalidKind(t *testing.T) { + file := projectconfig.ConfigFile{ + Tests: map[string]projectconfig.TestDefinition{ + "smoke": { + Type: "pytest", + Kind: projectconfig.TestKind("unknown-kind"), + Pytest: map[string]any{"working-dir": "tests"}, + }, + }, + } + + err := file.Validate() + require.Error(t, err) + require.ErrorIs(t, err, projectconfig.ErrUnknownTestKind) + assert.Contains(t, err.Error(), "unknown-kind") +} + +func TestProjectConfigFileValidation_LisaSelectionMissing(t *testing.T) { + file := projectconfig.ConfigFile{ + Tests: map[string]projectconfig.TestDefinition{ + "smoke": { + Type: "lisa", + Lisa: map[string]any{ + "source": map[string]any{"git-url": "https://example.com/lisa.git", "ref": "main"}, + }, + }, + }, + } + + err := file.Validate() + require.Error(t, err) + require.ErrorIs(t, err, projectconfig.ErrInvalidLisaSelection) + assert.Contains(t, err.Error(), "must set at least one LISA selector") +} + +func TestProjectConfigFileValidation_LisaSelectionCriteriaValid(t *testing.T) { + file := projectconfig.ConfigFile{ + Tests: map[string]projectconfig.TestDefinition{ + "smoke": { + Type: "lisa", + Lisa: map[string]any{ + "criteria": map[string]any{"priority": []any{1, 2}, "tags": []any{"vm", "smoke"}}, + }, + }, + "perf": { + Type: "lisa", + Lisa: map[string]any{ + "criteria": []any{ + map[string]any{"area": "network", "category": "performance"}, + map[string]any{"testcaseNames": []any{"case_a", "case_b"}}, + }, + }, + }, + }, + } + + assert.NoError(t, file.Validate()) +} + +func TestProjectConfigFileValidation_LisaSelectionUnsupportedCriteriaKey(t *testing.T) { + file := projectconfig.ConfigFile{ + Tests: map[string]projectconfig.TestDefinition{ + "smoke": { + Type: "lisa", + Lisa: map[string]any{ + "criteria": map[string]any{"suite": "smoke"}, + }, + }, + }, + } + + err := file.Validate() + require.Error(t, err) + require.ErrorIs(t, err, projectconfig.ErrInvalidLisaSelection) + assert.Contains(t, err.Error(), "unsupported selector") +} + +func TestProjectConfigFileValidation_UndefinedTestReferenceInGroup(t *testing.T) { + file := projectconfig.ConfigFile{ + TestGroups: map[string]projectconfig.TestGroup{ + "bvt": { + Tests: []projectconfig.TestRef{{Name: "does-not-exist"}}, + }, + }, + } + + err := file.Validate() + require.Error(t, err) + require.ErrorIs(t, err, projectconfig.ErrUndefinedTest) + assert.Contains(t, err.Error(), "does-not-exist") +} + +func TestProjectConfigFileValidation_UndefinedTestGroupReferenceInComponent(t *testing.T) { + file := projectconfig.ConfigFile{ + Components: map[string]projectconfig.ComponentConfig{ + "openssl": { + Tests: &projectconfig.ComponentTestsConfig{ + Tests: []projectconfig.TestRef{{Group: "missing-group"}}, + }, + }, + }, + } + + err := file.Validate() + require.Error(t, err) + require.ErrorIs(t, err, projectconfig.ErrUndefinedTestGroup) + assert.Contains(t, err.Error(), "missing-group") +} + +func TestProjectConfigFileValidation_InvalidTestReferenceShapeInImage(t *testing.T) { + file := projectconfig.ConfigFile{ + Images: map[string]projectconfig.ImageConfig{ + "base": { + Tests: &projectconfig.ImageTestsConfig{ + Tests: []projectconfig.TestRef{{Name: "smoke", Group: "bvt"}}, + }, + }, + }, + Tests: map[string]projectconfig.TestDefinition{ + "smoke": { + Type: "pytest", + Pytest: map[string]any{"working-dir": "tests"}, + }, + }, + TestGroups: map[string]projectconfig.TestGroup{ + "bvt": {Tests: []projectconfig.TestRef{{Name: "smoke"}}}, + }, + } + + err := file.Validate() + require.Error(t, err) + require.ErrorIs(t, err, projectconfig.ErrInvalidTestRef) + assert.Contains(t, err.Error(), "exactly one") +} + +func TestProjectConfigFileValidation_DuplicateTestReferenceInGroup(t *testing.T) { + file := projectconfig.ConfigFile{ + Tests: map[string]projectconfig.TestDefinition{ + "smoke": { + Type: "pytest", + Pytest: map[string]any{"working-dir": "tests"}, + }, + }, + TestGroups: map[string]projectconfig.TestGroup{ + "bvt": { + Tests: []projectconfig.TestRef{ + {Name: "smoke"}, + {Name: "smoke"}, + }, + }, + }, + } + + err := file.Validate() + require.Error(t, err) + require.ErrorIs(t, err, projectconfig.ErrDuplicateTestRef) + assert.Contains(t, err.Error(), "duplicates") + assert.Contains(t, err.Error(), "smoke") +} + +func TestProjectConfigFileValidation_DuplicateTestGroupReferenceInImage(t *testing.T) { + file := projectconfig.ConfigFile{ + Tests: map[string]projectconfig.TestDefinition{ + "smoke": { + Type: "pytest", + Pytest: map[string]any{"working-dir": "tests"}, + }, + }, + TestGroups: map[string]projectconfig.TestGroup{ + "bvt": { + Tests: []projectconfig.TestRef{{Name: "smoke"}}, + }, + }, + Images: map[string]projectconfig.ImageConfig{ + "base": { + Tests: &projectconfig.ImageTestsConfig{ + Tests: []projectconfig.TestRef{ + {Group: "bvt"}, + {Group: "bvt"}, + }, + }, + }, + }, + } + + err := file.Validate() + require.Error(t, err) + require.ErrorIs(t, err, projectconfig.ErrDuplicateTestRef) + assert.Contains(t, err.Error(), "duplicates") + assert.Contains(t, err.Error(), "bvt") +} + +func TestProjectConfigFileValidation_NestedTestGroupReferenceNotAllowed(t *testing.T) { + file := projectconfig.ConfigFile{ + Tests: map[string]projectconfig.TestDefinition{ + "smoke": { + Type: "pytest", + Pytest: map[string]any{"working-dir": "tests"}, + }, + }, + TestGroups: map[string]projectconfig.TestGroup{ + "a": {Tests: []projectconfig.TestRef{{Group: "b"}}}, + "b": {Tests: []projectconfig.TestRef{{Name: "smoke"}}}, + }, + } + + err := file.Validate() + require.Error(t, err) + require.ErrorIs(t, err, projectconfig.ErrNestedTestGroupReference) + assert.Contains(t, err.Error(), "is not allowed in [test-groups]") +} + func TestProjectConfigFileValidation_DefaultProjectInfo(t *testing.T) { file := projectconfig.ConfigFile{ Project: &projectconfig.ProjectInfo{}, diff --git a/internal/projectconfig/tests.go b/internal/projectconfig/tests.go index 601a92a0..7e06b0d5 100644 --- a/internal/projectconfig/tests.go +++ b/internal/projectconfig/tests.go @@ -4,12 +4,35 @@ package projectconfig import ( + "errors" "fmt" + "strconv" + "strings" "github.com/invopop/jsonschema" orderedmap "github.com/pb33f/ordered-map/v2" ) +// TestKind indicates what kind of behavior a test exercises. +type TestKind string + +const ( + TestKindFunctional TestKind = "functional" + TestKindPerformance TestKind = "performance" +) + +func (k TestKind) IsValid() bool { + switch k { + case "": + return true + case TestKindFunctional, + TestKindPerformance: + return true + default: + return false + } +} + func orderedMapWithConst(key string, value any) *orderedmap.OrderedMap[string, *jsonschema.Schema] { m := orderedmap.New[string, *jsonschema.Schema]() m.Set(key, &jsonschema.Schema{Const: value}) @@ -30,8 +53,8 @@ type TestDefinition struct { // Human-readable description. Description string `toml:"description,omitempty" json:"description,omitempty" jsonschema:"title=Description,description=Description of this test"` - // Kind hints at what the test exercises (e.g. functional, performance). - Kind []string `toml:"kind,omitempty" json:"kind,omitempty" jsonschema:"title=Kind,description=Test kind hints (e.g. functional or performance)"` + // Kind hints at what the test exercises. + Kind TestKind `toml:"kind,omitempty" json:"kind,omitempty" jsonschema:"title=Kind,description=Kind hint for the test,enum=functional,enum=performance"` // LongRunning hints to schedulers/policy that this test may take a long time. LongRunning bool `toml:"long-running,omitempty" json:"longRunning,omitempty" jsonschema:"title=Long running,description=Hints that this test may run for hours"` @@ -53,9 +76,9 @@ type TestGroup struct { // Human-readable description. Description string `toml:"description,omitempty" json:"description,omitempty" jsonschema:"title=Description,description=Description of this test group"` - // Tests is the ordered list of test or nested-group references that make up - // the group's membership. - Tests []TestRef `toml:"tests,omitempty" json:"tests,omitempty" jsonschema:"title=Tests,description=Member references (each is either {name=...} or {group=...})"` + // Tests is the ordered list of test references that make up the group's + // membership. Group refs are validated as invalid at load time. + Tests []TestRef `toml:"tests,omitempty" json:"tests,omitempty" jsonschema:"title=Tests,description=Ordered test references for this group (name only)"` } // TestRef is a reference to either a test (by name) or another group (by name). @@ -83,6 +106,10 @@ func (t TestDefinition) Validate(testName string) error { return fmt.Errorf("%w: test %#q is missing required field 'type'", ErrMissingTestField, testName) } + if !t.Kind.IsValid() { + return fmt.Errorf("%w: test %#q has invalid kind %#q", ErrUnknownTestKind, testName, t.Kind) + } + type testTypeRule struct { required string disallowed []string @@ -99,13 +126,13 @@ func (t TestDefinition) Validate(testName string) error { return fmt.Errorf("%w: %#q (test: %#q)", ErrUnknownTestType, t.Type, testName) } - subtableLengths := map[string]int{ - "pytest": len(t.Pytest), - "lisa": len(t.Lisa), - "tmt": len(t.Tmt), + subtablePresence := map[string]bool{ + "pytest": t.Pytest != nil, + "lisa": t.Lisa != nil, + "tmt": t.Tmt != nil, } - if subtableLengths[rule.required] == 0 { + if !subtablePresence[rule.required] { return fmt.Errorf( "%w: test %#q of type %#q requires a [%s] subtable", ErrMissingTestField, @@ -116,7 +143,7 @@ func (t TestDefinition) Validate(testName string) error { } for _, subtable := range rule.disallowed { - if subtableLengths[subtable] > 0 { + if subtablePresence[subtable] { return fmt.Errorf( "%w: test %#q of type %#q cannot include subtable '%s'", ErrMismatchedTestSubtable, @@ -127,9 +154,300 @@ func (t TestDefinition) Validate(testName string) error { } } + if t.Type == "lisa" { + if err := validateLisaSelection(t.Lisa, testName); err != nil { + return err + } + } + return nil } +// JSONSchemaExtend narrows [TestGroup.Tests] to name-only refs so editor-time +// validation matches runtime behavior (group refs in [test-groups] are rejected). +func (TestGroup) JSONSchemaExtend(schema *jsonschema.Schema) { + if schema == nil || schema.Properties == nil { + return + } + + testsProp, ok := schema.Properties.Get("tests") + if !ok || testsProp == nil { + return + } + + minLen := uint64(1) + itemProps := orderedmap.New[string, *jsonschema.Schema]() + itemProps.Set("name", &jsonschema.Schema{ + Type: "string", + MinLength: &minLen, + Pattern: "^\\S", + Description: "Name of a test", + }) + + testsProp.Items = &jsonschema.Schema{ + Type: "object", + Properties: itemProps, + Required: []string{"name"}, + Not: &jsonschema.Schema{ + Required: []string{"group"}, + }, + } +} + +func validateLisaSelection(lisa map[string]any, testName string) error { + hasSelector := false + + if rawCriteria, ok := lisa["criteria"]; ok { + hasSelector = true + + if err := validateLisaCriteria(rawCriteria, testName); err != nil { + return err + } + } + + if rawName, ok := lisa["testcaseName"]; ok { + hasSelector = true + + if !isNonEmptyString(rawName) { + return fmt.Errorf( + "%w: test %#q lisa.testcaseName must be a non-empty string", + ErrInvalidLisaSelection, + testName, + ) + } + } + + if rawName, ok := lisa["name"]; ok { + hasSelector = true + + if !isNonEmptyString(rawName) { + return fmt.Errorf( + "%w: test %#q lisa.name must be a non-empty string", + ErrInvalidLisaSelection, + testName, + ) + } + } + + if rawNames, ok := lisa["testcaseNames"]; ok { + hasSelector = true + + if err := validateStringList(rawNames, "lisa.testcaseNames", testName); err != nil { + return err + } + } + + if !hasSelector { + return fmt.Errorf( + "%w: test %#q of type %#q must set at least one LISA selector: criteria, testcaseName, testcaseNames, or name", + ErrInvalidLisaSelection, + testName, + "lisa", + ) + } + + return nil +} + +func validateLisaCriteria(rawCriteria any, testName string) error { + criteriaList, err := normalizeCriteriaList(rawCriteria) + if err != nil { + return fmt.Errorf("%w: test %#q lisa.criteria %w", ErrInvalidLisaSelection, testName, err) + } + + for idx, criteria := range criteriaList { + if err := validateSingleLisaCriteria(criteria, testName, idx); err != nil { + return err + } + } + + return nil +} + +func normalizeCriteriaList(rawCriteria any) ([]map[string]any, error) { + switch criteriaValue := rawCriteria.(type) { + case map[string]any: + if len(criteriaValue) == 0 { + return nil, errors.New("must not be empty") + } + + return []map[string]any{criteriaValue}, nil + case []any: + if len(criteriaValue) == 0 { + return nil, errors.New("must not be an empty list") + } + + result := make([]map[string]any, 0, len(criteriaValue)) + + for entryIndex, item := range criteriaValue { + criteriaMap, ok := item.(map[string]any) + if !ok { + return nil, fmt.Errorf("entry %d must be a table/object", entryIndex) + } + + if len(criteriaMap) == 0 { + return nil, fmt.Errorf("entry %d must not be empty", entryIndex) + } + + result = append(result, criteriaMap) + } + + return result, nil + default: + return nil, errors.New("must be a table or list of tables") + } +} + +func validateSingleLisaCriteria(criteria map[string]any, testName string, idx int) error { + allowedKeys := map[string]bool{ + "name": true, + "area": true, + "category": true, + "priority": true, + "tags": true, + "testcaseName": true, + "testcaseNames": true, + } + + hasSelector := false + + for key, value := range criteria { + if !allowedKeys[key] { + return fmt.Errorf( + "%w: test %#q lisa.criteria[%d] contains unsupported selector %#q", + ErrInvalidLisaSelection, + testName, + idx, + key, + ) + } + + switch key { + case "name", "area", "category", "testcaseName": + if !isNonEmptyString(value) { + return fmt.Errorf( + "%w: test %#q lisa.criteria[%d].%s must be a non-empty string", + ErrInvalidLisaSelection, + testName, + idx, + key, + ) + } + + hasSelector = true + case "priority": + if err := validateLisaPriority(value, testName, idx); err != nil { + return err + } + + hasSelector = true + case "tags", "testcaseNames": + fieldName := "lisa.criteria[" + strconv.Itoa(idx) + "]." + key + + if err := validateStringList(value, fieldName, testName); err != nil { + return err + } + + hasSelector = true + } + } + + if !hasSelector { + return fmt.Errorf( + "%w: test %#q lisa.criteria[%d] must include at least one selector", + ErrInvalidLisaSelection, + testName, + idx, + ) + } + + return nil +} + +func validateLisaPriority(value any, testName string, idx int) error { + if isLisaPriorityValue(value) { + return nil + } + + return fmt.Errorf( + "%w: test %#q lisa.criteria[%d].priority must be an integer 0..4 or a non-empty list of integers 0..4", + ErrInvalidLisaSelection, + testName, + idx, + ) +} + +func isLisaPriorityValue(value any) bool { + if parsed, ok := parseLisaPriority(value); ok { + return parsed >= 0 && parsed <= 4 + } + + priorityList, ok := value.([]any) + if !ok || len(priorityList) == 0 { + return false + } + + for _, item := range priorityList { + parsed, ok := parseLisaPriority(item) + if !ok || parsed < 0 || parsed > 4 { + return false + } + } + + return true +} + +func parseLisaPriority(value any) (int, bool) { + switch typed := value.(type) { + case int: + return typed, true + case int64: + return int(typed), true + case float64: + if typed != float64(int(typed)) { + return 0, false + } + + return int(typed), true + default: + return 0, false + } +} + +func validateStringList(value any, fieldName string, testName string) error { + items, ok := value.([]any) + if !ok || len(items) == 0 { + return fmt.Errorf( + "%w: test %#q %s must be a non-empty list of non-empty strings", + ErrInvalidLisaSelection, + testName, + fieldName, + ) + } + + for _, item := range items { + if !isNonEmptyString(item) { + return fmt.Errorf( + "%w: test %#q %s must be a non-empty list of non-empty strings", + ErrInvalidLisaSelection, + testName, + fieldName, + ) + } + } + + return nil +} + +func isNonEmptyString(value any) bool { + s, ok := value.(string) + if !ok { + return false + } + + return strings.TrimSpace(s) != "" +} + // JSONSchemaExtend tightens [TestDefinition] so the framework-specific subtable // must match the declared type. func (TestDefinition) JSONSchemaExtend(schema *jsonschema.Schema) { diff --git a/internal/projectconfig/testsuite.go b/internal/projectconfig/testsuite.go index 7a518932..2bf00427 100644 --- a/internal/projectconfig/testsuite.go +++ b/internal/projectconfig/testsuite.go @@ -30,9 +30,23 @@ var ( ErrMissingTestField = errors.New("missing required test field") // ErrUndefinedTestSuite is returned when an image references a test suite name that is not defined. ErrUndefinedTestSuite = errors.New("undefined test suite reference") + // ErrUndefinedTest is returned when a test reference points to a missing [tests] entry. + ErrUndefinedTest = errors.New("undefined test reference") + // ErrUndefinedTestGroup is returned when a test reference points to a missing [test-groups] entry. + ErrUndefinedTestGroup = errors.New("undefined test group reference") + // ErrInvalidTestRef is returned when a TestRef has neither or both of name/group set. + ErrInvalidTestRef = errors.New("invalid test reference") + // ErrDuplicateTestRef is returned when a list contains the same test ref more than once. + ErrDuplicateTestRef = errors.New("duplicate test reference") + // ErrNestedTestGroupReference is returned when a [test-groups] member uses a group ref. + ErrNestedTestGroupReference = errors.New("nested test group reference") // ErrMismatchedTestSubtable is returned when a test config has a subtable that does not // match its declared type. ErrMismatchedTestSubtable = errors.New("mismatched test subtable") + // ErrUnknownTestKind is returned for unrecognized test kinds. + ErrUnknownTestKind = errors.New("unknown test kind") + // ErrInvalidLisaSelection is returned when a lisa test has invalid or missing selectors. + ErrInvalidLisaSelection = errors.New("invalid lisa selection") // ErrInvalidInstallMode is returned when a [PytestConfig.Install] value is not recognized. ErrInvalidInstallMode = errors.New("invalid install mode") // ErrInvalidGitRef is returned when a git ref is not a valid hex commit SHA. diff --git a/scenario/__snapshots__/TestSnapshotsContainer_config_generate-schema_stdout_1.snap b/scenario/__snapshots__/TestSnapshotsContainer_config_generate-schema_stdout_1.snap index f17a8cf0..df072568 100755 --- a/scenario/__snapshots__/TestSnapshotsContainer_config_generate-schema_stdout_1.snap +++ b/scenario/__snapshots__/TestSnapshotsContainer_config_generate-schema_stdout_1.snap @@ -1532,12 +1532,13 @@ "description": "Description of this test" }, "kind": { - "items": { - "type": "string" - }, - "type": "array", + "type": "string", + "enum": [ + "functional", + "performance" + ], "title": "Kind", - "description": "Test kind hints (e.g. functional or performance)" + "description": "Kind hint for the test" }, "long-running": { "type": "boolean", @@ -1583,11 +1584,27 @@ }, "tests": { "items": { - "$ref": "#/$defs/TestRef" + "not": { + "required": [ + "group" + ] + }, + "properties": { + "name": { + "type": "string", + "minLength": 1, + "pattern": "^\\S", + "description": "Name of a test" + } + }, + "type": "object", + "required": [ + "name" + ] }, "type": "array", "title": "Tests", - "description": "Member references (each is either {name=...} or {group=...})" + "description": "Ordered test references for this group (name only)" } }, "additionalProperties": false, diff --git a/scenario/__snapshots__/TestSnapshots_config_generate-schema_stdout_1.snap b/scenario/__snapshots__/TestSnapshots_config_generate-schema_stdout_1.snap index f17a8cf0..df072568 100755 --- a/scenario/__snapshots__/TestSnapshots_config_generate-schema_stdout_1.snap +++ b/scenario/__snapshots__/TestSnapshots_config_generate-schema_stdout_1.snap @@ -1532,12 +1532,13 @@ "description": "Description of this test" }, "kind": { - "items": { - "type": "string" - }, - "type": "array", + "type": "string", + "enum": [ + "functional", + "performance" + ], "title": "Kind", - "description": "Test kind hints (e.g. functional or performance)" + "description": "Kind hint for the test" }, "long-running": { "type": "boolean", @@ -1583,11 +1584,27 @@ }, "tests": { "items": { - "$ref": "#/$defs/TestRef" + "not": { + "required": [ + "group" + ] + }, + "properties": { + "name": { + "type": "string", + "minLength": 1, + "pattern": "^\\S", + "description": "Name of a test" + } + }, + "type": "object", + "required": [ + "name" + ] }, "type": "array", "title": "Tests", - "description": "Member references (each is either {name=...} or {group=...})" + "description": "Ordered test references for this group (name only)" } }, "additionalProperties": false, diff --git a/schemas/azldev.schema.json b/schemas/azldev.schema.json index f17a8cf0..df072568 100644 --- a/schemas/azldev.schema.json +++ b/schemas/azldev.schema.json @@ -1532,12 +1532,13 @@ "description": "Description of this test" }, "kind": { - "items": { - "type": "string" - }, - "type": "array", + "type": "string", + "enum": [ + "functional", + "performance" + ], "title": "Kind", - "description": "Test kind hints (e.g. functional or performance)" + "description": "Kind hint for the test" }, "long-running": { "type": "boolean", @@ -1583,11 +1584,27 @@ }, "tests": { "items": { - "$ref": "#/$defs/TestRef" + "not": { + "required": [ + "group" + ] + }, + "properties": { + "name": { + "type": "string", + "minLength": 1, + "pattern": "^\\S", + "description": "Name of a test" + } + }, + "type": "object", + "required": [ + "name" + ] }, "type": "array", "title": "Tests", - "description": "Member references (each is either {name=...} or {group=...})" + "description": "Ordered test references for this group (name only)" } }, "additionalProperties": false, From c9f13eb8c2ae167a844b506c4160408c27e7df6f Mon Sep 17 00:00:00 2001 From: bhagyapathak Date: Wed, 15 Jul 2026 16:15:19 +0530 Subject: [PATCH 3/5] feat(projectconfig): accept [tests] and [test-groups] schema in config files --- internal/app/azldev/cmds/component/query.go | 17 ++ .../app/azldev/cmds/component/query_test.go | 89 ++++++++++ internal/app/azldev/cmds/image/test.go | 165 ++++++++++++------ .../azldev/cmds/image/test_internal_test.go | 73 ++++++++ internal/projectconfig/configfile.go | 19 +- internal/projectconfig/configfile_test.go | 132 +++++++++----- internal/projectconfig/loader.go | 38 ++++ internal/projectconfig/loader_test.go | 35 ++++ internal/projectconfig/project.go | 12 ++ internal/projectconfig/tests.go | 112 ++++++++++++ 10 files changed, 587 insertions(+), 105 deletions(-) create mode 100644 internal/app/azldev/cmds/image/test_internal_test.go diff --git a/internal/app/azldev/cmds/component/query.go b/internal/app/azldev/cmds/component/query.go index 59985a73..557d6094 100644 --- a/internal/app/azldev/cmds/component/query.go +++ b/internal/app/azldev/cmds/component/query.go @@ -56,6 +56,10 @@ slower than 'list' but more informative.`, // componentDetails encapsulates detailed information about a component. type componentDetails struct { specs.ComponentSpecDetails + + // ResolvedTests lists concrete test names after expanding any component + // tests.tests refs and test-groups. + ResolvedTests []string `json:"resolvedTests,omitempty" table:"-"` } // Queries env for component details, in accordance with options. Returns the found components. @@ -72,6 +76,7 @@ func QueryComponents( } allDetails := make([]*componentDetails, 0, comps.Len()) + cfg := env.Config() for _, comp := range comps.Components() { spec := comp.GetSpec() @@ -85,6 +90,18 @@ func QueryComponents( ComponentSpecDetails: *specInfo, } + if cfg != nil { + resolvedTests, err := cfg.ResolveComponentTests(comp.GetConfig()) + if err != nil { + return nil, fmt.Errorf("failed to resolve tests for component %q:\n%w", comp.GetName(), err) + } + + details.ResolvedTests = make([]string, 0, len(resolvedTests)) + for _, resolvedTest := range resolvedTests { + details.ResolvedTests = append(details.ResolvedTests, resolvedTest.Name) + } + } + allDetails = append(allDetails, details) } diff --git a/internal/app/azldev/cmds/component/query_test.go b/internal/app/azldev/cmds/component/query_test.go index 9d605e02..222c7f5a 100644 --- a/internal/app/azldev/cmds/component/query_test.go +++ b/internal/app/azldev/cmds/component/query_test.go @@ -80,3 +80,92 @@ func TestQueryComponents_OneComponent(t *testing.T) { result := results[0] assert.Equal(t, testComponentName, result.Name) } + +func TestQueryComponents_ResolvesComponentTests(t *testing.T) { + const ( + testComponentName = "test-component" + testSpecPath = "/path/to/spec" + ) + + testEnv := testutils.NewTestEnv(t) + testEnv.Config.Components[testComponentName] = projectconfig.ComponentConfig{ + Name: testComponentName, + Spec: projectconfig.SpecSource{ + SourceType: projectconfig.SpecSourceTypeLocal, + Path: testSpecPath, + }, + Tests: &projectconfig.ComponentTestsConfig{ + Tests: []projectconfig.TestRef{{Group: "runtime"}}, + }, + } + testEnv.Config.Tests = map[string]projectconfig.TestDefinition{ + "runtime-a": {Type: "pytest", Pytest: map[string]any{"test-paths": []string{"tests/a.py"}}}, + "runtime-b": {Type: "pytest", Pytest: map[string]any{"test-paths": []string{"tests/b.py"}}}, + } + testEnv.Config.TestGroups = map[string]projectconfig.TestGroup{ + "runtime": {Tests: []projectconfig.TestRef{{Name: "runtime-a"}, {Name: "runtime-b"}}}, + } + + // Pretend mock is present. + testEnv.CmdFactory.RegisterCommandInSearchPath(mock.MockBinary) + + // Mock the rpmspec command to return valid output. + testEnv.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { + return "name=test-component\nepoch=0\nversion=1.0.0\nrelease=1.azl4\n", nil + } + + options := component.QueryComponentsOptions{ + ComponentFilter: components.ComponentFilter{ + ComponentNamePatterns: []string{testComponentName}, + }, + } + + err := fileutils.WriteFile(testEnv.FS(), testSpecPath, []byte("test spec content"), fileperms.PublicFile) + require.NoError(t, err) + + results, err := component.QueryComponents(testEnv.Env, &options) + require.NoError(t, err) + require.Len(t, results, 1) + assert.Equal(t, []string{"runtime-a", "runtime-b"}, results[0].ResolvedTests) +} + +func TestQueryComponents_InvalidComponentTestRef(t *testing.T) { + const ( + testComponentName = "test-component" + testSpecPath = "/path/to/spec" + ) + + testEnv := testutils.NewTestEnv(t) + testEnv.Config.Components[testComponentName] = projectconfig.ComponentConfig{ + Name: testComponentName, + Spec: projectconfig.SpecSource{ + SourceType: projectconfig.SpecSourceTypeLocal, + Path: testSpecPath, + }, + Tests: &projectconfig.ComponentTestsConfig{ + Tests: []projectconfig.TestRef{{Name: "missing-test"}}, + }, + } + + // Pretend mock is present. + testEnv.CmdFactory.RegisterCommandInSearchPath(mock.MockBinary) + + // Mock the rpmspec command to return valid output. + testEnv.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { + return "name=test-component\nepoch=0\nversion=1.0.0\nrelease=1.azl4\n", nil + } + + options := component.QueryComponentsOptions{ + ComponentFilter: components.ComponentFilter{ + ComponentNamePatterns: []string{testComponentName}, + }, + } + + err := fileutils.WriteFile(testEnv.FS(), testSpecPath, []byte("test spec content"), fileperms.PublicFile) + require.NoError(t, err) + + _, err = component.QueryComponents(testEnv.Env, &options) + require.Error(t, err) + assert.ErrorContains(t, err, "failed to resolve tests for component") + assert.ErrorContains(t, err, "missing-test") +} diff --git a/internal/app/azldev/cmds/image/test.go b/internal/app/azldev/cmds/image/test.go index dee998fd..c6a7ed66 100644 --- a/internal/app/azldev/cmds/image/test.go +++ b/internal/app/azldev/cmds/image/test.go @@ -8,7 +8,6 @@ import ( "fmt" "log/slog" "path/filepath" - "slices" "sort" "strings" @@ -18,6 +17,7 @@ import ( "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" "github.com/samber/lo" "github.com/spf13/cobra" + "github.com/pelletier/go-toml/v2" ) // ImageTestOptions holds the options for the 'image test' command. @@ -26,8 +26,8 @@ type ImageTestOptions struct { // test suites and optionally resolve the image artifact path. ImageName string - // TestSuites optionally selects specific test suites to run. When empty, all test - // suites associated with the image are run. + // TestSuites optionally selects specific test names or test-group names to run. + // When empty, all tests associated with the image are run. TestSuites []string // ImagePath is an optional explicit path to the image file. When empty, the image @@ -49,15 +49,14 @@ func NewImageTestCmd() *cobra.Command { cmd := &cobra.Command{ Use: "test IMAGE_NAME", Short: "Run tests against an Azure Linux image", - Long: `Run tests against an Azure Linux image using test suites defined in the + Long: `Run tests against an Azure Linux image using test definitions declared in the project configuration. -Test suites are defined in the [test-suites] section of azldev.toml and referenced -by images via the [images.NAME.tests] subtable. Each test suite specifies a type -(pytest or lisa) and framework-specific configuration in a matching subtable. +Images may reference tests directly via [images.NAME.tests.tests] entries, or via +named [test-groups]. Legacy [test-suites] references are still supported. -By default, all test suites associated with the named image are run. Use ---test-suite to select specific suites (may be repeated). +By default, all tests associated with the named image are run. Use +--test-suite to select specific test names or test-group names (may be repeated). The image artifact can be specified explicitly with --image-path, or resolved automatically from the image name in the output directory. @@ -73,17 +72,17 @@ QEMU VM. azldev clones the LISA framework, generates a runbook from the suite's configured test cases, and runs it against the image. azldev generates an ephemeral SSH key pair to access the booted VM and removes it once the suite finishes.`, - Example: ` # Run all test suites for an image (artifact auto-resolved from output dir) + Example: ` # Run all tests for an image (artifact auto-resolved from output dir) azldev image test vm-base # Run all test suites with an explicit image path azldev image test vm-base --image-path ./out/images/vm-base/image.raw - # Run a specific test suite - azldev image test vm-base --test-suite common-vm-checks + # Run a specific test + azldev image test vm-base --test-suite static-image-checks - # Run multiple specific test suites - azldev image test vm-base --test-suite common-vm-checks --test-suite vm-base-checks + # Run multiple tests or a test-group + azldev image test vm-base --test-suite static-image-checks --test-suite vm-base-functional # Generate JUnit XML output azldev image test vm-base --junit-xml results.xml`, @@ -97,7 +96,7 @@ finishes.`, } cmd.Flags().StringSliceVar(&options.TestSuites, "test-suite", nil, - "Name of a test suite to run (may be repeated; defaults to all suites for the image)") + "Name of a test or test-group to run (may be repeated; defaults to all tests for the image)") cmd.Flags().StringVarP(&options.ImagePath, "image-path", "i", "", "Path to the disk image file (resolved from image name if not specified)") @@ -110,7 +109,7 @@ finishes.`, return cmd } -// runImageTest resolves which test suites to run and dispatches each one. +// runImageTest resolves which tests to run and dispatches each one. func runImageTest(env *azldev.Env, options *ImageTestOptions) error { cfg := env.Config() if cfg == nil { @@ -159,25 +158,31 @@ func runImageTest(env *azldev.Env, options *ImageTestOptions) error { options.JUnitXMLPath = absJUnitPath } - // Determine which test suites to run. - suiteNames := resolveTestSuiteNames(imageConfig, options.TestSuites) - - // Warn when explicitly requested suites are not referenced by the image config. - if len(options.TestSuites) > 0 { - warnUnassociatedSuites(options.ImageName, imageConfig, options.TestSuites) + resolvedTests, legacySuiteNames, err := resolveImageTestsToRun(cfg, imageConfig, options.TestSuites) + if err != nil { + return err } - if len(suiteNames) == 0 { - slog.Warn("No test suites to run for image", slog.String("image", options.ImageName)) + if len(resolvedTests) == 0 && len(legacySuiteNames) == 0 { + slog.Warn("No tests to run for image", slog.String("image", options.ImageName)) return nil } - // Resolve and run each test suite, continuing past failures so all suites get a chance - // to run. Config/resolution errors abort immediately since they indicate a broken setup. var testFailures []string - for _, suiteName := range suiteNames { + for _, resolvedTest := range resolvedTests { + if err := runResolvedTest(env, resolvedTest, imageConfig, options); err != nil { + slog.Error("Test failed", + slog.String("test", resolvedTest.Name), + slog.Any("error", err), + ) + + testFailures = append(testFailures, resolvedTest.Name) + } + } + + for _, suiteName := range legacySuiteNames { suiteConfig, err := resolveTestSuiteByName(cfg, suiteName) if err != nil { return err @@ -194,41 +199,99 @@ func runImageTest(env *azldev.Env, options *ImageTestOptions) error { } if len(testFailures) > 0 { - return fmt.Errorf("%d of %d test suite(s) failed: %s", - len(testFailures), len(suiteNames), strings.Join(testFailures, ", ")) + total := len(resolvedTests) + len(legacySuiteNames) + return fmt.Errorf("%d of %d test(s) failed: %s", + len(testFailures), total, strings.Join(testFailures, ", ")) } return nil } -// resolveTestSuiteNames determines which test suites to run. If explicit names are -// provided, they are used as-is. Otherwise, all test suites associated with the image -// are returned. -func resolveTestSuiteNames( - imageConfig *projectconfig.ImageConfig, explicitSuites []string, -) []string { - if len(explicitSuites) > 0 { - return explicitSuites + +func resolveImageTestsToRun( + cfg *projectconfig.ProjectConfig, + imageConfig *projectconfig.ImageConfig, + explicitSelectors []string, +) ([]projectconfig.ResolvedTest, []string, error) { + if imageConfig.Tests != nil && len(imageConfig.Tests.Tests) > 0 { + if len(explicitSelectors) > 0 { + resolvedTests, err := cfg.ResolveTestSelectors(explicitSelectors) + return resolvedTests, nil, err + } + + resolvedTests, err := cfg.ResolveImageTests(imageConfig) + return resolvedTests, nil, err + } + + if len(explicitSelectors) > 0 { + return nil, explicitSelectors, nil } - return imageConfig.TestNames() + return nil, imageConfig.TestNames(), nil } -// warnUnassociatedSuites logs a warning for each explicitly requested test suite -// that is not referenced by the image's test configuration. -func warnUnassociatedSuites( - imageName string, imageConfig *projectconfig.ImageConfig, explicitSuites []string, -) { - imageTestNames := imageConfig.TestNames() - - for _, name := range explicitSuites { - if !slices.Contains(imageTestNames, name) { - slog.Warn("Test suite is not associated with image", - slog.String("suite", name), - slog.String("image", imageName), - ) +func runResolvedTest( + env *azldev.Env, + resolvedTest projectconfig.ResolvedTest, + imageConfig *projectconfig.ImageConfig, + options *ImageTestOptions, +) error { + switch resolvedTest.Definition.Type { + case string(projectconfig.TestTypePytest): + suiteConfig, err := testDefinitionToSuiteConfig(resolvedTest) + if err != nil { + return err } + + return RunPytestSuite(env, suiteConfig, imageConfig, options) + + case string(projectconfig.TestTypeLisa): + return fmt.Errorf("LISA tests cannot be run locally via 'azldev image test'; test %#q must be run through the LISA infrastructure", resolvedTest.Name) + + case "tmt": + return fmt.Errorf("TMT tests cannot be run locally via 'azldev image test'; test %#q is metadata-only for external orchestration", resolvedTest.Name) + + default: + return fmt.Errorf("unsupported test type %#q for test %#q", resolvedTest.Definition.Type, resolvedTest.Name) + } +} + +func testDefinitionToSuiteConfig(resolvedTest projectconfig.ResolvedTest) (*projectconfig.TestSuiteConfig, error) { + pytestConfig, err := decodePytestConfig(resolvedTest.Definition.Pytest) + if err != nil { + return nil, fmt.Errorf("decode pytest config for test %#q:\n%w", resolvedTest.Name, err) + } + + suiteConfig := &projectconfig.TestSuiteConfig{ + Name: resolvedTest.Name, + Description: resolvedTest.Definition.Description, + Type: projectconfig.TestTypePytest, + Pytest: pytestConfig, + } + + if err := suiteConfig.Validate(); err != nil { + return nil, fmt.Errorf("invalid pytest test %#q:\n%w", resolvedTest.Name, err) } + + return suiteConfig, nil +} + +func decodePytestConfig(raw map[string]any) (*projectconfig.PytestConfig, error) { + if raw == nil { + return nil, fmt.Errorf("missing [pytest] subtable") + } + + bytes, err := toml.Marshal(raw) + if err != nil { + return nil, fmt.Errorf("marshal pytest config:\n%w", err) + } + + pytestConfig := &projectconfig.PytestConfig{} + if err := toml.Unmarshal(bytes, pytestConfig); err != nil { + return nil, fmt.Errorf("unmarshal pytest config:\n%w", err) + } + + return pytestConfig, nil } // resolveTestSuiteByName looks up a test suite by name in the project configuration. diff --git a/internal/app/azldev/cmds/image/test_internal_test.go b/internal/app/azldev/cmds/image/test_internal_test.go new file mode 100644 index 00000000..3d34c218 --- /dev/null +++ b/internal/app/azldev/cmds/image/test_internal_test.go @@ -0,0 +1,73 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package image + +import ( + "testing" + + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/testutils" + "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestResolveImageTestsToRun_UsesNewTestsRefs(t *testing.T) { + testEnv := testutils.NewTestEnv(t) + testEnv.Config.Tests = map[string]projectconfig.TestDefinition{ + "static-image-checks": {Type: "pytest", Pytest: map[string]any{"working-dir": "/project/tests"}}, + "functional_core": {Type: "lisa", Lisa: map[string]any{"criteria": map[string]any{"priority": []any{1}}}}, + } + testEnv.Config.TestGroups = map[string]projectconfig.TestGroup{ + "vm-base-functional": {Tests: []projectconfig.TestRef{{Name: "functional_core"}}}, + } + + imageCfg := &projectconfig.ImageConfig{ + Tests: &projectconfig.ImageTestsConfig{ + Tests: []projectconfig.TestRef{ + {Name: "static-image-checks"}, + {Group: "vm-base-functional"}, + }, + }, + } + + resolved, legacy, err := resolveImageTestsToRun(testEnv.Config, imageCfg, nil) + require.NoError(t, err) + assert.Empty(t, legacy) + require.Len(t, resolved, 2) + assert.Equal(t, "static-image-checks", resolved[0].Name) + assert.Equal(t, "functional_core", resolved[1].Name) +} + +func TestResolveImageTestsToRun_FallsBackToLegacyTestSuites(t *testing.T) { + testEnv := testutils.NewTestEnv(t) + imageCfg := &projectconfig.ImageConfig{ + Tests: &projectconfig.ImageTestsConfig{ + TestSuites: []projectconfig.TestSuiteRef{{Name: "smoke"}, {Name: "integration"}}, + }, + } + + resolved, legacy, err := resolveImageTestsToRun(testEnv.Config, imageCfg, nil) + require.NoError(t, err) + assert.Empty(t, resolved) + assert.Equal(t, []string{"smoke", "integration"}, legacy) +} + +func TestTestDefinitionToSuiteConfig_Pytest(t *testing.T) { + resolvedTest := projectconfig.ResolvedTest{ + Name: "static-image-checks", + Definition: projectconfig.TestDefinition{ + Type: "pytest", + Description: "offline validation", + Pytest: map[string]any{"working-dir": "/project/tests", "install": "pyproject"}, + }, + } + + suite, err := testDefinitionToSuiteConfig(resolvedTest) + require.NoError(t, err) + require.NotNil(t, suite) + assert.Equal(t, "static-image-checks", suite.Name) + require.NotNil(t, suite.Pytest) + assert.Equal(t, "/project/tests", suite.Pytest.WorkingDir) + assert.Equal(t, projectconfig.PytestInstallPyproject, suite.Pytest.Install) +} diff --git a/internal/projectconfig/configfile.go b/internal/projectconfig/configfile.go index 45b7a7a0..e0aea3ac 100644 --- a/internal/projectconfig/configfile.go +++ b/internal/projectconfig/configfile.go @@ -150,10 +150,6 @@ func (f ConfigFile) Validate() error { return err } - if err := validateNewTestReferences(f); err != nil { - return err - } - return nil } @@ -202,32 +198,32 @@ func validateComponentGroupMetadata(groups map[string]ComponentGroupConfig) erro return nil } -func validateNewTestReferences(cfgFile ConfigFile) error { - for groupName, group := range cfgFile.TestGroups { +func validateNewTestReferences(tests map[string]TestDefinition, groups map[string]TestGroup, components map[string]ComponentConfig, images map[string]ImageConfig) error { + for groupName, group := range groups { scope := fmt.Sprintf("test-group %#q tests", groupName) - if err := validateTestGroupMembers(scope, group.Tests, cfgFile.Tests); err != nil { + if err := validateTestGroupMembers(scope, group.Tests, tests); err != nil { return err } } - for componentName, component := range cfgFile.Components { + for componentName, component := range components { if component.Tests == nil { continue } scope := fmt.Sprintf("component %#q tests.tests", componentName) - if err := validateTestRefList(scope, component.Tests.Tests, cfgFile.Tests, cfgFile.TestGroups); err != nil { + if err := validateTestRefList(scope, component.Tests.Tests, tests, groups); err != nil { return err } } - for imageName, image := range cfgFile.Images { + for imageName, image := range images { if image.Tests == nil { continue } scope := fmt.Sprintf("image %#q tests.tests", imageName) - if err := validateTestRefList(scope, image.Tests.Tests, cfgFile.Tests, cfgFile.TestGroups); err != nil { + if err := validateTestRefList(scope, image.Tests.Tests, tests, groups); err != nil { return err } } @@ -371,7 +367,6 @@ func validateTestRefList( return nil } - // validateSourceFiles checks 'source-files' configuration for a component: // - All filenames must be unique. // - Hash type must be a supported algorithm when specified. diff --git a/internal/projectconfig/configfile_test.go b/internal/projectconfig/configfile_test.go index d1229a72..8c948e30 100644 --- a/internal/projectconfig/configfile_test.go +++ b/internal/projectconfig/configfile_test.go @@ -157,103 +157,99 @@ func TestProjectConfigFileValidation_LisaSelectionUnsupportedCriteriaKey(t *test assert.Contains(t, err.Error(), "unsupported selector") } -func TestProjectConfigFileValidation_UndefinedTestReferenceInGroup(t *testing.T) { - file := projectconfig.ConfigFile{ - TestGroups: map[string]projectconfig.TestGroup{ +func TestProjectConfigValidation_UndefinedTestReferenceInGroup(t *testing.T) { + cfg := projectconfig.NewProjectConfig() + cfg.TestGroups = map[string]projectconfig.TestGroup{ "bvt": { Tests: []projectconfig.TestRef{{Name: "does-not-exist"}}, }, - }, } - err := file.Validate() + err := cfg.Validate() require.Error(t, err) require.ErrorIs(t, err, projectconfig.ErrUndefinedTest) assert.Contains(t, err.Error(), "does-not-exist") } -func TestProjectConfigFileValidation_UndefinedTestGroupReferenceInComponent(t *testing.T) { - file := projectconfig.ConfigFile{ - Components: map[string]projectconfig.ComponentConfig{ +func TestProjectConfigValidation_UndefinedTestGroupReferenceInComponent(t *testing.T) { + cfg := projectconfig.NewProjectConfig() + cfg.Components = map[string]projectconfig.ComponentConfig{ "openssl": { Tests: &projectconfig.ComponentTestsConfig{ Tests: []projectconfig.TestRef{{Group: "missing-group"}}, }, }, - }, } - err := file.Validate() + err := cfg.Validate() require.Error(t, err) require.ErrorIs(t, err, projectconfig.ErrUndefinedTestGroup) assert.Contains(t, err.Error(), "missing-group") } -func TestProjectConfigFileValidation_InvalidTestReferenceShapeInImage(t *testing.T) { - file := projectconfig.ConfigFile{ - Images: map[string]projectconfig.ImageConfig{ +func TestProjectConfigValidation_InvalidTestReferenceShapeInImage(t *testing.T) { + cfg := projectconfig.NewProjectConfig() + cfg.Images = map[string]projectconfig.ImageConfig{ "base": { Tests: &projectconfig.ImageTestsConfig{ Tests: []projectconfig.TestRef{{Name: "smoke", Group: "bvt"}}, }, }, - }, - Tests: map[string]projectconfig.TestDefinition{ + } + cfg.Tests = map[string]projectconfig.TestDefinition{ "smoke": { Type: "pytest", Pytest: map[string]any{"working-dir": "tests"}, }, - }, - TestGroups: map[string]projectconfig.TestGroup{ + } + cfg.TestGroups = map[string]projectconfig.TestGroup{ "bvt": {Tests: []projectconfig.TestRef{{Name: "smoke"}}}, - }, } - err := file.Validate() + err := cfg.Validate() require.Error(t, err) require.ErrorIs(t, err, projectconfig.ErrInvalidTestRef) assert.Contains(t, err.Error(), "exactly one") } -func TestProjectConfigFileValidation_DuplicateTestReferenceInGroup(t *testing.T) { - file := projectconfig.ConfigFile{ - Tests: map[string]projectconfig.TestDefinition{ +func TestProjectConfigValidation_DuplicateTestReferenceInGroup(t *testing.T) { + cfg := projectconfig.NewProjectConfig() + cfg.Tests = map[string]projectconfig.TestDefinition{ "smoke": { Type: "pytest", Pytest: map[string]any{"working-dir": "tests"}, }, - }, - TestGroups: map[string]projectconfig.TestGroup{ + } + cfg.TestGroups = map[string]projectconfig.TestGroup{ "bvt": { Tests: []projectconfig.TestRef{ {Name: "smoke"}, {Name: "smoke"}, }, }, - }, } - err := file.Validate() + err := cfg.Validate() require.Error(t, err) require.ErrorIs(t, err, projectconfig.ErrDuplicateTestRef) assert.Contains(t, err.Error(), "duplicates") assert.Contains(t, err.Error(), "smoke") } -func TestProjectConfigFileValidation_DuplicateTestGroupReferenceInImage(t *testing.T) { - file := projectconfig.ConfigFile{ - Tests: map[string]projectconfig.TestDefinition{ +func TestProjectConfigValidation_DuplicateTestGroupReferenceInImage(t *testing.T) { + cfg := projectconfig.NewProjectConfig() + cfg.Tests = map[string]projectconfig.TestDefinition{ "smoke": { Type: "pytest", Pytest: map[string]any{"working-dir": "tests"}, }, - }, - TestGroups: map[string]projectconfig.TestGroup{ + } + cfg.TestGroups = map[string]projectconfig.TestGroup{ "bvt": { Tests: []projectconfig.TestRef{{Name: "smoke"}}, }, - }, - Images: map[string]projectconfig.ImageConfig{ + } + cfg.Images = map[string]projectconfig.ImageConfig{ "base": { Tests: &projectconfig.ImageTestsConfig{ Tests: []projectconfig.TestRef{ @@ -262,36 +258,88 @@ func TestProjectConfigFileValidation_DuplicateTestGroupReferenceInImage(t *testi }, }, }, - }, } - err := file.Validate() + err := cfg.Validate() require.Error(t, err) require.ErrorIs(t, err, projectconfig.ErrDuplicateTestRef) assert.Contains(t, err.Error(), "duplicates") assert.Contains(t, err.Error(), "bvt") } -func TestProjectConfigFileValidation_NestedTestGroupReferenceNotAllowed(t *testing.T) { - file := projectconfig.ConfigFile{ - Tests: map[string]projectconfig.TestDefinition{ +func TestProjectConfigValidation_NestedTestGroupReferenceNotAllowed(t *testing.T) { + cfg := projectconfig.NewProjectConfig() + cfg.Tests = map[string]projectconfig.TestDefinition{ "smoke": { Type: "pytest", Pytest: map[string]any{"working-dir": "tests"}, }, - }, - TestGroups: map[string]projectconfig.TestGroup{ + } + cfg.TestGroups = map[string]projectconfig.TestGroup{ "a": {Tests: []projectconfig.TestRef{{Group: "b"}}}, "b": {Tests: []projectconfig.TestRef{{Name: "smoke"}}}, - }, } - err := file.Validate() + err := cfg.Validate() require.Error(t, err) require.ErrorIs(t, err, projectconfig.ErrNestedTestGroupReference) assert.Contains(t, err.Error(), "is not allowed in [test-groups]") } +func TestProjectConfigResolveImageTests_ExpandsGroups(t *testing.T) { + cfg := projectconfig.NewProjectConfig() + cfg.Tests = map[string]projectconfig.TestDefinition{ + "static-image-checks": {Type: "pytest", Pytest: map[string]any{"working-dir": "tests"}}, + "functional_core": {Type: "lisa", Lisa: map[string]any{"criteria": map[string]any{"priority": []any{1}}}}, + "lisa_perf": {Type: "lisa", Lisa: map[string]any{"criteria": map[string]any{"area": "network", "category": "performance"}}}, + } + cfg.TestGroups = map[string]projectconfig.TestGroup{ + "vm-base-functional": {Tests: []projectconfig.TestRef{{Name: "functional_core"}}}, + "vm-base-performance": {Tests: []projectconfig.TestRef{{Name: "lisa_perf"}}}, + } + + imageCfg := &projectconfig.ImageConfig{ + Tests: &projectconfig.ImageTestsConfig{ + Tests: []projectconfig.TestRef{ + {Name: "static-image-checks"}, + {Group: "vm-base-functional"}, + {Group: "vm-base-performance"}, + }, + }, + } + + resolved, err := cfg.ResolveImageTests(imageCfg) + require.NoError(t, err) + require.Len(t, resolved, 3) + assert.Equal(t, []string{"static-image-checks", "functional_core", "lisa_perf"}, []string{ + resolved[0].Name, + resolved[1].Name, + resolved[2].Name, + }) +} + +func TestProjectConfigResolveComponentTests_ExpandsGroups(t *testing.T) { + cfg := projectconfig.NewProjectConfig() + cfg.Tests = map[string]projectconfig.TestDefinition{ + "bash-fedora-shell": {Type: "tmt", Tmt: map[string]any{"plan": "/plans/shell"}}, + } + cfg.TestGroups = map[string]projectconfig.TestGroup{ + "shell-tests": {Tests: []projectconfig.TestRef{{Name: "bash-fedora-shell"}}}, + } + + componentCfg := &projectconfig.ComponentConfig{ + Tests: &projectconfig.ComponentTestsConfig{ + Tests: []projectconfig.TestRef{{Group: "shell-tests"}}, + }, + } + + resolved, err := cfg.ResolveComponentTests(componentCfg) + require.NoError(t, err) + require.Len(t, resolved, 1) + assert.Equal(t, "bash-fedora-shell", resolved[0].Name) + assert.Equal(t, "tmt", resolved[0].Definition.Type) +} + func TestProjectConfigFileValidation_DefaultProjectInfo(t *testing.T) { file := projectconfig.ConfigFile{ Project: &projectconfig.ProjectInfo{}, diff --git a/internal/projectconfig/loader.go b/internal/projectconfig/loader.go index 31971f90..4a2f71b5 100644 --- a/internal/projectconfig/loader.go +++ b/internal/projectconfig/loader.go @@ -45,6 +45,8 @@ func loadAndResolveProjectConfig( GroupsByComponent: make(map[string][]string), PackageGroups: make(map[string]PackageGroupConfig), TestSuites: make(map[string]TestSuiteConfig), + Tests: make(map[string]TestDefinition), + TestGroups: make(map[string]TestGroup), } for _, configFilePath := range configFilePaths { @@ -146,6 +148,14 @@ func mergeConfigFile(resolvedCfg *ProjectConfig, loadedCfg *ConfigFile) error { return err } + if err := mergeTests(resolvedCfg, loadedCfg); err != nil { + return err + } + + if err := mergeTestGroups(resolvedCfg, loadedCfg); err != nil { + return err + } + if err := mergeResources(resolvedCfg, loadedCfg); err != nil { return err } @@ -319,6 +329,34 @@ func mergeTestSuites(resolvedCfg *ProjectConfig, loadedCfg *ConfigFile) error { return nil } +// mergeTests merges individual test definitions from a loaded config file into the +// resolved config. Duplicate test names are not allowed. +func mergeTests(resolvedCfg *ProjectConfig, loadedCfg *ConfigFile) error { + for testName, testDef := range loadedCfg.Tests { + if _, ok := resolvedCfg.Tests[testName]; ok { + return fmt.Errorf("%w: test %#q", ErrDuplicateTestSuites, testName) + } + + resolvedCfg.Tests[testName] = testDef.WithAbsolutePaths(loadedCfg.dir) + } + + return nil +} + +// mergeTestGroups merges named test groups from a loaded config file into the +// resolved config. Duplicate group names are not allowed. +func mergeTestGroups(resolvedCfg *ProjectConfig, loadedCfg *ConfigFile) error { + for groupName, group := range loadedCfg.TestGroups { + if _, ok := resolvedCfg.TestGroups[groupName]; ok { + return fmt.Errorf("%w: test group %#q", ErrDuplicateTestSuites, groupName) + } + + resolvedCfg.TestGroups[groupName] = group + } + + return nil +} + func loadProjectConfigWithIncludes( fs opctx.FS, filePath string, permissiveConfigParsing bool, seen map[string]bool, diff --git a/internal/projectconfig/loader_test.go b/internal/projectconfig/loader_test.go index 8083e89b..dfb0d7be 100644 --- a/internal/projectconfig/loader_test.go +++ b/internal/projectconfig/loader_test.go @@ -122,6 +122,41 @@ key = "value" assert.Equal(t, "/project/artifacts/logs", config.Project.LogDir) } +func TestLoadAndResolveProjectConfig_TestReferencesResolvedAcrossIncludedFiles(t *testing.T) { + testFiles := []struct { + path string + contents string + }{ + {testConfigPath, ` +includes = ["components.toml", "tests.toml"] +`}, + {"/project/components.toml", ` +[components.bash] +tests.tests = [{ name = "bash-fedora-shell" }] +`}, + {"/project/tests.toml", ` +[tests.bash-fedora-shell] +type = "tmt" +kind = "functional" + +[tests.bash-fedora-shell.tmt] +plan = "/plans/shell" +`}, + } + + ctx := testctx.NewCtx() + for _, testFile := range testFiles { + require.NoError(t, fileutils.MkdirAll(ctx.FS(), filepath.Dir(testFile.path))) + require.NoError(t, fileutils.WriteFile(ctx.FS(), testFile.path, []byte(testFile.contents), fileperms.PrivateFile)) + } + + config, err := loadAndResolveProjectConfig(ctx.FS(), false, testConfigPath) + require.NoError(t, err) + require.Contains(t, config.Components, "bash") + require.Contains(t, config.Tests, "bash-fedora-shell") + assert.Equal(t, TestKindFunctional, config.Tests["bash-fedora-shell"].Kind) +} + func TestLoadAndResolveProjectConfig_PermissiveParsing_IgnoresValidationError(t *testing.T) { // This config is structurally valid TOML but fails semantic validation: the // component group references a component that is not defined. diff --git a/internal/projectconfig/project.go b/internal/projectconfig/project.go index adb862aa..7ed2c3f6 100644 --- a/internal/projectconfig/project.go +++ b/internal/projectconfig/project.go @@ -47,6 +47,12 @@ type ProjectConfig struct { // Definitions of test suites. TestSuites map[string]TestSuiteConfig `toml:"test-suites,omitempty" json:"testSuites,omitempty" jsonschema:"title=Test Suites,description=Mapping of test suite names to configurations"` + // Definitions of individual tests. + Tests map[string]TestDefinition `toml:"tests,omitempty" json:"tests,omitempty" jsonschema:"title=Tests,description=Mapping of test names to configurations"` + + // Definitions of named test groups. + TestGroups map[string]TestGroup `toml:"test-groups,omitempty" json:"testGroups,omitempty" jsonschema:"title=Test Groups,description=Mapping of test group names to configurations"` + // Root config file path; not serialized. RootConfigFilePath string `toml:"-" json:"-"` // Map from component names to groups they belong to; not serialized. @@ -65,6 +71,8 @@ func NewProjectConfig() ProjectConfig { GroupsByComponent: make(map[string][]string), PackageGroups: make(map[string]PackageGroupConfig), TestSuites: make(map[string]TestSuiteConfig), + Tests: make(map[string]TestDefinition), + TestGroups: make(map[string]TestGroup), } } @@ -87,6 +95,10 @@ func (cfg *ProjectConfig) Validate() error { return err } + if err := validateNewTestReferences(cfg.Tests, cfg.TestGroups, cfg.Components, cfg.Images); err != nil { + return err + } + if err := validateRpmRepos(cfg.Resources.RpmRepos); err != nil { return err } diff --git a/internal/projectconfig/tests.go b/internal/projectconfig/tests.go index 7e06b0d5..680ab606 100644 --- a/internal/projectconfig/tests.go +++ b/internal/projectconfig/tests.go @@ -13,6 +13,12 @@ import ( orderedmap "github.com/pb33f/ordered-map/v2" ) +// ResolvedTest is a concrete test definition resolved from a direct [tests.X] +// reference or from expansion of a [test-groups.X] reference. +type ResolvedTest struct { + Name string + Definition TestDefinition +} // TestKind indicates what kind of behavior a test exercises. type TestKind string @@ -70,6 +76,20 @@ type TestDefinition struct { Pytest map[string]any `toml:"pytest,omitempty" json:"pytest,omitempty" jsonschema:"title=Pytest config,description=pytest-specific configuration"` } +// WithAbsolutePaths returns a copy of the test definition with any relative +// paths in framework-specific subtables converted to absolute paths. +func (t TestDefinition) WithAbsolutePaths(referenceDir string) TestDefinition { + result := t + result.Lisa = cloneStringAnyMap(t.Lisa) + result.Tmt = cloneStringAnyMap(t.Tmt) + result.Pytest = cloneStringAnyMap(t.Pytest) + + if workingDir, ok := result.Pytest["working-dir"].(string); ok { + result.Pytest["working-dir"] = makeAbsolute(referenceDir, workingDir) + } + + return result +} // TestGroup is a [test-groups.X] declaration: a named bundle of test references that // images or components can target via a single name. type TestGroup struct { @@ -100,6 +120,98 @@ type ComponentTestsConfig struct { Tests []TestRef `toml:"tests,omitempty" json:"tests,omitempty" jsonschema:"title=Tests,description=Per-component test or test-group references"` } +// ResolveTestRefs expands a list of [TestRef] entries into concrete tests. +func (cfg *ProjectConfig) ResolveTestRefs(refs []TestRef) ([]ResolvedTest, error) { + resolved := make([]ResolvedTest, 0, len(refs)) + + for _, ref := range refs { + switch { + case ref.Name != "": + testDef, ok := cfg.Tests[ref.Name] + if !ok { + return nil, fmt.Errorf("%w: %#q", ErrUndefinedTest, ref.Name) + } + + resolved = append(resolved, ResolvedTest{Name: ref.Name, Definition: testDef}) + + case ref.Group != "": + group, ok := cfg.TestGroups[ref.Group] + if !ok { + return nil, fmt.Errorf("%w: %#q", ErrUndefinedTestGroup, ref.Group) + } + + for _, groupRef := range group.Tests { + if groupRef.Group != "" { + return nil, fmt.Errorf("%w: %#q", ErrNestedTestGroupReference, ref.Group) + } + + testDef, ok := cfg.Tests[groupRef.Name] + if !ok { + return nil, fmt.Errorf("%w: %#q", ErrUndefinedTest, groupRef.Name) + } + + resolved = append(resolved, ResolvedTest{Name: groupRef.Name, Definition: testDef}) + } + } + } + + return resolved, nil +} + +// ResolveTestSelectors resolves user-provided selectors, where each selector may +// reference either a concrete test or a test group. +func (cfg *ProjectConfig) ResolveTestSelectors(selectors []string) ([]ResolvedTest, error) { + refs := make([]TestRef, 0, len(selectors)) + + for _, selector := range selectors { + _, isTest := cfg.Tests[selector] + _, isGroup := cfg.TestGroups[selector] + + switch { + case isTest && isGroup: + return nil, fmt.Errorf("ambiguous test selector %#q matches both [tests] and [test-groups]", selector) + case isTest: + refs = append(refs, TestRef{Name: selector}) + case isGroup: + refs = append(refs, TestRef{Group: selector}) + default: + return nil, fmt.Errorf("unknown test selector %#q", selector) + } + } + + return cfg.ResolveTestRefs(refs) +} + +// ResolveImageTests expands the new-style test refs associated with an image. +func (cfg *ProjectConfig) ResolveImageTests(image *ImageConfig) ([]ResolvedTest, error) { + if image == nil || image.Tests == nil { + return nil, nil + } + + return cfg.ResolveTestRefs(image.Tests.Tests) +} + +// ResolveComponentTests expands the new-style test refs associated with a component. +func (cfg *ProjectConfig) ResolveComponentTests(component *ComponentConfig) ([]ResolvedTest, error) { + if component == nil || component.Tests == nil { + return nil, nil + } + + return cfg.ResolveTestRefs(component.Tests.Tests) +} + +func cloneStringAnyMap(input map[string]any) map[string]any { + if input == nil { + return nil + } + + result := make(map[string]any, len(input)) + for key, value := range input { + result[key] = value + } + + return result +} // Validate checks that exactly one framework subtable is set and it matches Type. func (t TestDefinition) Validate(testName string) error { if t.Type == "" { From 520c625ab7e151e1ccf7ad0e1ae6159ef25778f4 Mon Sep 17 00:00:00 2001 From: bhagyapathak Date: Tue, 11 Aug 2026 13:11:00 +0530 Subject: [PATCH 4/5] fix: resolve remaining lint issues blocking CI - Break long error messages across multiple lines (test.go:270, 273) - Refactor function signature to comply with line length limit (configfile.go:201) - Reformat test data to fit line length constraint (configfile_test.go:294) - Wrap external package errors with fmt.Errorf (test.go:240, 244) - Use require assertion for error checks in tests (query_test.go:169) --- .../app/azldev/cmds/component/query_test.go | 4 +- internal/app/azldev/cmds/image/test.go | 105 ++++++++++++------ .../azldev/cmds/image/test_internal_test.go | 2 +- internal/projectconfig/configfile.go | 8 +- internal/projectconfig/configfile_test.go | 97 ++++++++-------- internal/projectconfig/tests.go | 3 + 6 files changed, 134 insertions(+), 85 deletions(-) diff --git a/internal/app/azldev/cmds/component/query_test.go b/internal/app/azldev/cmds/component/query_test.go index 222c7f5a..5882446c 100644 --- a/internal/app/azldev/cmds/component/query_test.go +++ b/internal/app/azldev/cmds/component/query_test.go @@ -166,6 +166,6 @@ func TestQueryComponents_InvalidComponentTestRef(t *testing.T) { _, err = component.QueryComponents(testEnv.Env, &options) require.Error(t, err) - assert.ErrorContains(t, err, "failed to resolve tests for component") - assert.ErrorContains(t, err, "missing-test") + require.ErrorContains(t, err, "failed to resolve tests for component") + require.ErrorContains(t, err, "missing-test") } diff --git a/internal/app/azldev/cmds/image/test.go b/internal/app/azldev/cmds/image/test.go index c6a7ed66..f9f28574 100644 --- a/internal/app/azldev/cmds/image/test.go +++ b/internal/app/azldev/cmds/image/test.go @@ -15,9 +15,9 @@ import ( "github.com/microsoft/azure-linux-dev-tools/internal/global/opctx" "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" + "github.com/pelletier/go-toml/v2" "github.com/samber/lo" "github.com/spf13/cobra" - "github.com/pelletier/go-toml/v2" ) // ImageTestOptions holds the options for the 'image test' command. @@ -116,59 +116,80 @@ func runImageTest(env *azldev.Env, options *ImageTestOptions) error { return errors.New("no project configuration loaded") } - // Resolve the image config from the positional argument. - imageConfig, err := ResolveImageByName(env, options.ImageName) + imageConfig, err := prepareImageTest(env, options) if err != nil { return err } - // Resolve image path: explicit --image-path takes precedence, otherwise resolve - // from the image name in the output directory. - imagePath := options.ImagePath - if imagePath == "" { - var resolveErr error + resolvedTests, legacySuiteNames, err := resolveImageTestsToRun(cfg, imageConfig, options.TestSuites) + if err != nil { + return err + } - imagePath, _, resolveErr = findImageArtifact(env, options.ImageName, "", AllImageFormats()) - if resolveErr != nil { - return resolveErr - } + if len(resolvedTests) == 0 && len(legacySuiteNames) == 0 { + slog.Warn("No tests to run for image", slog.String("image", options.ImageName)) - slog.Info("Resolved image artifact", - slog.String("image", options.ImageName), - slog.String("path", imagePath), - ) + return nil + } + + return runImageTests(env, cfg, imageConfig, options, resolvedTests, legacySuiteNames) +} + +func prepareImageTest(env *azldev.Env, options *ImageTestOptions) (*projectconfig.ImageConfig, error) { + imageConfig, err := ResolveImageByName(env, options.ImageName) + if err != nil { + return nil, err + } + + imagePath, err := resolveImageTestPath(env, options) + if err != nil { + return nil, err } - // Validate that the image file exists. if err := validateFileExists(env.FS(), imagePath); err != nil { - return fmt.Errorf("image path:\n%w", err) + return nil, fmt.Errorf("image path:\n%w", err) } options.ImagePath = imagePath - // Absolutize JUnitXMLPath against the user's CWD so pytest writes to the location the - // user expected — pytest itself resolves relative paths against its own working - // directory (the test suite's working-dir), which is rarely what the user intended. if options.JUnitXMLPath != "" && !filepath.IsAbs(options.JUnitXMLPath) { absJUnitPath, err := filepath.Abs(options.JUnitXMLPath) if err != nil { - return fmt.Errorf("failed to resolve --junit-xml path %#q:\n%w", options.JUnitXMLPath, err) + return nil, fmt.Errorf("failed to resolve --junit-xml path %#q:\n%w", options.JUnitXMLPath, err) } options.JUnitXMLPath = absJUnitPath } - resolvedTests, legacySuiteNames, err := resolveImageTestsToRun(cfg, imageConfig, options.TestSuites) + return imageConfig, nil +} + +func resolveImageTestPath(env *azldev.Env, options *ImageTestOptions) (string, error) { + if options.ImagePath != "" { + return options.ImagePath, nil + } + + imagePath, _, err := findImageArtifact(env, options.ImageName, "", AllImageFormats()) if err != nil { - return err + return "", err } - if len(resolvedTests) == 0 && len(legacySuiteNames) == 0 { - slog.Warn("No tests to run for image", slog.String("image", options.ImageName)) + slog.Info("Resolved image artifact", + slog.String("image", options.ImageName), + slog.String("path", imagePath), + ) - return nil - } + return imagePath, nil +} +func runImageTests( + env *azldev.Env, + cfg *projectconfig.ProjectConfig, + imageConfig *projectconfig.ImageConfig, + options *ImageTestOptions, + resolvedTests []projectconfig.ResolvedTest, + legacySuiteNames []string, +) error { var testFailures []string for _, resolvedTest := range resolvedTests { @@ -200,6 +221,7 @@ func runImageTest(env *azldev.Env, options *ImageTestOptions) error { if len(testFailures) > 0 { total := len(resolvedTests) + len(legacySuiteNames) + return fmt.Errorf("%d of %d test(s) failed: %s", len(testFailures), total, strings.Join(testFailures, ", ")) } @@ -207,7 +229,6 @@ func runImageTest(env *azldev.Env, options *ImageTestOptions) error { return nil } - func resolveImageTestsToRun( cfg *projectconfig.ProjectConfig, imageConfig *projectconfig.ImageConfig, @@ -216,11 +237,19 @@ func resolveImageTestsToRun( if imageConfig.Tests != nil && len(imageConfig.Tests.Tests) > 0 { if len(explicitSelectors) > 0 { resolvedTests, err := cfg.ResolveTestSelectors(explicitSelectors) - return resolvedTests, nil, err + if err != nil { + return nil, nil, fmt.Errorf("resolve test selectors: %w", err) + } + + return resolvedTests, nil, nil } resolvedTests, err := cfg.ResolveImageTests(imageConfig) - return resolvedTests, nil, err + if err != nil { + return nil, nil, fmt.Errorf("resolve image tests: %w", err) + } + + return resolvedTests, nil, nil } if len(explicitSelectors) > 0 { @@ -246,10 +275,18 @@ func runResolvedTest( return RunPytestSuite(env, suiteConfig, imageConfig, options) case string(projectconfig.TestTypeLisa): - return fmt.Errorf("LISA tests cannot be run locally via 'azldev image test'; test %#q must be run through the LISA infrastructure", resolvedTest.Name) + return fmt.Errorf( + "LISA tests cannot be run locally via 'azldev image test'; "+ + "test %#q must be run through the LISA infrastructure", + resolvedTest.Name, + ) case "tmt": - return fmt.Errorf("TMT tests cannot be run locally via 'azldev image test'; test %#q is metadata-only for external orchestration", resolvedTest.Name) + return fmt.Errorf( + "TMT tests cannot be run locally via 'azldev image test'; "+ + "test %#q is metadata-only for external orchestration", + resolvedTest.Name, + ) default: return fmt.Errorf("unsupported test type %#q for test %#q", resolvedTest.Definition.Type, resolvedTest.Name) @@ -278,7 +315,7 @@ func testDefinitionToSuiteConfig(resolvedTest projectconfig.ResolvedTest) (*proj func decodePytestConfig(raw map[string]any) (*projectconfig.PytestConfig, error) { if raw == nil { - return nil, fmt.Errorf("missing [pytest] subtable") + return nil, errors.New("missing [pytest] subtable") } bytes, err := toml.Marshal(raw) diff --git a/internal/app/azldev/cmds/image/test_internal_test.go b/internal/app/azldev/cmds/image/test_internal_test.go index 3d34c218..a60ad643 100644 --- a/internal/app/azldev/cmds/image/test_internal_test.go +++ b/internal/app/azldev/cmds/image/test_internal_test.go @@ -16,7 +16,7 @@ func TestResolveImageTestsToRun_UsesNewTestsRefs(t *testing.T) { testEnv := testutils.NewTestEnv(t) testEnv.Config.Tests = map[string]projectconfig.TestDefinition{ "static-image-checks": {Type: "pytest", Pytest: map[string]any{"working-dir": "/project/tests"}}, - "functional_core": {Type: "lisa", Lisa: map[string]any{"criteria": map[string]any{"priority": []any{1}}}}, + "functional_core": {Type: "lisa", Lisa: map[string]any{"criteria": map[string]any{"priority": []any{1}}}}, } testEnv.Config.TestGroups = map[string]projectconfig.TestGroup{ "vm-base-functional": {Tests: []projectconfig.TestRef{{Name: "functional_core"}}}, diff --git a/internal/projectconfig/configfile.go b/internal/projectconfig/configfile.go index e0aea3ac..d3be8891 100644 --- a/internal/projectconfig/configfile.go +++ b/internal/projectconfig/configfile.go @@ -198,7 +198,12 @@ func validateComponentGroupMetadata(groups map[string]ComponentGroupConfig) erro return nil } -func validateNewTestReferences(tests map[string]TestDefinition, groups map[string]TestGroup, components map[string]ComponentConfig, images map[string]ImageConfig) error { +func validateNewTestReferences( + tests map[string]TestDefinition, + groups map[string]TestGroup, + components map[string]ComponentConfig, + images map[string]ImageConfig, +) error { for groupName, group := range groups { scope := fmt.Sprintf("test-group %#q tests", groupName) if err := validateTestGroupMembers(scope, group.Tests, tests); err != nil { @@ -367,6 +372,7 @@ func validateTestRefList( return nil } + // validateSourceFiles checks 'source-files' configuration for a component: // - All filenames must be unique. // - Hash type must be a supported algorithm when specified. diff --git a/internal/projectconfig/configfile_test.go b/internal/projectconfig/configfile_test.go index 8c948e30..8a3fcc66 100644 --- a/internal/projectconfig/configfile_test.go +++ b/internal/projectconfig/configfile_test.go @@ -160,9 +160,9 @@ func TestProjectConfigFileValidation_LisaSelectionUnsupportedCriteriaKey(t *test func TestProjectConfigValidation_UndefinedTestReferenceInGroup(t *testing.T) { cfg := projectconfig.NewProjectConfig() cfg.TestGroups = map[string]projectconfig.TestGroup{ - "bvt": { - Tests: []projectconfig.TestRef{{Name: "does-not-exist"}}, - }, + "bvt": { + Tests: []projectconfig.TestRef{{Name: "does-not-exist"}}, + }, } err := cfg.Validate() @@ -174,11 +174,11 @@ func TestProjectConfigValidation_UndefinedTestReferenceInGroup(t *testing.T) { func TestProjectConfigValidation_UndefinedTestGroupReferenceInComponent(t *testing.T) { cfg := projectconfig.NewProjectConfig() cfg.Components = map[string]projectconfig.ComponentConfig{ - "openssl": { - Tests: &projectconfig.ComponentTestsConfig{ - Tests: []projectconfig.TestRef{{Group: "missing-group"}}, - }, + "openssl": { + Tests: &projectconfig.ComponentTestsConfig{ + Tests: []projectconfig.TestRef{{Group: "missing-group"}}, }, + }, } err := cfg.Validate() @@ -190,20 +190,20 @@ func TestProjectConfigValidation_UndefinedTestGroupReferenceInComponent(t *testi func TestProjectConfigValidation_InvalidTestReferenceShapeInImage(t *testing.T) { cfg := projectconfig.NewProjectConfig() cfg.Images = map[string]projectconfig.ImageConfig{ - "base": { - Tests: &projectconfig.ImageTestsConfig{ - Tests: []projectconfig.TestRef{{Name: "smoke", Group: "bvt"}}, - }, + "base": { + Tests: &projectconfig.ImageTestsConfig{ + Tests: []projectconfig.TestRef{{Name: "smoke", Group: "bvt"}}, }, + }, } cfg.Tests = map[string]projectconfig.TestDefinition{ - "smoke": { - Type: "pytest", - Pytest: map[string]any{"working-dir": "tests"}, - }, + "smoke": { + Type: "pytest", + Pytest: map[string]any{"working-dir": "tests"}, + }, } cfg.TestGroups = map[string]projectconfig.TestGroup{ - "bvt": {Tests: []projectconfig.TestRef{{Name: "smoke"}}}, + "bvt": {Tests: []projectconfig.TestRef{{Name: "smoke"}}}, } err := cfg.Validate() @@ -215,18 +215,18 @@ func TestProjectConfigValidation_InvalidTestReferenceShapeInImage(t *testing.T) func TestProjectConfigValidation_DuplicateTestReferenceInGroup(t *testing.T) { cfg := projectconfig.NewProjectConfig() cfg.Tests = map[string]projectconfig.TestDefinition{ - "smoke": { - Type: "pytest", - Pytest: map[string]any{"working-dir": "tests"}, - }, + "smoke": { + Type: "pytest", + Pytest: map[string]any{"working-dir": "tests"}, + }, } cfg.TestGroups = map[string]projectconfig.TestGroup{ - "bvt": { - Tests: []projectconfig.TestRef{ - {Name: "smoke"}, - {Name: "smoke"}, - }, + "bvt": { + Tests: []projectconfig.TestRef{ + {Name: "smoke"}, + {Name: "smoke"}, }, + }, } err := cfg.Validate() @@ -239,25 +239,25 @@ func TestProjectConfigValidation_DuplicateTestReferenceInGroup(t *testing.T) { func TestProjectConfigValidation_DuplicateTestGroupReferenceInImage(t *testing.T) { cfg := projectconfig.NewProjectConfig() cfg.Tests = map[string]projectconfig.TestDefinition{ - "smoke": { - Type: "pytest", - Pytest: map[string]any{"working-dir": "tests"}, - }, + "smoke": { + Type: "pytest", + Pytest: map[string]any{"working-dir": "tests"}, + }, } cfg.TestGroups = map[string]projectconfig.TestGroup{ - "bvt": { - Tests: []projectconfig.TestRef{{Name: "smoke"}}, - }, + "bvt": { + Tests: []projectconfig.TestRef{{Name: "smoke"}}, + }, } cfg.Images = map[string]projectconfig.ImageConfig{ - "base": { - Tests: &projectconfig.ImageTestsConfig{ - Tests: []projectconfig.TestRef{ - {Group: "bvt"}, - {Group: "bvt"}, - }, + "base": { + Tests: &projectconfig.ImageTestsConfig{ + Tests: []projectconfig.TestRef{ + {Group: "bvt"}, + {Group: "bvt"}, }, }, + }, } err := cfg.Validate() @@ -270,14 +270,14 @@ func TestProjectConfigValidation_DuplicateTestGroupReferenceInImage(t *testing.T func TestProjectConfigValidation_NestedTestGroupReferenceNotAllowed(t *testing.T) { cfg := projectconfig.NewProjectConfig() cfg.Tests = map[string]projectconfig.TestDefinition{ - "smoke": { - Type: "pytest", - Pytest: map[string]any{"working-dir": "tests"}, - }, + "smoke": { + Type: "pytest", + Pytest: map[string]any{"working-dir": "tests"}, + }, } cfg.TestGroups = map[string]projectconfig.TestGroup{ - "a": {Tests: []projectconfig.TestRef{{Group: "b"}}}, - "b": {Tests: []projectconfig.TestRef{{Name: "smoke"}}}, + "a": {Tests: []projectconfig.TestRef{{Group: "b"}}}, + "b": {Tests: []projectconfig.TestRef{{Name: "smoke"}}}, } err := cfg.Validate() @@ -290,11 +290,14 @@ func TestProjectConfigResolveImageTests_ExpandsGroups(t *testing.T) { cfg := projectconfig.NewProjectConfig() cfg.Tests = map[string]projectconfig.TestDefinition{ "static-image-checks": {Type: "pytest", Pytest: map[string]any{"working-dir": "tests"}}, - "functional_core": {Type: "lisa", Lisa: map[string]any{"criteria": map[string]any{"priority": []any{1}}}}, - "lisa_perf": {Type: "lisa", Lisa: map[string]any{"criteria": map[string]any{"area": "network", "category": "performance"}}}, + "functional_core": {Type: "lisa", Lisa: map[string]any{"criteria": map[string]any{"priority": []any{1}}}}, + "lisa_perf": { + Type: "lisa", + Lisa: map[string]any{"criteria": map[string]any{"area": "network", "category": "performance"}}, + }, } cfg.TestGroups = map[string]projectconfig.TestGroup{ - "vm-base-functional": {Tests: []projectconfig.TestRef{{Name: "functional_core"}}}, + "vm-base-functional": {Tests: []projectconfig.TestRef{{Name: "functional_core"}}}, "vm-base-performance": {Tests: []projectconfig.TestRef{{Name: "lisa_perf"}}}, } diff --git a/internal/projectconfig/tests.go b/internal/projectconfig/tests.go index 680ab606..e1795565 100644 --- a/internal/projectconfig/tests.go +++ b/internal/projectconfig/tests.go @@ -19,6 +19,7 @@ type ResolvedTest struct { Name string Definition TestDefinition } + // TestKind indicates what kind of behavior a test exercises. type TestKind string @@ -90,6 +91,7 @@ func (t TestDefinition) WithAbsolutePaths(referenceDir string) TestDefinition { return result } + // TestGroup is a [test-groups.X] declaration: a named bundle of test references that // images or components can target via a single name. type TestGroup struct { @@ -212,6 +214,7 @@ func cloneStringAnyMap(input map[string]any) map[string]any { return result } + // Validate checks that exactly one framework subtable is set and it matches Type. func (t TestDefinition) Validate(testName string) error { if t.Type == "" { From ae9c0392ac737e92f17e78e08b1de3fb7574aeb0 Mon Sep 17 00:00:00 2001 From: bhagyapathak Date: Tue, 11 Aug 2026 13:14:49 +0530 Subject: [PATCH 5/5] docs: regenerate CLI reference for test/test-groups schema Update azldev_image_test.md documentation to reflect new test and test-group references alongside legacy test-suite support. --- docs/user/reference/cli/azldev_image_test.md | 23 ++++++++-------- internal/projectconfig/configfile_test.go | 2 +- internal/projectconfig/tests.go | 28 ++++++++++---------- 3 files changed, 26 insertions(+), 27 deletions(-) diff --git a/docs/user/reference/cli/azldev_image_test.md b/docs/user/reference/cli/azldev_image_test.md index 1d4bf3f5..7cbf96ec 100644 --- a/docs/user/reference/cli/azldev_image_test.md +++ b/docs/user/reference/cli/azldev_image_test.md @@ -6,15 +6,14 @@ Run tests against an Azure Linux image ### Synopsis -Run tests against an Azure Linux image using test suites defined in the +Run tests against an Azure Linux image using test definitions declared in the project configuration. -Test suites are defined in the [test-suites] section of azldev.toml and referenced -by images via the [images.NAME.tests] subtable. Each test suite specifies a type -(pytest or lisa) and framework-specific configuration in a matching subtable. +Images may reference tests directly via [images.NAME.tests.tests] entries, or via +named [test-groups]. Legacy [test-suites] references are still supported. -By default, all test suites associated with the named image are run. Use ---test-suite to select specific suites (may be repeated). +By default, all tests associated with the named image are run. Use +--test-suite to select specific test names or test-group names (may be repeated). The image artifact can be specified explicitly with --image-path, or resolved automatically from the image name in the output directory. @@ -38,17 +37,17 @@ azldev image test IMAGE_NAME [flags] ### Examples ``` - # Run all test suites for an image (artifact auto-resolved from output dir) + # Run all tests for an image (artifact auto-resolved from output dir) azldev image test vm-base # Run all test suites with an explicit image path azldev image test vm-base --image-path ./out/images/vm-base/image.raw - # Run a specific test suite - azldev image test vm-base --test-suite common-vm-checks + # Run a specific test + azldev image test vm-base --test-suite static-image-checks - # Run multiple specific test suites - azldev image test vm-base --test-suite common-vm-checks --test-suite vm-base-checks + # Run multiple tests or a test-group + azldev image test vm-base --test-suite static-image-checks --test-suite vm-base-functional # Generate JUnit XML output azldev image test vm-base --junit-xml results.xml @@ -60,7 +59,7 @@ azldev image test IMAGE_NAME [flags] -h, --help help for test -i, --image-path string Path to the disk image file (resolved from image name if not specified) --junit-xml string Path for writing JUnit XML output - --test-suite strings Name of a test suite to run (may be repeated; defaults to all suites for the image) + --test-suite strings Name of a test or test-group to run (may be repeated; defaults to all tests for the image) ``` ### Options inherited from parent commands diff --git a/internal/projectconfig/configfile_test.go b/internal/projectconfig/configfile_test.go index 8a3fcc66..ca4345c2 100644 --- a/internal/projectconfig/configfile_test.go +++ b/internal/projectconfig/configfile_test.go @@ -129,7 +129,7 @@ func TestProjectConfigFileValidation_LisaSelectionCriteriaValid(t *testing.T) { Lisa: map[string]any{ "criteria": []any{ map[string]any{"area": "network", "category": "performance"}, - map[string]any{"testcaseNames": []any{"case_a", "case_b"}}, + map[string]any{"testcase-names": []any{"case_a", "case_b"}}, }, }, }, diff --git a/internal/projectconfig/tests.go b/internal/projectconfig/tests.go index e1795565..a997c544 100644 --- a/internal/projectconfig/tests.go +++ b/internal/projectconfig/tests.go @@ -320,12 +320,12 @@ func validateLisaSelection(lisa map[string]any, testName string) error { } } - if rawName, ok := lisa["testcaseName"]; ok { + if rawName, ok := lisa["testcase-name"]; ok { hasSelector = true if !isNonEmptyString(rawName) { return fmt.Errorf( - "%w: test %#q lisa.testcaseName must be a non-empty string", + "%w: test %#q lisa.testcase-name must be a non-empty string", ErrInvalidLisaSelection, testName, ) @@ -344,17 +344,17 @@ func validateLisaSelection(lisa map[string]any, testName string) error { } } - if rawNames, ok := lisa["testcaseNames"]; ok { + if rawNames, ok := lisa["testcase-names"]; ok { hasSelector = true - if err := validateStringList(rawNames, "lisa.testcaseNames", testName); err != nil { + if err := validateStringList(rawNames, "lisa.testcase-names", testName); err != nil { return err } } if !hasSelector { return fmt.Errorf( - "%w: test %#q of type %#q must set at least one LISA selector: criteria, testcaseName, testcaseNames, or name", + "%w: test %#q of type %#q must set at least one LISA selector: criteria, testcase-name, testcase-names, or name", ErrInvalidLisaSelection, testName, "lisa", @@ -415,13 +415,13 @@ func normalizeCriteriaList(rawCriteria any) ([]map[string]any, error) { func validateSingleLisaCriteria(criteria map[string]any, testName string, idx int) error { allowedKeys := map[string]bool{ - "name": true, - "area": true, - "category": true, - "priority": true, - "tags": true, - "testcaseName": true, - "testcaseNames": true, + "name": true, + "area": true, + "category": true, + "priority": true, + "tags": true, + "testcase-name": true, + "testcase-names": true, } hasSelector := false @@ -438,7 +438,7 @@ func validateSingleLisaCriteria(criteria map[string]any, testName string, idx in } switch key { - case "name", "area", "category", "testcaseName": + case "name", "area", "category", "testcase-name": if !isNonEmptyString(value) { return fmt.Errorf( "%w: test %#q lisa.criteria[%d].%s must be a non-empty string", @@ -456,7 +456,7 @@ func validateSingleLisaCriteria(criteria map[string]any, testName string, idx in } hasSelector = true - case "tags", "testcaseNames": + case "tags", "testcase-names": fieldName := "lisa.criteria[" + strconv.Itoa(idx) + "]." + key if err := validateStringList(value, fieldName, testName); err != nil {