diff --git a/api/generated/openapi/zz_generated.openapi.go b/api/generated/openapi/zz_generated.openapi.go index 70f67388e..bf1edc1a6 100644 --- a/api/generated/openapi/zz_generated.openapi.go +++ b/api/generated/openapi/zz_generated.openapi.go @@ -1050,20 +1050,20 @@ func schema_porch_api_porch_v1alpha1_PackageRevisionStatus(ref common.ReferenceC }, "selfLock": { SchemaProps: spec.SchemaProps{ - Description: "SelfLock identifies the location of the current package's data", + Description: "SelfLock identifies the location of the current package's data.", Ref: ref("github.com/kptdev/porch/api/porch/v1alpha1.Locator"), }, }, "publishedBy": { SchemaProps: spec.SchemaProps{ - Description: "PublishedBy is the identity of the user who approved the packagerevision.", + Description: "PublishedBy is the identity of the user who approved the package revision.", Type: []string{"string"}, Format: "", }, }, "publishTimestamp": { SchemaProps: spec.SchemaProps{ - Description: "PublishedAt is the time when the packagerevision were approved.", + Description: "PublishedAt is the time when the package revision was approved.", Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.Time"), }, }, @@ -1087,6 +1087,13 @@ func schema_porch_api_porch_v1alpha1_PackageRevisionStatus(ref common.ReferenceC }, }, }, + "resourcesSizeBytes": { + SchemaProps: spec.SchemaProps{ + Description: "ResourcesSizeBytes is the total file size, in bytes, of the package revision's resources.", + Type: []string{"integer"}, + Format: "int64", + }, + }, }, }, }, diff --git a/api/porch/types.go b/api/porch/types.go index 9c2739d2c..b2391b0f0 100644 --- a/api/porch/types.go +++ b/api/porch/types.go @@ -149,19 +149,22 @@ type PackageRevisionStatus struct { // UpstreamLock identifies the upstream data for this package. UpstreamLock *Locator `json:"upstreamLock,omitempty"` - // SelfLock identifies the location of the current package's data + // SelfLock identifies the location of the current package's data. SelfLock *Locator `json:"selfLock,omitempty"` - // PublishedBy is the identity of the user who approved the packagerevision. + // PublishedBy is the identity of the user who approved the package revision. PublishedBy string `json:"publishedBy,omitempty"` - // PublishedAt is the time when the packagerevision were approved. + // PublishedAt is the time when the package revision was approved. PublishedAt metav1.Time `json:"publishTimestamp,omitempty"` // Deployment is true if this is a deployment package (in a deployment repository). Deployment bool `json:"deployment,omitempty"` Conditions []Condition `json:"conditions,omitempty"` + + // ResourcesSizeBytes is the total file size, in bytes, of the package revision's resources. + ResourcesSizeBytes int64 `json:"resourcesSizeBytes,omitempty"` } type TaskType string diff --git a/api/porch/v1alpha1/types.go b/api/porch/v1alpha1/types.go index 0fa0ef8ea..75205c9f2 100644 --- a/api/porch/v1alpha1/types.go +++ b/api/porch/v1alpha1/types.go @@ -149,19 +149,22 @@ type PackageRevisionStatus struct { // UpstreamLock identifies the upstream data for this package. UpstreamLock *Locator `json:"upstreamLock,omitempty"` - // SelfLock identifies the location of the current package's data + // SelfLock identifies the location of the current package's data. SelfLock *Locator `json:"selfLock,omitempty"` - // PublishedBy is the identity of the user who approved the packagerevision. + // PublishedBy is the identity of the user who approved the package revision. PublishedBy string `json:"publishedBy,omitempty"` - // PublishedAt is the time when the packagerevision were approved. + // PublishedAt is the time when the package revision was approved. PublishedAt metav1.Time `json:"publishTimestamp,omitempty"` // Deployment is true if this is a deployment package (in a deployment repository). Deployment bool `json:"deployment,omitempty"` Conditions []Condition `json:"conditions,omitempty"` + + // ResourcesSizeBytes is the total file size, in bytes, of the package revision's resources. + ResourcesSizeBytes int64 `json:"resourcesSizeBytes,omitempty"` } type TaskType string diff --git a/api/porch/v1alpha1/zz_generated.conversion.go b/api/porch/v1alpha1/zz_generated.conversion.go index ae5168f1e..eb6b26088 100644 --- a/api/porch/v1alpha1/zz_generated.conversion.go +++ b/api/porch/v1alpha1/zz_generated.conversion.go @@ -932,6 +932,7 @@ func autoConvert_v1alpha1_PackageRevisionStatus_To_porch_PackageRevisionStatus(i out.PublishedAt = in.PublishedAt out.Deployment = in.Deployment out.Conditions = *(*[]porch.Condition)(unsafe.Pointer(&in.Conditions)) + out.ResourcesSizeBytes = in.ResourcesSizeBytes return nil } @@ -947,6 +948,7 @@ func autoConvert_porch_PackageRevisionStatus_To_v1alpha1_PackageRevisionStatus(i out.PublishedAt = in.PublishedAt out.Deployment = in.Deployment out.Conditions = *(*[]Condition)(unsafe.Pointer(&in.Conditions)) + out.ResourcesSizeBytes = in.ResourcesSizeBytes return nil } diff --git a/api/porch/v1alpha2/packagerevision_types.go b/api/porch/v1alpha2/packagerevision_types.go index 4a10267df..89d6feae5 100644 --- a/api/porch/v1alpha2/packagerevision_types.go +++ b/api/porch/v1alpha2/packagerevision_types.go @@ -169,10 +169,10 @@ type PackageRevisionStatus struct { // SelfLock identifies the location of the current package's data SelfLock *Locator `json:"selfLock,omitempty"` - // PublishedBy is the identity of the user who approved the packagerevision. + // PublishedBy is the identity of the user who approved the package revision. PublishedBy string `json:"publishedBy,omitempty"` - // PublishedAt is the time when the packagerevision were approved. + // PublishedAt is the time when the package revision was approved. // +optional PublishedAt *metav1.Time `json:"publishedAt,omitempty"` @@ -200,6 +200,9 @@ type PackageRevisionStatus struct { // +listType=map // +listMapKey=type Conditions []metav1.Condition `json:"conditions,omitempty"` + + // ResourcesSizeBytes is the total file size, in bytes, of the package revision's resources. + ResourcesSizeBytes int64 `json:"resourcesSizeBytes,omitempty"` } // PackageSource specifies how a package was created. diff --git a/api/porch/v1alpha2/porch.kpt.dev_packagerevisions.yaml b/api/porch/v1alpha2/porch.kpt.dev_packagerevisions.yaml index 23820a4be..07d0758dd 100644 --- a/api/porch/v1alpha2/porch.kpt.dev_packagerevisions.yaml +++ b/api/porch/v1alpha2/porch.kpt.dev_packagerevisions.yaml @@ -374,19 +374,24 @@ spec: type: object type: array publishedAt: - description: PublishedAt is the time when the packagerevision were + description: PublishedAt is the time when the package revision was approved. format: date-time type: string publishedBy: description: PublishedBy is the identity of the user who approved - the packagerevision. + the package revision. type: string renderingPrrResourceVersion: description: |- RenderingPrrResourceVersion tracks the PRR resourceVersion currently being rendered. Prevents concurrent renders. type: string + resourcesSizeBytes: + description: ResourcesSizeBytes is the total file size, in bytes, + of the package revision's resources. + format: int64 + type: integer revision: description: |- Revision identifies the version of the package. diff --git a/api/sql/porch-db-1.5.10-1.5.9.sql b/api/sql/porch-db-1.5.10-1.5.9.sql new file mode 100644 index 000000000..fd9494eb7 --- /dev/null +++ b/api/sql/porch-db-1.5.10-1.5.9.sql @@ -0,0 +1,18 @@ +/* +Copyright 2026 The kpt Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +ALTER TABLE package_revisions + DROP COLUMN IF EXISTS resources_size; \ No newline at end of file diff --git a/api/sql/porch-db-1.5.9-1.5.10.sql b/api/sql/porch-db-1.5.9-1.5.10.sql new file mode 100644 index 000000000..710d9240c --- /dev/null +++ b/api/sql/porch-db-1.5.9-1.5.10.sql @@ -0,0 +1,37 @@ +/* +Copyright 2026 The kpt Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +-- Add the resources_size column - stores the total size of a package revision's +-- resource files in bytes so it can be found without having to retrieve the full +-- resources. +ALTER TABLE package_revisions + ADD COLUMN IF NOT EXISTS resources_size BIGINT NOT NULL DEFAULT 0; + +-- In the event of an upgrade with repositories already synced, Porch's sync +-- routines (manual or background) will not detect the need to backfill +-- resource sizes for existing package revisions. +-- Go through them once, calculating their resource sizes, and backfill the +-- resources_size column. +UPDATE package_revisions pr +SET resources_size = r.total_size +FROM ( + SELECT k8s_name_space, k8s_name, revision, SUM(OCTET_LENGTH(resource_value)) AS total_size + FROM resources + GROUP BY k8s_name_space, k8s_name, revision +) r +WHERE pr.k8s_name_space = r.k8s_name_space + AND pr.k8s_name = r.k8s_name + AND pr.revision = r.revision; \ No newline at end of file diff --git a/api/sql/porch-db-1.5.9-1.6.0.sql b/api/sql/porch-db-1.5.9-1.6.0.sql new file mode 100644 index 000000000..710d9240c --- /dev/null +++ b/api/sql/porch-db-1.5.9-1.6.0.sql @@ -0,0 +1,37 @@ +/* +Copyright 2026 The kpt Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +-- Add the resources_size column - stores the total size of a package revision's +-- resource files in bytes so it can be found without having to retrieve the full +-- resources. +ALTER TABLE package_revisions + ADD COLUMN IF NOT EXISTS resources_size BIGINT NOT NULL DEFAULT 0; + +-- In the event of an upgrade with repositories already synced, Porch's sync +-- routines (manual or background) will not detect the need to backfill +-- resource sizes for existing package revisions. +-- Go through them once, calculating their resource sizes, and backfill the +-- resources_size column. +UPDATE package_revisions pr +SET resources_size = r.total_size +FROM ( + SELECT k8s_name_space, k8s_name, revision, SUM(OCTET_LENGTH(resource_value)) AS total_size + FROM resources + GROUP BY k8s_name_space, k8s_name, revision +) r +WHERE pr.k8s_name_space = r.k8s_name_space + AND pr.k8s_name = r.k8s_name + AND pr.revision = r.revision; \ No newline at end of file diff --git a/api/sql/porch-db-1.6.0-1.5.9.sql b/api/sql/porch-db-1.6.0-1.5.9.sql new file mode 100644 index 000000000..fd9494eb7 --- /dev/null +++ b/api/sql/porch-db-1.6.0-1.5.9.sql @@ -0,0 +1,18 @@ +/* +Copyright 2026 The kpt Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +ALTER TABLE package_revisions + DROP COLUMN IF EXISTS resources_size; \ No newline at end of file diff --git a/api/sql/porch-db.sql b/api/sql/porch-db.sql index a5ec725e1..c922c7080 100644 --- a/api/sql/porch-db.sql +++ b/api/sql/porch-db.sql @@ -1,5 +1,5 @@ /* -Copyright 2024-2025 The kpt Authors +Copyright 2024-2026 The kpt Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -86,6 +86,7 @@ CREATE TABLE IF NOT EXISTS package_revisions ( latest BOOLEAN NOT NULL DEFAULT FALSE, tasks TEXT NOT NULL, kptfile_status TEXT NOT NULL DEFAULT '{}', + resources_size BIGINT NOT NULL DEFAULT 0, PRIMARY KEY (k8s_name_space, k8s_name), CONSTRAINT fk_package FOREIGN KEY (k8s_name_space, package_k8s_name) diff --git a/controllers/packagevariants/pkg/controllers/packagevariant/packagevariant_controller-with-workspacename_test.go b/controllers/packagevariants/pkg/controllers/packagevariant/packagevariant_controller-with-workspacename_test.go index 44dabacbd..882ff3e96 100644 --- a/controllers/packagevariants/pkg/controllers/packagevariant/packagevariant_controller-with-workspacename_test.go +++ b/controllers/packagevariants/pkg/controllers/packagevariant/packagevariant_controller-with-workspacename_test.go @@ -1,4 +1,4 @@ -// Copyright 2022 The kpt Authors +// Copyright 2025 The kpt Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/deployments/porch/3-porch-postgres-bundle.yaml b/deployments/porch/3-porch-postgres-bundle.yaml index fd6d46f5a..11894d971 100644 --- a/deployments/porch/3-porch-postgres-bundle.yaml +++ b/deployments/porch/3-porch-postgres-bundle.yaml @@ -354,6 +354,7 @@ data: latest BOOLEAN NOT NULL DEFAULT FALSE, tasks TEXT NOT NULL, kptfile_status TEXT NOT NULL DEFAULT '{}', + resources_size BIGINT NOT NULL DEFAULT 0, PRIMARY KEY (k8s_name_space, k8s_name), CONSTRAINT fk_package FOREIGN KEY (k8s_name_space, package_k8s_name) diff --git a/docs/content/en/docs/2_concepts/package-revisions.md b/docs/content/en/docs/2_concepts/package-revisions.md index ba04af5c3..938e50ae2 100644 --- a/docs/content/en/docs/2_concepts/package-revisions.md +++ b/docs/content/en/docs/2_concepts/package-revisions.md @@ -122,8 +122,8 @@ spec: apiVersion: config.porch.kpt.dev/v1alpha1 kind: Repository metadata: -name: catalog-v4-0-0 -namespace: default + name: catalog-v4-0-0 + namespace: default spec: content: Package deployment: false @@ -138,14 +138,14 @@ spec: apiVersion: config.porch.kpt.dev/v1alpha1 kind: Repository metadata: -name: catalog-v3-0-0 -namespace: default + name: catalog-v3-0-0 + namespace: default spec: content: Package deployment: false type: git git: - branch: v3.0.0 + branch: v3 directory: / repo: https://github.com/nephio-project/catalog.git ``` diff --git a/docs/content/en/docs/5_architecture_and_components/package-cache/db-cache.md b/docs/content/en/docs/5_architecture_and_components/package-cache/db-cache.md index 43ee05422..a1978bdc2 100644 --- a/docs/content/en/docs/5_architecture_and_components/package-cache/db-cache.md +++ b/docs/content/en/docs/5_architecture_and_components/package-cache/db-cache.md @@ -44,7 +44,7 @@ PostgreSQL Database **Database Storage:** - Repository connections and metadata - Package metadata (names, paths) -- Package revision metadata (lifecycle, tasks, upstream locks) +- Package revision metadata (lifecycle, tasks, upstream locks, package resources size) - Package resources (full KRM resource content) - Timestamps and user tracking for all entities @@ -242,7 +242,9 @@ The DB Cache has a **database-first approach** to draft packages by default, but - Draft packages stored entirely in PostgreSQL - All draft modifications update database, not Git - UpdatePackageRevision operations modify database records -- ClosePackageRevisionDraft saves to database without Git interaction +- ClosePackageRevisionDraft: + - sums up package file size based on state of resources at this point + - saves to database without Git interaction **Default Git interaction pattern:** - **Draft creation**: No Git interaction (database only) diff --git a/docs/content/en/docs/7_cli_api/api-ref.md b/docs/content/en/docs/7_cli_api/api-ref.md index c28d17ed0..0fdf93785 100644 --- a/docs/content/en/docs/7_cli_api/api-ref.md +++ b/docs/content/en/docs/7_cli_api/api-ref.md @@ -362,11 +362,12 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | | `upstreamLock` _[Locator](#locator)_ | UpstreamLock identifies the upstream data for this package. | | | -| `selfLock` _[Locator](#locator)_ | SelfLock identifies the location of the current package's data | | | -| `publishedBy` _string_ | PublishedBy is the identity of the user who approved the packagerevision. | | | -| `publishTimestamp` _[Time](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.34/#time-v1-meta)_ | PublishedAt is the time when the packagerevision were approved. | | | +| `selfLock` _[Locator](#locator)_ | SelfLock identifies the location of the current package's data. | | | +| `publishedBy` _string_ | PublishedBy is the identity of the user who approved the package revision. | | | +| `publishTimestamp` _[Time](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.34/#time-v1-meta)_ | PublishedAt is the time when the package revision was approved. | | | | `deployment` _boolean_ | Deployment is true if this is a deployment package (in a deployment repository). | | | | `conditions` _[Condition](#condition) array_ | | | | +| `resourcesSizeBytes` _integer_ | ResourcesSizeBytes is the total file size, in bytes, of the package revision's resources. | | | #### PackageSpec @@ -635,5 +636,3 @@ _Appears in:_ | `type` _[RepositoryType](#repositorytype)_ | Type of the repository (i.e. git). If empty, `upstreamRef` will be used. | | | | `git` _[GitPackage](#gitpackage)_ | Git upstream package specification. Required if `type` is `git`. Must be unspecified if `type` is not `git`. | | | | `upstreamRef` _[PackageRevisionRef](#packagerevisionref)_ | UpstreamRef is the reference to the package from a registered repository rather than external package. | | | - - diff --git a/go.mod b/go.mod index 4c1bd2933..bcca49e43 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ replace k8s.io/apiserver v0.34.1 => ./third_party/k8s.io/apiserver-v0.34.1 require ( cloud.google.com/go/iam v1.5.3 - github.com/fergusstrange/embedded-postgres v1.32.0 + github.com/fergusstrange/embedded-postgres v1.34.0 github.com/fsnotify/fsnotify v1.9.0 github.com/go-git/go-billy/v5 v5.9.0 github.com/go-git/go-git/v5 v5.19.0 diff --git a/go.sum b/go.sum index 8413e1232..7650f65f6 100644 --- a/go.sum +++ b/go.sum @@ -110,8 +110,8 @@ github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f h1:Wl78ApPPB2 github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f/go.mod h1:OSYXu++VVOHnXeitef/D8n/6y4QV8uLHSFXX4NeXMGc= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= -github.com/fergusstrange/embedded-postgres v1.32.0 h1:kh2ozEvAx2A0LoIJZEGNwHmoFTEQD243KrHjifcYGMo= -github.com/fergusstrange/embedded-postgres v1.32.0/go.mod h1:w0YvnCgf19o6tskInrOOACtnqfVlOvluz3hlNLY7tRk= +github.com/fergusstrange/embedded-postgres v1.34.0 h1:c6RKhPKFsLVU+Tdxsx8q0UxCHsvZZ/iShAnljRBXs6s= +github.com/fergusstrange/embedded-postgres v1.34.0/go.mod h1:w0YvnCgf19o6tskInrOOACtnqfVlOvluz3hlNLY7tRk= github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= diff --git a/pkg/cache/dbcache/dbpackagerevision.go b/pkg/cache/dbcache/dbpackagerevision.go index 779f03295..a11806b4f 100644 --- a/pkg/cache/dbcache/dbpackagerevision.go +++ b/pkg/cache/dbcache/dbpackagerevision.go @@ -1,4 +1,4 @@ -// Copyright 2024-2025 The kpt Authors +// Copyright 2024-2026 The kpt Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -75,20 +75,21 @@ func extractFromKptfile(resources map[string]string) (kptfileStatus, []porchapi. } type dbPackageRevision struct { - repo *dbRepository - pkgRevKey repository.PackageRevisionKey - meta metav1.ObjectMeta - spec *porchapi.PackageRevisionSpec - updated time.Time - updatedBy string - lifecycle porchapi.PackageRevisionLifecycle - extPRID kptfile.Locator - latest bool - deployment bool - tasks []porchapi.Task - resources map[string]string - resourcesDirty bool - kptfileStatus kptfileStatus + repo *dbRepository + pkgRevKey repository.PackageRevisionKey + meta metav1.ObjectMeta + spec *porchapi.PackageRevisionSpec + updated time.Time + updatedBy string + lifecycle porchapi.PackageRevisionLifecycle + extPRID kptfile.Locator + latest bool + deployment bool + tasks []porchapi.Task + resources map[string]string + resourcesDirty bool + kptfileStatus kptfileStatus + resourcesSizeBytes int64 // gitDraftPR maintains the draft in the external git repository during editing (when pushDraftsToGit is true) gitPRDraft repository.PackageRevisionDraft @@ -236,10 +237,11 @@ func (pr *dbPackageRevision) GetPackageRevision(ctx context.Context) (*porchapi. _, selfLock, _ := pr.GetLock(ctx) status := porchapi.PackageRevisionStatus{ - UpstreamLock: repository.KptUpstreamLock2APIUpstreamLock(upstreamLock), - SelfLock: repository.KptUpstreamLock2APIUpstreamLock(selfLock), - Deployment: pr.deployment, - Conditions: pr.kptfileStatus.Conditions, + UpstreamLock: repository.KptUpstreamLock2APIUpstreamLock(upstreamLock), + SelfLock: repository.KptUpstreamLock2APIUpstreamLock(selfLock), + Deployment: pr.deployment, + Conditions: pr.kptfileStatus.Conditions, + ResourcesSizeBytes: pr.resourcesSizeBytes, } if porchapi.LifecycleIsPublished(pr.Lifecycle(ctx)) { @@ -347,17 +349,18 @@ func (pr *dbPackageRevision) ToMainPackageRevision(ctx context.Context) reposito Revision: -1, WorkspaceName: pr.Key().RKey().PlaceholderWSname, }, - meta: metav1.ObjectMeta{}, - spec: &porchapi.PackageRevisionSpec{}, - updated: time.Now(), - updatedBy: getCurrentUser(), - lifecycle: pr.lifecycle, - extPRID: pr.extPRID, - latest: false, - deployment: pr.deployment, - tasks: pr.tasks, - resources: pr.resources, - kptfileStatus: pr.kptfileStatus, + meta: metav1.ObjectMeta{}, + spec: &porchapi.PackageRevisionSpec{}, + updated: time.Now(), + updatedBy: getCurrentUser(), + lifecycle: pr.lifecycle, + extPRID: pr.extPRID, + latest: false, + deployment: pr.deployment, + tasks: pr.tasks, + resources: pr.resources, + kptfileStatus: pr.kptfileStatus, + resourcesSizeBytes: pr.resourcesSizeBytes, } mainPR.meta.CreationTimestamp = metav1.Time{Time: time.Now()} @@ -452,6 +455,7 @@ func (pr *dbPackageRevision) copyToThis(otherPr *dbPackageRevision) { pr.lifecycle = otherPr.lifecycle pr.tasks = otherPr.tasks pr.resources = otherPr.resources + pr.resourcesSizeBytes = otherPr.resourcesSizeBytes } func (pr *dbPackageRevision) UpdateResources(ctx context.Context, new *porchapi.PackageRevisionResources, change *porchapi.Task) error { diff --git a/pkg/cache/dbcache/dbpackagerevisionsql.go b/pkg/cache/dbcache/dbpackagerevisionsql.go index 7622bd6f7..284e10360 100644 --- a/pkg/cache/dbcache/dbpackagerevisionsql.go +++ b/pkg/cache/dbcache/dbpackagerevisionsql.go @@ -1,4 +1,4 @@ -// Copyright 2024-2025 The kpt Authors +// Copyright 2024-2026 The kpt Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -52,7 +52,8 @@ func pkgRevReadFromDB(ctx context.Context, prk repository.PackageRevisionKey, re package_revisions.ext_pr_id, package_revisions.latest, package_revisions.tasks, - package_revisions.kptfile_status + package_revisions.kptfile_status, + package_revisions.resources_size FROM package_revisions INNER JOIN packages ON package_revisions.k8s_name_space=packages.k8s_name_space AND package_revisions.package_k8s_name=packages.k8s_name INNER JOIN repositories @@ -124,7 +125,8 @@ func pkgRevListPRsFromDB(ctx context.Context, filter repository.ListPackageRevis package_revisions.ext_pr_id, package_revisions.latest, package_revisions.tasks, - package_revisions.kptfile_status + package_revisions.kptfile_status, + package_revisions.resources_size FROM package_revisions INNER JOIN packages ON package_revisions.k8s_name_space=packages.k8s_name_space AND package_revisions.package_k8s_name=packages.k8s_name @@ -173,7 +175,8 @@ func pkgRevReadPRsFromDB(ctx context.Context, pk repository.PackageKey) ([]*dbPa package_revisions.ext_pr_id, package_revisions.latest, package_revisions.tasks, - package_revisions.kptfile_status + package_revisions.kptfile_status, + package_revisions.resources_size FROM package_revisions INNER JOIN packages ON package_revisions.k8s_name_space=packages.k8s_name_space AND package_revisions.package_k8s_name=packages.k8s_name INNER JOIN repositories @@ -223,7 +226,8 @@ func pkgRevReadLatestPRFromDB(ctx context.Context, pk repository.PackageKey) (*d package_revisions.ext_pr_id, package_revisions.latest, package_revisions.tasks, - package_revisions.kptfile_status + package_revisions.kptfile_status, + package_revisions.resources_size FROM package_revisions INNER JOIN packages ON package_revisions.k8s_name_space=packages.k8s_name_space AND package_revisions.package_k8s_name=packages.k8s_name INNER JOIN repositories @@ -310,7 +314,8 @@ func pkgRevScanRowsFromDB(ctx context.Context, rows *sql.Rows) ([]*dbPackageRevi &extPRID, &pkgRev.latest, &tasks, - &kptfileStatusJSON) + &kptfileStatusJSON, + &pkgRev.resourcesSizeBytes) if err != nil { klog.Warningf("pkgRevScanRowsFromDB: scanning rows failed: %q", err) @@ -346,8 +351,8 @@ func pkgRevWriteToDB(ctx context.Context, pr *dbPackageRevision) error { klog.V(5).Infof("pkgRevWriteToDB: writing package revision %+v", pr.Key()) sqlStatement := ` - INSERT INTO package_revisions (k8s_name_space, k8s_name, package_k8s_name, revision, meta, spec, updated, updatedby, lifecycle, ext_pr_id, tasks, kptfile_status) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) + INSERT INTO package_revisions (k8s_name_space, k8s_name, package_k8s_name, revision, meta, spec, updated, updatedby, lifecycle, ext_pr_id, tasks, kptfile_status, resources_size) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13) ` klog.V(6).Infof("pkgRevWriteToDB: running query %q on package revision %+v", sqlStatement, pr) @@ -355,7 +360,7 @@ func pkgRevWriteToDB(ctx context.Context, pr *dbPackageRevision) error { if _, err := GetDB().db.Exec(ctx, sqlStatement, prk.K8SNS(), prk.K8SName(), - prk.PKey().K8SName(), prk.Revision, valueAsJSON(pr.meta), valueAsJSON(pr.spec), pr.updated, pr.updatedBy, pr.lifecycle, valueAsJSON(pr.extPRID), valueAsJSON(pr.tasks), valueAsJSON(pr.kptfileStatus)); err == nil { + prk.PKey().K8SName(), prk.Revision, valueAsJSON(pr.meta), valueAsJSON(pr.spec), pr.updated, pr.updatedBy, pr.lifecycle, valueAsJSON(pr.extPRID), valueAsJSON(pr.tasks), valueAsJSON(pr.kptfileStatus), pr.resourcesSizeBytes); err == nil { klog.V(5).Infof("pkgRevWriteToDB: query succeeded, row created") } else { klog.Warningf("pkgRevWriteToDB: query failed for %+v %q", pr.Key(), err) @@ -378,28 +383,29 @@ func pkgRevUpdateDB(ctx context.Context, pr *dbPackageRevision, updateResources klog.V(5).Infof("pkgRevUpdateDB: updating package revision %+v", pr.Key()) sqlStatement := ` - UPDATE package_revisions SET package_k8s_name=$3, revision=$4, meta=$5, spec=$6, updated=$7, updatedby=$8, lifecycle=$9, ext_pr_id=$10, tasks=$11, kptfile_status=$12 + UPDATE package_revisions SET package_k8s_name=$3, revision=$4, meta=$5, spec=$6, updated=$7, updatedby=$8, lifecycle=$9, ext_pr_id=$10, tasks=$11, kptfile_status=$12, resources_size=$13 WHERE k8s_name_space=$1 AND k8s_name=$2 ` if pr.pkgRevKey.Revision == -1 { sqlStatement = ` INSERT INTO package_revisions ( - k8s_name_space, k8s_name, package_k8s_name, revision, meta, spec, updated, updatedby, lifecycle, ext_pr_id, tasks, kptfile_status + k8s_name_space, k8s_name, package_k8s_name, revision, meta, spec, updated, updatedby, lifecycle, ext_pr_id, tasks, kptfile_status, resources_size ) VALUES ( - $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12 + $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13 ) ON CONFLICT (k8s_name_space, k8s_name) DO UPDATE SET package_k8s_name = EXCLUDED.package_k8s_name, meta = EXCLUDED.meta, - revision = EXCLUDED.revision, + revision = EXCLUDED.revision, spec = EXCLUDED.spec, updated = EXCLUDED.updated, updatedby = EXCLUDED.updatedby, lifecycle = EXCLUDED.lifecycle, ext_pr_id = EXCLUDED.ext_pr_id, tasks = EXCLUDED.tasks, - kptfile_status = EXCLUDED.kptfile_status; + kptfile_status = EXCLUDED.kptfile_status, + resources_size = EXCLUDED.resources_size; ` } @@ -408,7 +414,7 @@ func pkgRevUpdateDB(ctx context.Context, pr *dbPackageRevision, updateResources result, err := GetDB().db.Exec(ctx, sqlStatement, prk.K8SNS(), prk.K8SName(), - prk.PKey().K8SName(), prk.Revision, valueAsJSON(pr.meta), valueAsJSON(pr.spec), pr.updated, pr.updatedBy, pr.lifecycle, valueAsJSON(pr.extPRID), valueAsJSON(pr.tasks), valueAsJSON(pr.kptfileStatus)) + prk.PKey().K8SName(), prk.Revision, valueAsJSON(pr.meta), valueAsJSON(pr.spec), pr.updated, pr.updatedBy, pr.lifecycle, valueAsJSON(pr.extPRID), valueAsJSON(pr.tasks), valueAsJSON(pr.kptfileStatus), pr.resourcesSizeBytes) if err == nil { if rowsAffected, _ := result.RowsAffected(); rowsAffected == 1 { diff --git a/pkg/cache/dbcache/dbrepository.go b/pkg/cache/dbcache/dbrepository.go index 3f44cd11c..aad4c2b53 100644 --- a/pkg/cache/dbcache/dbrepository.go +++ b/pkg/cache/dbcache/dbrepository.go @@ -1,4 +1,4 @@ -// Copyright 2024-2025 The kpt Authors +// Copyright 2024-2026 The kpt Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -437,7 +437,7 @@ func (r *dbRepository) ListPackages(ctx context.Context, filter repository.ListP } func (r *dbRepository) Version(ctx context.Context) (string, error) { - _, span := tracer.Start(ctx, "cachedRepository::Version", trace.WithAttributes()) + _, span := tracer.Start(ctx, "dbRepository::Version", trace.WithAttributes()) defer span.End() return r.externalRepo.Version(ctx) @@ -476,6 +476,11 @@ func (r *dbRepository) ClosePackageRevisionDraft(ctx context.Context, prd reposi r.setCachedGitPR(dbPrd.Key().PkgKey, dbPrd.Key().WorkspaceName, gitPR) } + dbPrd.resourcesSizeBytes = 0 + for _, fileString := range dbPrd.resources { + dbPrd.resourcesSizeBytes += int64(len(fileString)) + } + pr, err := r.savePackageRevisionDraft(ctx, prd, version) if err != nil { return nil, err diff --git a/pkg/cache/dbcache/dbreposync.go b/pkg/cache/dbcache/dbreposync.go index e2e5a4a76..cd768b45c 100644 --- a/pkg/cache/dbcache/dbreposync.go +++ b/pkg/cache/dbcache/dbreposync.go @@ -198,8 +198,10 @@ func (s *repositorySync) cacheExternalPRs(ctx context.Context, externalPrMap map // Guard against nil return from GetResources (interface contract allows it). var resources map[string]string + var resourcesSize int64 if extPRResources == nil || extPRResources.Spec.Resources == nil { resources = make(map[string]string) + resourcesSize = 0 } else { // Filter out files with invalid UTF-8 or NUL bytes to avoid PostgreSQL TEXT errors. // Both resource_key and resource_value are TEXT columns, so both must be validated. @@ -211,6 +213,7 @@ func (s *repositorySync) cacheExternalPRs(ctx context.Context, externalPrMap map continue } resources[key] = val + resourcesSize += int64(len(val)) } } @@ -221,17 +224,18 @@ func (s *repositorySync) cacheExternalPRs(ctx context.Context, externalPrMap map _, extPRUpstreamLock, _ := extPR.GetLock(ctx) dbPR := dbPackageRevision{ - repo: s.repo, - pkgRevKey: extPRKey, - meta: extAPIPR.ObjectMeta, - spec: &extAPIPR.Spec, - updated: time.Now(), - lifecycle: extAPIPR.Spec.Lifecycle, - extPRID: extPRUpstreamLock, - tasks: extAPIPR.Spec.Tasks, - resources: resources, - deployment: s.repo.deployment, - kptfileStatus: extractKptfileStatus(resources), + repo: s.repo, + pkgRevKey: extPRKey, + meta: extAPIPR.ObjectMeta, + spec: &extAPIPR.Spec, + updated: time.Now(), + lifecycle: extAPIPR.Spec.Lifecycle, + extPRID: extPRUpstreamLock, + tasks: extAPIPR.Spec.Tasks, + resources: resources, + deployment: s.repo.deployment, + kptfileStatus: extractKptfileStatus(resources), + resourcesSizeBytes: resourcesSize, } _, err = s.repo.savePackageRevision(ctx, &dbPR, true) if err != nil { diff --git a/pkg/externalrepo/git/doc.go b/pkg/externalrepo/git/doc.go index 70e8a42a4..e6ed46880 100644 --- a/pkg/externalrepo/git/doc.go +++ b/pkg/externalrepo/git/doc.go @@ -28,7 +28,7 @@ // would add a layer of complexity where branches can become out of sync // and in need of reconciliation and conflict resolution. Instead, Porch // analyzes the remote references (refs/remotes/origin/branch...) to -// discover packges. These refs are never directly updated by Porch other +// discover packages. These refs are never directly updated by Porch other // than by push or fetch to/from remote. // Any intermediate commits Porch makes are either in 'detached HEAD' // mode, or using temporary branches (these will become relevant if/when diff --git a/test/e2e/api/repository_test.go b/test/e2e/api/repository_test.go index 1c61cd600..3787b2665 100644 --- a/test/e2e/api/repository_test.go +++ b/test/e2e/api/repository_test.go @@ -111,6 +111,8 @@ func (t *PorchSuite) TestGitRepositoryWithReleaseTagsAndDirectory() { if strings.HasPrefix(pr.Spec.PackageName, directory) { t.Errorf("package name %q should not include repo directory %q as prefix", pr.Spec.PackageName, directory) } + + t.validatePackageResourcesSize(&pr) } } diff --git a/test/e2e/api/rpkg_clone_test.go b/test/e2e/api/rpkg_clone_test.go index ed9d9838a..a682b4986 100644 --- a/test/e2e/api/rpkg_clone_test.go +++ b/test/e2e/api/rpkg_clone_test.go @@ -96,6 +96,7 @@ func (t *PorchSuite) TestCloneFromUpstream() { t.validateKptfileBasics(kptfile, istionsPackage) t.validateUpstreamLock(kptfile, testBlueprintsRepo) t.validateUpstream(kptfile, testBlueprintsRepo) + t.validatePackageResourcesSize(clonedPr) } func (t *PorchSuite) TestCloneIntoDeploymentRepository() { @@ -148,6 +149,7 @@ func (t *PorchSuite) TestCloneIntoDeploymentRepository() { t.validateKptfileBasics(kptfile, "istions-deployment") t.validateUpstreamLock(kptfile, testBlueprintsRepo) t.validateUpstream(kptfile, testBlueprintsRepo) + t.validatePackageResourcesSize(pr) // Check generated context var configmap corev1.ConfigMap @@ -262,6 +264,9 @@ data: upgradePr.Spec.Lifecycle = porchapi.PackageRevisionLifecyclePublished upgradePr = t.UpdateApprovalF(upgradePr) + // Check its package size + t.validatePackageResourcesSize(upgradePr) + basensMain := t.MustFindPackageRevision(&list, repository.PackageRevisionKey{PkgKey: repository.PackageKey{RepoKey: repository.RepositoryKey{Name: suiteutils.TestBlueprintsRepoName}, Package: basensPackage}, Revision: -1}) upgradePrTwo := t.CreatePackageSkeleton(gitRepository, "testns", testWorkspace+"-main-upgrade") diff --git a/test/e2e/api/rpkg_edit_test.go b/test/e2e/api/rpkg_edit_test.go index a6512eb2d..73664cabc 100644 --- a/test/e2e/api/rpkg_edit_test.go +++ b/test/e2e/api/rpkg_edit_test.go @@ -106,6 +106,9 @@ func (t *PorchSuite) TestEditPackageRevision() { assert.NotNil(t, tasks[0].Edit) assert.Equal(t, pr.Name, tasks[0].Edit.Source.Name) + // Check its package size + t.validatePackageResourcesSize(pr) + // Create a new revision with a placeholder package revision as the source. // This is not allowed. editPlaceholderPR := t.CreatePackageSkeleton(repository, packageName, workspace2) @@ -136,6 +139,9 @@ func (t *PorchSuite) TestUpdateResources() { // Create a new package (via init) pr := t.CreatePackageDraftF(repository, packageName, workspace) + // Check its package size + t.validatePackageResourcesSize(pr) + // Get the package resources var prResources porchapi.PackageRevisionResources t.GetF(client.ObjectKey{ @@ -156,10 +162,11 @@ func (t *PorchSuite) TestUpdateResources() { }, Name: "set-annotations", }) + t.SaveKptfileF(&prResources, kptfile) // Add a new resource - prResources.Spec.Resources["config-map.yaml"] = `apiVersion: v1 + addedResource := `apiVersion: v1 kind: ConfigMap metadata: name: update-resources-configmap @@ -167,6 +174,7 @@ metadata: data: value: Update Resources and Render ` + prResources.Spec.Resources["config-map.yaml"] = addedResource t.UpdateF(&prResources) // Re-fetch the resources to get the actual rendered result @@ -180,6 +188,13 @@ data: t.Fatalf("Updated config map config-map.yaml not found") } + // Check the PR's package size again to ensure the ResourcesSizeBytes reflects the added resource + t.GetF(client.ObjectKey{ + Namespace: t.Namespace, + Name: pr.Name, + }, pr) + t.validatePackageResourcesSize(pr) + renderStatus := prResources.Status.RenderStatus assert.Empty(t, renderStatus.Err, "render error must be empty for successful render operation.") assert.Zero(t, renderStatus.Result.ExitCode, "exit code must be zero for successful render operation.") diff --git a/test/e2e/api/rpkg_init_test.go b/test/e2e/api/rpkg_init_test.go index 78e1860a1..ef424f5a0 100644 --- a/test/e2e/api/rpkg_init_test.go +++ b/test/e2e/api/rpkg_init_test.go @@ -1,4 +1,4 @@ -// Copyright 2025 The kpt Authors +// Copyright 2025-2026 The kpt Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -19,6 +19,7 @@ import ( kptfilev1 "github.com/kptdev/kpt/pkg/api/kptfile/v1" porchapi "github.com/kptdev/porch/api/porch/v1alpha1" suiteutils "github.com/kptdev/porch/test/e2e/suiteutils" + "github.com/stretchr/testify/assert" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -40,6 +41,8 @@ func (t *PorchSuite) TestInitEmptyPackage() { t.validateKptFileMetadata(pr, packageName, &kptfilev1.PackageInfo{ Description: description, }) + + t.validatePackageResourcesSize(pr) } func (t *PorchSuite) TestInitTaskPackage() { @@ -91,3 +94,26 @@ func (t *PorchSuite) validateKptFileMetadata(pr *porchapi.PackageRevision, expec t.Fatalf("unexpected %s/%s package info (-want, +got) %s", pkg.Namespace, pkg.Name, cmp.Diff(want, got)) } } + +func (t *PorchSuite) validatePackageResourcesSize(pr *porchapi.PackageRevision) { + t.T().Helper() + if t.UsingDBCache { + t.Logf("DB cache is enabled: validating package resource size") + var pkg porchapi.PackageRevisionResources + t.GetF(client.ObjectKey{ + Namespace: t.Namespace, + Name: pr.Name, + }, &pkg) + + expectedResourcesSize := 0 + for _, file := range pkg.Spec.Resources { + expectedResourcesSize += len(file) + } + assert.EqualValues(t, expectedResourcesSize, pr.Status.ResourcesSizeBytes, + "Expected PackageRevision %s/%s resources size of %d bytes, got %d", + pr.Namespace, pr.Name, + expectedResourcesSize, pr.Status.ResourcesSizeBytes) + } else { + t.Logf("DB cache is disabled: skipping package resource size validation") + } +} diff --git a/test/e2e/api/rpkg_lifecycle_test.go b/test/e2e/api/rpkg_lifecycle_test.go index c08bc0e02..fc0f1016b 100644 --- a/test/e2e/api/rpkg_lifecycle_test.go +++ b/test/e2e/api/rpkg_lifecycle_test.go @@ -1,4 +1,4 @@ -// Copyright 2025 The kpt Authors +// Copyright 2025-2026 The kpt Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -21,6 +21,7 @@ import ( configapi "github.com/kptdev/porch/api/porchconfig/v1alpha1" suiteutils "github.com/kptdev/porch/test/e2e/suiteutils" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -45,9 +46,9 @@ func (t *PorchSuite) TestBasicLifecycle() { Workspace: defaultWorkspace, TargetState: porchapi.PackageRevisionLifecyclePublished, Validate: func(t *PorchSuite, pr *porchapi.PackageRevision) { - if pr.Spec.Revision != 1 { - t.Fatalf("Expected revision 1, got %d", pr.Spec.Revision) - } + require.Equal(t.T(), 1, pr.Spec.Revision, "Expected revision 1, got %d", pr.Spec.Revision) + + t.validatePackageResourcesSize(pr) }, }, { diff --git a/test/e2e/suiteutils/suite.go b/test/e2e/suiteutils/suite.go index 0a2cf917e..c8dad4407 100644 --- a/test/e2e/suiteutils/suite.go +++ b/test/e2e/suiteutils/suite.go @@ -85,6 +85,7 @@ type TestSuite struct { TestRunnerIsLocal bool // Tests running against local dev porch porchServerInCluster *bool // Cached result of IsPorchServerInCluster check repoControllerInCluster *bool // Cached result of IsRepoControllerInCluster check + UsingDBCache bool // Tests running against Porch with database cache } func (t *TestSuite) SetupSuite() { @@ -141,6 +142,9 @@ func (t *TestSuite) Initialize() { }) t.Namespace = namespace + + t.checkIfUsingDBCache() + c := t.Client t.Cleanup(func() { if err := c.Delete(t.GetContext(), &coreapi.Namespace{ @@ -155,6 +159,10 @@ func (t *TestSuite) Initialize() { }) } +func (t *TestSuite) checkIfUsingDBCache() { + t.UsingDBCache = (os.Getenv("DB_CACHE") != "") +} + func (t *TestSuite) PorchServerServiceKey() client.ObjectKey { porchApiService := aggregatorv1.APIService{} apiServiceName := porchapi.SchemeGroupVersion.Version + "." + porchapi.SchemeGroupVersion.Group