From 171fc894547e0fdb0dc9e38a82cdd18754c8a845 Mon Sep 17 00:00:00 2001 From: Forrest Babcock Date: Thu, 14 May 2026 15:59:23 -0400 Subject: [PATCH] TRT-1989: model changes to prep for partitioning --- cmd/sippy/seed_data.go | 24 ++++++----- pkg/api/job_runs_test.go | 9 ++-- pkg/dataloader/prowloader/prow.go | 46 ++++++++++++--------- pkg/db/models/prow.go | 17 +++++++- pkg/sippyserver/pr_new_tests_worker_test.go | 10 +++-- 5 files changed, 67 insertions(+), 39 deletions(-) diff --git a/cmd/sippy/seed_data.go b/cmd/sippy/seed_data.go index d3fd762234..6550bf7729 100644 --- a/cmd/sippy/seed_data.go +++ b/cmd/sippy/seed_data.go @@ -574,10 +574,11 @@ func seedRunsForJob(dbc *db.DB, suite *models.Suite, prowJob models.ProwJob, jrK for i := range totalRuns { timestamp := start.Add(time.Duration(i) * interval) run := models.ProwJobRun{ - ProwJobID: prowJob.ID, - Cluster: "build01", - Timestamp: timestamp, - Duration: 3 * time.Hour, + ProwJobID: prowJob.ID, + ProwJobRelease: prowJob.Release, + Cluster: "build01", + Timestamp: timestamp, + Duration: 3 * time.Hour, } if err := dbc.DB.Create(&run).Error; err != nil { return 0, 0, fmt.Errorf("failed to create ProwJobRun: %w", err) @@ -616,12 +617,15 @@ func seedRunsForJob(dbc *db.DB, suite *models.Suite, prowJob models.ProwJob, jrK } result := models.ProwJobRunTest{ - ProwJobRunID: runIDs[i], - TestID: testID, - SuiteID: &suite.ID, - Status: status, - Duration: 5.0, - CreatedAt: start.Add(time.Duration(i) * interval), + ProwJobRunID: runIDs[i], + ProwJobID: prowJob.ID, + ProwJobRunRelease: prowJob.Release, + ProwJobRunTimestamp: start.Add(time.Duration(i) * interval), + TestID: testID, + SuiteID: &suite.ID, + Status: status, + Duration: 5.0, + CreatedAt: start.Add(time.Duration(i) * interval), } if err := dbc.DB.Create(&result).Error; err != nil { return 0, 0, fmt.Errorf("failed to create ProwJobRunTest: %w", err) diff --git a/pkg/api/job_runs_test.go b/pkg/api/job_runs_test.go index 9f3cf2edb6..58e7ba0092 100644 --- a/pkg/api/job_runs_test.go +++ b/pkg/api/job_runs_test.go @@ -220,9 +220,11 @@ func TestRunJobAnalysis(t *testing.T) { for _, t := range tests { fakeProwJobRun.Tests = append(fakeProwJobRun.Tests, models.ProwJobRunTest{ - Test: models.Test{Name: t.Name}, - Suite: models.Suite{Name: t.SuiteName}, - Status: 12, + ProwJobID: fakeProwJobRun.ProwJobID, + ProwJobRunRelease: fakeProwJobRun.ProwJobRelease, + Test: models.Test{Name: t.Name}, + Suite: models.Suite{Name: t.SuiteName}, + Status: 12, }) } @@ -289,6 +291,7 @@ func buildFakeProwJobRun() *models.ProwJobRun { }, }, ProwJobID: 1000000000, + ProwJobRelease: "4.12", URL: "https://example.com/run/1000000000", Tests: []models.ProwJobRunTest{}, // will be populated in the test cases TestCount: 5, diff --git a/pkg/dataloader/prowloader/prow.go b/pkg/dataloader/prowloader/prow.go index f01cbea16a..33f7f2ad20 100644 --- a/pkg/dataloader/prowloader/prow.go +++ b/pkg/dataloader/prowloader/prow.go @@ -930,7 +930,7 @@ func (pl *ProwLoader) createOrUpdateProwJob(ctx context.Context, pj *prow.ProwJo } func (pl *ProwLoader) processGCSBucketJobRun(ctx context.Context, pj *prow.ProwJob, id uint64, path string, junitMatches []string, dbProwJob *models.ProwJob) error { - tests, failures, overallResult, err := pl.prowJobRunTestsFromGCS(ctx, pj, uint(id), path, junitMatches) + tests, failures, overallResult, err := pl.prowJobRunTestsFromGCS(ctx, pj, uint(id), dbProwJob.ID, dbProwJob.Release, path, junitMatches) if err != nil { return err } @@ -959,19 +959,20 @@ func (pl *ProwLoader) processGCSBucketJobRun(ctx context.Context, pj *prow.ProwJ Model: gorm.Model{ ID: uint(id), }, - Cluster: pj.Spec.Cluster, - Duration: duration, - ProwJob: *dbProwJob, - ProwJobID: dbProwJob.ID, - URL: pj.Status.URL, - GCSBucket: pj.Spec.DecorationConfig.GCSConfiguration.Bucket, - Timestamp: pj.Status.StartTime, - OverallResult: overallResult, - PullRequests: pulls, - TestFailures: failures, - Succeeded: overallResult == sippyprocessingv1.JobSucceeded, - Labels: labels, - Annotations: annotations, + Cluster: pj.Spec.Cluster, + Duration: duration, + ProwJob: *dbProwJob, + ProwJobID: dbProwJob.ID, + ProwJobRelease: dbProwJob.Release, + URL: pj.Status.URL, + GCSBucket: pj.Spec.DecorationConfig.GCSConfiguration.Bucket, + Timestamp: pj.Status.StartTime, + OverallResult: overallResult, + PullRequests: pulls, + TestFailures: failures, + Succeeded: overallResult == sippyprocessingv1.JobSucceeded, + Labels: labels, + Annotations: annotations, }).Error if err != nil { return err @@ -1187,7 +1188,7 @@ func (pl *ProwLoader) findSuite(name string) *uint { return id } -func (pl *ProwLoader) prowJobRunTestsFromGCS(ctx context.Context, pj *prow.ProwJob, id uint, path string, junitPaths []string) ([]*models.ProwJobRunTest, int, sippyprocessingv1.JobOverallResult, error) { +func (pl *ProwLoader) prowJobRunTestsFromGCS(ctx context.Context, pj *prow.ProwJob, id, prowJobID uint, prowJobRelease, path string, junitPaths []string) ([]*models.ProwJobRunTest, int, sippyprocessingv1.JobOverallResult, error) { failures := 0 bkt := pl.gcsClient.Bucket(pj.Spec.DecorationConfig.GCSConfiguration.Bucket) @@ -1206,7 +1207,7 @@ func (pl *ProwLoader) prowJobRunTestsFromGCS(ctx context.Context, pj *prow.ProwJ continue } - pl.extractTestCases(suite, suiteID, testCases) + pl.extractTestCases(suite, suiteID, testCases, prowJobRelease, pj.Status.StartTime) } syntheticSuite, jobResult := testconversion.ConvertProwJobRunToSyntheticTests(*pj, testCases, pl.syntheticTestManager) @@ -1216,7 +1217,7 @@ func (pl *ProwLoader) prowJobRunTestsFromGCS(ctx context.Context, pj *prow.ProwJ // this shouldn't happen but if it does we want to know panic("synthetic suite is missing from the database") } - pl.extractTestCases(syntheticSuite, suiteID, testCases) + pl.extractTestCases(syntheticSuite, suiteID, testCases, prowJobRelease, pj.Status.StartTime) log.Infof("synthetic suite had %d tests", syntheticSuite.NumTests) results := make([]*models.ProwJobRunTest, 0) @@ -1226,6 +1227,9 @@ func (pl *ProwLoader) prowJobRunTestsFromGCS(ctx context.Context, pj *prow.ProwJ } testCases[k].ProwJobRunID = id + testCases[k].ProwJobID = prowJobID + testCases[k].ProwJobRunRelease = prowJobRelease + testCases[k].ProwJobRunTimestamp = pj.Status.StartTime results = append(results, testCases[k]) if testCases[k].Status == 12 { failures++ @@ -1235,7 +1239,7 @@ func (pl *ProwLoader) prowJobRunTestsFromGCS(ctx context.Context, pj *prow.ProwJ return results, failures, jobResult, nil } -func (pl *ProwLoader) extractTestCases(suite *junit.TestSuite, suiteID *uint, testCases map[string]*models.ProwJobRunTest) { +func (pl *ProwLoader) extractTestCases(suite *junit.TestSuite, suiteID *uint, testCases map[string]*models.ProwJobRunTest, prowJobRelease string, prowJobStartTime time.Time) { for _, tc := range suite.TestCases { if testidentification.IsIgnoredTest(tc.Name) { @@ -1250,7 +1254,9 @@ func (pl *ProwLoader) extractTestCases(suite *junit.TestSuite, suiteID *uint, te status = sippyprocessingv1.TestStatusSuccess default: failureOutput = &models.ProwJobRunTestOutput{ - Output: tc.FailureOutput.Output, + Output: tc.FailureOutput.Output, + ProwJobRunTestTimestamp: prowJobStartTime, + ProwJobRunTestRelease: prowJobRelease, } } @@ -1283,6 +1289,6 @@ func (pl *ProwLoader) extractTestCases(suite *junit.TestSuite, suiteID *uint, te } for _, c := range suite.Children { - pl.extractTestCases(c, suiteID, testCases) + pl.extractTestCases(c, suiteID, testCases, prowJobRelease, prowJobStartTime) } } diff --git a/pkg/db/models/prow.go b/pkg/db/models/prow.go index 18fd49e037..646ec96080 100644 --- a/pkg/db/models/prow.go +++ b/pkg/db/models/prow.go @@ -39,6 +39,8 @@ type ProwJobRun struct { // ProwJob is a link to the prow job this run belongs to. ProwJob ProwJob ProwJobID uint `gorm:"index"` + // Used for partitioning + ProwJobRelease string `gorm:"varchar(10)"` // Cluster is the cluster where the prow job was run. Cluster string @@ -87,8 +89,15 @@ type ProwJobRunTest struct { gorm.Model ProwJobRunID uint `gorm:"index"` ProwJobRun ProwJobRun - TestID uint `gorm:"index;index:idx_prow_job_run_tests_test_id_status"` - Test Test + // used for variants + // skips joining on ProwJobRunID just to get ProwJobID + ProwJobID uint + // used for partitioning + ProwJobRunTimestamp time.Time `gorm:"expression:DATE(timestamp AT TIME ZONE 'UTC')"` + // used for partitioning + ProwJobRunRelease string `gorm:"varchar(10)"` + TestID uint `gorm:"index;index:idx_prow_job_run_tests_test_id_status"` + Test Test // SuiteID may be nil if no suite name could be parsed from the testgrid test name. SuiteID *uint `gorm:"index"` Suite Suite @@ -107,6 +116,10 @@ type ProwJobRunTestOutput struct { ProwJobRunTestID uint `gorm:"index"` // Output stores the output of a ProwJobRunTest. Output string + // used for partitioning + ProwJobRunTestTimestamp time.Time `gorm:"expression:DATE(timestamp AT TIME ZONE 'UTC')"` + // used for partitioning + ProwJobRunTestRelease string `gorm:"varchar(10)"` } // Suite defines a junit testsuite. Used to differentiate the same test being run in different suites in ProwJobRunTest. diff --git a/pkg/sippyserver/pr_new_tests_worker_test.go b/pkg/sippyserver/pr_new_tests_worker_test.go index a193fbe06a..7a134d41ab 100644 --- a/pkg/sippyserver/pr_new_tests_worker_test.go +++ b/pkg/sippyserver/pr_new_tests_worker_test.go @@ -233,10 +233,11 @@ func TestUnit_getNewTestsForJobRun(t *testing.T) { name: "successful fetch", fetchJobRun: func(dbc *db.DB, jobRunID int64, unknownTests bool, preloads []string, logger *logrus.Entry) (*models.ProwJobRun, error) { pjr := models.ProwJobRun{ + ProwJobRelease: "4.12", Tests: []models.ProwJobRunTest{ - {Test: models.Test{Name: "test1"}, Status: int(v1.TestStatusSuccess)}, - {Test: models.Test{Name: "test2"}, Status: int(v1.TestStatusFailure)}, - {Test: models.Test{Name: "test3"}, Status: int(v1.TestStatusFlake)}, + {ProwJobID: 1, ProwJobRunRelease: "4.12", Test: models.Test{Name: "test1"}, Status: int(v1.TestStatusSuccess)}, + {ProwJobID: 1, ProwJobRunRelease: "4.12", Test: models.Test{Name: "test2"}, Status: int(v1.TestStatusFailure)}, + {ProwJobID: 1, ProwJobRunRelease: "4.12", Test: models.Test{Name: "test3"}, Status: int(v1.TestStatusFlake)}, }, } pjr.ID = 12345 // Gorm model ID for some reason can't be put in the struct literal @@ -252,8 +253,9 @@ func TestUnit_getNewTestsForJobRun(t *testing.T) { name: "error on filtering", fetchJobRun: func(dbc *db.DB, jobRunID int64, unknownTests bool, preloads []string, logger *logrus.Entry) (*models.ProwJobRun, error) { pjr := models.ProwJobRun{ + ProwJobRelease: "4.12", Tests: []models.ProwJobRunTest{ - {Test: models.Test{Name: "test1"}, Status: int(v1.TestStatusSuccess)}, + {ProwJobID: 1, ProwJobRunRelease: "4.12", Test: models.Test{Name: "test1"}, Status: int(v1.TestStatusSuccess)}, }, } pjr.ID = 12345 // Gorm model ID for some reason can't be put in the struct literal