Add annotation field to /api/jobs/runs from BQ

pkg/api/job_runs.go

@@ -106,12 +106,41 @@ func JobsRunsReportFromDB(dbc *db.DB, filterOpts *filter.FilterOptions, release
 	}
 
 	res := q.Scan(&jobsResult)
+	if res.Error != nil {
+		return nil, res.Error
+	}
+
+	// Fetch annotations separately to avoid bloating the materialized view.
+	if len(jobsResult) > 0 {
+		ids := make([]int, len(jobsResult))
+		for i, jr := range jobsResult {
+			ids[i] = jr.ID
+		}
+		var annotations []models.ProwJobRunAnnotation
+		if err := dbc.DB.Where("prow_job_run_id IN ?", ids).Find(&annotations).Error; err != nil {
+			return nil, err
+		}
+		annotationsByRun := make(map[string]apitype.AnnotationMap)
+		for _, a := range annotations {
+			annotationID := strconv.FormatUint(uint64(a.ProwJobRunID), 10)
+			if annotationsByRun[annotationID] == nil {
+				annotationsByRun[annotationID] = make(apitype.AnnotationMap)
+			}
+			annotationsByRun[annotationID][a.Key] = a.Value
+		}
+		for i := range jobsResult {
+			if am, ok := annotationsByRun[strconv.Itoa(jobsResult[i].ID)]; ok {
+				jobsResult[i].Annotations = am
+			}
+		}
+	}
+
 	return &apitype.PaginationResult{
 		Rows:      jobsResult,
 		TotalRows: rowCount,
 		PageSize:  pagination.PerPage,
 		Page:      pagination.Page,
-	}, res.Error
+	}, nil
 }
 
 // FetchJobRun returns a single job run loaded from postgres and populated with the ProwJob and test results.

pkg/apis/api/types.go

@@ -356,8 +356,12 @@ type JobRun struct {
 	PullRequestSHA        string              `json:"pull_request_sha"`
 	PullRequestAuthor     string              `json:"pull_request_author"`
 	Labels                pq.StringArray      `json:"labels" gorm:"type:text[]"`
+	Annotations           AnnotationMap       `json:"annotations,omitempty"`
 }
 
+// AnnotationMap is a map[string]string for job run annotations.
+type AnnotationMap map[string]string
+
 func (run JobRun) GetFieldType(param string) ColumnType {
 	switch param {
 	case "name":

pkg/apis/prow/prow.go

@@ -92,6 +92,7 @@ type ProwJobStatus struct {
 }
 
 type ProwJob struct {
-	Spec   ProwJobSpec   `json:"spec,omitempty"`
-	Status ProwJobStatus `json:"status,omitempty"`
+	Spec        ProwJobSpec       `json:"spec,omitempty"`
+	Status      ProwJobStatus     `json:"status,omitempty"`
+	Annotations map[string]string `json:"annotations,omitempty"`
 }

pkg/dataloader/prowloader/bigqueryjobs.go

@@ -2,6 +2,7 @@ package prowloader
 
 import (
 	"strconv"
+	"strings"
 	"time"
 
 	"cloud.google.com/go/bigquery"
@@ -43,6 +44,7 @@ func (pl *ProwLoader) fetchProwJobsFromOpenShiftBigQuery() ([]prow.ProwJob, []er
 			prowjob_type,
 			prowjob_cluster,
 			prowjob_url,
+			prowjob_annotations,
 			pr_sha,
 			pr_author,
 			pr_number,
@@ -103,6 +105,21 @@ func (pl *ProwLoader) fetchProwJobsFromOpenShiftBigQuery() ([]prow.ProwJob, []er
 			// Do not return an error as that will cause the job to fail.
 			continue
 		}
+		// Filter out annotations with excluded prefixes
+		filteredAnnotations := make(map[string]string)
+		for _, a := range bqjr.Annotations {
+			parts := strings.SplitN(a, "=", 2)
+			if len(parts) != 2 {
+				continue
+			}
+			key := parts[0]
+			value := parts[1]
+			if strings.HasPrefix(key, "prow.k8s.io") || strings.HasPrefix(key, "ci.openshift.io") {
+				continue
+			}
+			filteredAnnotations[key] = value
+		}
+
 		prowJobs[bqjr.BuildID] = prow.ProwJob{
 			Spec: prow.ProwJobSpec{
 				Type:    bqjr.Type,
@@ -122,6 +139,7 @@ func (pl *ProwLoader) fetchProwJobsFromOpenShiftBigQuery() ([]prow.ProwJob, []er
 				URL:            bqjr.URL,
 				BuildID:        bqjr.BuildID,
 			},
+			Annotations: filteredAnnotations,
 		}
 		count++
 	}
@@ -152,4 +170,5 @@ type bigqueryProwJobRun struct {
 	PROrg          bigquery.NullString    `bigquery:"org"`
 	PRRepo         bigquery.NullString    `bigquery:"repo"`
 	GCSBucket      bigquery.NullString    `bigquery:"gcs_bucket"`
+	Annotations    []string               `bigquery:"prowjob_annotations"`
 }

pkg/dataloader/prowloader/prow.go

@@ -921,6 +921,14 @@ func (pl *ProwLoader) processGCSBucketJobRun(ctx context.Context, pj *prow.ProwJ
 		return err
 	}
 
+	var annotations []models.ProwJobRunAnnotation
+	for k, v := range pj.Annotations {
+		annotations = append(annotations, models.ProwJobRunAnnotation{
+			Key:   k,
+			Value: v,
+		})
+	}
+
 	var duration time.Duration
 	if pj.Status.CompletionTime != nil {
 		duration = pj.Status.CompletionTime.Sub(pj.Status.StartTime)
@@ -942,6 +950,7 @@ func (pl *ProwLoader) processGCSBucketJobRun(ctx context.Context, pj *prow.ProwJ
 		TestFailures:  failures,
 		Succeeded:     overallResult == sippyprocessingv1.JobSucceeded,
 		Labels:        labels,
+		Annotations:   annotations,
 	}).Error
 	if err != nil {
 		return err

pkg/db/db.go

@@ -74,6 +74,7 @@ func (d *DB) UpdateSchema(reportEnd *time.Time) error {
 		&models.ReleaseJobRun{},
 		&models.ProwJob{},
 		&models.ProwJobRun{},
+		&models.ProwJobRunAnnotation{},
 		&models.Test{},
 		&models.Suite{},
 		&models.ProwJobRunTest{},

pkg/db/models/prow.go

@@ -46,8 +46,9 @@ type ProwJobRun struct {
 	GCSBucket    string
 	URL          string
 	TestFailures int
-	Tests        []ProwJobRunTest  `gorm:"constraint:OnDelete:CASCADE;"`
-	PullRequests []ProwPullRequest `gorm:"many2many:prow_job_run_prow_pull_requests;constraint:OnDelete:CASCADE;"`
+	Tests        []ProwJobRunTest       `gorm:"constraint:OnDelete:CASCADE;"`
+	PullRequests []ProwPullRequest      `gorm:"many2many:prow_job_run_prow_pull_requests;constraint:OnDelete:CASCADE;"`
+	Annotations  []ProwJobRunAnnotation `gorm:"constraint:OnDelete:CASCADE;"`
 	Failed       bool
 	// InfrastructureFailure is true if the job run failed, for reasons which appear to be related to test/CI infra.
 	InfrastructureFailure bool
@@ -65,6 +66,14 @@ type ProwJobRun struct {
 	ClusterData ClusterData `gorm:"-"`
 }
 
+// ProwJobRunAnnotation stores a single key-value annotation for a ProwJobRun.
+type ProwJobRunAnnotation struct {
+	gorm.Model
+	ProwJobRunID uint   `gorm:"index;uniqueIndex:idx_prow_job_run_annotations_key"`
+	Key          string `gorm:"uniqueIndex:idx_prow_job_run_annotations_key"`
+	Value        string
+}
+
 type Test struct {
 	gorm.Model
 	Name           string          `gorm:"uniqueIndex"`