Skip to content

Support a metric being a string value as well as a numeric value #639

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 3 commits into
base: main
Choose a base branch
from
+246 −6
Draft

Support a metric being a string value as well as a numeric value #639

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
aba3a09
Support a metric being a string value.
epugh Sep 16, 2025
0977eec
Add unit tests for string values in metrics
epugh Sep 16, 2025
71b1b56
Merge remote-tracking branch 'upstream/main' into support_string_valu…
epugh Sep 17, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
118 changes: 118 additions & 0 deletions 0001-Support-String-values-for-metrics.-Sometimes-you-wan.patch
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,118 @@
From 0af047adb9424d73c06090908bb6b4d019d6a71a Mon Sep 17 00:00:00 2001
From: Eric Pugh <[email protected]>
Date: Wed, 10 Sep 2025 08:54:02 -0700
Subject: [PATCH] Support String values for metrics. Sometimes you want a
label!

---
.../experiment/metrics/metrics_summary.tsx | 16 +++++++++++++---
.../views/evaluation_experiment_view.tsx | 3 +++
.../views/hybrid_optimizer_experiment_view.tsx | 5 ++++-
.../views/pairwise_experiment_view.tsx | 3 +++
public/types/index.ts | 4 ++--
5 files changed, 25 insertions(+), 6 deletions(-)

diff --git a/public/components/experiment/metrics/metrics_summary.tsx b/public/components/experiment/metrics/metrics_summary.tsx
index d36603f..348feda 100644
--- a/public/components/experiment/metrics/metrics_summary.tsx
+++ b/public/components/experiment/metrics/metrics_summary.tsx
@@ -30,10 +30,20 @@ interface MetricsSummaryPanelProps {
}

export const MetricsSummaryPanel: React.FC<MetricsSummaryPanelProps> = ({ metrics }) => {
- const formatValue = (values: number[] | undefined) => {
+ const formatValue = (values: (number | string)[] | undefined) => {
if (!values || values.length === 0) return '-';
- // Calculate average of the values
- const avg = values.reduce((sum, val) => sum + val, 0) / values.length;
+
+ // Check if any value is a string
+ const stringValues = values.filter(val => typeof val === 'string');
+ if (stringValues.length > 0) {
+ // If we have string values, return the first one
+ return stringValues[0];
+ }
+
+ // For numeric values, calculate average as before
+ const numericValues = values.filter(val => typeof val === 'number') as number[];
+ if (numericValues.length === 0) return '-';
+ const avg = numericValues.reduce((sum, val) => sum + val, 0) / numericValues.length;
return avg.toFixed(2).replace(/\.00$/, '');
};
// tool tip texts
diff --git a/public/components/experiment/views/evaluation_experiment_view.tsx b/public/components/experiment/views/evaluation_experiment_view.tsx
index 3dac7c5..649e21f 100644
--- a/public/components/experiment/views/evaluation_experiment_view.tsx
+++ b/public/components/experiment/views/evaluation_experiment_view.tsx
@@ -248,6 +248,9 @@ export const EvaluationExperimentView: React.FC<EvaluationExperimentViewProps> =
sortable: true,
render: (value) => {
if (value !== undefined && value !== null) {
+ if (typeof value === 'string') {
+ return value; // Return string values directly
+ }
return new Intl.NumberFormat(undefined, {
minimumFractionDigits: 2,
maximumFractionDigits: 2,
diff --git a/public/components/experiment/views/hybrid_optimizer_experiment_view.tsx b/public/components/experiment/views/hybrid_optimizer_experiment_view.tsx
index 36a5f6b..89eceed 100644
--- a/public/components/experiment/views/hybrid_optimizer_experiment_view.tsx
+++ b/public/components/experiment/views/hybrid_optimizer_experiment_view.tsx
@@ -31,7 +31,7 @@ import {
} from '../../../../common';

interface VariantEvaluation {
- metrics: Record<string, number>;
+ metrics: Record<string, number | string>;
}

interface QueryVariantEvaluations {
@@ -262,6 +262,9 @@ export const HybridOptimizerExperimentView: React.FC<HybridOptimizerExperimentVi
sortable: true,
render: (value) => {
if (value !== undefined && value !== null) {
+ if (typeof value === 'string') {
+ return value; // Return string values directly
+ }
return new Intl.NumberFormat(undefined, {
minimumFractionDigits: 2,
maximumFractionDigits: 2,
diff --git a/public/components/experiment/views/pairwise_experiment_view.tsx b/public/components/experiment/views/pairwise_experiment_view.tsx
index 0c81ac2..4537d54 100644
--- a/public/components/experiment/views/pairwise_experiment_view.tsx
+++ b/public/components/experiment/views/pairwise_experiment_view.tsx
@@ -185,6 +185,9 @@ export const PairwiseExperimentView: React.FC<PairwiseExperimentViewProps> = ({
sortable: true,
render: (value) => {
if (value !== undefined && value !== null) {
+ if (typeof value === 'string') {
+ return value; // Return string values directly
+ }
return new Intl.NumberFormat(undefined, {
minimumFractionDigits: 2,
maximumFractionDigits: 2,
diff --git a/public/types/index.ts b/public/types/index.ts
index 97b7eef..8c6c5f7 100644
--- a/public/types/index.ts
+++ b/public/types/index.ts
@@ -159,7 +159,7 @@ export const printType = (type: string) => {
};

export interface Metrics {
- [key: string]: number;
+ [key: string]: number | string;
}

export type MetricsCollection = Metrics[];
@@ -197,7 +197,7 @@ export function combineResults(...results: Array<ParseResult<any>>): ParseResult
return errors.length > 0 ? { success: false, errors } : { success: true, data: values };
}

-export const parseMetrics = (metricsArray: Array<{ metric: string; value: number }>): Metrics => {
+export const parseMetrics = (metricsArray: Array<{ metric: string; value: number | string }>): Metrics => {
return Object.fromEntries(
metricsArray.map(({ metric, value }) => [metric, value])
) as Metrics;
--
2.45.1

29 changes: 29 additions & 0 deletions public/components/experiment/__tests__/metrics_summary.test.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,4 +37,33 @@ describe('MetricsSummaryPanel', () => {

expect(screen.getByText('Metrics Summary')).toBeInTheDocument();
});

it('handles string values in metrics', () => {
const mockMetrics = [
{ 'ndcg@10': 0.85, 'status': 'Excellent' },
{ 'ndcg@10': 0.9, 'status': 'Excellent' },
];

render(<MetricsSummaryPanel metrics={mockMetrics} />);

expect(screen.getByText('ndcg@10')).toBeInTheDocument();
expect(screen.getByText('status')).toBeInTheDocument();
expect(screen.getByText('0.88')).toBeInTheDocument(); // Average of ndcg@10
expect(screen.getByText('Excellent')).toBeInTheDocument(); // String value
});

it('handles mixed numeric and string values', () => {
const mockMetrics = [
{ 'metric1': 0.5, 'metric2': 'Value1' },
{ 'metric1': 0.7, 'metric2': 'Value2' },
{ 'metric1': 0.9, 'metric2': 'Value1' }
];

render(<MetricsSummaryPanel metrics={mockMetrics} />);

expect(screen.getByText('metric1')).toBeInTheDocument();
expect(screen.getByText('metric2')).toBeInTheDocument();
expect(screen.getByText('0.70')).toBeInTheDocument(); // Average of metric1
expect(screen.getByText('Value1')).toBeInTheDocument(); // First string value is used
});
});
16 changes: 13 additions & 3 deletions public/components/experiment/metrics/metrics_summary.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,10 +30,20 @@ interface MetricsSummaryPanelProps {
}

export const MetricsSummaryPanel: React.FC<MetricsSummaryPanelProps> = ({ metrics }) => {
const formatValue = (values: number[] | undefined) => {
const formatValue = (values: (number | string)[] | undefined) => {
if (!values || values.length === 0) return '-';
// Calculate average of the values
const avg = values.reduce((sum, val) => sum + val, 0) / values.length;

// Check if any value is a string
const stringValues = values.filter(val => typeof val === 'string');
if (stringValues.length > 0) {
// If we have string values, return the first one
return stringValues[0];
}

// For numeric values, calculate average as before
const numericValues = values.filter(val => typeof val === 'number') as number[];
if (numericValues.length === 0) return '-';
const avg = numericValues.reduce((sum, val) => sum + val, 0) / numericValues.length;
return avg.toFixed(2).replace(/\.00$/, '');
};
// tool tip texts
Expand Down
3 changes: 3 additions & 0 deletions public/components/experiment/views/evaluation_experiment_view.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -248,6 +248,9 @@ export const EvaluationExperimentView: React.FC<EvaluationExperimentViewProps> =
sortable: true,
render: (value) => {
if (value !== undefined && value !== null) {
if (typeof value === 'string') {
return value; // Return string values directly
}
return new Intl.NumberFormat(undefined, {
minimumFractionDigits: 2,
maximumFractionDigits: 2,
Expand Down
5 changes: 4 additions & 1 deletion public/components/experiment/views/hybrid_optimizer_experiment_view.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ import {
} from '../../../../common';

interface VariantEvaluation {
metrics: Record<string, number>;
metrics: Record<string, number | string>;
}

interface QueryVariantEvaluations {
Expand Down Expand Up @@ -262,6 +262,9 @@ export const HybridOptimizerExperimentView: React.FC<HybridOptimizerExperimentVi
sortable: true,
render: (value) => {
if (value !== undefined && value !== null) {
if (typeof value === 'string') {
return value; // Return string values directly
}
return new Intl.NumberFormat(undefined, {
minimumFractionDigits: 2,
maximumFractionDigits: 2,
Expand Down
3 changes: 3 additions & 0 deletions public/components/experiment/views/pairwise_experiment_view.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -185,6 +185,9 @@ export const PairwiseExperimentView: React.FC<PairwiseExperimentViewProps> = ({
sortable: true,
render: (value) => {
if (value !== undefined && value !== null) {
if (typeof value === 'string') {
return value; // Return string values directly
}
return new Intl.NumberFormat(undefined, {
minimumFractionDigits: 2,
maximumFractionDigits: 2,
Expand Down
74 changes: 74 additions & 0 deletions public/types/__tests__/index.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
/*
* Copyright OpenSearch Contributors
* SPDX-License-Identifier: Apache-2.0
*/

import { parseMetrics, Metrics } from '../index';

describe('Types utilities', () => {
describe('parseMetrics', () => {
it('parses an array with numeric values', () => {
const metricsArray = [
{ metric: 'ndcg@10', value: 0.85 },
{ metric: 'precision@5', value: 0.75 }
];

const result = parseMetrics(metricsArray);

expect(result).toEqual({
'ndcg@10': 0.85,
'precision@5': 0.75
});
});

it('parses an array with string values', () => {
const metricsArray = [
{ metric: 'status', value: 'Excellent' },
{ metric: 'tier', value: 'Premium' }
];

const result = parseMetrics(metricsArray);

expect(result).toEqual({
'status': 'Excellent',
'tier': 'Premium'
});
});

it('parses an array with mixed numeric and string values', () => {
const metricsArray = [
{ metric: 'ndcg@10', value: 0.85 },
{ metric: 'status', value: 'Excellent' },
{ metric: 'precision@5', value: 0.75 }
];

const result = parseMetrics(metricsArray);

expect(result).toEqual({
'ndcg@10': 0.85,
'status': 'Excellent',
'precision@5': 0.75
});
});

it('handles empty arrays', () => {
const metricsArray: Array<{ metric: string; value: number | string }> = [];

const result = parseMetrics(metricsArray);

expect(result).toEqual({});
});

it('preserves the original types of values', () => {
const metricsArray = [
{ metric: 'score', value: 1 },
{ metric: 'label', value: '1' }
];

const result = parseMetrics(metricsArray);

expect(typeof result.score).toBe('number');
expect(typeof result.label).toBe('string');
});
});
});
4 changes: 2 additions & 2 deletions public/types/index.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -159,7 +159,7 @@ export const printType = (type: string) => {
};

export interface Metrics {
[key: string]: number;
[key: string]: number | string;
}

export type MetricsCollection = Metrics[];
Expand Down Expand Up @@ -197,7 +197,7 @@ export function combineResults(...results: Array<ParseResult<any>>): ParseResult
return errors.length > 0 ? { success: false, errors } : { success: true, data: values };
}

export const parseMetrics = (metricsArray: Array<{ metric: string; value: number }>): Metrics => {
export const parseMetrics = (metricsArray: Array<{ metric: string; value: number | string }>): Metrics => {
return Object.fromEntries(
metricsArray.map(({ metric, value }) => [metric, value])
) as Metrics;
Expand Down
Loading