Skip to content

Commit 38305b7

Browse files
rzeyde-varadamartint
rzeyde-varada
authored and
martint
committed
Allow exposing custom metrics from the connector
1 parent d159c6c commit 38305b7

16 files changed

+228
-1
lines changed

‎core/trino-main/src/main/java/io/trino/operator/OperatorContext.java

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
import io.trino.operator.OperationTimer.OperationTiming;
2929
import io.trino.spi.Page;
3030
import io.trino.spi.TrinoException;
31+
import io.trino.spi.metrics.Metrics;
3132
import io.trino.sql.planner.plan.PlanNodeId;
3233

3334
import javax.annotation.Nullable;
@@ -83,6 +84,7 @@ public class OperatorContext
8384
private final CounterStat outputPositions = new CounterStat();
8485

8586
private final AtomicLong dynamicFilterSplitsProcessed = new AtomicLong();
87+
private final AtomicReference<Metrics> metrics = new AtomicReference<>(Metrics.EMPTY); // this is not incremental, but gets overwritten by the latest value.
8688

8789
private final AtomicLong physicalWrittenDataSize = new AtomicLong();
8890

@@ -219,6 +221,16 @@ public void recordDynamicFilterSplitProcessed(long dynamicFilterSplits)
219221
dynamicFilterSplitsProcessed.getAndAdd(dynamicFilterSplits);
220222
}
221223

224+
/**
225+
* Overwrites the metrics with the latest one.
226+
*
227+
* @param metrics Latest operator's metrics.
228+
*/
229+
public void setLatestMetrics(Metrics metrics)
230+
{
231+
this.metrics.set(metrics);
232+
}
233+
222234
public void recordPhysicalWrittenData(long sizeInBytes)
223235
{
224236
physicalWrittenDataSize.getAndAdd(sizeInBytes);
@@ -531,6 +543,7 @@ public OperatorStats getOperatorStats()
531543
outputPositions.getTotalCount(),
532544

533545
dynamicFilterSplitsProcessed.get(),
546+
metrics.get(),
534547

535548
succinctBytes(physicalWrittenDataSize.get()),
536549

‎core/trino-main/src/main/java/io/trino/operator/OperatorStats.java

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
import io.airlift.units.DataSize;
2020
import io.airlift.units.Duration;
2121
import io.trino.spi.Mergeable;
22+
import io.trino.spi.metrics.Metrics;
2223
import io.trino.sql.planner.plan.PlanNodeId;
2324

2425
import javax.annotation.Nullable;
@@ -63,6 +64,7 @@ public class OperatorStats
6364
private final long outputPositions;
6465

6566
private final long dynamicFilterSplitsProcessed;
67+
private final Metrics metrics;
6668

6769
private final DataSize physicalWrittenDataSize;
6870

@@ -115,6 +117,7 @@ public OperatorStats(
115117
@JsonProperty("outputPositions") long outputPositions,
116118

117119
@JsonProperty("dynamicFilterSplitsProcessed") long dynamicFilterSplitsProcessed,
120+
@JsonProperty("metrics") Metrics metrics,
118121

119122
@JsonProperty("physicalWrittenDataSize") DataSize physicalWrittenDataSize,
120123

@@ -169,6 +172,7 @@ public OperatorStats(
169172
this.outputPositions = outputPositions;
170173

171174
this.dynamicFilterSplitsProcessed = dynamicFilterSplitsProcessed;
175+
this.metrics = requireNonNull(metrics);
172176

173177
this.physicalWrittenDataSize = requireNonNull(physicalWrittenDataSize, "physicalWrittenDataSize is null");
174178

@@ -332,6 +336,12 @@ public long getDynamicFilterSplitsProcessed()
332336
return dynamicFilterSplitsProcessed;
333337
}
334338

339+
@JsonProperty
340+
public Metrics getMetrics()
341+
{
342+
return metrics;
343+
}
344+
335345
@JsonProperty
336346
public DataSize getPhysicalWrittenDataSize()
337347
{
@@ -451,6 +461,7 @@ public OperatorStats add(Iterable<OperatorStats> operators)
451461
long outputPositions = this.outputPositions;
452462

453463
long dynamicFilterSplitsProcessed = this.dynamicFilterSplitsProcessed;
464+
Metrics.Accumulator metricsAccumulator = Metrics.accumulator().add(this.getMetrics());
454465

455466
long physicalWrittenDataSize = this.physicalWrittenDataSize.toBytes();
456467

@@ -498,6 +509,7 @@ public OperatorStats add(Iterable<OperatorStats> operators)
498509
outputPositions += operator.getOutputPositions();
499510

500511
dynamicFilterSplitsProcessed += operator.getDynamicFilterSplitsProcessed();
512+
metricsAccumulator.add(operator.getMetrics());
501513

502514
physicalWrittenDataSize += operator.getPhysicalWrittenDataSize().toBytes();
503515

@@ -557,6 +569,7 @@ public OperatorStats add(Iterable<OperatorStats> operators)
557569
outputPositions,
558570

559571
dynamicFilterSplitsProcessed,
572+
metricsAccumulator.get(),
560573

561574
succinctBytes(physicalWrittenDataSize),
562575

@@ -623,6 +636,7 @@ public OperatorStats summarize()
623636
outputDataSize,
624637
outputPositions,
625638
dynamicFilterSplitsProcessed,
639+
metrics,
626640
physicalWrittenDataSize,
627641
blockedWall,
628642
finishCalls,

‎core/trino-main/src/main/java/io/trino/operator/ScanFilterAndProjectOperator.java

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@
3939
import io.trino.spi.connector.RecordCursor;
4040
import io.trino.spi.connector.RecordPageSource;
4141
import io.trino.spi.connector.UpdatablePageSource;
42+
import io.trino.spi.metrics.Metrics;
4243
import io.trino.spi.type.Type;
4344
import io.trino.split.EmptySplit;
4445
import io.trino.split.PageSourceProvider;
@@ -79,6 +80,7 @@ public class ScanFilterAndProjectOperator
7980
private long physicalBytes;
8081
private long readTimeNanos;
8182
private long dynamicFilterSplitsProcessed;
83+
private Metrics metrics = Metrics.EMPTY;
8284

8385
private ScanFilterAndProjectOperator(
8486
Session session,
@@ -160,6 +162,12 @@ public long getDynamicFilterSplitsProcessed()
160162
return dynamicFilterSplitsProcessed;
161163
}
162164

165+
@Override
166+
public Metrics getConnectorMetrics()
167+
{
168+
return metrics;
169+
}
170+
163171
@Override
164172
public WorkProcessor<Page> getOutputPages()
165173
{
@@ -172,6 +180,7 @@ public void close()
172180
if (pageSource != null) {
173181
try {
174182
pageSource.close();
183+
metrics = pageSource.getMetrics();
175184
}
176185
catch (IOException e) {
177186
throw new UncheckedIOException(e);
@@ -394,6 +403,7 @@ public ProcessState<Page> process()
394403
processedPositions += page.getPositionCount();
395404
physicalBytes = pageSource.getCompletedBytes();
396405
readTimeNanos = pageSource.getReadTimeNanos();
406+
metrics = pageSource.getMetrics();
397407

398408
return ProcessState.ofResult(page);
399409
}

‎core/trino-main/src/main/java/io/trino/operator/TableScanOperator.java

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -247,6 +247,7 @@ public void finish()
247247
throw new UncheckedIOException(e);
248248
}
249249
systemMemoryContext.setBytes(source.getSystemMemoryUsage());
250+
operatorContext.setLatestMetrics(source.getMetrics());
250251
}
251252
}
252253

@@ -322,7 +323,7 @@ public Page getOutput()
322323

323324
// updating system memory usage should happen after page is loaded.
324325
systemMemoryContext.setBytes(source.getSystemMemoryUsage());
325-
326+
operatorContext.setLatestMetrics(source.getMetrics());
326327
return page;
327328
}
328329
}

‎core/trino-main/src/main/java/io/trino/operator/TableScanWorkProcessorOperator.java

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@
3232
import io.trino.spi.connector.DynamicFilter;
3333
import io.trino.spi.connector.EmptyPageSource;
3434
import io.trino.spi.connector.UpdatablePageSource;
35+
import io.trino.spi.metrics.Metrics;
3536
import io.trino.split.EmptySplit;
3637
import io.trino.split.PageSourceProvider;
3738

@@ -117,6 +118,12 @@ public long getDynamicFilterSplitsProcessed()
117118
return splitToPages.getDynamicFilterSplitsProcessed();
118119
}
119120

121+
@Override
122+
public Metrics getConnectorMetrics()
123+
{
124+
return splitToPages.source.getMetrics();
125+
}
126+
120127
@Override
121128
public Duration getReadTime()
122129
{

‎core/trino-main/src/main/java/io/trino/operator/WorkProcessorPipelineSourceOperator.java

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
import io.trino.operator.WorkProcessor.ProcessState;
3030
import io.trino.spi.Page;
3131
import io.trino.spi.connector.UpdatablePageSource;
32+
import io.trino.spi.metrics.Metrics;
3233
import io.trino.spi.type.Type;
3334
import io.trino.sql.planner.LocalExecutionPlanner.OperatorFactoryWithTypes;
3435
import io.trino.sql.planner.plan.PlanNodeId;
@@ -39,6 +40,7 @@
3940
import java.util.List;
4041
import java.util.Optional;
4142
import java.util.concurrent.atomic.AtomicLong;
43+
import java.util.concurrent.atomic.AtomicReference;
4244
import java.util.function.Supplier;
4345

4446
import static com.google.common.base.Preconditions.checkState;
@@ -232,11 +234,14 @@ private void workProcessorOperatorStateMonitor(WorkProcessor.ProcessState<Page>
232234
long deltaReadTimeNanos = deltaAndSet(context.readTimeNanos, sourceOperator.getReadTime().roundTo(NANOSECONDS));
233235

234236
long deltaDynamicFilterSplitsProcessed = deltaAndSet(context.dynamicFilterSplitsProcessed, sourceOperator.getDynamicFilterSplitsProcessed());
237+
Metrics metrics = sourceOperator.getConnectorMetrics();
238+
context.connectorMetrics.set(metrics);
235239

236240
operatorContext.recordPhysicalInputWithTiming(deltaPhysicalInputDataSize, deltaPhysicalInputPositions, deltaReadTimeNanos);
237241
operatorContext.recordNetworkInput(deltaInternalNetworkInputDataSize, deltaInternalNetworkInputPositions);
238242
operatorContext.recordProcessedInput(deltaInputDataSize, deltaInputPositions);
239243
operatorContext.recordDynamicFilterSplitProcessed(deltaDynamicFilterSplitsProcessed);
244+
operatorContext.setLatestMetrics(metrics);
240245
}
241246

242247
if (state.getType() == FINISHED) {
@@ -337,6 +342,7 @@ private List<OperatorStats> getNestedOperatorStats()
337342
context.outputPositions.get(),
338343

339344
context.dynamicFilterSplitsProcessed.get(),
345+
context.connectorMetrics.get(),
340346

341347
DataSize.ofBytes(0),
342348

@@ -676,6 +682,7 @@ private static class WorkProcessorOperatorContext
676682
final AtomicLong outputPositions = new AtomicLong();
677683

678684
final AtomicLong dynamicFilterSplitsProcessed = new AtomicLong();
685+
final AtomicReference<Metrics> connectorMetrics = new AtomicReference<>(Metrics.EMPTY);
679686

680687
final AtomicLong peakUserMemoryReservation = new AtomicLong();
681688
final AtomicLong peakSystemMemoryReservation = new AtomicLong();

‎core/trino-main/src/main/java/io/trino/operator/WorkProcessorSourceOperator.java

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
import io.airlift.units.DataSize;
1717
import io.airlift.units.Duration;
1818
import io.trino.spi.connector.UpdatablePageSource;
19+
import io.trino.spi.metrics.Metrics;
1920

2021
import java.util.Optional;
2122
import java.util.function.Supplier;
@@ -67,4 +68,6 @@ default long getDynamicFilterSplitsProcessed()
6768
{
6869
return 0;
6970
}
71+
72+
Metrics getConnectorMetrics();
7073
}

‎core/trino-main/src/main/java/io/trino/operator/WorkProcessorSourceOperatorAdapter.java

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
import io.trino.metadata.Split;
2222
import io.trino.spi.Page;
2323
import io.trino.spi.connector.UpdatablePageSource;
24+
import io.trino.spi.metrics.Metrics;
2425
import io.trino.sql.planner.plan.PlanNodeId;
2526

2627
import java.util.ArrayList;
@@ -175,6 +176,7 @@ public boolean isFinished()
175176
public void close()
176177
throws Exception
177178
{
179+
operatorContext.setLatestMetrics(sourceOperator.getConnectorMetrics());
178180
sourceOperator.close();
179181
}
180182

@@ -191,6 +193,7 @@ private void updateOperatorStats()
191193
long currentInputPositions = sourceOperator.getInputPositions();
192194

193195
long currentDynamicFilterSplitsProcessed = sourceOperator.getDynamicFilterSplitsProcessed();
196+
Metrics currentMetrics = sourceOperator.getConnectorMetrics();
194197

195198
if (currentPhysicalInputBytes != previousPhysicalInputBytes
196199
|| currentPhysicalInputPositions != previousPhysicalInputPositions
@@ -229,6 +232,8 @@ private void updateOperatorStats()
229232
operatorContext.recordDynamicFilterSplitProcessed(currentDynamicFilterSplitsProcessed - previousDynamicFilterSplitsProcessed);
230233
previousDynamicFilterSplitsProcessed = currentDynamicFilterSplitsProcessed;
231234
}
235+
236+
operatorContext.setLatestMetrics(currentMetrics);
232237
}
233238

234239
private static class SplitBuffer

‎core/trino-main/src/test/java/io/trino/execution/TestQueryStats.java

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
import io.trino.operator.OperatorStats;
2323
import io.trino.operator.TableWriterOperator;
2424
import io.trino.spi.eventlistener.StageGcStatistics;
25+
import io.trino.spi.metrics.Metrics;
2526
import io.trino.sql.planner.plan.PlanNodeId;
2627
import org.joda.time.DateTime;
2728
import org.testng.annotations.Test;
@@ -62,6 +63,7 @@ public class TestQueryStats
6263
succinctBytes(116L),
6364
117L,
6465
1833,
66+
Metrics.EMPTY,
6567
succinctBytes(118L),
6668
new Duration(119, NANOSECONDS),
6769
120L,
@@ -101,6 +103,7 @@ public class TestQueryStats
101103
succinctBytes(216L),
102104
217L,
103105
2833,
106+
Metrics.EMPTY,
104107
succinctBytes(218L),
105108
new Duration(219, NANOSECONDS),
106109
220L,
@@ -140,6 +143,7 @@ public class TestQueryStats
140143
succinctBytes(316L),
141144
317L,
142145
3833,
146+
Metrics.EMPTY,
143147
succinctBytes(318L),
144148
new Duration(319, NANOSECONDS),
145149
320L,

‎core/trino-main/src/test/java/io/trino/operator/TestOperatorStats.java

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,11 +13,13 @@
1313
*/
1414
package io.trino.operator;
1515

16+
import com.google.common.collect.ImmutableMap;
1617
import io.airlift.json.JsonCodec;
1718
import io.airlift.units.DataSize;
1819
import io.airlift.units.Duration;
1920
import io.trino.connector.CatalogName;
2021
import io.trino.operator.PartitionedOutputOperator.PartitionedOutputInfo;
22+
import io.trino.spi.metrics.Metrics;
2123
import io.trino.sql.planner.plan.PlanNodeId;
2224
import org.testng.annotations.Test;
2325

@@ -59,6 +61,7 @@ public class TestOperatorStats
5961
DataSize.ofBytes(12),
6062
13,
6163
533,
64+
Metrics.EMPTY,
6265

6366
DataSize.ofBytes(14),
6467

@@ -106,6 +109,7 @@ public class TestOperatorStats
106109
DataSize.ofBytes(12),
107110
13,
108111
533,
112+
Metrics.EMPTY,
109113

110114
DataSize.ofBytes(14),
111115

@@ -163,6 +167,7 @@ public static void assertExpectedOperatorStats(OperatorStats actual)
163167
assertEquals(actual.getOutputPositions(), 13);
164168

165169
assertEquals(actual.getDynamicFilterSplitsProcessed(), 533);
170+
assertEquals(actual.getMetrics().getMetrics(), ImmutableMap.of());
166171

167172
assertEquals(actual.getPhysicalWrittenDataSize(), DataSize.ofBytes(14));
168173

‎core/trino-main/src/test/java/io/trino/operator/TestWorkProcessorPipelineSourceOperator.java

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
import io.trino.operator.WorkProcessorAssertion.Transform;
2727
import io.trino.spi.Page;
2828
import io.trino.spi.connector.UpdatablePageSource;
29+
import io.trino.spi.metrics.Metrics;
2930
import io.trino.sql.planner.LocalExecutionPlanner.OperatorFactoryWithTypes;
3031
import io.trino.sql.planner.plan.PlanNodeId;
3132
import org.testng.annotations.AfterClass;
@@ -392,6 +393,12 @@ public Duration getReadTime()
392393
return new Duration(7, NANOSECONDS);
393394
}
394395

396+
@Override
397+
public Metrics getConnectorMetrics()
398+
{
399+
return Metrics.EMPTY;
400+
}
401+
395402
@Override
396403
public WorkProcessor<Page> getOutputPages()
397404
{

0 commit comments

Comments
 (0)