Fix oldest non-terminal tx db query deadlock

Author: silaslenihan

pkg/txmgr/evm_tx_store.go

@@ -1645,10 +1645,15 @@ func (o *evmTxStore) OldestNonTerminalTxAgeSeconds(ctx context.Context, chainID
 	ctx, cancel = o.stopCh.Ctx(ctx)
 	defer cancel()
 	err = o.q.GetContext(ctx, &seconds, `
-SELECT GREATEST(0::float8, COALESCE(EXTRACT(EPOCH FROM (NOW() - MIN(created_at))), 0::float8))
-FROM evm.txes
-WHERE evm_chain_id = $1 AND state NOT IN ('finalized', 'fatal_error')`,
+SELECT GREATEST(0::float8, COALESCE(EXTRACT(EPOCH FROM (NOW() - t.created_at)), 0::float8)) AS seconds
+FROM evm.txes AS t
+WHERE t.evm_chain_id = $1 AND t.state NOT IN ('finalized', 'fatal_error')
+ORDER BY t.created_at ASC NULLS LAST
+LIMIT 1`,
 		chainID.String())
+	if errors.Is(err, sql.ErrNoRows) {
+		return 0, nil
+	}
 	if err != nil {
 		return 0, fmt.Errorf("failed to OldestNonTerminalTxAgeSeconds: %w", err)
 	}

pkg/txmgr/finalizer.go

@@ -37,6 +37,8 @@ var (
 const (
 	processHeadTimeout            = 10 * time.Minute
 	attemptsCacheRefreshThreshold = 5
+	// Bound metrics query time so a contended shared test DB cannot block the finalizer head path.
+	oldestNonTerminalTxAgeMetricQueryTimeout = 5 * time.Second
 )
 
 type finalizerTxStore interface {
@@ -213,7 +215,9 @@ func (f *evmFinalizer) ProcessHead(ctx context.Context, head *types.Head) error
 }
 
 func (f *evmFinalizer) observeOldestNonTerminalTxAge(ctx context.Context) {
-	age, err := f.txStore.OldestNonTerminalTxAgeSeconds(ctx, f.chainID)
+	ctxObs, cancel := context.WithTimeout(ctx, oldestNonTerminalTxAgeMetricQueryTimeout)
+	defer cancel()
+	age, err := f.txStore.OldestNonTerminalTxAgeSeconds(ctxObs, f.chainID)
 	if err != nil {
 		f.lggr.Errorw("Failed to load oldest non-terminal transaction age for metrics", "err", err)
 		return