fix(pbs): correct submit_block v2 content negotiation and fail loud on v1-only relays

Content negotiation (Builder API):
- Missing Accept defaults the response to JSON instead of inheriting the request
  Content-Type; request and response encodings are independent.
- v2 succeeds with an empty 202 and has no body to negotiate, so it skips Accept
  entirely -- a bad Accept no longer 406s a v2 submission before it reaches a relay.
- An unrecognized request Content-Type returns 415 instead of 400.
- Always request SSZ from the relay (JSON fallback) rather than mirroring the caller's
  format: PBS decodes and re-validates the payload and the route re-encodes to the BN's
  Accept regardless, so SSZ is the fastest wire format on the relay hop.

Fail loud on v1-only relays:
- Drop the v2->v1 fallback. It returned the relay's v1 payload as a 200 body, but a v2
  caller (Lighthouse, Prysm) requires exactly 202 and never reads the body, so the block
  was silently dropped. PBS cannot publish the payload itself, so a relay that 404s v2
  now fails loud (RELAY_V2_UNSUPPORTED) and other relays still get a chance.

Author: JasonVranek

crates/common/src/wire.rs

@@ -254,9 +254,10 @@ pub fn get_accept_types(
         return Err(AcceptedEncodingsError::UnsupportedAcceptType)
     }
 
-    // No accept header (or only q=0 rejections): fall back to the request
-    // Content-Type, which mirrors the historical behavior.
-    Ok(AcceptedEncodings::single(get_content_type(req_headers)))
+    // No Accept header (or only q=0 rejections): per the Builder API a missing
+    // Accept means JSON, and request/response encodings are independent — so do
+    // NOT inherit the request Content-Type (an SSZ request still gets JSON).
+    Ok(AcceptedEncodings::single(NO_PREFERENCE_DEFAULT))
 }
 
 fn essence_encoding(mt: &MediaType) -> Option<EncodingType> {
@@ -468,6 +469,17 @@ mod test {
         assert_eq!(result, AcceptedEncodings::single(EncodingType::Json));
     }
 
+    /// A missing Accept header defaults to JSON even when the request body is
+    /// SSZ: request and response encodings are independent, so the response
+    /// encoding MUST NOT inherit the request Content-Type.
+    #[test]
+    fn test_missing_accept_header_ignores_ssz_content_type() {
+        let mut headers = HeaderMap::new();
+        headers.append(CONTENT_TYPE, HeaderValue::from_str(APPLICATION_OCTET_STREAM).unwrap());
+        let result = get_accept_types(&headers).unwrap();
+        assert_eq!(result, AcceptedEncodings::single(EncodingType::Json));
+    }
+
     /// Test accepting JSON
     #[test]
     fn test_accept_header_json() {

crates/pbs/src/error.rs

@@ -26,6 +26,9 @@ impl PbsClientError {
             PbsClientError::NoResponse => StatusCode::BAD_GATEWAY,
             PbsClientError::NoPayload => StatusCode::BAD_GATEWAY,
             PbsClientError::Internal => StatusCode::INTERNAL_SERVER_ERROR,
+            PbsClientError::DecodeError(BodyDeserializeError::UnsupportedMediaType) => {
+                StatusCode::UNSUPPORTED_MEDIA_TYPE
+            }
             PbsClientError::DecodeError(_) => StatusCode::BAD_REQUEST,
             PbsClientError::HeaderError(_) => StatusCode::NOT_ACCEPTABLE,
         }
@@ -45,3 +48,24 @@ impl IntoResponse for PbsClientError {
         (self.status_code(), msg).into_response()
     }
 }
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[test]
+    fn unsupported_media_type_maps_to_415() {
+        assert_eq!(
+            PbsClientError::DecodeError(BodyDeserializeError::UnsupportedMediaType).status_code(),
+            StatusCode::UNSUPPORTED_MEDIA_TYPE,
+        );
+    }
+
+    #[test]
+    fn other_decode_errors_map_to_400() {
+        assert_eq!(
+            PbsClientError::DecodeError(BodyDeserializeError::MissingVersionHeader).status_code(),
+            StatusCode::BAD_REQUEST,
+        );
+    }
+}

crates/pbs/src/metrics.rs

@@ -61,12 +61,13 @@ lazy_static! {
         PBS_METRICS_REGISTRY
     ).unwrap();
 
-    /// Count of v2 submit_block requests that fell back to the v1 endpoint
-    /// because the relay returned 404 on v2. A high value indicates the relay
-    /// fleet has not been upgraded to support submitBlindedBlockV2.
-    pub static ref V2_FALLBACK_TO_V1: IntCounterVec = register_int_counter_vec_with_registry!(
-        "pbs_submit_block_v2_fallback_to_v1_total",
-        "Count of v2 submit_block requests that fell back to v1 because the relay did not support v2",
+    /// Count of v2 submit_block requests that could not be served because the
+    /// relay returned 404 on the v2 endpoint. A non-zero value means the relay
+    /// fleet has not been upgraded to support submitBlindedBlockV2 and those
+    /// blocks were not submitted.
+    pub static ref RELAY_V2_UNSUPPORTED: IntCounterVec = register_int_counter_vec_with_registry!(
+        "pbs_submit_block_v2_unsupported_total",
+        "Count of v2 submit_block requests a relay could not serve because it does not support v2",
         &["relay_id"],
         PBS_METRICS_REGISTRY
     ).unwrap();

crates/pbs/src/mev_boost/submit_block.rs

@@ -15,8 +15,7 @@ use cb_common::{
     utils::utcnow_ms,
     wire::{
         AcceptedEncodings, CONSENSUS_VERSION_HEADER, EncodingType, build_outbound_accept,
-        get_accept_types, get_user_agent_with_version, parse_response_encoding_and_fork,
-        read_chunked_body_with_max,
+        get_user_agent_with_version, parse_response_encoding_and_fork, read_chunked_body_with_max,
     },
 };
 use futures::{FutureExt, future::select_ok};
@@ -25,13 +24,13 @@ use reqwest::{
     header::{ACCEPT, CONTENT_TYPE, USER_AGENT},
 };
 use ssz::Encode;
-use tracing::{debug, error, warn};
+use tracing::{debug, warn};
 use url::Url;
 
 use crate::{
     TIMEOUT_ERROR_CODE_STR,
     constants::{MAX_SIZE_SUBMIT_BLOCK_RESPONSE, SUBMIT_BLINDED_BLOCK_ENDPOINT_TAG},
-    metrics::{RELAY_LATENCY, RELAY_STATUS_CODE, V2_FALLBACK_TO_V1},
+    metrics::{RELAY_LATENCY, RELAY_STATUS_CODE, RELAY_V2_UNSUPPORTED},
     state::{BuilderApiState, PbsState},
 };
 
@@ -55,8 +54,8 @@ struct SubmitBlockResponseInfo {
     /// ACCEPTED/OK paths where no body is returned.
     content_type: Option<EncodingType>,
 
-    /// Which fork the response bid is for (if provided as a header, rather than
-    /// part of the body)
+    /// Which fork the response payload is for (if provided as a header, rather
+    /// than part of the body)
     fork: Option<ForkName>,
 
     /// The status code of the response, for logging
@@ -82,21 +81,21 @@ pub async fn submit_block<S: BuilderApiState>(
     send_headers.insert(HEADER_START_TIME_UNIX_MS, HeaderValue::from(utcnow_ms()));
     send_headers.insert(USER_AGENT, get_user_agent_with_version(&req_headers)?);
 
-    // Forward the caller's Accept preference to the relay so the relay
-    // returns data in the format the BN expects, avoiding decode→re-encode.
-    // Always offer both encodings as fallback so a format-limited relay
-    // still returns a bid (PBS converts if needed).
-    let caller_accept = get_accept_types(&req_headers).inspect_err(|err| {
-        error!(%err, "error parsing accept header");
-    })?;
-    let relay_accept = AcceptedEncodings {
-        primary: caller_accept.primary,
-        fallback: Some(match caller_accept.primary {
-            EncodingType::Ssz => EncodingType::Json,
-            EncodingType::Json => EncodingType::Ssz,
-        }),
-    };
-    send_headers.insert(ACCEPT, build_outbound_accept(relay_accept));
+    // PBS always decodes and re-validates the relay payload, then the route
+    // re-encodes it to the BN's Accept. So always request SSZ from the relay
+    // (smaller on the wire, faster to decode than JSON); JSON is the fallback for
+    // a relay that can't do SSZ. The BN's own format preference is applied later
+    // by the route, not here. Skip for v2, whose success is an empty 202 with no
+    // body to negotiate.
+    if api_version == BuilderApiVersion::V1 {
+        send_headers.insert(
+            ACCEPT,
+            build_outbound_accept(AcceptedEncodings {
+                primary: EncodingType::Ssz,
+                fallback: Some(EncodingType::Json),
+            }),
+        );
+    }
 
     // Send requests to all relays concurrently
     let proposal_info =
@@ -130,11 +129,11 @@ async fn submit_block_with_timeout(
     relay: RelayClient,
     timeout_ms: u64,
 ) -> Result<Option<SubmitBlindedBlockResponse>, PbsError> {
-    let mut url = Arc::new(relay.submit_block_url(proposal_info.api_version)?);
+    let url = Arc::new(relay.submit_block_url(proposal_info.api_version)?);
     let mut remaining_timeout_ms = timeout_ms;
     let mut retry = 0;
     let mut backoff = Duration::from_millis(250);
-    let mut request_api_version = proposal_info.api_version;
+    let request_api_version = proposal_info.api_version;
 
     loop {
         let start_request = Instant::now();
@@ -148,23 +147,7 @@ async fn submit_block_with_timeout(
         )
         .await
         {
-            Ok(response) => {
-                // If the original request was for v2 but we had to fall back to v1, the
-                // V1 response body (execution payload + blobs bundle) MUST be forwarded
-                // back to the beacon node so the proposer can broadcast. Returning an
-                // empty 202 here would cause silent block loss because the BN never
-                // receives the unblinded payload.
-                if request_api_version == BuilderApiVersion::V1 &&
-                    proposal_info.api_version != request_api_version
-                {
-                    warn!(
-                        relay_id = relay.id.as_ref(),
-                        "v2 submit_block fell back to v1; forwarding v1 payload to beacon node"
-                    );
-                    V2_FALLBACK_TO_V1.with_label_values(&[relay.id.as_ref()]).inc();
-                }
-                return Ok(response);
-            }
+            Ok(response) => return Ok(response),
 
             Err(err) if err.should_retry() => {
                 tokio::time::sleep(backoff).await;
@@ -178,15 +161,18 @@ async fn submit_block_with_timeout(
                 }
             }
 
-            Err(err)
-                if err.is_not_found() && matches!(request_api_version, BuilderApiVersion::V2) =>
-            {
+            // A relay that 404s the v2 endpoint cannot serve a v2 submission. In
+            // v2 the relay itself publishes the block after an empty 202, so a v1
+            // payload is useless here: the beacon node expects 202 and will not
+            // read a 200 body, so forwarding it would silently drop the block.
+            // Fail loud instead, and let another relay (if any) serve v2.
+            Err(err) if err.is_not_found() && request_api_version == BuilderApiVersion::V2 => {
                 warn!(
                     relay_id = relay.id.as_ref(),
-                    "relay does not support v2 endpoint, retrying with v1"
+                    "relay does not support the v2 submit_block endpoint; cannot serve this submission"
                 );
-                url = Arc::new(relay.submit_block_url(BuilderApiVersion::V1)?);
-                request_api_version = BuilderApiVersion::V1;
+                RELAY_V2_UNSUPPORTED.with_label_values(&[relay.id.as_ref()]).inc();
+                return Err(err);
             }
 
             Err(err) => return Err(err),

crates/pbs/src/routes/submit_block.rs

@@ -10,8 +10,8 @@ use cb_common::{
     pbs::{BuilderApiVersion, GetPayloadInfo},
     utils::{timestamp_of_slot_start_millis, utcnow_ms},
     wire::{
-        AcceptedEncodingsError, CONSENSUS_VERSION_HEADER, EncodingType, deserialize_body,
-        get_accept_types, get_user_agent,
+        AcceptedEncodings, AcceptedEncodingsError, CONSENSUS_VERSION_HEADER, EncodingType,
+        NO_PREFERENCE_DEFAULT, deserialize_body, get_accept_types, get_user_agent,
     },
 };
 use reqwest::{StatusCode, header::CONTENT_TYPE};
@@ -63,9 +63,16 @@ async fn handle_submit_block_impl<S: BuilderApiState, A: BuilderApi<S>>(
     let block_hash = signed_blinded_block.block_hash();
     let slot_start_ms = timestamp_of_slot_start_millis(slot.into(), state.config.chain);
     let ua = get_user_agent(&req_headers);
-    let accept_types = get_accept_types(&req_headers).inspect_err(|err| {
-        error!(%err, "error parsing accept header");
-    })?;
+    // v1 enforces Accept (a bad one is a 406). v2 succeeds with an empty 202 and
+    // has no response body to negotiate, so skip Accept for it entirely: a bad
+    // Accept must not 406 a v2 submission before it reaches a relay.
+    let accept_types = if api_version == BuilderApiVersion::V1 {
+        get_accept_types(&req_headers).inspect_err(|err| {
+            error!(%err, "error parsing accept header");
+        })?
+    } else {
+        AcceptedEncodings::single(NO_PREFERENCE_DEFAULT)
+    };
     // Honor caller q-value preference: pick the highest-priority encoding that
     // we can actually produce. Server preference for tiebreaks is SSZ first.
     let response_encoding = accept_types.preferred(&[EncodingType::Ssz, EncodingType::Json]);

tests/tests/pbs_post_blinded_blocks.rs

@@ -6,7 +6,7 @@ use cb_common::{
     },
     signer::random_secret,
     types::Chain,
-    wire::EncodingType,
+    wire::{CONSENSUS_VERSION_HEADER, EncodingType},
 };
 use cb_pbs::{DefaultBuilderApi, PbsService, PbsState};
 use cb_tests::{
@@ -18,7 +18,10 @@ use cb_tests::{
 };
 use eyre::Result;
 use lh_types::ForkVersionDecode;
-use reqwest::{Response, StatusCode};
+use reqwest::{
+    Response, StatusCode,
+    header::{ACCEPT, CONTENT_TYPE},
+};
 use tracing::info;
 
 #[tokio::test]
@@ -61,31 +64,24 @@ async fn test_submit_block_v2() -> Result<()> {
     Ok(())
 }
 
-// Test that when submitting a block using v2 to a relay that does not support
-// v2, PBS falls back to v1 and forwards the v1 response body to the beacon
-// node (a 200 with the execution payload), rather than swallowing the payload
-// and replying 202 with an empty body — which would cause silent block loss.
+// A v2 submission to a relay that does not support v2 must fail loud, not fake
+// success. v2's contract is an empty 202 after which the relay publishes the
+// block; a v1 payload is useless because the beacon node expects 202 and will
+// not read a 200 body. So PBS returns an error (no relay could serve v2) rather
+// than forwarding a v1 body the caller silently drops.
 #[tokio::test]
 async fn test_submit_block_v2_without_relay_support() -> Result<()> {
-    let res = submit_block_impl(
+    let _res = submit_block_impl(
         BuilderApiVersion::V2,
         vec![EncodingType::Json],
         HashSet::from([EncodingType::Ssz, EncodingType::Json]),
         EncodingType::Json,
-        1,
-        StatusCode::OK,
+        0, // relay 404s v2 and there is no v1 fallback, so it never receives a submit
+        StatusCode::BAD_GATEWAY,
         true,
         false,
     )
     .await?;
-    // Payload must be forwarded so the BN can broadcast.
-    let signed_blinded_block = load_test_signed_blinded_block();
-    let response_body = serde_json::from_slice::<SubmitBlindedBlockResponse>(&res.bytes().await?)?;
-    assert_eq!(
-        response_body.data.execution_payload.block_hash(),
-        signed_blinded_block.block_hash().into(),
-        "v2->v1 fallback must forward the execution payload to the BN"
-    );
     Ok(())
 }
 
@@ -330,6 +326,52 @@ async fn test_submit_block_rejects_fork_mismatch() -> Result<()> {
     Ok(())
 }
 
+// A v2 submission does no content negotiation (its success is an empty 202), so
+// an unsupported Accept header must NOT 406 the request before it reaches a
+// relay. Only v1, which returns a body, negotiates on Accept.
+#[tokio::test]
+async fn test_submit_block_v2_ignores_unsupported_accept() -> Result<()> {
+    setup_test_env();
+    let signer = random_secret();
+    let pubkey = signer.public_key();
+    let chain = Chain::Holesky;
+    let pbs_listener = get_free_listener().await;
+    let relay_listener = get_free_listener().await;
+    let pbs_port = pbs_listener.local_addr().unwrap().port();
+    let relay_port = relay_listener.local_addr().unwrap().port();
+
+    let mock_relay = generate_mock_relay(relay_port, pubkey)?;
+    // Default mock supports v2.
+    let mock_state = Arc::new(MockRelayState::new(chain, signer));
+    tokio::spawn(start_mock_relay_service_with_listener(mock_state.clone(), relay_listener));
+
+    let config = to_pbs_config(chain, get_pbs_config(pbs_port), vec![mock_relay]);
+    let state = PbsState::new(config, PathBuf::new());
+    drop(pbs_listener);
+    tokio::spawn(PbsService::run::<(), DefaultBuilderApi>(state));
+    tokio::time::sleep(Duration::from_millis(100)).await;
+
+    let mock_validator = MockValidator::new(pbs_port)?;
+    let url = mock_validator.comm_boost.submit_block_url(BuilderApiVersion::V2).unwrap();
+    let body = serde_json::to_vec(&load_test_signed_blinded_block()).unwrap();
+    let res = mock_validator
+        .comm_boost
+        .client
+        .post(url)
+        .body(body)
+        .header(CONTENT_TYPE, EncodingType::Json.to_string())
+        .header(CONSENSUS_VERSION_HEADER, ForkName::Electra.to_string())
+        .header(ACCEPT, "application/garbage")
+        .send()
+        .await?;
+
+    // Reaches the relay and returns the v2 empty-202 rather than a 406.
+    assert_ne!(res.status(), StatusCode::NOT_ACCEPTABLE);
+    assert_eq!(res.status(), StatusCode::ACCEPTED);
+    assert_eq!(mock_state.received_submit_block(), 1);
+    Ok(())
+}
+
 #[allow(clippy::too_many_arguments)]
 async fn submit_block_impl(
     api_version: BuilderApiVersion,