Fenrir fixes

ChangeLog.md

@@ -29,6 +29,26 @@
       because [MQTT-3.1.3.5] defines Password as Binary Data, not a UTF-8
       string. A binary password containing bytes that are not valid UTF-8
       (e.g., `0xC0`, `0xFF`) would otherwise be incorrectly rejected.
+    - `MqttClient_Publish` / `MqttClient_Publish_ex` now return the new
+      `MQTT_CODE_ERROR_PUBLISH_REJECTED` (-21) when a v5 broker rejects a
+      QoS>0 PUBLISH via a PUBACK (QoS 1), PUBREC, or PUBCOMP (QoS 2) reason
+      code >= 0x80 (e.g. Not authorized, Quota exceeded, Topic Name invalid,
+      Payload format invalid). Previously these were reported as
+      `MQTT_CODE_SUCCESS`, so the application proceeded as if the broker had
+      accepted the message. The specific reason is available in
+      `MqttPublish.resp.reason_code`. For QoS 2, a PUBREC reason code >= 0x80
+      now ends the exchange without sending PUBREL per [MQTT-4.3.3] instead of
+      timing out. v3.1.1 publishes are unaffected, as is the return value of
+      the fire-and-forget `MqttClient_Publish_WriteOnly` call itself. Callers
+      that treat any non-success return as fatal may need to handle this code.
+      In `WOLFMQTT_MULTITHREAD` builds where a dedicated thread drives reads,
+      that reading thread now returns `MQTT_CODE_ERROR_PUBLISH_REJECTED` when it
+      processes a QoS 2 PUBREC rejection (instead of advancing the handshake
+      with an illegal PUBREL); the originating write-only publish's pending
+      response is not auto-completed in that case, so it blocks until
+      `cmd_timeout_ms`. A QoS 1 PUBACK or QoS 2 PUBCOMP rejection is NOT
+      detected on the write-only path (the publish appears successful), matching
+      prior behavior; use `MqttClient_Publish`/`_ex` for reliable detection.
 
 ### v2.0.0 (03/20/2026)
 Release 2.0.0 has been developed according to wolfSSL's development and QA

src/mqtt_client.c

@@ -1030,6 +1030,28 @@ static int MqttClient_HandlePacket(MqttClient* client,
                 break;
             }
 
+        #ifdef WOLFMQTT_V5
+            /* A v5 broker rejects a QoS 2 PUBLISH at the PUBREC stage with a
+             * reason code >= 0x80 (e.g. not authorized, quota exceeded, topic
+             * name invalid, payload format invalid). Per [MQTT-4.3.3] the
+             * exchange is then complete and the sender MUST NOT send a PUBREL.
+             * Surface the rejection instead of advancing the handshake, which
+             * would emit an illegal PUBREL and then block waiting for a PUBCOMP
+             * the broker will never send. The QoS 1 PUBACK and the QoS 2
+             * PUBCOMP reason codes are checked by the caller after the wait.
+             * Note (WOLFMQTT_MULTITHREAD): when a separate thread drives reads
+             * and processes this PUBREC, it receives this error directly and
+             * the publishing thread's PUBCOMP pending response is not marked
+             * done, so that publish blocks until cmd_timeout_ms. This matches
+             * the pre-existing behavior (which left the publisher waiting on a
+             * PUBCOMP after an illegal PUBREL) and is not made worse here. */
+            if (packet_type == MQTT_PACKET_TYPE_PUBLISH_REC &&
+                client->protocol_level >= MQTT_CONNECT_PROTOCOL_LEVEL_5 &&
+                (((MqttPublishResp*)packet_obj)->reason_code & 0x80)) {
+                return MQTT_TRACE_ERROR(MQTT_CODE_ERROR_PUBLISH_REJECTED);
+            }
+        #endif
+
             /* Populate information needed for ack */
             resp->packet_type = packet_type+1; /* next ack */
             resp->packet_id = packet_id;
@@ -2324,6 +2346,26 @@ static int MqttPublishMsg(MqttClient *client, MqttPublish *publish,
                     /* Wait for publish response packet */
                     rc = MqttClient_WaitType(client, &publish->resp, resp_type,
                         publish->packet_id, client->cmd_timeout_ms);
+
+                #ifdef WOLFMQTT_V5
+                    /* A v5 broker can acknowledge a QoS>0 PUBLISH at the
+                     * protocol layer yet still reject the message via a
+                     * PUBACK/PUBCOMP reason code >= 0x80 (e.g. not authorized,
+                     * quota exceeded, topic name invalid, payload format
+                     * invalid). Surface that as an error so the caller does not
+                     * treat a rejected message as delivered. Mirrors the
+                     * CONNECT/SUBSCRIBE/UNSUBSCRIBE rejection handling. The
+                     * protocol_level guard avoids misreading a stale byte for
+                     * v3.1.1 ACKs, which carry no reason code (same guard the
+                     * PUBREC check in MqttClient_HandlePacket uses). */
+                    if (rc == MQTT_CODE_SUCCESS &&
+                        client->protocol_level >=
+                            MQTT_CONNECT_PROTOCOL_LEVEL_5 &&
+                        (publish->resp.reason_code & 0x80)) {
+                        rc = MQTT_TRACE_ERROR(
+                            MQTT_CODE_ERROR_PUBLISH_REJECTED);
+                    }
+                #endif
                 }
 
             #if defined(WOLFMQTT_NONBLOCK) || defined(WOLFMQTT_MULTITHREAD)
@@ -3202,6 +3244,8 @@ const char* MqttClient_ReturnCodeToString(int return_code)
             return "Error (Broker rejected subscription)";
         case MQTT_CODE_ERROR_UNSUBSCRIBE_REJECTED:
             return "Error (Broker rejected unsubscribe)";
+        case MQTT_CODE_ERROR_PUBLISH_REJECTED:
+            return "Error (Broker rejected publish)";
 #if defined(ENABLE_MQTT_CURL)
         case MQTT_CODE_ERROR_CURL:
             return "Error (libcurl)";