Improve provider hashing for batch_chat_*() (#802)

Rather than hashing the complete provider object, we should just hash the most critical components. This trades a little safety for convenience; I don't think there are likely to be too many cases where you re-call `batch_chat()` with a subtly different provider object and this makes it more likely that you can retrieve results stored with a different version of ellmer. I've also added an escape hatch so you can use chats saved by ellmer 0.3.0 in ellmer 0.4.0

Extracted out from #799.

Author: hadley

NEWS.md

@@ -1,5 +1,7 @@
 # ellmer (development version)
 
+* `batch_*()` no longer hashes properties of the provider besides the `name`, `model`, and `base_url`. This should provide some protection from accidentally reusing the same `.json` file with different providers, while still allowing you to use the same batch file across ellmer versions.
+* `batch_*()` have a new `ignore_hash` argument that allows you to opt out of the check if you're confident the difference only arises because ellmer itself has changed.
 * Turns now have a `@duration` slot. The slot is `NA` for user turns and a numeric giving the total time to complete the request for assistant turns (@simonpcouch, #798).
 * New `chat_openai_responses()` to use the new OpenAI responses API (#365).
 * `parallel_chat_structured()` now returns a tibble, since this does a better job of printing more complex data frames (#787).

R/batch-chat.R

@@ -37,6 +37,9 @@
 #'   it will return `NULL` if the batch is not complete, and you can retrieve
 #'   the results later by re-running `batch_chat()` when
 #'   `batch_chat_completed()` is `TRUE`.
+#' @param ignore_hash If `TRUE`, will only warn rather than error when the hash
+#'   doesn't match. You can use this if ellmer has changed the hash structure
+#'   and you're confident that you're reusing the same inputs.
 #' @returns
 #' For `batch_chat()`, a list of [Chat] objects, one for each prompt.
 #' For `batch_chat_test()`, a character vector of text responses.
@@ -78,14 +81,15 @@
 #' data
 #' }
 #' @export
-batch_chat <- function(chat, prompts, path, wait = TRUE) {
+batch_chat <- function(chat, prompts, path, wait = TRUE, ignore_hash = FALSE) {
   chat <- as_chat(chat)
 
   job <- BatchJob$new(
     chat = chat,
     prompts = prompts,
     path = path,
-    wait = wait
+    wait = wait,
+    ignore_hash = ignore_hash
   )
   job$step_until_done()
 
@@ -101,9 +105,21 @@ batch_chat <- function(chat, prompts, path, wait = TRUE) {
 
 #' @export
 #' @rdname batch_chat
-batch_chat_text <- function(chat, prompts, path, wait = TRUE) {
+batch_chat_text <- function(
+  chat,
+  prompts,
+  path,
+  wait = TRUE,
+  ignore_hash = FALSE
+) {
   chat <- as_chat(chat)
-  chats <- batch_chat(chat, prompts, path, wait = wait)
+  chats <- batch_chat(
+    chat,
+    prompts,
+    path,
+    wait = wait,
+    ignore_hash = ignore_hash
+  )
   map_chr(chats, \(chat) if (is.null(chat)) NA else chat$last_turn()@text)
 }
 
@@ -116,6 +132,7 @@ batch_chat_structured <- function(
   path,
   type,
   wait = TRUE,
+  ignore_hash = FALSE,
   convert = TRUE,
   include_tokens = FALSE,
   include_cost = FALSE
@@ -129,7 +146,8 @@ batch_chat_structured <- function(
     prompts = prompts,
     type = wrap_type_if_needed(type, needs_wrapper),
     path = path,
-    wait = wait
+    wait = wait,
+    ignore_hash = ignore_hash
   )
   job$step_until_done()
   turns <- job$result_turns()
@@ -169,6 +187,7 @@ BatchJob <- R6::R6Class(
     user_turns = NULL,
     path = NULL,
     should_wait = TRUE,
+    ignore_hash = FALSE,
     type = NULL,
 
     # Internal state
@@ -184,6 +203,7 @@ BatchJob <- R6::R6Class(
       path,
       type = NULL,
       wait = TRUE,
+      ignore_hash = FALSE,
       call = caller_env(2)
     ) {
       self$provider <- chat$get_provider()
@@ -192,12 +212,14 @@ BatchJob <- R6::R6Class(
       user_turns <- as_user_turns(prompts, call = call)
       check_string(path, allow_empty = FALSE, call = call)
       check_bool(wait, call = call)
+      check_bool(ignore_hash, call = call)
 
       self$chat <- chat
       self$user_turns <- user_turns
       self$type <- type
       self$path <- path
       self$should_wait <- wait
+      self$ignore_hash <- ignore_hash
 
       if (file.exists(path)) {
         state <- jsonlite::read_json(path)
@@ -332,25 +354,31 @@ BatchJob <- R6::R6Class(
       }
       differences <- names(new_hash)[!same]
 
-      cli::cli_abort(
-        c(
-          "{differences} {?does/do}n't match stored value{?s}.",
-          i = "Do you need to pick a different {.arg path}?"
-        ),
-        call = call
-      )
+      if (self$ignore_hash) {
+        cli::cli_warn(
+          c("!" = "{differences} {?does/do}n't match stored value{?s}."),
+          call = call
+        )
+      } else {
+        cli::cli_abort(
+          c(
+            "{differences} {?does/do}n't match stored value{?s}.",
+            i = "Do you need to pick a different {.arg path}?",
+            i = "Or set {.code ignore_hash = TRUE} to ignore this check?"
+          ),
+          call = call
+        )
+      }
     }
   )
 )
 
 provider_hash <- function(x) {
-  props <- props(x)
-
-  # Backward compatible hashing after introduction of new properties
-  if (length(props$extra_headers) == 0) {
-    props$extra_headers <- NULL
-  }
-  props
+  list(
+    name = x@name,
+    model = x@model,
+    base_url = x@base_url
+  )
 }
 
 check_has_batch_support <- function(provider, call = caller_env()) {

man/batch_chat.Rd

@@ -6,6 +6,7 @@
       Error in `batch_chat()`:
       ! provider, prompts, and user_turns don't match stored values.
       i Do you need to pick a different `path`?
+      i Or set `ignore_hash = TRUE` to ignore this check?
 
 ---
 
@@ -15,6 +16,15 @@
       Error in `batch_chat_structured()`:
       ! provider, prompts, and user_turns don't match stored values.
       i Do you need to pick a different `path`?
+      i Or set `ignore_hash = TRUE` to ignore this check?
+
+# can override hash check
+
+    Code
+      . <- batch_chat(chat, prompts, path, ignore_hash = TRUE)
+    Condition
+      Warning in `batch_chat()`:
+      ! prompts and user_turns don't match stored values.
 
 # informative error for bad inputs
 

tests/testthat/_snaps/batch-chat.md

@@ -2,20 +2,21 @@
   "version": 1,
   "stage": "done",
   "batch": {
-    "id": "batch_6826575e8994819083aca31bb6d307de",
+    "id": "batch_68f802ddb59481908fc70271e9eff0f6",
     "object": "batch",
     "endpoint": "/v1/chat/completions",
+    "model": "gpt-4.1-nano-2025-04-14",
     "errors": {},
-    "input_file_id": "file-M1HWkC3RwUDPgaWtZYjjcy",
+    "input_file_id": "file-JPA9WjqJ5dYmauWGMnEijt",
     "completion_window": "24h",
     "status": "completed",
-    "output_file_id": "file-SdRQAoNeNzaFYr3YFb5hcp",
+    "output_file_id": "file-5JPzmFxLH7MtjLzL2TpjWN",
     "error_file_id": {},
-    "created_at": 1747343198,
-    "in_progress_at": 1747343199,
-    "expires_at": 1747429598,
-    "finalizing_at": 1747343399,
-    "completed_at": 1747343400,
+    "created_at": 1761084125,
+    "in_progress_at": 1761084128,
+    "expires_at": 1761170525,
+    "finalizing_at": 1761084149,
+    "completed_at": 1761084152,
     "failed_at": {},
     "expired_at": {},
     "cancelling_at": {},
@@ -25,16 +26,27 @@
       "completed": 4,
       "failed": 0
     },
+    "usage": {
+      "input_tokens": 81,
+      "output_tokens": 7,
+      "total_tokens": 88,
+      "input_tokens_details": {
+        "cached_tokens": 0
+      },
+      "output_tokens_details": {
+        "reasoning_tokens": 0
+      }
+    },
     "metadata": {}
   },
   "results": [
     {
       "status_code": 200,
-      "request_id": "6c41082f8debff173736a0b52e07d948",
+      "request_id": "98d76687bb4841866c5e3660bc7aec00",
       "body": {
-        "id": "chatcmpl-BXa7llR1CJFH3KbFBLJf4ZvvYEUj4",
+        "id": "chatcmpl-CTEikVO7HRPuFV6CqMKpuwCp0iSKN",
         "object": "chat.completion",
-        "created": 1747343385,
+        "created": 1761084134,
         "model": "gpt-4.1-nano-2025-04-14",
         "choices": [
           {
@@ -65,16 +77,16 @@
           }
         },
         "service_tier": "default",
-        "system_fingerprint": "fp_eede8f0d45"
+        "system_fingerprint": "fp_7c233bf9d1"
       }
     },
     {
       "status_code": 200,
-      "request_id": "881526127f188f503beb9e98d7933d01",
+      "request_id": "5c4a190dd9079087f0ee2d1cc6ecbbed",
       "body": {
-        "id": "chatcmpl-BXa6CU6GypSA9ODF6TKhpkHPP8jlI",
+        "id": "chatcmpl-CTEioj4PdLMO0B9dVarkET9n9cXzu",
         "object": "chat.completion",
-        "created": 1747343288,
+        "created": 1761084138,
         "model": "gpt-4.1-nano-2025-04-14",
         "choices": [
           {
@@ -105,16 +117,16 @@
           }
         },
         "service_tier": "default",
-        "system_fingerprint": "fp_eede8f0d45"
+        "system_fingerprint": "fp_f12167b370"
       }
     },
     {
       "status_code": 200,
-      "request_id": "d35aa9cb5535dd1ea70e1c5e3c469791",
+      "request_id": "dde5e47eb23802d49bbc415a2b76599e",
       "body": {
-        "id": "chatcmpl-BXa6316yijIIu48fQeazyXljyjTXk",
+        "id": "chatcmpl-CTEisUiHnFjkZp7qyVIt9icqbf7dk",
         "object": "chat.completion",
-        "created": 1747343279,
+        "created": 1761084142,
         "model": "gpt-4.1-nano-2025-04-14",
         "choices": [
           {
@@ -131,8 +143,8 @@
         ],
         "usage": {
           "prompt_tokens": 20,
-          "completion_tokens": 3,
-          "total_tokens": 23,
+          "completion_tokens": 2,
+          "total_tokens": 22,
           "prompt_tokens_details": {
             "cached_tokens": 0,
             "audio_tokens": 0
@@ -145,16 +157,16 @@
           }
         },
         "service_tier": "default",
-        "system_fingerprint": "fp_eede8f0d45"
+        "system_fingerprint": "fp_f12167b370"
       }
     },
     {
       "status_code": 200,
-      "request_id": "fb17b486b4053f56120614c6fbffe571",
+      "request_id": "a905ebc6ccd6c29b1f5ee6d9d2839db4",
       "body": {
-        "id": "chatcmpl-BXa6aPBtnciow6qc6gjEPMJ7DjgY7",
+        "id": "chatcmpl-CTEivElBhxFZjwzSP0QqcnJoZQzGx",
         "object": "chat.completion",
-        "created": 1747343312,
+        "created": 1761084145,
         "model": "gpt-4.1-nano-2025-04-14",
         "choices": [
           {
@@ -171,8 +183,8 @@
         ],
         "usage": {
           "prompt_tokens": 20,
-          "completion_tokens": 2,
-          "total_tokens": 22,
+          "completion_tokens": 1,
+          "total_tokens": 21,
           "prompt_tokens_details": {
             "cached_tokens": 0,
             "audio_tokens": 0
@@ -185,13 +197,13 @@
           }
         },
         "service_tier": "default",
-        "system_fingerprint": "fp_eede8f0d45"
+        "system_fingerprint": "fp_f12167b370"
       }
     }
   ],
-  "started_at": 1747343197,
+  "started_at": 1761084124,
   "hash": {
-    "provider": "de1de607a44b8964f051074a91a3787e",
+    "provider": "959dc7923e4dc3760a7690794dc41c2a",
     "prompts": "b8eafe281e3cc5113058d9722be3e295",
     "user_turns": "8c1302088b9bc30258d1191db0d35705"
   }