Compute token costs

Fixes #203

Author: hadley

.Rbuildignore

@@ -13,3 +13,4 @@ _cache/
 ^CRAN-SUBMISSION$
 ^[\.]?air\.toml$
 ^\.vscode$
+^data-raw$

DESCRIPTION

@@ -97,3 +97,5 @@ Collate:
     'utils-merge.R'
     'utils.R'
     'zzz.R'
+Depends: 
+    R (>= 3.5)

NAMESPACE

@@ -1,6 +1,8 @@
 # Generated by roxygen2: do not edit by hand
 
+S3method(format,ellmer_price)
 S3method(print,Chat)
+S3method(print,ellmer_price)
 export(Content)
 export(ContentImage)
 export(ContentImageInline)

R/chat.R

@@ -119,32 +119,59 @@ Chat <- R6::R6Class(
       assistant_turns <- keep(turns, function(x) x@role == "assistant")
 
       n <- length(assistant_turns)
-      tokens <- t(vapply(
+      tokens_acc <- t(vapply(
         assistant_turns,
         function(turn) turn@tokens,
         double(2)
       ))
+
+      tokens <- tokens_acc
       if (n > 1) {
         # Compute just the new tokens
         tokens[-1, 1] <- tokens[seq(2, n), 1] -
           (tokens[seq(1, n - 1), 1] + tokens[seq(1, n - 1), 2])
       }
       # collapse into a single vector
       tokens_v <- c(t(tokens))
+      tokens_acc_v <- c(t(tokens_acc))
 
       tokens_df <- data.frame(
         role = rep(c("user", "assistant"), times = n),
-        tokens = tokens_v
+        tokens = tokens_v,
+        tokens_total = tokens_acc_v
       )
 
       if (include_system_prompt && private$has_system_prompt()) {
         # How do we compute this?
-        tokens_df <- rbind(data.frame(role = "system", tokens = 0), tokens_df)
+        tokens_df <- rbind(
+          data.frame(role = "system", tokens = 0, tokens_total = 0),
+          tokens_df
+        )
       }
 
       tokens_df
     },
 
+    #' @description The total price of all turns in this chat.
+    get_price = function() {
+      turns <- self$get_turns(include_system_prompt = FALSE)
+      assistant_turns <- keep(turns, function(x) x@role == "assistant")
+
+      n <- length(assistant_turns)
+      tokens <- t(vapply(
+        assistant_turns,
+        function(turn) turn@tokens,
+        double(2)
+      ))
+
+      find_price(
+        private$provider@name,
+        private$provider@model,
+        sum(tokens[, 1]),
+        sum(tokens[, 2])
+      )
+    },
+
     #' @description The last turn returned by the assistant.
     #' @param role Optionally, specify a role to find the last turn with
     #'   for the role.
@@ -645,16 +672,31 @@ print.Chat <- function(x, ...) {
   turns <- x$get_turns(include_system_prompt = TRUE)
 
   tokens <- x$tokens(include_system_prompt = TRUE)
-  tokens_user <- sum(tokens$tokens[tokens$role == "user"])
-  tokens_assistant <- sum(tokens$tokens[tokens$role == "assistant"])
-
-  cat(paste_c(
-    "<Chat",
-    c(" ", provider@name, "/", provider@model),
-    c(" turns=", length(turns)),
-    c(" tokens=", tokens_user, "/", tokens_assistant),
+
+  tokens_user <- sum(tokens$tokens_total[tokens$role == "user"])
+  tokens_assistant <- sum(tokens$tokens_total[tokens$role == "assistant"])
+  price <- find_price(
+    provider@name,
+    provider@model,
+    tokens_user,
+    tokens_assistant
+  )
+
+  cat(
+    paste_c(
+      "<Chat",
+      c(" ", provider@name, "/", provider@model),
+      c(" turns=", length(turns)),
+      c(
+        " tokens=",
+        tokens_user,
+        "/",
+        tokens_assistant,
+        if (!is.na(price)) c("/", format(price))
+      )
+    ),
     ">\n"
-  ))
+  )
 
   for (i in seq_along(turns)) {
     turn <- turns[[i]]

R/provider-bedrock.R

@@ -263,8 +263,11 @@ method(value_turn, ProviderBedrock) <- function(
     }
   })
 
-  tokens <- c(result$usage$inputTokens, result$usage$outputTokens)
-  tokens_log(provider, tokens)
+  tokens <- tokens_log(
+    provider,
+    input = result$usage$inputTokens,
+    output = result$usage$outputTokens
+  )
 
   Turn(result$output$message$role, contents, json = result, tokens = tokens)
 }

R/provider-claude.R

@@ -278,8 +278,11 @@ method(value_turn, ProviderClaude) <- function(
     }
   })
 
-  tokens <- c(result$usage$input_tokens, result$usage$output_tokens)
-  tokens_log(provider, tokens)
+  tokens <- tokens_log(
+    provider,
+    input = result$usage$input_tokens,
+    output = result$usage$output_tokens
+  )
 
   Turn(result$role, contents, json = result, tokens = tokens)
 }
@@ -377,6 +380,11 @@ method(as_json, list(ProviderClaude, ToolDef)) <- function(provider, x) {
   )
 }
 
+# Pricing ----------------------------------------------------------------------
+
+method(standardise_model, ProviderClaude) <- function(provider, model) {
+  gsub("-(latest|\\d{8})$", "", model)
+}
 
 # Helpers ----------------------------------------------------------------
 

R/provider-gemini.R

@@ -202,11 +202,11 @@ method(value_turn, ProviderGemini) <- function(
   })
   contents <- compact(contents)
   usage <- result$usageMetadata
-  tokens <- c(
-    usage$promptTokenCount %||% NA_integer_,
-    usage$candidatesTokenCount %||% NA_integer_
+  tokens <- tokens_log(
+    provider,
+    input = usage$promptTokenCount,
+    output = usage$candidatesTokenCount
   )
-  tokens_log(provider, tokens)
 
   Turn("assistant", contents, json = result, tokens = tokens)
 }

R/provider-openai.R

@@ -237,12 +237,11 @@ method(value_turn, ProviderOpenAI) <- function(
     })
     content <- c(content, calls)
   }
-  tokens <- c(
-    result$usage$prompt_tokens %||% NA_integer_,
-    result$usage$completion_tokens %||% NA_integer_
+  tokens <- tokens_log(
+    provider,
+    input = result$usage$prompt_tokens,
+    output = result$usage$completion_tokens
   )
-  tokens_log(provider, tokens)
-
   Turn(message$role, content, json = result, tokens = tokens)
 }
 

R/provider-snowflake.R

@@ -142,11 +142,11 @@ method(value_turn, ProviderSnowflake) <- function(
 ) {
   deltas <- compact(sapply(result$choices, function(x) x$delta$content))
   content <- list(as_content(paste(deltas, collapse = "")))
-  tokens <- c(
-    result$usage$prompt_tokens %||% NA_integer_,
-    result$usage$completion_tokens %||% NA_integer_
+  tokens <- tokens_log(
+    provider,
+    input = result$usage$prompt_tokens,
+    output = result$usage$completion_tokens
   )
-  tokens_log(provider, tokens)
   Turn(
     # Snowflake's response format seems to omit the role.
     "assistant",

R/provider.R

@@ -113,3 +113,17 @@ method(as_json, list(Provider, class_list)) <- function(provider, x) {
 method(as_json, list(Provider, ContentJson)) <- function(provider, x) {
   as_json(provider, ContentText("<structured data/>"))
 }
+
+# Pricing ---------------------------------------------------------------------
+
+standardise_model <- new_generic(
+  "standardise_model",
+  "provider",
+  function(provider, model) {
+    S7_dispatch()
+  }
+)
+
+method(standardise_model, Provider) <- function(provider, model) {
+  model
+}

R/sysdata.rda

@@ -1,52 +1,92 @@
-tokens_log <- function(provider, tokens) {
-  # TODO: probably should make this store in a data frame, but will tackle
-  # when implementing token costs.
+on_load(
+  the$tokens <- tokens_row(character(), character(), numeric(), numeric())
+)
 
-  name <- paste0(provider@name, "/", provider@model)
+tokens_log <- function(provider, input = NULL, output = NULL) {
+  input <- input %||% 0
+  output <- output %||% 0
 
-  if (is.null(the$tokens)) {
-    the$tokens <- list()
-  }
-  if (is.null(the$tokens[[name]])) {
-    the$tokens[[name]] <- c(0, 0)
+  model <- standardise_model(provider, provider@model)
+
+  name <- function(provider, model) paste0(provider, "/", model)
+  i <- tokens_match(provider@name, model, the$tokens$provider, the$tokens$model)
+
+  if (is.na(i)) {
+    new_row <- tokens_row(provider@name, model, input, output)
+    the$tokens <- rbind(the$tokens, new_row)
+  } else {
+    the$tokens$input[i] <- the$tokens$input[i] + input
+    the$tokens$output[i] <- the$tokens$output[i] + output
   }
 
-  tokens[is.na(tokens)] <- 0
-  the$tokens[[name]] <- the$tokens[[name]] + tokens
-  invisible()
+  # Returns value to be passed to Turn
+  c(input, output)
+}
+
+tokens_row <- function(provider, model, input, output) {
+  data.frame(provider = provider, model = model, input = input, output = output)
 }
 
+tokens_match <- function(
+  provider_needle,
+  model_needle,
+  provider_haystack,
+  model_haystack
+) {
+  match(
+    paste0(provider_needle, "/", model_needle),
+    paste0(provider_haystack, "/", model_haystack)
+  )
+}
+
+
 local_tokens <- function(frame = parent.frame()) {
   old <- the$tokens
-  the$tokens <- NULL
+  the$tokens <- tokens_row(character(), character(), numeric(), numeric())
 
   defer(the$tokens <- old, env = frame)
 }
 
-tokens_set <- function() {
-  the$tokens <- NULL
-  invisible()
-}
-
 #' Report on token usage in the current session
 #'
 #' Call this function to find out the cumulative number of tokens that you
-#' have sent and recieved in the current session.
+#' have sent and recieved in the current session. The price will be shown
+#' if known.
 #'
 #' @export
 #' @return A data frame
 #' @examples
 #' token_usage()
 token_usage <- function() {
-  if (is.null(the$tokens)) {
+  if (nrow(the$tokens) == 0) {
     cli::cli_inform(c(x = "No recorded usage in this session"))
-    return(invisible(
-      data.frame(name = character(), input = numeric(), output = numeric())
-    ))
+    return(invisible(the$tokens))
   }
 
-  rows <- map2(names(the$tokens), the$tokens, function(name, tokens) {
-    data.frame(name = name, input = tokens[[1]], output = tokens[[2]])
-  })
-  do.call("rbind", rows)
+  out <- the$tokens
+  out$price <- find_price(out$provider, out$model, out$input, out$output)
+  out
+}
+
+# Pricing ----------------------------------------------------------------------
+
+find_price <- function(provider, model, input, output) {
+  idx <- tokens_match(provider, model, prices$provider, prices$model)
+
+  input_price <- input * prices$input[idx] / 1e6
+  output_price <- output * prices$output[idx] / 1e6
+  ellmer_price(input_price + output_price)
+}
+
+ellmer_price <- function(x) {
+  structure(x, class = c("ellmer_price", "numeric"))
+}
+#' @export
+format.ellmer_price <- function(x, ...) {
+  paste0("$", format(unclass(x), nsmall = 2))
+}
+#' @export
+print.ellmer_price <- function(x, ...) {
+  print(format(x), quote = FALSE)
+  invisible(x)
 }

R/tokens.R

@@ -0,0 +1,24 @@
+model,cached_input,input,output
+gpt-4.5-preview,37.50,75.00,150.00
+gpt-4.5-preview-2025-02-27,37.50,75.00,150.00
+gpt-4o,1.25,2.50,10.00
+gpt-4o-2024-11-20,1.25,2.50,10.00
+gpt-4o-2024-08-06,1.25,2.50,10.00
+gpt-4o-2024-05-13,,5.00,15.00
+gpt-4o-mini,0.075,0.15,0.60
+gpt-4o-mini-2024-07-18,0.075,0.15,0.60
+o1,7.50,15.00,60.00
+o1-2024-12-17,7.50,15.00,60.00
+o1-preview-2024-09-12,7.50,15.00,60.00
+o1-pro,,150.00,600.00
+o1-pro-2025-03-19,,150.00,600.00
+o3-mini,0.55,1.10,4.40
+o3-mini-2025-01-31,0.55,1.10,4.40
+o1-mini,0.55,1.10,4.40
+o1-mini-2024-09-12,0.55,1.10,4.40
+gpt-4o-mini-search-preview,,0.15,0.60
+gpt-4o-mini-search-preview-2025-03-11,,0.15,0.60
+gpt-4o-search-preview,,2.50,10.00
+gpt-4o-search-preview-2025-03-11,,2.50,10.00
+computer-use-preview,,3.00,12.00
+computer-use-preview-2025-03-11,,3.00,12.00