[TAN-8053] Add segments count sms

back/db/migrate/20260709105146_add_twilio_billing_to_sms_deliveries.email_campaigns.rb

@@ -0,0 +1,14 @@
+# This migration comes from email_campaigns (originally 20260709130000)
+# Twilio is the source of truth for what a message actually cost. These columns are
+# backfilled by Sms::FetchMessageJob once a delivery reaches a terminal status; they are
+# null until then. Names mirror Twilio's Message resource.
+#
+# `price` is signed the way Twilio reports it (negative = charged to the account) and has
+# five decimal places, e.g. -0.00750.
+class AddTwilioBillingToSmsDeliveries < ActiveRecord::Migration[7.2]
+  def change
+    add_column :sms_deliveries, :num_segments, :integer
+    add_column :sms_deliveries, :price, :decimal, precision: 10, scale: 5
+    add_column :sms_deliveries, :price_unit, :string
+  end
+end

back/db/structure.sql

@@ -3725,7 +3725,10 @@ CREATE TABLE public.sms_deliveries (
     status character varying NOT NULL,
     error_message character varying,
     created_at timestamp(6) without time zone NOT NULL,
-    updated_at timestamp(6) without time zone NOT NULL
+    updated_at timestamp(6) without time zone NOT NULL,
+    num_segments integer,
+    price numeric(10,5),
+    price_unit character varying
 );
 
 
@@ -8922,6 +8925,7 @@ ALTER TABLE ONLY public.project_reviews
 SET search_path TO public,shared_extensions;
 
 INSERT INTO "schema_migrations" (version) VALUES
+('20260709105146'),
 ('20260701113056'),
 ('20260630140754'),
 ('20260625093937'),

back/engines/free/email_campaigns/app/controllers/email_campaigns/web_api/v1/sms/callbacks_controller.rb

@@ -30,7 +30,17 @@ def create
 
       # advance_status! is a no-op when the callback arrives out of order; we
       # still return 200 so the provider stops retrying.
-      delivery.advance_status!(parsed[:status])
+      advanced = delivery.advance_status!(parsed[:status])
+
+      # The message is now final, so what it cost is settled. The provider never sends
+      # that in a callback, so read it back once, on the transition into a terminal
+      # status
+      # The delay gives the provider time to populate the segment count and price.
+      if advanced && delivery.terminal?
+        job = EmailCampaigns::Sms::FetchMessageJob
+        job.set(wait: job::SETTLE_DELAY).perform_later(delivery.id)
+      end
+
       head :ok
     end
 

back/engines/free/email_campaigns/app/jobs/email_campaigns/sms/fetch_message_job.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module EmailCampaigns
+  module Sms
+    # Reads back from the provider what a message actually cost, once the delivery has
+    # reached a terminal status. The provider, not us, is the source of truth here: the
+    # segment count is only settled once a sender has been assigned, and the price only
+    # once the message has been handed to a carrier.
+    #
+    # Enqueued by the status-callback controller, because no provider callback carries
+    # this information -- Twilio's outbound status callback has no NumSegments or Price
+    # parameter, so it has to be fetched.
+    class FetchMessageJob < ApplicationJob
+      # A Messaging Service picks a sender and analyses the message *after* the send call
+      # returns, and only then are the segment count and price populated. Twilio therefore
+      # recommends waiting 10-30 seconds after sending before reading a message back:
+      # https://support.twilio.com/hc/en-us/articles/360034857114
+      SETTLE_DELAY = 60.seconds
+
+      def run(delivery_id)
+        delivery = Delivery.find(delivery_id)
+        return if delivery.message_sid.blank?
+
+        message = provider.fetch_message(delivery.message_sid)
+
+        delivery.update!(
+          num_segments: message[:num_segments],
+          price: message[:price],
+          price_unit: message[:price_unit]
+        )
+      end
+
+      def handle_error(error)
+        case error
+        when *ProviderError::RETRYABLE_ERRORS
+          super
+        else
+          expire
+        end
+      end
+
+      private
+
+      def provider
+        Providers::Twilio.new
+      end
+    end
+  end
+end

back/engines/free/email_campaigns/app/models/email_campaigns/campaigns/sms_manual.rb

@@ -39,6 +39,11 @@ class Campaigns::SmsManual < Campaigns::BaseSms
     include RecipientConfigurable
     include LifecycleStageRestrictable
 
+    # SMS is billed per segment, so a long message multiplies the cost of a campaign by
+    # its recipient count. This caps that blast radius. Raise it deliberately.
+    # The admin UI blocks the same limit while composing; this is the guard on the API.
+    MAX_SEGMENTS = 8
+
     allow_lifecycle_stages except: %w[trial churned]
 
     recipient_filter :user_filter_no_invitees
@@ -48,6 +53,8 @@ class Campaigns::SmsManual < Campaigns::BaseSms
     validates :subject_multiloc, presence: true, multiloc: { presence: true }
     validates :body_multiloc, presence: true, multiloc: { presence: true }
 
+    validate :body_within_segment_limit, on: :send
+
     def self.recipient_role_multiloc_key
       'email_campaigns.admin_labels.recipient_role.registered_users'
     end
@@ -96,6 +103,24 @@ def unique_campaigns_per_context?
 
     private
 
+    # Each recipient is sent the body in their own language, so every translation is its
+    # own message and has to fit within the limit on its own. A language that falls
+    # outside GSM-7 (Arabic, Cyrillic, Greek, or a stray emoji) fits far fewer characters
+    # per segment, so one translation can exceed the limit while the others do not.
+    def body_within_segment_limit
+      over_limit = body_multiloc.to_h.filter_map do |locale, body|
+        segments = Sms::SegmentedMessage.new(body).segments_count
+        locale if segments > MAX_SEGMENTS
+      end
+      return if over_limit.empty?
+
+      errors.add(
+        :body_multiloc,
+        :too_many_segments,
+        message: "is longer than #{MAX_SEGMENTS} SMS segments for: #{over_limit.join(', ')}"
+      )
+    end
+
     def user_filter_no_invitees(users_scope, _options = {})
       users_scope.active
     end

back/engines/free/email_campaigns/app/models/email_campaigns/sms/delivery.rb

@@ -13,6 +13,9 @@
 #  error_message :string
 #  created_at    :datetime         not null
 #  updated_at    :datetime         not null
+#  num_segments  :integer
+#  price         :decimal(10, 5)
+#  price_unit    :string
 #
 # Indexes
 #
@@ -50,14 +53,18 @@ def self.status_counts(campaign_id)
       validates :body, presence: true
       validates :status, inclusion: { in: STATUSES }
 
+      def terminal?
+        TERMINAL_STATUSES.include?(status)
+      end
+
       # Moves the delivery to `new_status` only when that represents forward
       # progress, so out-of-order provider callbacks (Twilio warns these can arrive
       # in any order) never regress it. A delivery already in a terminal status is
       # frozen there. Persists the change.
       # @return [Boolean] whether the status actually advanced
       def advance_status!(new_status)
         raise ArgumentError, "unknown status: #{new_status.inspect}" unless STATUSES.include?(new_status)
-        return false if TERMINAL_STATUSES.include?(status)
+        return false if terminal?
         return false if STATUSES.index(new_status) <= STATUSES.index(status)
 
         update!(status: new_status)

back/engines/free/email_campaigns/app/services/email_campaigns/sms/providers/base.rb

@@ -12,6 +12,18 @@ def send(to:, body:)
           raise NotImplementedError
         end
 
+        # Reads back what a message actually cost. The provider, not us, is the source of
+        # truth for this: the segment count and price are only settled once the message
+        # has been handed to a carrier.
+        # @param message_sid [String] the provider's identifier for the message
+        # @return [Hash{Symbol => Integer, BigDecimal, String, nil}]
+        #   { num_segments:, price:, price_unit: } -- price and price_unit may be nil when
+        #   the provider has not billed the message yet
+        # @raise [Error] when the message cannot be read
+        def fetch_message(message_sid)
+          raise NotImplementedError
+        end
+
         # @param request [ActionDispatch::Request] the inbound callback request
         # @return [Boolean] whether the request genuinely originates from the provider
         def verify_signature(request)

back/engines/free/email_campaigns/app/services/email_campaigns/sms/providers/twilio.rb

@@ -37,6 +37,20 @@ def send(to:, body:)
           raise ProviderError, e.message
         end
 
+        def fetch_message(message_sid)
+          message = client.api.v2010.messages(message_sid).fetch
+
+          {
+            num_segments: message.num_segments&.to_i,
+            price: message.price.presence&.to_d,
+            price_unit: message.price_unit.presence
+          }
+        rescue ::Twilio::REST::RestError => e
+          raise error_for(e)
+        rescue ::Twilio::REST::TwilioError => e
+          raise ProviderError, e.message
+        end
+
         def verify_signature(request)
           validator = ::Twilio::Security::RequestValidator.new(config['twilio_auth_token'])
           signature = request.headers['X-Twilio-Signature']

back/engines/free/email_campaigns/app/services/email_campaigns/sms/segmented_message.rb

@@ -0,0 +1,109 @@
+# frozen_string_literal: true
+
+module EmailCampaigns
+  module Sms
+    # Works out how many SMS segments a message body costs, and in which encoding it
+    # will be sent. SMS is billed per segment, so a 161-character message costs twice
+    # as much as a 160-character one, and a single emoji can double the cost of an
+    # entire message by forcing it out of GSM-7 into UCS-2.
+    #
+    #   message = SegmentedMessage.new('Hello 😀')
+    #   message.segments_count      # => 1
+    #   message.encoding_name       # => 'UCS-2'
+    #   message.non_gsm_characters  # => ['😀']
+    #
+    # Mirrors Twilio's reference implementation, which the admin UI runs client-side to
+    # show a live counter (npm `sms-segments-calculator`). The two must agree, since we
+    # persist this value on Sms::Delivery; segmented_message_spec.rb pins the boundaries.
+    #
+    # Note this is an estimate of what Twilio will bill. If a tenant has enabled Smart
+    # Encoding on their Messaging Service (a console-side setting we cannot read), Twilio
+    # transliterates characters like curly quotes into GSM-7 before sending, and may bill
+    # fewer segments than we count. We never under-count.
+    class SegmentedMessage
+      GSM_7 = 'GSM-7'
+      UCS_2 = 'UCS-2'
+
+      # GSM 03.38 basic set. One septet each.
+      GSM_BASIC = "@£$¥èéùìòÇ\nØø\rÅåΔ_ΦΓΛΩΠΨΣΘΞ ÆæßÉ !\"#¤%&'()*+,-./" \
+                  '0123456789:;<=>?¡ABCDEFGHIJKLMNOPQRSTUVWXYZÄÖÑܧ¿' \
+                  'abcdefghijklmnopqrstuvwxyzäöñüà'
+
+      # GSM 03.38 extension set. Sent as an escape byte followed by the character, so
+      # each costs *two* septets and counts as two characters towards the limit.
+      GSM_EXTENDED = "\f^{}\\[~]|€"
+
+      # A segment carries 140 octets = 1120 bits of user data: 160 GSM-7 septets, or
+      # 70 UCS-2 code units.
+      SINGLE_SEGMENT_CAPACITY = { GSM_7 => 160, UCS_2 => 70 }.freeze
+
+      # Once a message spans several segments, every segment reserves a 6-octet User
+      # Data Header so the handset can reassemble them in order. That leaves
+      # 1120 - 48 = 1072 bits: 153 GSM-7 septets, or 67 UCS-2 code units.
+      CONCATENATED_SEGMENT_CAPACITY = { GSM_7 => 153, UCS_2 => 67 }.freeze
+
+      attr_reader :encoding_name, :graphemes
+
+      def initialize(body)
+        # Grapheme clusters, not characters: a combining pair like "e" + U+0301 occupies
+        # two code units but cannot be split across a segment boundary.
+        @graphemes = body.to_s.grapheme_clusters
+        @encoding_name = @graphemes.all? { |grapheme| gsm_7?(grapheme) } ? GSM_7 : UCS_2
+      end
+
+      # The number of SMS this body will be billed as.
+      def segments_count
+        sizes = @graphemes.map { |grapheme| size_in_units(grapheme) }
+        return 1 if sizes.sum <= SINGLE_SEGMENT_CAPACITY.fetch(encoding_name)
+
+        pack_into_segments(sizes, CONCATENATED_SEGMENT_CAPACITY.fetch(encoding_name))
+      end
+
+      # Informational character count. Every character counts as one, except those in the
+      # GSM-7 extension set, which count as two.
+      def number_of_characters
+        return @graphemes.size if encoding_name == UCS_2
+
+        @graphemes.sum { |grapheme| GSM_EXTENDED.include?(grapheme) ? 2 : 1 }
+      end
+
+      # The characters that are not representable in GSM-7, and therefore forced the whole
+      # message into UCS-2. Surfaced to admins so they can swap them out.
+      def non_gsm_characters
+        @graphemes.reject { |grapheme| gsm_7?(grapheme) }
+      end
+
+      private
+
+      # Fills segments greedily. A character never straddles a boundary, so a segment with
+      # one septet free cannot take a two-septet "€" -- it leaves the septet unused and
+      # starts a new segment. That is why this packs rather than dividing by the capacity.
+      def pack_into_segments(sizes, capacity)
+        segments = 1
+        used = 0
+
+        sizes.each do |size|
+          if used + size > capacity
+            segments += 1
+            used = 0
+          end
+          used += size
+        end
+
+        segments
+      end
+
+      def gsm_7?(grapheme)
+        grapheme.length == 1 && (GSM_BASIC.include?(grapheme) || GSM_EXTENDED.include?(grapheme))
+      end
+
+      # A character's cost: septets under GSM-7, UTF-16 code units under UCS-2 (where an
+      # astral character such as an emoji is a surrogate pair, costing two).
+      def size_in_units(grapheme)
+        return grapheme.encode(Encoding::UTF_16BE).bytesize / 2 if encoding_name == UCS_2
+
+        GSM_EXTENDED.include?(grapheme) ? 2 : 1
+      end
+    end
+  end
+end

back/engines/free/email_campaigns/db/migrate/20260709130000_add_twilio_billing_to_sms_deliveries.rb

@@ -0,0 +1,13 @@
+# Twilio is the source of truth for what a message actually cost. These columns are
+# backfilled by Sms::FetchMessageJob once a delivery reaches a terminal status; they are
+# null until then. Names mirror Twilio's Message resource.
+#
+# `price` is signed the way Twilio reports it (negative = charged to the account) and has
+# five decimal places, e.g. -0.00750.
+class AddTwilioBillingToSmsDeliveries < ActiveRecord::Migration[7.2]
+  def change
+    add_column :sms_deliveries, :num_segments, :integer
+    add_column :sms_deliveries, :price, :decimal, precision: 10, scale: 5
+    add_column :sms_deliveries, :price_unit, :string
+  end
+end

back/engines/free/email_campaigns/spec/acceptance/sms_callbacks_spec.rb

@@ -34,6 +34,38 @@
       expect(delivery.reload.status).to eq 'delivered'
     end
 
+    # The provider needs a moment to populate the segment count and price, so the read is
+    # deferred rather than fired the instant the callback lands.
+    example 'reads back what the message cost once it reaches a terminal status' do
+      freeze_time do
+        expect { do_request(callback_params) }
+          .to have_enqueued_job(EmailCampaigns::Sms::FetchMessageJob)
+          .with(delivery.id)
+          .at(EmailCampaigns::Sms::FetchMessageJob::SETTLE_DELAY.from_now)
+      end
+    end
+
+    context 'when the delivery is not yet terminal' do
+      let(:message_status) { 'sent' }
+      let!(:delivery) do
+        EmailCampaigns::Sms::Delivery.create!(body: 'hi', status: 'queued', message_sid: 'SM_123')
+      end
+
+      example 'does not read the message back yet, since the price is not settled' do
+        expect { do_request(callback_params) }
+          .not_to have_enqueued_job(EmailCampaigns::Sms::FetchMessageJob)
+      end
+    end
+
+    context 'when a duplicate terminal callback arrives' do
+      before { delivery.advance_status!('delivered') }
+
+      example 'does not enqueue a second fetch' do
+        expect { do_request(callback_params) }
+          .not_to have_enqueued_job(EmailCampaigns::Sms::FetchMessageJob)
+      end
+    end
+
     context 'when the signature is invalid' do
       let(:signature_valid) { false }