From 71eecb79ebe39d33f829c759a7e3dbd4eca63b3b Mon Sep 17 00:00:00 2001 From: Christophe Brun Date: Tue, 1 Sep 2026 17:43:50 +0200 Subject: [PATCH 1/6] feat(datasource intercom): tickets and conversations (lot 1) (#383) PRD-1112, first lot of the Intercom datasource. Six read-only collections: conversations and tickets on the cursor tier, admins, teams, ticket types and ticket states read whole. Rows, record details, exact counts and the conversation thread work. Server-side filtering (lot 2), writes and actions (lot 3), contacts and companies (lot 4) are refused with a message naming the lot that will answer them, rather than approximated. 254 specs, 100% line coverage, rubocop clean. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/build.yml | 3 +- .releaserc.js | 7 +- .rubocop.yml | 6 + .../.gitignore | 8 + .../forest_admin_datasource_intercom/.rspec | 3 + .../forest_admin_datasource_intercom/Gemfile | 16 + .../Gemfile-test | 19 + .../README.md | 235 +++++++++ .../forest_admin_datasource_intercom/Rakefile | 6 + .../forest_admin_datasource_intercom.gemspec | 36 ++ .../lib/forest_admin_datasource_intercom.rb | 59 +++ .../client.rb | 390 ++++++++++++++ .../collections/admin.rb | 52 ++ .../collections/base_collection.rb | 93 ++++ .../collections/contact_identity.rb | 78 +++ .../collections/conversation.rb | 157 ++++++ .../collections/conversation/serializer.rb | 75 +++ .../collections/conversation/timeline.rb | 73 +++ .../collections/cursor_collection.rb | 246 +++++++++ .../collections/fetch_all_collection.rb | 196 +++++++ .../collections/team.rb | 36 ++ .../collections/ticket.rb | 132 +++++ .../collections/ticket/derived_columns.rb | 127 +++++ .../collections/ticket/serializer.rb | 81 +++ .../collections/ticket_state.rb | 40 ++ .../collections/ticket_type.rb | 47 ++ .../configuration.rb | 112 ++++ .../datasource.rb | 48 ++ .../pagination/cursor_walker.rb | 126 +++++ .../rate_limiter.rb | 169 ++++++ .../retry_policy.rb | 72 +++ .../schema/ticket_attributes_introspector.rb | 142 +++++ .../throttle.rb | 19 + .../version.rb | 3 + .../client_spec.rb | 487 ++++++++++++++++++ .../collections/admin_spec.rb | 61 +++ .../collections/conversation_spec.rb | 480 +++++++++++++++++ .../collections/fetch_all_collection_spec.rb | 252 +++++++++ .../collections/team_spec.rb | 41 ++ .../collections/ticket_spec.rb | 307 +++++++++++ .../collections/ticket_state_spec.rb | 38 ++ .../collections/ticket_type_spec.rb | 45 ++ .../configuration_spec.rb | 98 ++++ .../datasource_spec.rb | 59 +++ .../pagination/cursor_walker_spec.rb | 173 +++++++ .../rate_limiter_spec.rb | 150 ++++++ .../retry_policy_spec.rb | 49 ++ .../ticket_attributes_introspector_spec.rb | 164 ++++++ .../throttle_spec.rb | 40 ++ .../forest_admin_datasource_intercom_spec.rb | 49 ++ .../spec/spec_helper.rb | 59 +++ 51 files changed, 5461 insertions(+), 3 deletions(-) create mode 100644 packages/forest_admin_datasource_intercom/.gitignore create mode 100644 packages/forest_admin_datasource_intercom/.rspec create mode 100644 packages/forest_admin_datasource_intercom/Gemfile create mode 100644 packages/forest_admin_datasource_intercom/Gemfile-test create mode 100644 packages/forest_admin_datasource_intercom/README.md create mode 100644 packages/forest_admin_datasource_intercom/Rakefile create mode 100644 packages/forest_admin_datasource_intercom/forest_admin_datasource_intercom.gemspec create mode 100644 packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom.rb create mode 100644 packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/client.rb create mode 100644 packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/admin.rb create mode 100644 packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/base_collection.rb create mode 100644 packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/contact_identity.rb create mode 100644 packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/conversation.rb create mode 100644 packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/conversation/serializer.rb create mode 100644 packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/conversation/timeline.rb create mode 100644 packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/cursor_collection.rb create mode 100644 packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/fetch_all_collection.rb create mode 100644 packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/team.rb create mode 100644 packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/ticket.rb create mode 100644 packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/ticket/derived_columns.rb create mode 100644 packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/ticket/serializer.rb create mode 100644 packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/ticket_state.rb create mode 100644 packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/ticket_type.rb create mode 100644 packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/configuration.rb create mode 100644 packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/datasource.rb create mode 100644 packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/pagination/cursor_walker.rb create mode 100644 packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/rate_limiter.rb create mode 100644 packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/retry_policy.rb create mode 100644 packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/schema/ticket_attributes_introspector.rb create mode 100644 packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/throttle.rb create mode 100644 packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/version.rb create mode 100644 packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/client_spec.rb create mode 100644 packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/admin_spec.rb create mode 100644 packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/conversation_spec.rb create mode 100644 packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/fetch_all_collection_spec.rb create mode 100644 packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/team_spec.rb create mode 100644 packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/ticket_spec.rb create mode 100644 packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/ticket_state_spec.rb create mode 100644 packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/ticket_type_spec.rb create mode 100644 packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/configuration_spec.rb create mode 100644 packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/datasource_spec.rb create mode 100644 packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/pagination/cursor_walker_spec.rb create mode 100644 packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/rate_limiter_spec.rb create mode 100644 packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/retry_policy_spec.rb create mode 100644 packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/schema/ticket_attributes_introspector_spec.rb create mode 100644 packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/throttle_spec.rb create mode 100644 packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom_spec.rb create mode 100644 packages/forest_admin_datasource_intercom/spec/spec_helper.rb diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f4fe543c2..a88277e4b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -70,6 +70,7 @@ jobs: - forest_admin_datasource_snowflake - forest_admin_datasource_mambu_payments - forest_admin_datasource_graphql_hasura + - forest_admin_datasource_intercom services: mongodb: image: mongo:latest @@ -153,7 +154,7 @@ jobs: with: verbose: true oidc: true - files: ${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_agent/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_datasource_active_record/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_datasource_customizer/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_datasource_toolkit/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_rails/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_datasource_mongoid/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_rpc_agent/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_datasource_rpc/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_datasource_zendesk/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_datasource_snowflake/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_datasource_mambu_payments/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_datasource_graphql_hasura/coverage.json + files: ${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_agent/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_datasource_active_record/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_datasource_customizer/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_datasource_toolkit/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_rails/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_datasource_mongoid/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_rpc_agent/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_datasource_rpc/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_datasource_zendesk/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_datasource_snowflake/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_datasource_mambu_payments/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_datasource_graphql_hasura/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_datasource_intercom/coverage.json deploy: name: Release package diff --git a/.releaserc.js b/.releaserc.js index 76e57c8f0..479152e69 100644 --- a/.releaserc.js +++ b/.releaserc.js @@ -31,7 +31,8 @@ module.exports = { 'sed -i \'s/VERSION = ".*"/VERSION = "${nextRelease.version}"/g\' packages/forest_admin_datasource_zendesk/lib/forest_admin_datasource_zendesk/version.rb; '+ 'sed -i \'s/VERSION = ".*"/VERSION = "${nextRelease.version}"/g\' packages/forest_admin_datasource_snowflake/lib/forest_admin_datasource_snowflake/version.rb; '+ 'sed -i \'s/VERSION = ".*"/VERSION = "${nextRelease.version}"/g\' packages/forest_admin_datasource_mambu_payments/lib/forest_admin_datasource_mambu_payments/version.rb; '+ - 'sed -i \'s/VERSION = ".*"/VERSION = "${nextRelease.version}"/g\' packages/forest_admin_datasource_graphql_hasura/lib/forest_admin_datasource_graphql_hasura/version.rb; ', + 'sed -i \'s/VERSION = ".*"/VERSION = "${nextRelease.version}"/g\' packages/forest_admin_datasource_graphql_hasura/lib/forest_admin_datasource_graphql_hasura/version.rb; '+ + 'sed -i \'s/VERSION = ".*"/VERSION = "${nextRelease.version}"/g\' packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/version.rb; ', successCmd: '( cd packages/forest_admin_agent && gem build && gem push forest_admin_agent-*.gem );' + '( cd packages/forest_admin_datasource_active_record && gem build && gem push forest_admin_datasource_active_record-*.gem );' + @@ -45,7 +46,8 @@ module.exports = { '( cd packages/forest_admin_datasource_zendesk && gem build && gem push forest_admin_datasource_zendesk-*.gem );' + '( cd packages/forest_admin_datasource_snowflake && gem build && gem push forest_admin_datasource_snowflake-*.gem );' + '( cd packages/forest_admin_datasource_mambu_payments && gem build && gem push forest_admin_datasource_mambu_payments-*.gem );' + - '( cd packages/forest_admin_datasource_graphql_hasura && gem build && gem push forest_admin_datasource_graphql_hasura-*.gem );' , + '( cd packages/forest_admin_datasource_graphql_hasura && gem build && gem push forest_admin_datasource_graphql_hasura-*.gem );' + + '( cd packages/forest_admin_datasource_intercom && gem build && gem push forest_admin_datasource_intercom-*.gem );' , }, ], [ @@ -68,6 +70,7 @@ module.exports = { 'packages/forest_admin_datasource_snowflake/lib/forest_admin_datasource_snowflake/version.rb', 'packages/forest_admin_datasource_mambu_payments/lib/forest_admin_datasource_mambu_payments/version.rb', 'packages/forest_admin_datasource_graphql_hasura/lib/forest_admin_datasource_graphql_hasura/version.rb', + 'packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/version.rb', 'package.json' ], }, diff --git a/.rubocop.yml b/.rubocop.yml index c6985f27d..b2cc222be 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -42,6 +42,7 @@ Gemspec/RequireMFA: - 'packages/forest_admin_datasource_snowflake/forest_admin_datasource_snowflake.gemspec' - 'packages/forest_admin_datasource_mambu_payments/forest_admin_datasource_mambu_payments.gemspec' - 'packages/forest_admin_datasource_graphql_hasura/forest_admin_datasource_graphql_hasura.gemspec' + - 'packages/forest_admin_datasource_intercom/forest_admin_datasource_intercom.gemspec' # Offense count: 1 # This cop supports unsafe autocorrection (--autocorrect-all). @@ -133,6 +134,7 @@ Style/MutableConstant: - 'packages/forest_admin_datasource_zendesk/lib/forest_admin_datasource_zendesk/version.rb' - 'packages/forest_admin_datasource_snowflake/lib/forest_admin_datasource_snowflake/version.rb' - 'packages/forest_admin_datasource_mambu_payments/lib/forest_admin_datasource_mambu_payments/version.rb' + - 'packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/version.rb' # Offense count: 38 # This cop supports safe autocorrection (--autocorrect). @@ -217,6 +219,7 @@ Style/StringLiterals: - 'packages/forest_admin_datasource_zendesk/lib/forest_admin_datasource_zendesk/version.rb' - 'packages/forest_admin_datasource_snowflake/lib/forest_admin_datasource_snowflake/version.rb' - 'packages/forest_admin_datasource_mambu_payments/lib/forest_admin_datasource_mambu_payments/version.rb' + - 'packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/version.rb' # Offense count: 1 # This cop supports safe autocorrection (--autocorrect). @@ -261,6 +264,8 @@ Metrics/ParameterLists: - 'packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/capture.rb' - 'packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/store.rb' - 'packages/forest_admin_datasource_graphql_hasura/lib/forest_admin_datasource_graphql_hasura/configuration.rb' + - 'packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/configuration.rb' + - 'packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/client.rb' - 'packages/forest_admin_datasource_zendesk/lib/forest_admin_datasource_zendesk/collections/base_collection.rb' - 'packages/forest_admin_datasource_snowflake/lib/forest_admin_datasource_snowflake/datasource.rb' - 'packages/forest_admin_agent/lib/forest_admin_agent/routes/query_handler.rb' @@ -296,6 +301,7 @@ Metrics/ModuleLength: - 'packages/forest_admin_datasource_mambu_payments/spec/**/*' - 'packages/forest_admin_rails/spec/**/*' - 'packages/forest_admin_rpc_agent/spec/**/*' + - 'packages/forest_admin_datasource_intercom/spec/**/*' - 'packages/forest_admin_datasource_mongoid/lib/forest_admin_datasource_mongoid/utils/helpers.rb' Metrics/MethodLength: diff --git a/packages/forest_admin_datasource_intercom/.gitignore b/packages/forest_admin_datasource_intercom/.gitignore new file mode 100644 index 000000000..06cfcfb83 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/.gitignore @@ -0,0 +1,8 @@ +*.gem +.bundle/ +Gemfile.lock +Gemfile-test.lock +coverage/ +pkg/ +tmp/ +.rspec_status diff --git a/packages/forest_admin_datasource_intercom/.rspec b/packages/forest_admin_datasource_intercom/.rspec new file mode 100644 index 000000000..34c5164d9 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/.rspec @@ -0,0 +1,3 @@ +--format documentation +--color +--require spec_helper diff --git a/packages/forest_admin_datasource_intercom/Gemfile b/packages/forest_admin_datasource_intercom/Gemfile new file mode 100644 index 000000000..c229ff1d5 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/Gemfile @@ -0,0 +1,16 @@ +source 'https://rubygems.org' + +gemspec + +gem 'forest_admin_datasource_customizer' +gem 'forest_admin_datasource_toolkit' +gem 'rake', '~> 13.0' +gem 'rubocop', '1.86.1' +gem 'rubocop-performance', '1.26.1' +gem 'rubocop-rspec', '3.9.0' + +group :development, :test do + gem 'rspec', '~> 3.0' + gem 'simplecov', '~> 0.22', require: false + gem 'webmock', '~> 3.0' +end diff --git a/packages/forest_admin_datasource_intercom/Gemfile-test b/packages/forest_admin_datasource_intercom/Gemfile-test new file mode 100644 index 000000000..8b433e2b5 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/Gemfile-test @@ -0,0 +1,19 @@ +source 'https://rubygems.org' + +# Specify your gem's dependencies in forest_admin_datasource_intercom.gemspec +gemspec + +gem 'rake', '~> 13.0' +gem 'rubocop', '1.86.1' +gem 'rubocop-performance', '1.26.1' +gem 'rubocop-rspec', '3.9.0' + +group :development, :test do + gem 'forest_admin_datasource_customizer', path: '../forest_admin_datasource_customizer' + gem 'forest_admin_datasource_toolkit', path: '../forest_admin_datasource_toolkit' + gem 'rspec', '~> 3.0' + gem 'simplecov', '~> 0.22', require: false + gem 'simplecov-html', '~> 0.12.3' + gem 'simplecov_json_formatter', '~> 0.1.4' + gem 'webmock', '~> 3.0' +end diff --git a/packages/forest_admin_datasource_intercom/README.md b/packages/forest_admin_datasource_intercom/README.md new file mode 100644 index 000000000..3505508e0 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/README.md @@ -0,0 +1,235 @@ +# Forest — Intercom datasource + +Surface [Intercom](https://www.intercom.com) conversations, tickets, teammates, teams, ticket types +and ticket states as Forest collections. + +This is **lot 1: read only**. Rows, record details, exact counts and the conversation thread work; +server-side filtering, writes, business actions, contacts and companies arrive in the lots after it +— see "What is not here yet". + +## Installation + +```ruby +# Gemfile +gem 'forest_admin_datasource_intercom' +``` + +## Usage + +```ruby +# app/lib/forest_admin_rails/create_agent.rb +ForestAdminAgent::Builder::AgentFactory.instance.add_datasource( + ForestAdminDatasourceIntercom::Datasource.new( + access_token: ENV['INTERCOM_ACCESS_TOKEN'], + region: :eu # :us (default), :eu or :au + ) +) +``` + +The token is the access token of a private app, created in Intercom's Developer Hub under +*Configure › Authentication*. OAuth is out of scope: it belongs to a control plane distributing a +connector, not to an agent reading one workspace. + +`Client#me` is the health check — it returns the admin the token belongs to, and is the one call +that verifies the pinned API version was honoured. + +### Configuration + +| Option | Default | What it is for | +| --- | --- | --- | +| `access_token` | — | Required. The private app's bearer token. | +| `region` | `:us` | `:us`, `:eu`, `:au`. A workspace answers in its own region only. | +| `base_url` | from `region` | Wins over `region`. For an egress proxy or a mock server. | +| `api_version` | `'2.16'` | Sent as `Intercom-Version` on every request. | +| `open_timeout` / `timeout` | `5` / `30` | A request that already has a page on screen. | +| `boot_open_timeout` / `boot_timeout` | `3` / `10` | The one read performed while the agent starts. | +| `retry_policy` | `RetryPolicy.new` | Statuses, verbs and backoff. | +| `boot_retry_policy` | `RetryPolicy.boot` | One quick retry; gives up rather than waiting a 429 out. | +| `rate_limiter` | `RateLimiter.new` | `nil` takes the pacing out of the stack. | + +**Pin the region explicitly.** `api.intercom.io` does route to the right one, but a workspace under +GDPR wants its requests reaching the European host and nothing else. + +**The version is pinned on purpose.** Without the header a request follows the workspace's own +default version, which an operator can change on Intercom's side — and the payloads change shape +underneath. Intercom echoes the version it served, so `me` compares the two and logs a warning when +the pin was not honoured, rather than raising: running against a version we did not ask for still +beats not running. + +### Token permissions + +A read-only token is enough, and is what to recommend for this lot. A permission the token lacks +costs **columns or a collection, never the boot of the agent**: the ticket-type introspection +degrades to no attribute column, and a collection whose endpoint answers 403 fails its own page. + +## Collections + +| Collection | Endpoint | Paginated | Countable | +| --- | --- | --- | --- | +| `IntercomConversation` | `GET /conversations`, `GET /conversations/{id}` | cursor | yes, exactly | +| `IntercomTicket` | `POST /tickets/search`, `GET /tickets/{id}` | cursor | yes, exactly | +| `IntercomAdmin` | `GET /admins` | read whole | yes, exactly | +| `IntercomTeam` | `GET /teams` | read whole | yes, exactly | +| `IntercomTicketType` | `GET /ticket_types` | read whole | yes, exactly | +| `IntercomTicketState` | `GET /ticket_states` | read whole | yes, exactly | + +Two tiers, and they behave differently on purpose. + +**Read whole** — admins, teams, ticket types, ticket states. Their endpoints answer in one response, +so filtering, sorting, paging and counting them in memory is *exact*: the records in hand are every +record Intercom holds. These are the only collections that can be filtered, sorted and grouped in +this lot, and the only ones a chart may group by. The cost is bandwidth, not correctness. + +**Cursor** — conversations and tickets. What is in hand is a page of something far larger, so +nothing is filtered or sorted in memory. Three routes and no fourth: no condition walks the listing, +`id equals X` reads the record through its own endpoint, and **anything else is refused** with a +message naming the lot that will answer it. + +## What the API cannot do, and what this does about it + +Where Forest asks for something Intercom has no equivalent for, this datasource **refuses with a +message naming the reason** rather than answering something that looks right and is not. Those +arrive as a 400 carrying the text. + +- **No offset pagination.** Intercom hands out the page after a cursor and documents that jumping to + page N is unsupported, so reaching page 20 costs 20 sequential requests. The walk is capped at 50 + pages / 7 500 records and every truncation is logged, naming the window it stopped in. +- **Duplicates on a moving dataset.** Intercom documents that records modified between two paginated + requests can be served twice; the walk deduplicates by id. The missed counterpart is inherent to + cursor pagination and cannot be repaired — it is documented rather than papered over. +- **A sort is accepted and ignored.** Measured: `sort` on these endpoints raises nothing and changes + nothing. Since the lack of support is undetectable at runtime, no column is declared sortable and + a requested order is reported in the log. The rows come back in the order the API imposes. +- **No aggregate endpoint.** Counting is free and exact — `total_count` counts what the query names, + not what a page held — so the record counter is one request. Anything beyond a count is refused on + the cursor collections: grouping over the pages a walk collected would look exact while answering + a fraction. +- **`per_page` is refused past 150**, with `invalid_per_page` and no silent downgrade, so the page + size is bounded before the request leaves. Tickets are bounded far lower still: **25**, because + the search response carries the whole timeline of every ticket and Intercom offers no field + selection. Provisional, pending measurement against real response sizes. +- **No `GET /tickets` at all.** Even an unfiltered ticket list goes through `POST /tickets/search` + with a predicate matching everything. +- **The envelope key is not always `data`.** Measured: `/tickets/search` answers under `tickets`, + `/admins` under `admins`, `/teams` under `teams`. A response carrying neither the expected key nor + `data` is refused rather than read as an empty page. + +## Conversations + +The row carries what a queue is read for: state, priority, assignee and team ids, the company, the +tags, and the lifecycle Intercom keeps in `statistics` — `closed_at`, `closed_by_id`, +`first_contact_reply_at`, `last_contact_reply_at`, `last_admin_reply_at`, `reopen_count`. + +**The timeline opens on `source`, not on the parts.** The message that started the conversation +lives there; a thread built from the parts alone opens on the first reply and loses what the +customer actually asked. Every entry keeps its `part_type` — an assignment, a note and a reply are +different events. + +Intercom returns the parts **only when retrieving a single conversation**, so: + +- a record detail gets its timeline for free; +- a list view asking for the `timeline` column pays one request per row, bounded to 10. The rows past + that keep a `nil`, which reads as *unknown* — never as an empty thread. + +A conversation is capped at its **500 most recent parts**; a very long thread is therefore partial, +and says so nowhere but here. + +Contact name and e-mail are denormalized onto the row by **one bulk read per page**, not one per +row, and only when the projection names them. A failure there costs those two columns, not the page. + +## Tickets + +A ticket carries **no `statistics` block** — measured against a workspace of 81 142 tickets — so +neither a closure date nor a last responder exists as a field. Both are derived from the parts, +which ride along in the search response whether or not anything asks for them, and therefore cost +nothing: + +| Column | Derived from | +| --- | --- | +| `closed_at`, `closed_by_name` | the last transition into a state of category `resolved` | +| `last_reply_at`, `last_responder_name`, `last_responder_type` | the last `comment` part | + +Four things to know about them: + +- a ticket is not "closed" on Intercom, it enters a **resolved** state; +- the state-change event is matched on its **prefix**, not on `ticket_state_updated_by_admin`: a + workspace running workflows closes tickets through other variants, and an invisible closure is + worse than an absent column; +- a transition whose target equals the previous state is ignored — measured, they exist; +- **a resolved ticket showing no closure date may have been closed all the same**: past the 500-part + ceiling the transition falls out of the window. That case is detected and logged, since a Date + column cannot say "unknown". + +Both columns are **display only**, and not temporarily: `/tickets/search` filters on neither and +ignores a sort, so neither advertises an operator. + +The attributes a workspace declares on its ticket types are introspected once at boot and published +as the **union** of every type's, keyed by name the way the payload is. Filtering one is a different +matter: Intercom filters an attribute by id (`ticket_attribute.{id}`), and the same name carries a +different id from one ticket type to the next — measured, `_default_title_` is `14162161` on one +type and `14162165` on another. A union column has no single id to translate to, so filtering on a +ticket attribute means one collection per ticket type. The ids are kept per type for the lot that +will need them. + +## Rate limits + +Intercom meters the app and, above it, the whole workspace — 25 000 requests a minute shared with +every other private app the customer runs — and allocates that budget in **10-second windows**: the +measured `x-ratelimit-limit` is 1667, not 10 000. A burst therefore takes a 429 while the minute's +budget is barely touched, which is why what matters is the instantaneous rate. + +The limiter is driven by the headers Intercom returns on every response rather than by a table: it +waits out the reset when the window is spent, and counts its own in-flight requests down so several +of them do not go out on the same stale figure. A reset further out than a window is a clock +disagreement rather than a window emptying — the request goes through and the log says so, once per +window. + +It sits **in front of** the 429 retry, not instead of it: the retry remains the defence against the +part of the workspace budget spent by traffic this process cannot see. Pass `rate_limiter: nil` to +meter on your own side instead. + +## Privacy + +The body of a conversation is raw personal data, and this datasource is built on that assumption. + +- **Nothing logs a body.** Logs carry the operation, the counts and Intercom's request id — never + content. A response that fails to parse is reported by name, never quoted: a JSON parser opens its + message with the characters it choked on, and on a 200 those are the payload. +- **`display_as=plaintext` on every conversation read.** The bodies are HTML written by end + customers; rendering third-party HTML inside Forest is neither safe nor useful. +- **The regional host is configurable** so a workspace's data stays in its region. +- Ticket list pages carry customer message bodies whether or not anything asks for them — Intercom + offers no field selection. Restrict the body columns with Forest's field-level permissions where + that matters. + +## Boot-time introspection + +Constructing the datasource performs exactly **one** read: `GET /ticket_types`, for the attribute +columns of `IntercomTicket`. It runs on the boot connection — short timeouts, one quick retry — so a +slow Intercom cannot turn a Rails boot into minutes the operator sits through, and it degrades to no +attribute column rather than to a failed boot. + +Everything else is read when a collection is listed, so an agent boots whatever Intercom is doing. + +## What is not here yet + +| Lot | What it brings | +| --- | --- | +| 2 | Filter translation into Intercom's search DSL, free-text search, per-endpoint operator tables, UTC date bounds | +| 3 | Writes and business actions: reply, close, snooze, reopen, assign, tag, convert | +| 4 | Contacts and companies, and the relations promoted from today's denormalized columns | +| 5 | Notes, tags, segments | +| 6 | Bounded group-by and the reporting export | + +## Development + +```bash +cd packages/forest_admin_datasource_intercom +BUNDLE_GEMFILE=Gemfile-test bundle install +BUNDLE_GEMFILE=Gemfile-test bundle exec rspec +bundle exec rubocop # from the repository root +``` + +Specs stub the HTTP layer with WebMock. Every payload they feed in is **hand-written from the +OpenAPI 2.16 specification**, never captured from a workspace: a conversation body is personal data, +and a fixture is read by everyone who clones the repository. diff --git a/packages/forest_admin_datasource_intercom/Rakefile b/packages/forest_admin_datasource_intercom/Rakefile new file mode 100644 index 000000000..4c774a2bf --- /dev/null +++ b/packages/forest_admin_datasource_intercom/Rakefile @@ -0,0 +1,6 @@ +require 'bundler/gem_tasks' +require 'rspec/core/rake_task' + +RSpec::Core::RakeTask.new(:spec) + +task default: :spec diff --git a/packages/forest_admin_datasource_intercom/forest_admin_datasource_intercom.gemspec b/packages/forest_admin_datasource_intercom/forest_admin_datasource_intercom.gemspec new file mode 100644 index 000000000..14ce48bb4 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/forest_admin_datasource_intercom.gemspec @@ -0,0 +1,36 @@ +lib = File.expand_path('lib', __dir__) +$LOAD_PATH.unshift lib unless $LOAD_PATH.include?(lib) + +require_relative 'lib/forest_admin_datasource_intercom/version' + +Gem::Specification.new do |spec| + spec.name = 'forest_admin_datasource_intercom' + spec.version = ForestAdminDatasourceIntercom::VERSION + spec.authors = ['Forest Admin'] + spec.email = ['contact@forestadmin.com'] + spec.homepage = 'https://www.forestadmin.com' + spec.summary = 'Intercom datasource for Forest Admin Ruby agent.' + spec.description = 'Surface Intercom conversations, tickets, contacts and companies as Forest Admin collections.' + spec.license = 'GPL-3.0' + spec.required_ruby_version = '>= 3.0.0' + + spec.metadata['homepage_uri'] = spec.homepage + spec.metadata['source_code_uri'] = 'https://github.com/ForestAdmin/agent-ruby' + spec.metadata['changelog_uri'] = 'https://github.com/ForestAdmin/agent-ruby/blob/main/CHANGELOG.md' + spec.metadata['rubygems_mfa_required'] = 'false' + + spec.files = Dir.chdir(__dir__) do + `git ls-files -z`.split("\x0").reject do |f| + (File.expand_path(f) == __FILE__) || + f.start_with?(*%w[bin/ test/ spec/ features/ .git .circleci appveyor Gemfile]) + end + end + spec.bindir = 'exe' + spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) } + spec.require_paths = ['lib'] + + spec.add_dependency 'activesupport', '>= 6.1' + spec.add_dependency 'faraday', '~> 2.0' + spec.add_dependency 'faraday-retry', '~> 2.0' + spec.add_dependency 'zeitwerk', '~> 2.3' +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom.rb new file mode 100644 index 000000000..43d8fe9c9 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom.rb @@ -0,0 +1,59 @@ +require_relative 'forest_admin_datasource_intercom/version' +require 'cgi/escape' +require 'json' +require 'logger' +require 'set' +require 'time' +require 'uri' +require 'zeitwerk' +require 'faraday' +require 'faraday/retry' +require 'forest_admin_datasource_toolkit' + +loader = Zeitwerk::Loader.for_gem +loader.setup + +module ForestAdminDatasourceIntercom + class Error < StandardError; end + class ConfigurationError < Error; end + + # A filter Intercom cannot express exactly: an operator its search DSL refuses + # on that field, a tree deeper than the two levels it allows, or a group past + # its fifteen filters. It descends from the toolkit's ValidationError rather + # than from this package's Error so the agent answers 400 carrying the message + # instead of a 500 "Unexpected error" -- each one names something the operator + # set and can change, and the message is the only place they learn which. + # + # This datasource refuses rather than approximates: a result that looks + # filtered and is not is worse than an explicit refusal. + class UnsupportedOperatorError < ForestAdminDatasourceToolkit::Exceptions::ValidationError; end + + # Raised when an Intercom API call fails. Carries the HTTP status and the + # parsed response body so callers -- smart actions in particular -- can + # surface Intercom's own error message instead of a generic string. + class APIError < Error + attr_reader :status, :body + + def initialize(message, status: nil, body: nil) + super(message) + @status = status + @body = body + end + end + + class << self + attr_writer :logger + + def logger + @logger ||= default_logger + end + + private + + def default_logger + return Rails.logger if defined?(Rails) && Rails.respond_to?(:logger) && Rails.logger + + Logger.new($stderr).tap { |l| l.progname = 'forest_admin_datasource_intercom' } + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/client.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/client.rb new file mode 100644 index 000000000..e9b278e3a --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/client.rb @@ -0,0 +1,390 @@ +module ForestAdminDatasourceIntercom + # Every call to Intercom goes through here. Hand-written on Faraday rather + # than through the official `intercom` gem: that one maps the JSON onto + # objects, and what this datasource needs is the parts it hides -- the raw + # payload, because a custom attribute is a key nobody declared in advance; + # the quota headers, because the pacing is driven by them; and Intercom's own + # error body, because that text is what an operator reads when an action + # fails. + # Long by line count only: the public surface is one method per endpoint, and + # the rest is the envelope and error handling every one of them shares. + class Client # rubocop:disable Metrics/ClassLength + # `per_page=200` is refused with `invalid_per_page` -- "must be an integer + # between 0 and 150". There is no silent downgrade, so a page size is bounded + # before it is sent or the list view breaks rather than shrinks. + MAX_PER_PAGE = 150 + + # Bounds `fetch_all`, which asks for a whole reference collection rather than + # a window: those endpoints answer in one response, so reaching this many + # pages means Intercom started paginating on its own and the read is spending + # more than the answer is worth. + MAX_COLLECTED_PAGES = 10 + + # `next_cursor` is nil as soon as Intercom stops advertising a next page, so + # callers never have to know how the absence is spelled on the wire. + # `total_count` is exact, filter included, which is what makes Forest's + # record counter and its "number of" charts one request each. + Page = Struct.new(:records, :next_cursor, :total_count, keyword_init: true) + + def initialize(configuration) + @configuration = configuration + end + + # Health check: the admin the token belongs to, plus its workspace. Enough + # to prove the credentials are usable, and the one call that verifies the + # pinned API version was honoured -- Intercom echoes the version it served + # in a response header. + # + # `boot: true` runs it on the short-timeout connection, for a caller + # checking the token while the agent is still starting. + def me(boot: false) + must_succeed('me') do + response = get('me', boot: boot) + verify_pinned_version(response) + response.body + end + end + + # One page of a cursor-paginated listing. `starting_after` is what the + # previous page advertised; nil asks for the first one. + # + # `CursorWalker` is what turns the offset/limit window a list view asks for + # into a sequence of these. + # + # `list_key` is the key the endpoint puts its records under, and it is not + # `data` everywhere: `/tickets/search` answers under `tickets` (measured), so + # a caller names its own and `data` stays the fallback. + def list_page(path, per_page:, starting_after: nil, params: {}, list_key: 'data', boot: false) + query = params.merge('per_page' => self.class.bounded_per_page(per_page)) + query['starting_after'] = starting_after unless blank?(starting_after) + + must_succeed(path) { to_page(get(path, query, boot: boot).body, path, list_key) } + end + + # One page of a search endpoint. The query is written by the caller rather + # than translated from a Forest filter -- that translation is lot 2 -- so + # what goes on the wire is what the caller asked for. + def search_page(path, query:, per_page:, starting_after: nil, list_key: 'data') + pagination = { 'per_page' => self.class.bounded_per_page(per_page) } + pagination['starting_after'] = starting_after unless blank?(starting_after) + body = { 'query' => query, 'pagination' => pagination } + + must_succeed(path) { to_page(post(path, body).body, path, list_key) } + end + + # One record from its own endpoint. Raises on a 404 like on any other + # failure: what a missing record means -- a stale link, a record outside the + # token's scope, a deletion -- is the caller's to decide, not the client's. + def fetch_record(path, id, params: {}, boot: false) + operation = "#{path}/#{id}" + + must_succeed(operation) do + body = get("#{path}/#{Faraday::Utils.escape(id)}", params, boot: boot).body + body.is_a?(Hash) ? body : refuse_body_shape(operation, 'the response is not a record') + end + end + + # Every record of an endpoint that answers in one response: the reference + # collections -- admins, teams, ticket types, ticket states -- whose paths + # declare no pagination parameter at all. + # + # `list_key` is the key the endpoint puts its records under. Intercom is not + # consistent about it: `/admins` answers `{"type": "admin.list", "admins": + # [...]}` where `/ticket_types` answers the `data` envelope every paginated + # listing uses, so the collection names its own and `data` is the fallback. + # + # A cursor is followed if one is advertised, defensively: no pagination + # parameter in the specification is not a promise that a large workspace + # answers in one response, and a truncated reference collection would show + # an operator a state list missing its last states. + def fetch_all(path, list_key: 'data', boot: false) + must_succeed(path) { collect_pages(path, list_key: list_key, boot: boot) } + end + + # The page size Intercom accepts, whatever was asked for. + def self.bounded_per_page(size) + value = size.to_i + return 1 if value < 1 + + [value, MAX_PER_PAGE].min + end + + # The client holds the connections whose headers carry the access token in + # clear, and Faraday prints those headers on `inspect`. + def inspect + "#<#{self.class.name} url=#{@configuration.url.inspect}>" + end + + private + + # The raw response rather than its body: the quota headers are read by the + # throttle, and the version echo by `verify_pinned_version`. + def get(path, params = nil, boot: false) + (boot ? boot_connection : connection).get(path, params) + end + + def post(path, body, boot: false) + (boot ? boot_connection : connection).post(path, body) + end + + # Intercom serves the version its workspace defaults to when the pin is not + # honoured, and the payloads differ between versions. The echo is the only + # way to notice, and noticing at boot is worth more than a schema that + # drifts silently -- so this reports rather than raises: the agent still runs + # against a version it did not ask for, which is better than not running. + def verify_pinned_version(response) + served = response.headers['intercom-version'] if response.respond_to?(:headers) + return if served.nil? || served.to_s == @configuration.api_version + + ForestAdminDatasourceIntercom.logger.warn( + "[forest_admin_datasource_intercom] asked Intercom for API version #{@configuration.api_version} " \ + "and it served #{served}. Payload shapes may differ from the ones this datasource expects; check the " \ + "workspace's default version in the Developer Hub." + ) + end + + def collect_pages(path, list_key:, boot:) + records = [] + cursor = nil + pages = 0 + + loop do + body = get(path, cursor.nil? ? nil : { 'starting_after' => cursor }, boot: boot).body + records.concat(extract_entities(body, path, list_key)) + pages += 1 + cursor = next_cursor(body, path) + break if cursor.nil? + + if pages >= MAX_COLLECTED_PAGES + log_collection_cap(path, pages, records.size) + break + end + end + + records + end + + # The records under the key the endpoint uses, or under `data`. Anything + # else is refused rather than read as an empty page: `Array()` would turn the + # envelope into `[key, value]` pairs a collection would serialize into rows + # holding nothing -- a page that looks answered and is empty -- and a + # reference collection silently read as empty is a state column with no + # values and an assignee shown as a raw id. + def extract_entities(body, operation, list_key) + return [] if body.nil? || body == '' + + entities = body.is_a?(Hash) ? (body[list_key] || body['data']) : nil + return entities if entities.is_a?(Array) + + detail = list_key == 'data' ? "'data' is not a list" : "neither '#{list_key}' nor 'data' is a list" + refuse_body_shape(operation, detail) + end + + def log_collection_cap(path, pages, collected) + ForestAdminDatasourceIntercom.logger.warn( + "[forest_admin_datasource_intercom] Stopped reading #{path} after #{pages} page(s) / " \ + "#{collected} record(s); the rest is left out. This endpoint is read whole on purpose, so a workspace " \ + 'this large needs the collection bounded rather than listed.' + ) + end + + # Intercom wraps a listing in `{ "type": "list", "data": [...], + # "total_count": N, "pages": { "next": { "starting_after": "..." } } }`. + def to_page(body, operation, list_key) + Page.new(records: extract_entities(body, operation, list_key), + next_cursor: next_cursor(body, operation), + total_count: extract_count(body)) + end + + # Absent on the last page, which is how the walk knows it is done. An older + # API version spells it as a url instead of an object -- and one can be + # served despite the pin, which is what the version echo warns about -- so + # the cursor is read out of its query string rather than the page being + # taken for the last one. + # + # Anything else is refused, an advertised page whose cursor cannot be read + # included: taking it for the last page would truncate the answer silently, + # which is worse than a failure naming what it could not read. + def next_cursor(body, operation) + advertised = body.is_a?(Hash) && body['pages'].is_a?(Hash) ? body['pages']['next'] : nil + return nil if advertised.nil? + + cursor = case advertised + when Hash then advertised['starting_after'] + when String then cursor_from_url(advertised) + end + + presence(cursor) || refuse_body_shape(operation, "'pages.next' carries no cursor this can follow") + end + + def cursor_from_url(url) + Faraday::Utils.parse_query(URI.parse(url).query.to_s)['starting_after'] + rescue URI::InvalidURIError + nil + end + + # nil rather than 0 when Intercom sends no count: zero is an answer, and + # this is the absence of one. + def extract_count(body) + count = body['total_count'] if body.is_a?(Hash) + count.is_a?(Numeric) ? count.to_i : nil + end + + def refuse_body_shape(operation, detail) + raise APIError.new("Intercom API call failed: #{operation}: unexpected response shape, #{detail}", status: nil) + end + + def presence(value) + blank?(value) ? nil : value + end + + def blank?(value) + value.nil? || value.to_s.empty? + end + + # `JSON::ParserError` alongside Faraday's own errors: on its own it would + # reach the catch-all below, whose message is the exception's -- and a JSON + # parser opens its message with what it choked on. + def must_succeed(operation) + yield + rescue Faraday::Error, JSON::ParserError => e + raise api_error(operation, e) + rescue APIError + # Already mapped, with its status intact; re-wrapping would erase it -- + # a 404 read as "no such record" rather than as a failure, above all. + raise + rescue StandardError => e + raise APIError, "Intercom API call failed: #{operation}: #{e.class}: #{e.message}" + end + + # Builds an APIError preserving the HTTP status and Intercom's own error body + # so a smart action can show the operator the real reason instead of + # "failed". + def api_error(operation, error) + response = response_of(error) + status = response[:status] + body = parse_body(response[:body]) + + APIError.new("Intercom API call failed: #{operation}: #{failure_detail(error, status, body)}", + status: status, body: body) + end + + # Faraday hands the status and the body back in a plain hash on most errors + # and in its own `Env` on a parsing error; both answer `[]`. + def response_of(error) + response = error.respond_to?(:response) ? error.response : nil + return { status: nil, body: nil } unless response.respond_to?(:[]) + + { status: response[:status], body: response[:body] } + end + + # A body that could not be parsed is named, never quoted: the parser's own + # message opens with the characters it choked on, and on a 200 those are the + # payload -- a conversation body, most of the time (R10). + def failure_detail(error, status, body) + return unreadable_detail(status) if parse_failure?(error) + return "#{error.class}: #{error.message}" unless status + + "HTTP #{status} #{error_message(body)}".strip + end + + def unreadable_detail(status) + detail = 'the response could not be read as JSON' + status ? "#{detail} (HTTP #{status})" : detail + end + + def parse_failure?(error) + error.is_a?(Faraday::ParsingError) || error.is_a?(JSON::ParserError) + end + + # Intercom answers a failure with `{ "type": "error.list", "request_id": + # "...", "errors": [{ "code": ..., "message": ... }] }`. The request id is + # what its support asks for first, so it is appended after the truncation + # rather than being what a long body pushes out. + # + # A body of any other shape is *not* echoed here. This message travels into + # the interface and into whatever collects the agent's errors, and the body + # of a response that failed to parse is a payload rather than an error -- + # conversation bodies included (R10). Its size is reported instead, and the + # body itself stays on the exception for whoever inspects one. + def error_message(parsed) + return "(unreadable body, #{parsed.to_s.bytesize} bytes)" unless parsed.is_a?(Hash) + + message = join_errors(parsed['errors']) + message = parsed.to_json if message.empty? + + append_request_id(message[0, 500], parsed['request_id']) + end + + def join_errors(errors) + Array(errors).filter_map do |error| + next error unless error.is_a?(Hash) + + [error['code'], error['message']].compact.join(': ') + end.join('; ') + end + + def append_request_id(message, request_id) + return message unless request_id + + "#{message} (request_id: #{request_id})" + end + + def parse_body(body) + return body unless body.is_a?(String) && !body.empty? + + JSON.parse(body) + rescue JSON::ParserError + body + end + + def connection + @connection ||= build_connection( + retry_policy: @configuration.retry_policy, + timeout: @configuration.timeout, + open_timeout: @configuration.open_timeout + ) + end + + # For what is read while the datasource is being constructed -- the + # custom-attribute introspection above all: short timeouts and one quick + # retry, so a slow Intercom cannot turn a Rails boot into minutes of + # waiting. Memoized separately from `connection`, which keeps the patience + # every later request is entitled to. + def boot_connection + @boot_connection ||= build_connection( + retry_policy: @configuration.boot_retry_policy, + timeout: @configuration.boot_timeout, + open_timeout: @configuration.boot_open_timeout + ) + end + + # Middleware order is deliberate: `raise_error` sits outside the JSON parser + # so it raises with an already-parsed body, and `retry` sits innermost so it + # inspects raw statuses -- behind `raise_error` it would never see a 429. + # + # The throttle goes inside `retry`, which is what makes a replay wait for + # the window like a first attempt, and what lets the 429's own headers reach + # the limiter: outside it, the middleware would run once for a request that + # reached Intercom three times. + # + # How long a request may take is the caller's to state; everything else is + # the same on every connection this builds, the limiter included -- a second + # limiter would meter in a window of its own and spend the budget twice. + def build_connection(retry_policy:, timeout:, open_timeout:) + Faraday.new(url: @configuration.url) do |f| + f.request :json + f.response :raise_error + f.response :json + f.request :retry, **retry_policy.to_faraday_options + f.use Throttle, limiter: @configuration.rate_limiter if @configuration.rate_limiter + f.headers['Authorization'] = "Bearer #{@configuration.access_token}" + f.headers['Accept'] = 'application/json' + f.headers['Intercom-Version'] = @configuration.api_version + f.headers['User-Agent'] = "forest_admin_datasource_intercom/#{VERSION}" + f.options.open_timeout = open_timeout + f.options.timeout = timeout + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/admin.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/admin.rb new file mode 100644 index 000000000..45704ea82 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/admin.rb @@ -0,0 +1,52 @@ +module ForestAdminDatasourceIntercom + module Collections + # The teammates of the workspace: who a conversation or a ticket is assigned + # to. Without this collection an assignee is a raw id on every row. + class Admin < FetchAllCollection + def initialize(datasource) + super(datasource, 'IntercomAdmin') + end + + protected + + # `/admins` puts its records under `admins`, not under the `data` envelope + # the paginated listings use. + def fetch_all + client.fetch_all('admins', list_key: 'admins') + end + + def serialize(admin) + attrs = admin.is_a?(Hash) ? admin : {} + + { 'id' => stringify_id(attrs['id']), + 'name' => attrs['name'], + 'email' => attrs['email'], + 'job_title' => attrs['job_title'], + 'away_mode_enabled' => attrs['away_mode_enabled'], + 'away_mode_reassign' => attrs['away_mode_reassign'], + 'has_inbox_seat' => attrs['has_inbox_seat'], + 'team_ids' => Array(attrs['team_ids']).map { |id| stringify_id(id) } } + end + + private + + def define_schema + add_column('id', 'String', is_primary_key: true) + add_column('name', 'String') + add_column('email', 'String') + add_column('job_title', 'String') + # Whether the teammate is away, and whether their conversations get + # reassigned while they are: the two an ops lead looks at before + # assigning anything. + add_column('away_mode_enabled', 'Boolean') + add_column('away_mode_reassign', 'Boolean') + add_column('has_inbox_seat', 'Boolean') + # A list, so neither filterable nor sortable. It stays a plain column + # rather than a relation: Intercom carries the membership on the admin + # and on the team both, so declaring it twice would give the schema two + # sides of a many-to-many with no join collection to hold it. + add_column('team_ids', 'Json') + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/base_collection.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/base_collection.rb new file mode 100644 index 000000000..9c4731f27 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/base_collection.rb @@ -0,0 +1,93 @@ +module ForestAdminDatasourceIntercom + module Collections + # What every Intercom collection shares: how a schema is declared, how a + # record is narrowed to the projection asked for, and how a window is cut + # out of records already in hand. + # + # Read-only for now. The writes and the business actions arrive with lot 3, + # and the relations with lot 4, once Contacts and Companies exist -- a + # relation whose target collection is missing is a schema the agent refuses + # to boot on. + class BaseCollection < ForestAdminDatasourceToolkit::Collection + ColumnSchema = ForestAdminDatasourceToolkit::Schema::ColumnSchema + Operators = ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Operators + Equivalent = ForestAdminDatasourceToolkit::Components::Query::ConditionTree::ConditionTreeEquivalent + Leaf = ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Nodes::ConditionTreeLeaf + + def initialize(datasource, name) + super + define_schema + end + + def client + datasource.client + end + + protected + + def define_schema = raise(NotImplementedError, "#{self.class} did not implement define_schema") + + # A record narrowed to what was asked for. A projection naming a field the + # record does not carry yields nil rather than nothing at all: the agent + # asked for a column, and an absent key would read as a record missing it. + def project(record, projection) + fields = Array(projection) + return record if fields.empty? + + fields.to_h { |field| [field, record[field]] } + end + + # The window a list view asked for, cut out of records already in hand. + # + # A filter with no page -- or a page naming no limit -- asks for every + # record it matched, and there is nothing to cut. How far the read that + # collected them went is a different question, answered by the walker and + # its caps. + def page_window(records, filter) + page = filter&.page + return records if page.nil? + + offset = page.offset.to_i.clamp(0, nil) + limit = page.limit.to_i + return records.drop(offset) unless limit.positive? + + records[offset, limit] || [] + end + + # The timezone in-memory date comparisons are evaluated in. The caller's, + # since that is whose "today" the filter was written against. + def timezone_for(caller) + caller.respond_to?(:timezone) ? caller.timezone : nil + end + + # Ids reach this datasource as strings -- a filter value from Forest, a + # segment, a url -- while Intercom types them inconsistently: a team id is + # a string, the same team's id inside `admin_ids` is a number. Left as it + # comes, an integer id would never match the string the filter carries. + def stringify_id(value) + value&.to_s + end + + # Intercom nests its lists twice -- `{"type": "contact.list", "contacts": + # [...]}` -- and answers a null instead of an empty list when there is + # nothing. + def nested_list(container, key) + return [] unless container.is_a?(Hash) + + list = container[key] + list.is_a?(Array) ? list : [] + end + + # Intercom dates travel as epoch seconds; Forest reads a Date column as an + # ISO8601 string, and a filter carries one too, so comparing the two is the + # ordering itself. UTC deliberately: that is where Intercom stores and + # truncates, and rendering a local time here would hide the very shift that + # makes a day-granular date filter wrong. + def stamp(seconds) + return nil unless seconds.is_a?(Numeric) && seconds.positive? + + Time.at(seconds).utc.iso8601 + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/contact_identity.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/contact_identity.rb new file mode 100644 index 000000000..2bc4fab54 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/contact_identity.rb @@ -0,0 +1,78 @@ +module ForestAdminDatasourceIntercom + module Collections + # The contact of a conversation or of a ticket, denormalized onto the row. + # + # Intercom nests only the ids -- `{"type": "contact.list", "contacts": + # [{"id": "..."}]}` -- so a name and an e-mail cost a read. That read is done + # once per page, for every row at once, and never per row: a page of 25 rows + # is one request, not 25. + # + # It stays a pair of columns rather than a relation because the Contacts + # collection arrives in lot 4, and a relation whose target collection is + # missing is a schema the agent refuses to boot on. + module ContactIdentity + COLUMNS = %w[contact_name contact_email].freeze + + # How many ids one `id in [...]` read carries. A page holds fewer than this + # in practice; the chunk keeps the request bounded if it ever does not. + CONTACT_CHUNK = 100 + + private + + def define_contact_columns + add_column('contact_ids', 'Json') + add_column('contact_count', 'Number') + add_column('contact_name', 'String') + add_column('contact_email', 'String') + end + + # A group conversation, or a ticket opened for several people, has more + # than one contact: the row names the first and counts them, rather than + # presenting one of several as the one. + def contact_columns_for(attrs) + ids = nested_list(attrs['contacts'], 'contacts').filter_map { |contact| stringify_id(contact['id']) } + + { 'contact_ids' => ids, 'contact_count' => ids.size, + # Filled by the bulk read below, and left nil when the projection did + # not ask for them. + 'contact_name' => nil, 'contact_email' => nil } + end + + def first_contact_id(record) + contact = nested_list((record || {})['contacts'], 'contacts').first + contact.is_a?(Hash) ? stringify_id(contact['id']) : nil + end + + def embed_contact_identity(records, rows, projection) + return unless (COLUMNS & projection).any? + + identities = contact_identities(records) + records.each_with_index do |record, index| + identity = identities[first_contact_id(record)] || {} + rows[index]['contact_name'] = identity['name'] if rows[index].key?('contact_name') + rows[index]['contact_email'] = identity['email'] if rows[index].key?('contact_email') + end + end + + # A failure costs the two columns and nothing else: an identity that could + # not be read is not a page that could not be served. + def contact_identities(records) + ids = records.filter_map { |record| first_contact_id(record) }.uniq + return {} if ids.empty? + + ids.each_slice(CONTACT_CHUNK).with_object({}) do |chunk, indexed| + page = client.search_page('contacts/search', per_page: chunk.size, + query: { 'field' => 'id', 'operator' => 'IN', + 'value' => chunk }) + page.records.each { |contact| indexed[contact['id'].to_s] = contact } + end + rescue APIError => e + ForestAdminDatasourceIntercom.logger.warn( + "[forest_admin_datasource_intercom] #{name} could not read the contacts of this page (HTTP " \ + "#{e.status || "-"}); the name and e-mail columns are left empty for it." + ) + {} + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/conversation.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/conversation.rb new file mode 100644 index 000000000..9c0d13fc6 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/conversation.rb @@ -0,0 +1,157 @@ +module ForestAdminDatasourceIntercom + module Collections + # The conversations of the workspace: what a support team actually works on. + # + # Read through `GET /conversations`, whose records Intercom puts under + # `conversations` rather than under the `data` envelope -- `/tickets/search` + # does the same with `tickets`, so the key is named rather than assumed. + # + # `display_as=plaintext` on every read: the bodies are HTML written by end + # customers, and rendering third-party HTML inside Forest is neither safe nor + # useful (R10). + # Long by line count only: most of it declares the columns, one call each. + class Conversation < CursorCollection + include ContactIdentity + include Conversation::Serializer + include Conversation::Timeline + + # How many conversations of one page may have their timeline read. The + # parts are absent from the listing response -- Intercom returns them only + # when retrieving a single conversation -- so a timeline asked for in a + # list view costs one request per row. Bounded rather than turned into a + # page the operator waits half a minute for; rows past the cap are left at + # nil, which reads as "unknown", never as "this conversation is empty". + MAX_TIMELINE_READS = 10 + + def initialize(datasource) + super(datasource, 'IntercomConversation') + end + + protected + + def list_endpoint = 'conversations' + def list_key = 'conversations' + def read_params = { 'display_as' => 'plaintext' } + + # The contact identity and the timeline, each read only when the projection + # names it: neither is on the conversation payload, and a page that never + # asked for them must not pay for them. + def enrich(records, rows, projection) + wanted = Array(projection).map(&:to_s) + + embed_contact_identity(records, rows, wanted) + embed_timeline(records, rows, wanted) + end + + private + + def define_schema + add_column('id', 'String', is_primary_key: true) + add_column('title', 'String') + # Left plain strings rather than enums: the values a workspace really + # serves are worth measuring before the interface offers them as a + # closed list, and nothing filters on them in this lot anyway. + add_column('state', 'String') + add_column('priority', 'String') + add_column('open', 'Boolean') + add_column('read', 'Boolean') + add_column('created_at', 'Date') + add_column('updated_at', 'Date') + add_column('waiting_since', 'Date') + add_column('snoozed_until', 'Date') + add_column('admin_assignee_id', 'String') + add_column('team_assignee_id', 'String') + # The conversation carries its company as a whole object, so the account + # name is free here -- unlike on a ticket, which carries the id alone. + add_column('company_id', 'String') + add_column('company_name', 'String') + define_contact_columns + define_source_columns + define_statistics_columns + add_column('tag_names', 'Json') + add_column('ai_agent_participated', 'Boolean') + add_column('timeline', 'Json') + end + + # The contact identity is denormalized onto the row rather than declared as + # a relation: the Contacts collection arrives in lot 4, and a relation whose + # target collection is missing is a schema the agent refuses to boot on. + # + # A group conversation has several contacts; the row carries the first and + # says how many there are, rather than pretending there is one. + def define_contact_columns + add_column('contact_ids', 'Json') + add_column('contact_count', 'Number') + add_column('contact_name', 'String') + add_column('contact_email', 'String') + end + + # The message that opened the conversation lives in `source`, not in the + # parts. A timeline built from the parts alone loses it, which is the one + # message nobody opens a conversation without wanting to read. + def define_source_columns + add_column('source_type', 'String') + add_column('source_subject', 'String') + add_column('source_body', 'String') + add_column('source_author_name', 'String') + add_column('source_author_email', 'String') + add_column('source_delivered_as', 'String') + end + + # Intercom keeps the lifecycle of a conversation in `statistics`, which is + # where the closure date and the reply timestamps come from. Flattened onto + # the row: they cost nothing, they are exact, and they are what an ops lead + # reads a queue for. + def define_statistics_columns + add_column('closed_at', 'Date') + add_column('first_closed_at', 'Date') + add_column('closed_by_id', 'String') + add_column('first_contact_reply_at', 'Date') + add_column('last_contact_reply_at', 'Date') + add_column('last_admin_reply_at', 'Date') + add_column('reopen_count', 'Number') + add_column('part_count', 'Number') + end + + # A record read through the record endpoint already carries its parts, so + # its timeline is free; one read from the listing does not, and pays a + # request. Rows past the cap keep the nil the projection put there. + def embed_timeline(records, rows, projection) + return unless projection.include?('timeline') + + budget = MAX_TIMELINE_READS + missing = 0 + + records.each_with_index do |record, index| + if parts_of(record) + rows[index]['timeline'] = build_timeline(record) + elsif budget.positive? + budget -= 1 + detail = read_detail(record['id']) + rows[index]['timeline'] = detail && build_timeline(detail) + else + missing += 1 + end + end + + warn_truncated_timelines(missing) if missing.positive? + end + + def read_detail(id) + client.fetch_record(list_endpoint, id, params: read_params) + rescue APIError => e + raise unless e.status == 404 + + nil + end + + def warn_truncated_timelines(missing) + ForestAdminDatasourceIntercom.logger.warn( + "[forest_admin_datasource_intercom] #{name} left the timeline of #{missing} row(s) unread: Intercom " \ + 'returns the parts only when retrieving one conversation, so a list view pays a request per row and ' \ + "this reads at most #{MAX_TIMELINE_READS}. Those rows show no timeline rather than an empty one." + ) + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/conversation/serializer.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/conversation/serializer.rb new file mode 100644 index 000000000..e241b168c --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/conversation/serializer.rb @@ -0,0 +1,75 @@ +module ForestAdminDatasourceIntercom + module Collections + class Conversation < CursorCollection + # One Intercom conversation flattened into the row the schema declares. + # Nothing here reads a sub-resource: every value comes from the payload the + # listing already returned. + module Serializer + protected + + def serialize(conversation) + attrs = conversation.is_a?(Hash) ? conversation : {} + + native(attrs) + .merge(contact_columns_for(attrs)) + .merge(source_of(attrs['source'])) + .merge(statistics_of(attrs['statistics'])) + end + + private + + def native(attrs) + company = attrs['company'].is_a?(Hash) ? attrs['company'] : {} + + { + 'id' => stringify_id(attrs['id']), + 'title' => attrs['title'], + 'state' => attrs['state'], + 'priority' => attrs['priority'], + 'open' => attrs['open'], + 'read' => attrs['read'], + 'created_at' => stamp(attrs['created_at']), + 'updated_at' => stamp(attrs['updated_at']), + 'waiting_since' => stamp(attrs['waiting_since']), + 'snoozed_until' => stamp(attrs['snoozed_until']), + 'admin_assignee_id' => stringify_id(attrs['admin_assignee_id']), + 'team_assignee_id' => stringify_id(attrs['team_assignee_id']), + 'company_id' => stringify_id(company['id']), + 'company_name' => company['name'], + 'tag_names' => nested_list(attrs['tags'], 'tags').filter_map { |tag| tag['name'] if tag.is_a?(Hash) }, + 'ai_agent_participated' => attrs['ai_agent_participated'] + } + end + + def source_of(source) + attrs = source.is_a?(Hash) ? source : {} + author = attrs['author'].is_a?(Hash) ? attrs['author'] : {} + + { 'source_type' => attrs['type'], + 'source_subject' => attrs['subject'], + # Plaintext, because `display_as=plaintext` rides on every read: the + # bodies are HTML written by end customers. + 'source_body' => attrs['body'], + 'source_author_name' => author['name'], + 'source_author_email' => author['email'], + 'source_delivered_as' => attrs['delivered_as'] } + end + + # `statistics` is null on a conversation Intercom has computed nothing + # for yet; every column then reads as absent rather than as zero. + def statistics_of(statistics) + attrs = statistics.is_a?(Hash) ? statistics : {} + + { 'closed_at' => stamp(attrs['last_close_at']), + 'first_closed_at' => stamp(attrs['first_close_at']), + 'closed_by_id' => stringify_id(attrs['last_closed_by_id']), + 'first_contact_reply_at' => stamp(attrs['first_contact_reply_at']), + 'last_contact_reply_at' => stamp(attrs['last_contact_reply_at']), + 'last_admin_reply_at' => stamp(attrs['last_admin_reply_at']), + 'reopen_count' => attrs['count_reopens'], + 'part_count' => attrs['count_conversation_parts'] } + end + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/conversation/timeline.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/conversation/timeline.rb new file mode 100644 index 000000000..9b757f6c8 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/conversation/timeline.rb @@ -0,0 +1,73 @@ +module ForestAdminDatasourceIntercom + module Collections + class Conversation < CursorCollection + # The thread of a conversation, as a structured list the record view can + # render: who said what, when, and through which kind of event. + # + # Two things this exists to get right. The opening message lives in + # `source`, not in the parts -- a timeline built from the parts alone opens + # on the first reply and loses what the customer actually asked. And + # `part_type` is kept on every entry: an assignment, a note and a reply are + # not the same event, and a thread that flattens them reads as a + # conversation that never happened the way it did. + # + # Intercom caps a conversation at its 500 most recent parts; the entry + # count is therefore what is in hand, not necessarily what exists. + module Timeline + # The pseudo type of the opening entry. Not an Intercom part type: it is + # the source, and calling it `comment` would make it indistinguishable + # from the replies that follow. + SOURCE_PART_TYPE = 'conversation_started'.freeze + + private + + def build_timeline(conversation) + attrs = conversation.is_a?(Hash) ? conversation : {} + entries = [source_entry(attrs)].compact + + entries + (parts_of(attrs) || []).map { |part| part_entry(part) } + end + + # nil rather than an empty list when the payload carries no parts at all: + # a listing response has none, and reading that as "this conversation is + # empty" is exactly the answer that looks complete without being it. + def parts_of(conversation) + container = (conversation || {})['conversation_parts'] + return nil unless container.is_a?(Hash) + + parts = container['conversation_parts'] + parts.is_a?(Array) ? parts : nil + end + + def source_entry(attrs) + source = attrs['source'] + return nil unless source.is_a?(Hash) + + entry(part_type: SOURCE_PART_TYPE, created_at: attrs['created_at'], author: source['author'], + body: source['body'], attachments: source['attachments']) + .merge('id' => stringify_id(source['id'])) + end + + def part_entry(part) + attrs = part.is_a?(Hash) ? part : {} + + entry(part_type: attrs['part_type'], created_at: attrs['created_at'], author: attrs['author'], + body: attrs['body'], attachments: attrs['attachments']) + .merge('id' => stringify_id(attrs['id']), 'redacted' => attrs['redacted']) + end + + def entry(part_type:, created_at:, author:, body:, attachments:) + writer = author.is_a?(Hash) ? author : {} + + { 'part_type' => part_type, + 'created_at' => stamp(created_at), + 'author_type' => writer['type'], + 'author_name' => writer['name'], + 'author_email' => writer['email'], + 'body' => body, + 'attachment_count' => Array(attachments).size } + end + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/cursor_collection.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/cursor_collection.rb new file mode 100644 index 000000000..eb8fa4721 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/cursor_collection.rb @@ -0,0 +1,246 @@ +module ForestAdminDatasourceIntercom + module Collections + # Base for the collections Intercom paginates by cursor: conversations and + # tickets. The opposite tier of `FetchAllCollection` in every way -- what is + # in hand is a page of something far larger, so nothing may be filtered, + # sorted or counted in memory without answering a fraction as if it were the + # whole. + # + # Three routes, and no fourth: + # + # * no condition at all -- a list view -- walks the listing endpoint; + # * `id equals X` reads the record through its own endpoint, which is what a + # record detail is; + # * anything else is **refused**. Translating a Forest condition tree into + # Intercom's search DSL is lot 2, and until it exists a filter that cannot + # be honoured has to say so: an unfiltered page served in answer to a + # filter is the one failure this datasource is built to avoid. + # + # Counting is the exception that costs nothing: `total_count` is exact on + # every response, filter included, so the record counter is one request. + # Long by line count only: half of it is the refusals, and a refusal that + # does not say what to do instead is a refusal an operator cannot act on. + class CursorCollection < BaseCollection # rubocop:disable Metrics/ClassLength + Aggregation = ForestAdminDatasourceToolkit::Components::Query::Aggregation + + # How many records an `id in [...]` read may fetch. One request per id -- + # Intercom has no "read these records" endpoint -- so the fan-out is + # bounded rather than turned into a rate limit halfway through a page. + MAX_ID_READS = 25 + + # Countable, and exactly: unlike the pages a walk collected, `total_count` + # is the whole dataset the filter names. + def initialize(datasource, name) + super + enable_count + end + + def list(_caller, filter, projection) + warn_ignored_sort(filter&.sort) + + records = fetch_records(filter) + rows = records.map { |record| project(serialize(record), projection) } + enrich(records, rows, projection) + rows + end + + # Count only, and never a group: Intercom exposes no aggregate endpoint, + # and grouping over the pages a walk happened to collect would look exact + # while answering a fraction. Refused here rather than through the + # contract's NotImplementedError, which reads as an oversight. + def aggregate(_caller, filter, aggregation, _limit = nil) + refuse_unsupported_aggregation!(aggregation) + + [{ 'group' => {}, 'value' => count_records(filter) }] + end + + protected + + # The listing endpoint, its record key, and the parameters every read of + # this collection carries. + def list_endpoint = raise(NotImplementedError, "#{self.class} did not implement list_endpoint") + def record_endpoint = list_endpoint + def list_key = 'data' + def read_params = {} + + # One Intercom entity flattened into a record matching the schema. + def serialize(_entity) = raise(NotImplementedError, "#{self.class} did not implement serialize") + + # Hook for what a row needs beyond its own payload. Left empty here: what + # it costs is the collection's business, not this base's. + def enrich(_records, _rows, _projection); end + + # Bounded per collection rather than by the API maximum: Intercom offers no + # field selection, so a collection whose rows carry their whole timeline + # pays for it by the page. See Ticket. + def max_page_size = Client::MAX_PER_PAGE + + # One page of the collection. A listing for conversations, a search for + # tickets -- Intercom exposes no `GET /tickets` at all -- so the endpoint + # and its shape belong to the collection, while walking it does not. + def read_page(per_page:, cursor:) + client.list_page(list_endpoint, per_page: [per_page, max_page_size].min, + starting_after: cursor, params: read_params, list_key: list_key) + end + + # A column of this tier advertises no filter and no sort, because the + # collection can honour neither -- except on the primary key, which is + # answered by the record endpoint rather than by a filter. A schema that + # advertised more would put filters in the interface that the read then + # refuses. Read-only for the same reason, on the write side. + def add_column(name, type, is_primary_key: false) + operators = is_primary_key ? [Operators::EQUAL, Operators::IN] : [] + add_field(name, ColumnSchema.new(column_type: type, + filter_operators: operators, + is_primary_key: is_primary_key, + is_read_only: true, + is_sortable: false, + is_groupable: false)) + end + + def walker + @walker ||= Pagination::CursorWalker.new + end + + private + + def fetch_records(filter) + ids = id_lookup(filter) + return records_by_ids(ids) if ids + + refuse_filter!(filter) unless browsing?(filter) + + listed_records(filter) + end + + def browsing?(filter) + filter.nil? || (filter.condition_tree.nil? && blank_search?(filter)) + end + + def blank_search?(filter) + search = filter.respond_to?(:search) ? filter.search : nil + search.nil? || search.to_s.strip.empty? + end + + # A record detail is `id equals X`, and a bulk read of related records is + # `id in [...]`. Only a bare leaf on the primary key takes this route: an + # `and` also carrying a scope names a narrower set than the ids do, and + # answering it with the ids alone would serve records the scope excludes. + def id_lookup(filter) + tree = filter&.condition_tree + return nil unless tree.is_a?(Leaf) && tree.field.to_s == primary_key + return nil unless blank_search?(filter) + + case tree.operator + when Operators::EQUAL then [tree.value].compact.map(&:to_s) + when Operators::IN then Array(tree.value).compact.map(&:to_s) + end + end + + def primary_key + @primary_key ||= fields.find do |_name, field| + field.respond_to?(:is_primary_key) && field.is_primary_key + end&.first + end + + # A record the operator can no longer reach -- deleted, or outside the + # token's scope -- reads as "no record" rather than as a failed page. + def records_by_ids(ids) + wanted = ids.first(MAX_ID_READS) + warn_truncated_ids(ids.size) if ids.size > wanted.size + + wanted.filter_map do |id| + client.fetch_record(record_endpoint, id, params: read_params) + rescue APIError => e + raise unless e.status == 404 + + nil + end + end + + def listed_records(filter) + offset, limit = translate_page(filter&.page) + + walker.walk(offset: offset, limit: limit) { |per_page, cursor| read_page(per_page: per_page, cursor: cursor) } + end + + # A filter with no page asks for every record it matched; the walker reads + # that as the nil limit it bounds with its own caps. + def translate_page(page) + return [0, nil] if page.nil? + + limit = page.limit.to_i + [page.offset.to_i.clamp(0, nil), limit.positive? ? limit : nil] + end + + # Exact, and one request: `total_count` counts what the filter names, not + # what a page happened to hold. An id lookup counts the records it found, + # which is cheaper still. + def count_records(filter) + ids = id_lookup(filter) + return records_by_ids(ids).size if ids + + refuse_filter!(filter) unless browsing?(filter) + + page = read_page(per_page: 1, cursor: nil) + return page.total_count if page.total_count + + raise UnsupportedOperatorError, + "#{name} cannot be counted: Intercom answered this listing without a total_count, and counting the " \ + 'pages the agent walked would answer a fraction of the collection as if it were the whole of it.' + end + + def refuse_unsupported_aggregation!(aggregation) + return if aggregation.is_a?(Aggregation) && aggregation.operation.to_s.casecmp('count').zero? && + Array(aggregation.groups).empty? && aggregation.field.nil? + + raise UnsupportedOperatorError, + "#{name} can only be counted: Intercom exposes no aggregate endpoint, and grouping or summing the " \ + 'pages the agent walked would answer a fraction of the collection as if it were the whole of it. ' \ + 'Chart it on a collection read whole, or wait for the bounded group-by of the reporting lot.' + end + + def refuse_filter!(filter) + detail = if filter&.condition_tree + 'a condition on this collection' + else + 'a free-text search' + end + + raise UnsupportedOperatorError, + "#{name} cannot answer #{detail} yet: it reads Intercom's listing endpoint, which takes no filter. " \ + 'Server-side filtering goes through the search endpoint and arrives with the filter translation. ' \ + 'Until then, remove the condition, the scope or the segment carrying it rather than being served a ' \ + 'page that would look filtered without being it.' + end + + # Intercom accepts a `sort` on these endpoints and ignores it without a + # word -- measured -- so an order the operator asked for and did not get + # has to be reported here or nowhere. The ascending primary-key sort the + # agent injects when a request names none is not one of those. + def warn_ignored_sort(sort) + clauses = Array(sort) + return if clauses.empty? || default_pk_sort?(clauses) + + ForestAdminDatasourceIntercom.logger.warn( + "[forest_admin_datasource_intercom] #{name} was asked to sort on " \ + "#{clauses.map { |clause| clause[:field] || clause["field"] }.join(", ")}, and Intercom ignores a sort on " \ + 'this endpoint without reporting it. The rows come back in the order the API imposes.' + ) + end + + def default_pk_sort?(clauses) + clauses.size == 1 && + (clauses.first[:field] || clauses.first['field']).to_s == primary_key && + (clauses.first[:ascending] || clauses.first['ascending']) != false + end + + def warn_truncated_ids(asked) + ForestAdminDatasourceIntercom.logger.warn( + "[forest_admin_datasource_intercom] #{name} was asked for #{asked} records by id and read the first " \ + "#{MAX_ID_READS}: Intercom reads them one request each. The result is truncated." + ) + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/fetch_all_collection.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/fetch_all_collection.rb new file mode 100644 index 000000000..ceeefe5f5 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/fetch_all_collection.rb @@ -0,0 +1,196 @@ +module ForestAdminDatasourceIntercom + module Collections + # Base for the reference collections Intercom hands back whole in a single + # response: admins, teams, ticket types, ticket states. Their endpoints + # declare no pagination parameter, no filter and no sort. + # + # Paradoxically this is the most capable tier of the datasource. Filtering, + # sorting, paginating and counting that response in memory is *exact* rather + # than approximate, because the records in hand are every record Intercom + # holds: a window cut out of them carries the rows a server-side query would + # have returned. It is the one place where an in-memory pass does not risk + # the thing this datasource refuses everywhere else -- a result that looks + # filtered without being filtered -- which only arises when what one holds + # is a single page of something larger. The cost is bandwidth, not + # correctness. + # + # Each read re-reads the endpoint, so an operator sees what Intercom holds + # now rather than what it held when the process booted. One request per list + # against a 10 000-a-minute budget is not a figure any list view approaches. + class FetchAllCollection < BaseCollection + # The filters a column may advertise, per column type. Restricted to what + # the toolkit can evaluate in memory, since the in-memory pass is the only + # pass there is here: an operator with no equivalence makes `match` answer + # nil, which `apply` reads as "no match" and would empty the page instead + # of filtering it. + OPERATOR_CANDIDATES = { + 'String' => [Operators::EQUAL, Operators::NOT_EQUAL, Operators::IN, Operators::NOT_IN, + Operators::PRESENT, Operators::BLANK, Operators::CONTAINS, Operators::I_CONTAINS, + Operators::NOT_CONTAINS, Operators::STARTS_WITH, Operators::ENDS_WITH], + 'Boolean' => [Operators::EQUAL, Operators::NOT_EQUAL, Operators::IN, Operators::NOT_IN, + Operators::PRESENT, Operators::BLANK] + }.freeze + + # The operators `ConditionTreeLeaf#match` evaluates natively; anything else + # needs an equivalence for the column's type to be evaluable at all. + IN_MEMORY_OPERATORS = [Operators::IN, Operators::EQUAL, Operators::LESS_THAN, Operators::GREATER_THAN, + Operators::MATCH, Operators::STARTS_WITH, Operators::ENDS_WITH, + Operators::LONGER_THAN, Operators::SHORTER_THAN, Operators::INCLUDES_ALL, + Operators::NOT_IN, Operators::NOT_EQUAL, Operators::NOT_CONTAINS].freeze + + # Candidates are re-checked against the toolkit rather than trusted, so an + # equivalence it stops providing takes the filter out of the schema instead + # of turning every page using it into an empty one. + def self.operators_for(column_type) + Array(OPERATOR_CANDIDATES[column_type]).select do |operator| + Equivalent.equivalent_tree?(operator, IN_MEMORY_OPERATORS, column_type) + end + end + + # Countable, unlike the cursor-paginated collections: the count answered + # here is taken over every record Intercom holds rather than over the pages + # a walk happened to collect. + def initialize(datasource, name) + super + enable_count + end + + def list(caller, filter, projection) + records = sort_in_memory(filtered_records(caller, filter), filter&.sort) + + page_window(records, filter).map { |record| project(record, projection) } + end + + # Exact, like the filter and the sort above it, which is why these columns + # stay groupable. + # + # Rows come back keyed with strings because that is how the agent reads + # them, while `Aggregation#apply` hands them back keyed with symbols. + def aggregate(caller, filter, aggregation, limit = nil) + aggregation.apply(filtered_records(caller, filter), timezone_for(caller), limit) + .map { |row| { 'group' => row[:group], 'value' => row[:value] } } + end + + protected + + # Scalar columns are sortable and groupable, the in-memory pass honouring + # anything asked of them. A Json column is neither, nor filterable: it + # holds a list, and what a filter on it would mean has no in-memory + # counterpart. + # + # Every column is read-only: this lot writes nothing, and an editable + # column would offer a Save that reaches an `update` the collection does + # not implement. + def add_column(name, type, is_primary_key: false) + add_field(name, ColumnSchema.new(column_type: type, + filter_operators: self.class.operators_for(type), + is_primary_key: is_primary_key, + is_read_only: true, + is_sortable: type != 'Json', + is_groupable: type != 'Json')) + end + + # Every record of the collection, straight from its endpoint. + def fetch_all = raise(NotImplementedError, "#{self.class} did not implement fetch_all") + + # One Intercom entity flattened into a record matching the schema. + def serialize(_entity) = raise(NotImplementedError, "#{self.class} did not implement serialize") + + private + + # The complete dataset, serialized and narrowed to the rows the filter + # keeps: what `list` pages and what `aggregate` counts are the same rows. + def filtered_records(caller, filter) + records = fetch_all.map { |entity| serialize(entity) } + tree = filter&.condition_tree + return records if tree.nil? + + refuse_unevaluable!(tree) + tree.apply(records, self, timezone_for(caller)) + end + + # A condition this collection cannot evaluate is refused rather than + # applied. `match` answers nil for an operator with no in-memory + # equivalence and `apply` reads that as "no match", so an unevaluable + # condition would hand back an empty page that looks like a filter + # matching nothing -- indistinguishable, to the operator, from a real + # answer. The schema advertises no such operator; a scope, a segment or a + # customizer can still send one. + def refuse_unevaluable!(tree) + offender = nil + tree.some_leaf do |leaf| + offender = leaf unless evaluable?(leaf) + !offender.nil? + end + return if offender.nil? + + raise UnsupportedOperatorError, + "#{name} cannot filter '#{offender.field}' with '#{offender.operator}': it is read whole from " \ + 'Intercom and filtered in memory, which supports only the operators its columns advertise. ' \ + 'Change the condition, or the scope or segment carrying it.' + end + + def evaluable?(leaf) + schema = fields[leaf.field] + schema.is_a?(ColumnSchema) && schema.filter_operators.include?(leaf.operator) + end + + # Neither Ruby's `sort` nor the toolkit's `Sort#apply` can be used as is: + # `sort` is not stable, and `<=>` answers nil on a null, on two booleans + # and on mixed types, which leaves the comparator undefined and the order + # arbitrary. Ties therefore fall back to the position Intercom returned the + # record in. + # + # Every requested order is honoured, the ascending primary-key sort the + # agent injects when a request names none included, so there is no + # unsortable order to report here -- unlike the cursor collections, where + # Intercom ignores a sort without saying so. + def sort_in_memory(records, sort) + clauses = sort_clauses(sort) + return records if clauses.empty? + + records.each_with_index.sort do |(left, left_index), (right, right_index)| + compare_clauses(left, right, clauses).nonzero? || (left_index <=> right_index) + end.map(&:first) + end + + # A sort clause naming a field this collection does not carry is dropped: + # ordering by a column that is not there would compare nil to nil on every + # row and leave the order to the tie-break. + def sort_clauses(sort) + Array(sort).filter_map do |clause| + field = clause[:field] || clause['field'] + next unless fields.key?(field) + + # `key?` rather than `||`: a descending clause carries `false`, which an + # `||` fallback would read as "absent" and turn back into ascending. + ascending = clause.key?(:ascending) ? clause[:ascending] : clause['ascending'] + [field, ascending != false] + end + end + + def compare_clauses(left, right, clauses) + clauses.each do |field, ascending| + comparison = compare_values(left[field], right[field]) + next if comparison.zero? + + return ascending ? comparison : -comparison + end + + 0 + end + + # Nulls sort last ascending and first descending, the way a database orders + # them; values `<=>` cannot compare -- two booleans, for one -- are + # compared through their string form rather than left undefined, which puts + # `false` before `true`, again like a database. + def compare_values(left, right) + return 0 if left.nil? && right.nil? + return 1 if left.nil? + return -1 if right.nil? + + (left <=> right) || (left.to_s <=> right.to_s) + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/team.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/team.rb new file mode 100644 index 000000000..2286bfa46 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/team.rb @@ -0,0 +1,36 @@ +module ForestAdminDatasourceIntercom + module Collections + # The inbox teams a conversation can be assigned to, rather than a single + # teammate. + class Team < FetchAllCollection + def initialize(datasource) + super(datasource, 'IntercomTeam') + end + + protected + + # Like `/admins`, `/teams` uses its own key instead of the `data` envelope. + def fetch_all + client.fetch_all('teams', list_key: 'teams') + end + + def serialize(team) + attrs = team.is_a?(Hash) ? team : {} + + { 'id' => stringify_id(attrs['id']), + 'name' => attrs['name'], + 'admin_ids' => Array(attrs['admin_ids']).map { |id| stringify_id(id) } } + end + + private + + def define_schema + add_column('id', 'String', is_primary_key: true) + add_column('name', 'String') + # Intercom types these as numbers here and as strings on the admin + # itself; they are stringified so both sides carry the same id. + add_column('admin_ids', 'Json') + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/ticket.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/ticket.rb new file mode 100644 index 000000000..69dcf6953 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/ticket.rb @@ -0,0 +1,132 @@ +module ForestAdminDatasourceIntercom + module Collections + # The tickets of the workspace. + # + # Read through `POST /tickets/search`: Intercom exposes no `GET /tickets` at + # all, so even an unfiltered list view goes through the search endpoint with + # a predicate that matches everything. Its records come back under `tickets` + # rather than under the `data` envelope -- measured. + # + # The response carries the whole timeline of every ticket, and there is no + # way to ask it not to: Intercom offers no field selection. Measured, one + # ticket carried 155 parts, so a page of 150 would move some 23 000 part + # objects, customer message bodies included. Two consequences run through + # this class: the page size is bounded far below what the API accepts, and + # everything derived from those parts is free, since they are paid for + # whether or not anything asks. + class Ticket < CursorCollection + include ContactIdentity + include Ticket::Serializer + include Ticket::DerivedColumns + + # Intercom accepts 150. This is not that: it is what keeps one page of + # tickets, timelines included, a response an agent can hold and an operator + # can wait for. Provisional until measured against real response sizes on + # the customer's workspace. + MAX_TICKETS_PER_PAGE = 25 + + # `/tickets/search` demands a query, so a list view sends the least noisy + # predicate that matches everything. Every ticket has a creation date, and + # a bound at the epoch keeps whatever the day-granular truncation does to + # it harmless. + MATCH_EVERY_TICKET = { 'field' => 'created_at', 'operator' => '>', 'value' => '0' }.freeze + + def initialize(datasource, attributes: []) + @attributes = attributes + super(datasource, 'IntercomTicket') + end + + protected + + def list_endpoint = 'tickets/search' + def record_endpoint = 'tickets' + def list_key = 'tickets' + def max_page_size = MAX_TICKETS_PER_PAGE + + # A search rather than a listing, which is the whole reason this hook + # exists. + def read_page(per_page:, cursor:) + client.search_page(list_endpoint, query: MATCH_EVERY_TICKET, list_key: list_key, + per_page: [per_page, max_page_size].min, starting_after: cursor) + end + + def enrich(records, rows, projection) + wanted = Array(projection).map(&:to_s) + + embed_contact_identity(records, rows, wanted) + embed_derived_columns(records, rows, wanted) + end + + private + + def define_schema + add_column('id', 'String', is_primary_key: true) + # The number the support team says out loud, next to the id the API + # answers by. + add_column('ticket_id', 'String') + # `request` / `task` / `tracker` on the wire, never the labels the + # Intercom interface shows -- the same mismatch a filter on it will have + # to respect. + add_column('category', 'String') + add_column('open', 'Boolean') + add_column('is_shared', 'Boolean') + add_column('created_at', 'Date') + add_column('updated_at', 'Date') + add_column('admin_assignee_id', 'String') + add_column('team_assignee_id', 'String') + # The ticket carries its company as an id alone, unlike a conversation + # which carries the whole object: the account name would cost a request + # per row, so it is not offered here. Measured: the id is Intercom's own, + # not the customer's external one, which is what a relation will have to + # target in lot 4. + add_column('company_id', 'String') + define_state_columns + define_type_columns + define_contact_columns + define_derived_columns + add_column('part_count', 'Number') + register_attribute_columns + end + + # The state arrives embedded as a whole object, so its labels cost nothing. + # `IntercomTicketState` remains a collection of its own -- it is the list of + # what a state can be -- but a row does not depend on it to be readable. + def define_state_columns + add_column('state_id', 'String') + add_column('state_category', 'String') + add_column('state_label', 'String') + add_column('state_external_label', 'String') + add_column('previous_state_id', 'String') + end + + def define_type_columns + add_column('ticket_type_id', 'String') + add_column('ticket_type_name', 'String') + end + + # The attribute columns of every ticket type, in union. Read at boot by + # `TicketAttributesIntrospector`, which is also where a workspace's own + # name is turned into one a Forest query string can carry. An attribute + # landing on a native column is skipped rather than overwriting it. + def register_attribute_columns + @attribute_columns = @attributes.reject { |attribute| collides?(attribute) } + @attribute_columns.each { |attribute| add_column(attribute.column_name, attribute.column_type) } + end + + def collides?(attribute) + return false unless fields.key?(attribute.column_name) + + ForestAdminDatasourceIntercom.logger.warn( + "[forest_admin_datasource_intercom] #{name} skips the ticket attribute #{attribute.name.inspect}: a " \ + "native column already carries the name #{attribute.column_name.inspect}, and overwriting it would show " \ + 'the attribute where the operator expects the ticket field.' + ) + true + end + + def attribute_columns + @attribute_columns || [] + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/ticket/derived_columns.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/ticket/derived_columns.rb new file mode 100644 index 000000000..41911f280 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/ticket/derived_columns.rb @@ -0,0 +1,127 @@ +module ForestAdminDatasourceIntercom + module Collections + class Ticket < CursorCollection + # The two columns a support queue is read for and that Intercom does not + # carry: when the ticket was closed, and who spoke last. + # + # A ticket has no `statistics` block -- measured against a workspace of 81 + # 142 tickets, confirming the specification -- so neither exists as a + # field. Both are derived from the parts, and the parts arrive with the + # search response whether or not anything asks for them, so both cost + # nothing: this is the one place where deriving a column is cheaper than + # reading one. + # + # Display only, and that is not a temporary state: `/tickets/search` + # filters on neither and ignores a sort without reporting it, so a column + # advertising either would put in the interface what the read cannot + # honour. + module DerivedColumns + # A ticket is not "closed" on Intercom, it enters a state whose category + # is resolved. + RESOLVED = 'resolved'.freeze + + # Matched on the prefix, never on the full `ticket_state_updated_by_admin` + # the sample showed: a workspace running workflows closes tickets through + # other variants of the same event, and a closure nobody can see is worse + # than a column nobody offers. + STATE_CHANGE_PREFIX = 'ticket_state_updated'.freeze + + # A reply to the customer. A `note` is an internal touch, not an answer: + # counting it would name as "last responder" someone who never wrote to + # the person waiting. + REPLY_PART = 'comment'.freeze + + private + + def define_derived_columns + add_column('closed_at', 'Date') + add_column('closed_by_name', 'String') + add_column('last_reply_at', 'Date') + add_column('last_responder_name', 'String') + # `admin` or `contact`: whether the last word came from the team or + # from the customer is what tells a queue who owes the next one. + add_column('last_responder_type', 'String') + end + + def derived_columns_for(attrs) + parts = parts_of(attrs) + closure = last_closure(parts) + reply = last_reply(parts) + + { 'closed_at' => stamp(closure&.dig('created_at')), + 'closed_by_name' => author_of(closure)['name'], + 'last_reply_at' => stamp(reply&.dig('created_at')), + 'last_responder_name' => author_of(reply)['name'], + 'last_responder_type' => author_of(reply)['type'] } + end + + # The hook of the base's `enrich`: nothing to read here, since + # `serialize` already derived everything. What is left is telling the + # operator when a value is missing because the timeline was truncated + # rather than because the event never happened. + def embed_derived_columns(records, _rows, projection) + return unless (%w[closed_at closed_by_name] & projection).any? + + unknown = records.count { |record| closure_unknown?(record) } + warn_unknown_closures(unknown) if unknown.positive? + end + + # A resolved ticket with no closure in hand, on a timeline Intercom + # truncated: the date is *unknown*, not absent. A Date column cannot say + # that, so the log does. + def closure_unknown?(record) + state = record['ticket_state'].is_a?(Hash) ? record['ticket_state'] : {} + return false unless state['category'] == RESOLVED + + last_closure(parts_of(record)).nil? && truncated?(record) + end + + # Intercom keeps the 500 most recent parts of a ticket. Past that, the + # transition that closed it may have fallen out of the window. + def truncated?(record) + total = parts_total(record) + total ? total > parts_of(record).size : false + end + + def last_closure(parts) + closures = parts.select { |part| state_change?(part) && part['ticket_state'] == RESOLVED } + + closures.max_by { |part| part['created_at'].to_i } + end + + # A part can record a transition to the state the ticket was already in + # -- measured -- and that is not an event. + def state_change?(part) + part['part_type'].to_s.start_with?(STATE_CHANGE_PREFIX) && + part['ticket_state'] != part['previous_ticket_state'] + end + + def last_reply(parts) + parts.select { |part| part['part_type'] == REPLY_PART }.max_by { |part| part['created_at'].to_i } + end + + def author_of(part) + author = (part || {})['author'] + author.is_a?(Hash) ? author : {} + end + + def parts_of(record) + nested_list((record || {})['ticket_parts'], 'ticket_parts') + end + + def parts_total(record) + container = (record || {})['ticket_parts'] + container.is_a?(Hash) ? container['total_count'] : nil + end + + def warn_unknown_closures(unknown) + ForestAdminDatasourceIntercom.logger.warn( + "[forest_admin_datasource_intercom] #{name}: #{unknown} resolved ticket(s) of this page show no " \ + 'closure date because Intercom truncated their timeline at 500 parts, not because they were never ' \ + 'closed. The column is unknown for those rows.' + ) + end + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/ticket/serializer.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/ticket/serializer.rb new file mode 100644 index 000000000..3fb91030a --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/ticket/serializer.rb @@ -0,0 +1,81 @@ +module ForestAdminDatasourceIntercom + module Collections + class Ticket < CursorCollection + # One Intercom ticket flattened into the row the schema declares. Nothing + # here reads a sub-resource: the state, the type and the attributes all + # travel with the ticket. + module Serializer + protected + + def serialize(ticket) + attrs = ticket.is_a?(Hash) ? ticket : {} + + native(attrs) + .merge(state_of(attrs)) + .merge(type_of(attrs['ticket_type'])) + .merge(contact_columns_for(attrs)) + .merge(attribute_values_of(attrs['ticket_attributes'])) + .merge(derived_columns_for(attrs)) + end + + private + + def native(attrs) + { 'id' => stringify_id(attrs['id']), + 'ticket_id' => stringify_id(attrs['ticket_id']), + 'category' => attrs['category'], + 'open' => attrs['open'], + 'is_shared' => attrs['is_shared'], + 'created_at' => stamp(attrs['created_at']), + 'updated_at' => stamp(attrs['updated_at']), + 'admin_assignee_id' => stringify_id(attrs['admin_assignee_id']), + 'team_assignee_id' => stringify_id(attrs['team_assignee_id']), + 'company_id' => stringify_id(attrs['company_id']), + 'part_count' => parts_total(attrs) } + end + + def state_of(attrs) + state = attrs['ticket_state'].is_a?(Hash) ? attrs['ticket_state'] : {} + + { 'state_id' => stringify_id(state['id']), + 'state_category' => state['category'], + 'state_label' => state['internal_label'], + 'state_external_label' => state['external_label'], + 'previous_state_id' => stringify_id(attrs['previous_ticket_state_id']) } + end + + def type_of(ticket_type) + attrs = ticket_type.is_a?(Hash) ? ticket_type : {} + + { 'ticket_type_id' => stringify_id(attrs['id']), 'ticket_type_name' => attrs['name'] } + end + + # Intercom keys the values by attribute **name**, which is what lets a + # single collection display the union of every type's attributes -- and + # what stops it from filtering on them, since the filter is written by id + # and the id differs from one type to the next. + # + # The value is read under the name the workspace gave it and written + # under the column name the schema publishes; the two differ whenever the + # first could not travel through a Forest query string. + # + # A ticket of another type simply does not carry the key: the column + # reads as absent rather than as empty. + def attribute_values_of(values) + held = values.is_a?(Hash) ? values : {} + + attribute_columns.to_h { |attribute| [attribute.column_name, coerce(held[attribute.name], attribute)] } + end + + # A date attribute comes back as epoch seconds like every other Intercom + # date; the rest is handed over as it came. + def coerce(value, attribute) + return nil if value.nil? + return stamp(value) if attribute.column_type == 'Date' && value.is_a?(Numeric) + + value + end + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/ticket_state.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/ticket_state.rb new file mode 100644 index 000000000..f9845d7e7 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/ticket_state.rb @@ -0,0 +1,40 @@ +module ForestAdminDatasourceIntercom + module Collections + # The ticket states of the workspace. A ticket carries its state as an id, so + # without this collection a support queue reads as a column of numbers. + class TicketState < FetchAllCollection + def initialize(datasource) + super(datasource, 'IntercomTicketState') + end + + protected + + def fetch_all + client.fetch_all('ticket_states') + end + + # Two labels rather than one: `internal_label` is what the support team + # sees, `external_label` what the customer is shown. An operator reading a + # queue needs the first, and needs to know what the second says. + def serialize(ticket_state) + attrs = ticket_state.is_a?(Hash) ? ticket_state : {} + + { 'id' => stringify_id(attrs['id']), + 'category' => attrs['category'], + 'internal_label' => attrs['internal_label'], + 'external_label' => attrs['external_label'], + 'archived' => attrs['archived'] } + end + + private + + def define_schema + add_column('id', 'String', is_primary_key: true) + add_column('category', 'String') + add_column('internal_label', 'String') + add_column('external_label', 'String') + add_column('archived', 'Boolean') + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/ticket_type.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/ticket_type.rb new file mode 100644 index 000000000..7b6367198 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/ticket_type.rb @@ -0,0 +1,47 @@ +module ForestAdminDatasourceIntercom + module Collections + # The ticket types the workspace defines. They are what makes a ticket's type + # readable, and they are also where the ticket attributes are declared -- + # which is what the ticket collection reads them for. + class TicketType < FetchAllCollection + def initialize(datasource) + super(datasource, 'IntercomTicketType') + end + + protected + + def fetch_all + client.fetch_all('ticket_types') + end + + # `ticket_type_attributes` is deliberately left out: it is a nested list of + # attribute definitions, useful to the ticket collection and meaningless as + # a column. An attribute of the same name carries a different id from one + # type to the next (measured), which is exactly why the ticket collection + # has to read the definitions rather than assume them. + def serialize(ticket_type) + attrs = ticket_type.is_a?(Hash) ? ticket_type : {} + + { 'id' => stringify_id(attrs['id']), + 'name' => attrs['name'], + 'description' => attrs['description'], + 'category' => attrs['category'], + 'icon' => attrs['icon'], + 'archived' => attrs['archived'] } + end + + private + + def define_schema + add_column('id', 'String', is_primary_key: true) + add_column('name', 'String') + add_column('description', 'String') + # `request` / `task` / `tracker` on the wire, not the labels the Intercom + # interface shows -- the same mismatch the ticket filter has to respect. + add_column('category', 'String') + add_column('icon', 'String') + add_column('archived', 'Boolean') + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/configuration.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/configuration.rb new file mode 100644 index 000000000..65a3013c8 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/configuration.rb @@ -0,0 +1,112 @@ +module ForestAdminDatasourceIntercom + class Configuration + # A workspace is hosted in one region and answers in that region only. The + # host is therefore a configuration parameter rather than a constant: + # `api.intercom.io` does route to the right region, but a customer under + # GDPR wants its requests to reach the European host and nothing else. + REGION_HOSTS = { + us: 'https://api.intercom.io', + eu: 'https://api.eu.intercom.io', + au: 'https://api.au.intercom.io' + }.freeze + + DEFAULT_REGION = :us + + # Without an explicit version a request follows the workspace's own default, + # which an operator can change on Intercom's side -- and the payloads change + # shape under us. Pinned to what the spike ran against; 2.14 and 2.16 both + # answered, and the response echoes the version back, so `Client#me` + # verifies at boot that the pin was honoured. + DEFAULT_API_VERSION = '2.16'.freeze + + attr_reader :access_token, :region, :base_url, :api_version, :open_timeout, :timeout, + :retry_policy, :rate_limiter, :boot_open_timeout, :boot_timeout, :boot_retry_policy + + # `rate_limiter: nil` takes the pacing out of the stack, leaving the 429 + # retry as the only rate-limit handling. For a deployment that meters on its + # own side, or one that would rather see the 429. + # + # The `boot_` trio governs what the datasource reads while it is being + # constructed -- the custom-attribute introspection above all -- where the + # wait is a Rails boot the operator sits through rather than a request that + # has already returned a page. + def initialize(access_token:, region: nil, base_url: nil, api_version: DEFAULT_API_VERSION, + open_timeout: 5, timeout: 30, retry_policy: RetryPolicy.new, + rate_limiter: RateLimiter.new, boot_open_timeout: 3, boot_timeout: 10, + boot_retry_policy: RetryPolicy.boot) + @access_token = access_token + @region = (region || DEFAULT_REGION).to_s.downcase.to_sym + @base_url = base_url + @api_version = api_version.to_s + @open_timeout = open_timeout + @timeout = timeout + @retry_policy = retry_policy + @rate_limiter = rate_limiter + @boot_open_timeout = boot_open_timeout + @boot_timeout = boot_timeout + @boot_retry_policy = boot_retry_policy + validate! + end + + # An explicit `base_url` wins over the region: it is what points the client + # at a mock server or an egress proxy, neither of which is a region. + def url + @url ||= (@base_url || REGION_HOSTS.fetch(@region)).chomp('/') + end + + # Whatever precedes the endpoint in the path, for a base url mounted under a + # subpath. Empty against the API itself. + def base_path + @base_path ||= URI.parse(url).path + end + + # `access_token` is a bearer credential, and nothing prints a Configuration + # on purpose: what reaches an `inspect` is a Rails error page, or a + # `logger.debug` of something holding one. The default would put the token + # in clear there. `Client` and `Datasource` mask their own for the same + # reason -- together they cut every path from an object this package hands + # out to the credential. + def inspect + "#<#{self.class.name} url=#{url.inspect} api_version=#{@api_version.inspect} access_token=[FILTERED]>" + end + + private + + def validate! + raise ConfigurationError, 'ForestAdminDatasourceIntercom missing required config: access_token' if + blank?(@access_token) + + validate_region! + validate_base_url! + raise ConfigurationError, 'ForestAdminDatasourceIntercom api_version cannot be empty' if blank?(@api_version) + end + + def validate_region! + return if @base_url || REGION_HOSTS.key?(@region) + + raise ConfigurationError, + "ForestAdminDatasourceIntercom unknown region #{@region.inspect}: " \ + "expected one of #{REGION_HOSTS.keys.map(&:inspect).join(", ")}, or an explicit base_url." + end + + # A base url that is not absolute makes Faraday resolve every path against + # the process's working directory instead of Intercom, which surfaces much + # later as a connection failure naming nothing. + def validate_base_url! + return if @base_url.nil? + + uri = URI.parse(@base_url) + return if uri.is_a?(URI::HTTP) && !blank?(uri.host) + + raise ConfigurationError, + "ForestAdminDatasourceIntercom base_url must be an absolute http(s) url, got #{@base_url.inspect}" + rescue URI::InvalidURIError + raise ConfigurationError, + "ForestAdminDatasourceIntercom base_url is not a valid url: #{@base_url.inspect}" + end + + def blank?(value) + value.nil? || value.to_s.strip.empty? + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/datasource.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/datasource.rb new file mode 100644 index 000000000..f9ccf242a --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/datasource.rb @@ -0,0 +1,48 @@ +module ForestAdminDatasourceIntercom + class Datasource < ForestAdminDatasourceToolkit::Datasource + attr_reader :client, :configuration + + def initialize(access_token:, **options) + super() + @configuration = Configuration.new(access_token: access_token, **options) + @client = Client.new(@configuration) + + register_collections + end + + # The datasource is what a Rails error page or a `logger.debug` is likeliest + # to print, and it holds the client whose connections carry the access token. + # Every collection will reach that token the same way, through the + # `@datasource` the toolkit's Collection keeps, so cutting the chain here + # covers them too -- and spares the recursive dump the default `inspect` + # walks into, a datasource and its collections pointing at each other. + def inspect + "#<#{self.class.name} collections=#{collections.keys.inspect}>" + end + + private + + # The reference collections first: they are what turns an assignee id into a + # teammate and a state id into a label. No request is made here -- each + # collection reads its endpoint when it is listed, so a datasource boots + # whatever Intercom is doing, and a workspace the token cannot read costs + # rows rather than the agent. + def register_collections + add_collection(Collections::Admin.new(self)) + add_collection(Collections::Team.new(self)) + add_collection(Collections::TicketType.new(self)) + add_collection(Collections::TicketState.new(self)) + add_collection(Collections::Conversation.new(self)) + # The one boot-time read of the datasource: the attributes a workspace + # defines on its ticket types, which are columns of the Tickets collection + # and cannot be discovered from a ticket payload -- a ticket carries the + # values of its own type only. It degrades to no attribute column rather + # than to a failed boot. + add_collection(Collections::Ticket.new(self, attributes: ticket_attributes)) + end + + def ticket_attributes + Schema::TicketAttributesIntrospector.new(@client).attributes + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/pagination/cursor_walker.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/pagination/cursor_walker.rb new file mode 100644 index 000000000..5132f0d8a --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/pagination/cursor_walker.rb @@ -0,0 +1,126 @@ +module ForestAdminDatasourceIntercom + module Pagination + # Forest asks for an offset/limit window; Intercom only knows how to hand + # out the page after a cursor, and documents that jumping to page N is not + # supported. Bridging the two means walking pages until the window is + # covered, then slicing it out. Reaching page 20 therefore costs 20 + # requests -- sequential ones, a cursor only being known once the page + # before it came back. + # + # The walk is capped for that reason rather than for the quota's: 10 000 + # requests a minute is generous enough that the caps below are about what an + # operator is willing to wait for, and about the fact that page 200 of a + # list view answers no real question (R9). Every truncation is logged -- + # never silent, since a page that looks like the whole answer and is not is + # the failure this datasource exists to avoid. + class CursorWalker + # 50 pages of 150 records. Intercom's quota lets these be generous: the + # walk is bounded by patience, and by the point past which a list view is + # not being read but scraped. + MAX_PAGES = 50 + MAX_RECORDS = 7_500 + + def initialize(max_pages: MAX_PAGES, max_records: MAX_RECORDS) + @max_pages = max_pages + @max_records = max_records + end + + # Yields `(per_page, cursor)` and expects a Client::Page back. + # + # A nil limit asks for every record past the offset: the walk then runs + # until Intercom says there is no page left, or until a cap stops it. That + # distinction is the whole point of accepting nil rather than a huge limit + # standing in for "everything": a walk told to collect a thousand records + # stops at a thousand having covered the window it was given, and reports + # nothing, while a walk told to collect everything and stopped by a cap + # knows it is handing back less than it was asked for, and says so. + def walk(offset:, limit:, &page_source) + offset = offset.to_i.clamp(0, nil) + limit = limit&.to_i + return [] if limit && !limit.positive? + + records = collect(offset, limit, &page_source) + + limit ? (records[offset, limit] || []) : records.drop(offset) + end + + private + + # The walk itself: pages are collected until the window is covered, the + # source says there is nothing left, or a cap stops it. The slicing is + # `walk`'s; this only decides how far to go. + def collect(offset, limit) + needed = limit && (offset + limit) + records = [] + cursor = nil + seen_ids = Set.new + seen_cursors = Set.new + pages = 0 + + loop do + page = yield(batch_size(needed, records.size), cursor) + records.concat(fresh(page.records, seen_ids)) + pages += 1 + + break if stop?(page, seen_cursors) + break if needed && records.size >= needed + + if capped?(pages, records.size) + log_truncation(offset: offset, limit: limit, pages: pages, collected: records.size) + break + end + + cursor = page.next_cursor + end + + records + end + + # Intercom documents that "if items are modified between paginated + # requests it is possible to see duplicate or missed records" -- and + # conversations move constantly, so a deep walk over them will see the + # same record twice. A duplicate is dropped here rather than being served + # as two rows carrying one id, which is what a list view would render as + # two identical lines and a record count that never adds up. The missing + # counterpart is inherent to cursor pagination and is documented instead. + # + # A record with no id is kept: it is not this walk's business to decide + # that a payload it does not recognise is not a record. + def fresh(records, seen_ids) + records.select { |record| record['id'].nil? || seen_ids.add?(record['id']) } + end + + # An empty page, a cursor that does not move and a cursor already followed + # all stop the walk. Intercom does none of the three today -- `pages.next` + # is simply absent on the last page -- but a walk driven by a remote value + # stops on its own terms rather than on the caps only: a cycle wider than + # one page would otherwise collect the same pages until a cap cut it + # short. + def stop?(page, seen_cursors) + page.next_cursor.nil? || page.records.empty? || !seen_cursors.add?(page.next_cursor) + end + + def capped?(pages, collected) + pages >= @max_pages || collected >= @max_records + end + + # Bounded by the record budget left, and by the window still missing when + # there is one, so the walk never collects past @max_records. `Client` + # bounds it again to what Intercom accepts. + def batch_size(needed, collected) + budget = @max_records - collected + budget = [needed - collected, budget].min if needed + Client.bounded_per_page(budget) + end + + def log_truncation(offset:, limit:, pages:, collected:) + window = limit ? "offset=#{offset} limit=#{limit}" : "every record past offset=#{offset}" + ForestAdminDatasourceIntercom.logger.warn( + "[forest_admin_datasource_intercom] Stopped paginating after #{pages} page(s) / " \ + "#{collected} record(s) while fetching #{window}; results are truncated. " \ + 'Narrow the filter to reach records past this point.' + ) + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/rate_limiter.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/rate_limiter.rb new file mode 100644 index 000000000..6b37a5794 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/rate_limiter.rb @@ -0,0 +1,169 @@ +module ForestAdminDatasourceIntercom + # Paces requests on what Intercom says is left of the current window, so the + # budget is spent rather than exceeded. + # + # Intercom meters the app and, above it, the whole workspace -- 25 000 + # requests a minute shared with every other private app the customer runs -- + # and it allocates that budget in 10-second windows: the measured + # `x-ratelimit-limit` is 1667, not 10 000. A burst of 3 000 requests in two + # seconds therefore takes a 429 while the minute's budget is barely touched, + # which is why what matters here is the instantaneous rate and not a volume + # per minute. + # + # Unlike an API that only documents its budgets, Intercom reports the state of + # the window on every response, so this is driven by those headers rather than + # by a table: what is left, and when it refills. This sits in front of the 429 + # retry rather than replacing it -- the retry stays the backstop for the part + # of the workspace budget spent by traffic this process never sees. + # + # One limiter per Configuration, hence per token, since that is what Intercom + # meters. + class RateLimiter + # Intercom's allocation window. Only used as the ceiling below: the reset + # instant itself always comes from the response. + WINDOW = 10.0 + + # How far past the reset a request may be held. A little over one window, so + # a full window can be waited out, and no more: past this the wait is not + # Intercom's window emptying but a clock disagreeing. + DEFAULT_MAX_WAIT = 12.0 + + attr_reader :max_wait + + def initialize(max_wait: DEFAULT_MAX_WAIT, now: nil, sleeper: nil) + @max_wait = max_wait.to_f + # Wall clock rather than monotonic on purpose: `X-RateLimit-Reset` is an + # absolute epoch second on Intercom's clock, so the two have to be + # comparable. `clamp_wait` is what keeps a skewed clock from turning that + # comparison into a long sleep. + @now = now || -> { Time.now.to_f } + @sleeper = sleeper || ->(seconds) { sleep(seconds) } + @mutex = Mutex.new + @remaining = nil + @reset_at = nil + @limit = nil + @warned_at = nil + end + + # Blocks until the current window has room, then returns. Called once per + # attempt, retries included: a replayed request spends the budget a first + # one did. + def acquire + wait, declined = @mutex.synchronize { plan_wait } + + warn_saturated(declined) if declined + return if wait <= 0 + + @sleeper.call(wait) + end + + # What a response says about the window it was answered in. Called on every + # response, the 429 included -- that one carries the most useful reset of + # all. + def observe(headers) + limit = integer_header(headers, 'x-ratelimit-limit') + remaining = integer_header(headers, 'x-ratelimit-remaining') + reset_at = integer_header(headers, 'x-ratelimit-reset') + return if remaining.nil? && reset_at.nil? + + @mutex.synchronize { record(limit, remaining, reset_at) } + end + + private + + # A local decrement per request on top of what the headers report: several + # requests can be in flight before any of them comes back, and a `remaining` + # that only ever moves on a response lets all of them through on the same + # stale figure. + # + # Returns the wait the caller owes and, when the reset is too far out to be + # waited for, the wait that was declined -- nil otherwise. Both are settled + # here, the second reading shared state like the first: it is nil on every + # bypass but the first of a window, so the log line is not repeated. + def plan_wait + @remaining -= 1 if @remaining + return [0, nil] unless exhausted? + + wait = clamp_wait(@reset_at - @now.call) + return [0, nil] if wait <= 0 + return [0, first_warning? ? wait : nil] if wait > @max_wait + + [wait, nil] + end + + # Nothing left in a window that has not refilled yet. An unknown state -- + # before the first response -- is not exhaustion: the first request is what + # discovers the budget. + def exhausted? + !@remaining.nil? && @remaining <= 0 && !@reset_at.nil? + end + + # Intercom's reset is a timestamp from its clock, and the two clocks can + # disagree by more than the window is long. A wait longer than a window plus + # its own slack is that disagreement rather than a window emptying, so it is + # cut back to something a caller can afford to wait. + def clamp_wait(seconds) + return 0.0 if seconds <= 0 + + [seconds, WINDOW + @max_wait].min + end + + # A window is adopted whole. A response answered in an older window than the + # one already recorded is ignored: replies come back out of order, and one + # from the previous window would otherwise resurrect a budget already spent. + # + # Within the same window the smaller `remaining` wins, so the local + # decrements of in-flight requests are not undone by a response that left + # Intercom before they were made. + def record(limit, remaining, reset_at) + return if reset_at && @reset_at && reset_at < @reset_at + + new_window = reset_at && (@reset_at.nil? || reset_at > @reset_at) + @reset_at = reset_at if reset_at + @limit = limit if limit + return if remaining.nil? + + @remaining = new_window || @remaining.nil? ? remaining : [remaining, @remaining].min + end + + # One line per window. What the warning reports is a saturation that lasts, + # so a line per request puts one on every request it describes -- hundreds + # of them, burying the first, which is the only one the operator needed. + def first_warning? + now = @now.call + return false if @warned_at && now - @warned_at < WINDOW + + @warned_at = now + true + end + + def warn_saturated(wait) + ForestAdminDatasourceIntercom.logger.warn( + "[forest_admin_datasource_intercom] the Intercom rate-limit window is spent (limit #{@limit || "unknown"} " \ + "per #{WINDOW.round}s) and its reset is #{wait.round(1)}s out, past the #{@max_wait.round(1)}s this waits. " \ + 'Letting the request through -- Intercom may answer 429, which the client retries. A reset this far out ' \ + "usually means this host's clock disagrees with Intercom's. Reported once per #{WINDOW.round}s." + ) + end + + def integer_header(headers, name) + value = header_value(headers, name) + return nil if value.nil? || value.to_s.strip.empty? + + Integer(value.to_s.strip, exception: false) + end + + # Faraday hands over headers that look themselves up case-insensitively, but + # what reaches here is whatever the middleware was given -- a plain hash + # included -- and HTTP header names are case-insensitive on the wire. + def header_value(headers, name) + return nil if headers.nil? + + direct = headers[name] + return direct unless direct.nil? + return nil unless headers.respond_to?(:find) + + headers.find { |key, _value| key.to_s.downcase == name }&.last + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/retry_policy.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/retry_policy.rb new file mode 100644 index 000000000..249b74eb2 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/retry_policy.rb @@ -0,0 +1,72 @@ +module ForestAdminDatasourceIntercom + # Everything governing how the client reacts to a failed request, in one + # place: which statuses and exceptions are worth another attempt, on which + # verbs, and how long to wait. + class RetryPolicy + # Intercom allocates its quota in 10-second windows, so a 429 is recovered + # from within one of them -- unlike an API metering by the minute. The cap + # still has to cover a whole window: faraday-retry gives up outright when + # Retry-After exceeds max_interval, which would turn the 429 retry into an + # immediate give-up exactly when it matters. + DEFAULT_MAX_INTERVAL = 12 + + STATUSES = [429, 500, 502, 503, 504].freeze + + # faraday-retry's defaults plus ConnectionFailed: a dropped connection is + # exactly the transient failure a resilient client should absorb, and it is + # not retried out of the box. + EXCEPTIONS = [ + Errno::ETIMEDOUT, 'Timeout::Error', Faraday::TimeoutError, + Faraday::RetriableResponse, Faraday::ConnectionFailed + ].freeze + + # The verbs that change nothing, so any transient failure is worth another + # attempt. Narrower than faraday-retry's idempotent default: a 502 or a + # dropped connection on the way back from a POST Intercom did perform would + # be replayed into a second reply on the conversation, or a second ticket. + # + # A 429 stays safe to retry on any verb, Intercom having rejected the + # request before processing it, and travels through retry_if rather than + # through this list: faraday-retry ORs the two, so `methods` can only widen + # the set, never restrict it. + RETRYABLE_METHODS = %i[get head options].freeze + RETRY_IF = ->(env, _exception) { env[:status] == 429 } + + # The cap for a call that must not hold the boot, deliberately below a + # rate-limit window where DEFAULT_MAX_INTERVAL sits above it: a Retry-After + # past the cap makes faraday-retry abandon outright, which is what turns a + # 429 at boot into an immediate give-up rather than a window of waiting per + # attempt. + BOOT_MAX_INTERVAL = 2 + + BACKOFF_FACTOR = 2 + + attr_reader :max_retries, :interval, :max_interval + + # One retry rather than none, for what is read once and never revisited: a + # transient failure there costs its result for the whole life of the + # process, and half a second absorbs the hiccup without waiting a 429 out. + def self.boot + new(max_retries: 1, interval: 0.5, max_interval: BOOT_MAX_INTERVAL) + end + + def initialize(max_retries: 3, interval: 0.5, max_interval: DEFAULT_MAX_INTERVAL) + @max_retries = max_retries + @interval = interval + @max_interval = max_interval + end + + def to_faraday_options + { + max: @max_retries, + interval: @interval, + max_interval: @max_interval, + backoff_factor: BACKOFF_FACTOR, + retry_statuses: STATUSES, + exceptions: EXCEPTIONS, + methods: RETRYABLE_METHODS, + retry_if: RETRY_IF + } + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/schema/ticket_attributes_introspector.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/schema/ticket_attributes_introspector.rb new file mode 100644 index 000000000..85f597e5f --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/schema/ticket_attributes_introspector.rb @@ -0,0 +1,142 @@ +module ForestAdminDatasourceIntercom + module Schema + # The attributes a workspace defines on its ticket types, read once while the + # datasource is being constructed. + # + # They are declared **per ticket type**, so a single Tickets collection can + # only carry their union -- and that union is for display. Measured on a real + # workspace: two types share the names `_default_title_` and + # `_default_description_` while carrying different attribute ids (14162161 + # against 14162165), and Intercom filters an attribute by id + # (`ticket_attribute.{id}`), never by name. A union column therefore has no + # single id to translate to unless the type of the row is known, which is why + # these ship unfilterable and why filtering on one means a collection per + # ticket type (R7). + # + # The ids are kept per type all the same: they are exactly what the filter + # translation of the next lot will need, and reading them again would cost a + # second boot-time round trip. + class TicketAttributesIntrospector + # Intercom's attribute data types, mapped onto what Forest can render. A + # `list` is a single choice among values the workspace defined, so it reads + # as a string rather than as a Json blob; `files` is a list of attachments + # and has no scalar form at all. + COLUMN_TYPES = { + 'string' => 'String', 'list' => 'String', 'integer' => 'Number', 'decimal' => 'Number', + 'boolean' => 'Boolean', 'datetime' => 'Date', 'date' => 'Date', 'files' => 'Json' + }.freeze + + DEFAULT_COLUMN_TYPE = 'String'.freeze + + # What a column name may not contain, and it has nothing to do with + # Intercom: Forest lists the fields of a request in a **comma-separated** + # query parameter, and uses a colon to name a field through a relation. + # A workspace names its ticket attributes in free text -- measured, one is + # called `ID de l'objet en question (immo, facture, user)` -- and a comma + # in there splits the projection into fields no collection has, which the + # agent rejects as a 400 before the page is ever read. + UNSAFE_IN_A_COLUMN_NAME = /[,:]/ + + # `name` is the key the payload uses, `column_name` the one the schema + # publishes; they differ whenever the workspace's own name cannot travel + # through Forest's query string. + Attribute = Struct.new(:name, :column_name, :column_type, :data_type, :ids_by_ticket_type, + keyword_init: true) + + def initialize(client) + @client = client + end + + # The union, one entry per attribute name. Degrades to nothing rather than + # to a failure: a token without the ticket-types permission costs the + # attribute columns, never the boot of the agent. + def attributes + @attributes ||= build + rescue APIError => e + ForestAdminDatasourceIntercom.logger.warn( + "[forest_admin_datasource_intercom] could not read the ticket types (HTTP #{e.status || "-"}); " \ + 'the Tickets collection boots without its attribute columns.' + ) + @attributes = [] + end + + private + + def build + # Read on the boot connection: this happens while Rails is starting, and + # a slow Intercom must not turn that into minutes the operator sits + # through. + @client.fetch_all('ticket_types', boot: true) + .each_with_object({}) { |ticket_type, union| collect(ticket_type, union) } + .values + end + + def collect(ticket_type, union) + type_id = ticket_type['id'].to_s + definitions(ticket_type).each do |definition| + entry = entry_for(definition, union) + next if entry.nil? + + entry.ids_by_ticket_type[type_id] = definition['id'].to_s + end + end + + # The union is keyed by column name rather than by the workspace's own, + # since that is what has to be unique in a schema. Two different attributes + # landing on one column would otherwise share an entry, and the second's + # values would be read under the first's name -- wrong values rather than + # missing ones, which is worse. + def entry_for(definition, union) + name = definition['name'].to_s + # An archived attribute is not offered any more, and a nameless one has + # nothing to be a column of. + return nil if name.empty? || definition['archived'] + + column = column_name_for(name) + return nil if column.empty? + + entry = union[column] + return union[column] = attribute_from(name, column, definition) if entry.nil? + return entry if entry.name == name + + warn_collision(name, entry.name, column) + nil + end + + def attribute_from(name, column, definition) + Attribute.new(name: name, column_name: column, column_type: column_type_for(definition), + data_type: definition['data_type'], ids_by_ticket_type: {}) + end + + # Intercom hands these back HTML-escaped -- `Ce que j'ai vérifié` -- + # which is an artefact of where they were typed, not part of the name. + def column_name_for(name) + CGI.unescapeHTML(name).gsub(UNSAFE_IN_A_COLUMN_NAME, ' ').squeeze(' ').strip + end + + def warn_collision(name, kept, column) + ForestAdminDatasourceIntercom.logger.warn( + "[forest_admin_datasource_intercom] the ticket attribute #{name.inspect} is left out: it reads as the " \ + "column #{column.inspect}, which #{kept.inspect} already carries. Rename one of them in Intercom to " \ + 'publish both.' + ) + end + + def definitions(ticket_type) + return [] unless ticket_type.is_a?(Hash) + + container = ticket_type['ticket_type_attributes'] + return [] unless container.is_a?(Hash) + + list = container['data'] + list.is_a?(Array) ? list : [] + end + + # An unknown data type reads as a string rather than being dropped: showing + # the value Intercom sent beats hiding a column because its type is new. + def column_type_for(definition) + COLUMN_TYPES.fetch(definition['data_type'].to_s, DEFAULT_COLUMN_TYPE) + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/throttle.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/throttle.rb new file mode 100644 index 000000000..fa8fdad3b --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/throttle.rb @@ -0,0 +1,19 @@ +module ForestAdminDatasourceIntercom + # Holds a request until the rate-limit window has room, and feeds the window + # back what the response says about it. A middleware rather than a call in + # each client method: there is one code path for every request here, where the + # client has one per endpoint, and this one also covers the requests the client + # never issues itself -- the replays `retry` performs. + class Throttle < Faraday::Middleware + def initialize(app, limiter:) + super(app) + @limiter = limiter + end + + def call(env) + @limiter.acquire + + @app.call(env).on_complete { |response_env| @limiter.observe(response_env[:response_headers]) } + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/version.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/version.rb new file mode 100644 index 000000000..bcf5c5876 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/version.rb @@ -0,0 +1,3 @@ +module ForestAdminDatasourceIntercom + VERSION = "0.1.0" +end diff --git a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/client_spec.rb b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/client_spec.rb new file mode 100644 index 000000000..65a9c9d56 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/client_spec.rb @@ -0,0 +1,487 @@ +module ForestAdminDatasourceIntercom + RSpec.describe Client do + subject(:client) { described_class.new(configuration) } + + let(:retry_policy) { RetryPolicy.new(max_retries: 2, interval: 0) } + let(:configuration) { Configuration.new(access_token: 's3cr3t', retry_policy: retry_policy, rate_limiter: nil) } + let(:base) { configuration.url } + + def json(payload, status = 200, headers = {}) + { status: status, + body: payload.is_a?(String) ? payload : payload.to_json, + headers: { 'Content-Type' => 'application/json' }.merge(headers) } + end + + describe 'authentication and version pinning' do + before { stub_request(:get, "#{base}/me").to_return(json({ 'type' => 'admin' })) } + + it 'sends the access token as a bearer token' do + client.me + + expect(WebMock).to have_requested(:get, "#{base}/me") + .with(headers: { 'Authorization' => 'Bearer s3cr3t', 'Accept' => 'application/json' }) + end + + # Without the header the request follows the workspace's own default + # version, which an operator can change on Intercom's side. + it 'pins the API version on every request' do + client.me + + expect(WebMock).to have_requested(:get, "#{base}/me").with(headers: { 'Intercom-Version' => '2.16' }) + end + + it 'advertises a versioned user agent' do + client.me + + expect(WebMock).to have_requested(:get, "#{base}/me") + .with(headers: { 'User-Agent' => "forest_admin_datasource_intercom/#{VERSION}" }) + end + end + + describe '#me' do + it 'returns the admin the token belongs to' do + stub_request(:get, "#{base}/me").to_return(json('type' => 'admin', 'id' => '1', 'email' => 'a@b.test')) + + expect(client.me).to include('id' => '1', 'email' => 'a@b.test') + end + + it 'reaches the regional host it was configured for' do + eu = described_class.new(Configuration.new(access_token: 's3cr3t', region: :eu, rate_limiter: nil)) + stub_request(:get, 'https://api.eu.intercom.io/me').to_return(json('type' => 'admin')) + + eu.me + + expect(WebMock).to have_requested(:get, 'https://api.eu.intercom.io/me') + end + end + + describe 'the version Intercom actually served' do + # Intercom echoes the version it served. A mismatch means the payloads may + # not be the ones this datasource expects, which is worth saying out loud + # -- and worth saying rather than raising: running against a version we + # did not ask for beats not running. + it 'warns when it differs from the pinned one' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + stub_request(:get, "#{base}/me").to_return(json({ 'type' => 'admin' }, 200, 'intercom-version' => '2.14')) + + client.me + + expect(ForestAdminDatasourceIntercom.logger).to have_received(:warn).with(/asked.*2\.16.*served 2\.14/m) + end + + it 'stays quiet when the pin was honoured' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + stub_request(:get, "#{base}/me").to_return(json({ 'type' => 'admin' }, 200, 'intercom-version' => '2.16')) + + client.me + + expect(ForestAdminDatasourceIntercom.logger).not_to have_received(:warn) + end + + it 'stays quiet when Intercom echoes nothing' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + stub_request(:get, "#{base}/me").to_return(json('type' => 'admin')) + + client.me + + expect(ForestAdminDatasourceIntercom.logger).not_to have_received(:warn) + end + end + + describe 'failures' do + it "carries Intercom's status, parsed body and error text" do + body = { 'type' => 'error.list', 'request_id' => 'req_1', + 'errors' => [{ 'code' => 'unauthorized', 'message' => 'Access Token Invalid' }] } + stub_request(:get, "#{base}/me").to_return(json(body, 401)) + + expect { client.me }.to raise_error(APIError) { |error| + expect(error.message).to eq('Intercom API call failed: me: HTTP 401 unauthorized: ' \ + 'Access Token Invalid (request_id: req_1)') + expect(error.status).to eq(401) + expect(error.body).to eq(body) + } + end + + it 'joins the several errors one response can carry' do + body = { 'errors' => [{ 'code' => 'parameter_invalid', 'message' => 'per_page' }, + { 'code' => 'parameter_invalid', 'message' => 'starting_after' }] } + stub_request(:get, "#{base}/me").to_return(json(body, 400)) + + expect { client.me }.to raise_error(APIError, /per_page; parameter_invalid: starting_after/) + end + + it 'falls back to the whole body when the shape is not the documented one' do + stub_request(:get, "#{base}/me").to_return(json({ 'oops' => true }, 500)) + + expect { client.me }.to raise_error(APIError, /\{"oops":true\}/) + end + + it 'keeps a body that is not JSON at all, which is what a gateway answers' do + stub_request(:get, "#{base}/me").to_return(status: 502, body: 'bad gateway') + + expect { client.me }.to raise_error(APIError) { |error| + expect(error.status).to eq(502) + expect(error.body).to eq('bad gateway') + } + end + + # No status to report: the request never reached Intercom, so there is + # nothing of its to surface. + it 'reports a dropped connection without a status' do + stub_request(:get, "#{base}/me").to_raise(Faraday::ConnectionFailed.new('closed')) + + expect { client.me }.to raise_error(APIError) { |error| + expect(error.message).to include('Faraday::ConnectionFailed') + expect(error.status).to be_nil + } + end + + it 'replays a 429 rather than surfacing it' do + stub_request(:get, "#{base}/me") + .to_return(json({ 'errors' => [{ 'code' => 'rate_limit_exceeded' }] }, 429)) + .then.to_return(json('type' => 'admin')) + + expect(client.me).to eq('type' => 'admin') + end + + it 'gives up on a 429 that outlasts the retries, saying which endpoint' do + stub_request(:get, "#{base}/me").to_return(json({ 'errors' => [{ 'code' => 'rate_limit_exceeded' }] }, 429)) + + expect { client.me }.to raise_error(APIError, /me: HTTP 429 rate_limit_exceeded/) + end + + # A body that failed to parse is a payload, not an error: on a 200 it is + # customer content, and this message is shown in the interface and + # collected by whatever watches the agent (R10). + it 'names a body it could not read rather than quoting it' do + stub_request(:get, "#{base}/me") + .to_return(json('Bonjour, voici mon RIB FR76 3000 4000 0500 0012 3456 789')) + + expect { client.me }.to raise_error(APIError) { |error| + expect(error.message).to eq('Intercom API call failed: me: the response could not be read as JSON') + # Faraday hands a parsing error an unfinished response, so there is no + # body to keep here -- which suits this one: the point is that the + # payload does not travel with the error. + expect(error.body).to be_nil + } + end + + # The same guard, one level down: a parser raising on its own would + # otherwise reach the catch-all, whose message is the exception's -- and + # a JSON parser quotes what it choked on. + it 'says as little when a parser raises outside Faraday' do + stub_request(:get, "#{base}/me").to_raise(JSON::ParserError.new("unexpected token 'mon RIB FR76'")) + + expect { client.me }.to raise_error(APIError) { |error| + expect(error.message).to include('could not be read as JSON') + expect(error.message).not_to include('RIB') + } + end + + # Whatever else goes wrong on the way, a caller of this client only ever + # has to rescue APIError -- and the message names the operation, since a + # failure with no endpoint in it is a failure nobody can place. + it 'still names the operation when the failure is not one it expected' do + stub_request(:get, "#{base}/me").to_raise(ArgumentError.new('unexpected')) + + expect { client.me }.to raise_error(APIError, /me: ArgumentError: unexpected/) + end + end + + describe '#list_page' do + def list_body(data, next_page: nil, total: 2) + body = { 'type' => 'list', 'data' => data, 'total_count' => total, + 'pages' => { 'type' => 'pages', 'page' => 1, 'per_page' => 50 } } + body['pages']['next'] = next_page unless next_page.nil? + body + end + + it 'reads the records, the next cursor and the exact count off one response' do + body = list_body([{ 'id' => '1' }], next_page: { 'starting_after' => 'cursor_2' }) + stub_request(:get, "#{base}/conversations").with(query: { 'per_page' => '150' }).to_return(json(body)) + + page = client.list_page('conversations', per_page: 150) + + expect(page.records).to eq([{ 'id' => '1' }]) + expect(page.next_cursor).to eq('cursor_2') + expect(page.total_count).to eq(2) + end + + it 'sends the cursor the previous page advertised' do + stub_request(:get, "#{base}/conversations") + .with(query: { 'per_page' => '50', 'starting_after' => 'cursor_2' }) + .to_return(json(list_body([]))) + + client.list_page('conversations', per_page: 50, starting_after: 'cursor_2') + + expect(WebMock).to have_requested(:get, "#{base}/conversations") + .with(query: { 'per_page' => '50', 'starting_after' => 'cursor_2' }) + end + + it 'bounds the page size before sending it, Intercom refusing rather than clamping' do + stub_request(:get, "#{base}/conversations").with(query: { 'per_page' => '150' }) + .to_return(json(list_body([]))) + + client.list_page('conversations', per_page: 500) + + expect(WebMock).to have_requested(:get, "#{base}/conversations").with(query: { 'per_page' => '150' }) + end + + it 'carries the parameters an endpoint of its own needs' do + stub_request(:get, "#{base}/conversations") + .with(query: { 'per_page' => '150', 'display_as' => 'plaintext' }).to_return(json(list_body([]))) + + client.list_page('conversations', per_page: 150, params: { 'display_as' => 'plaintext' }) + + expect(WebMock).to have_requested(:get, "#{base}/conversations") + .with(query: hash_including('display_as' => 'plaintext')) + end + + # The last page simply carries no `pages.next`, which is what stops a walk. + it 'reports no next cursor on the last page' do + stub_request(:get, "#{base}/conversations").with(query: hash_including({})) + .to_return(json(list_body([{ 'id' => '1' }]))) + + expect(client.list_page('conversations', per_page: 150).next_cursor).to be_nil + end + + # An older API version spells `pages.next` as a url, and one can be served + # despite the pin -- reading the cursor out of it beats taking the page for + # the last one and truncating the answer. + it 'reads the cursor out of a next page spelled as a url' do + url = "#{base}/conversations?per_page=50&starting_after=cursor_9" + stub_request(:get, "#{base}/conversations").with(query: hash_including({})) + .to_return(json(list_body([], next_page: url))) + + expect(client.list_page('conversations', per_page: 50).next_cursor).to eq('cursor_9') + end + + # An advertised page taken for the last one is a silently truncated + # answer, so every unreadable shape is refused rather than dropped. + it 'refuses a next-page url carrying no cursor' do + body = list_body([], next_page: "#{base}/conversations?per_page=50") + stub_request(:get, "#{base}/conversations").with(query: hash_including({})).to_return(json(body)) + + expect { client.list_page('conversations', per_page: 50) } + .to raise_error(APIError, /pages\.next' carries no cursor/) + end + + it 'refuses a next-page url it cannot parse' do + stub_request(:get, "#{base}/conversations").with(query: hash_including({})) + .to_return(json(list_body([], next_page: 'http://[bad'))) + + expect { client.list_page('conversations', per_page: 50) } + .to raise_error(APIError, /pages\.next' carries no cursor/) + end + + it 'refuses a next page it can read neither way, rather than truncating silently' do + stub_request(:get, "#{base}/conversations").with(query: hash_including({})) + .to_return(json(list_body([], next_page: 42))) + + expect { client.list_page('conversations', per_page: 50) } + .to raise_error(APIError, /unexpected response shape.*pages\.next/m) + end + + # `Array()` on the envelope would hand the collection rows built out of + # [key, value] pairs: a page that looks answered and holds nothing. + it 'refuses a response whose data is not a list' do + stub_request(:get, "#{base}/conversations").with(query: hash_including({})) + .to_return(json({ 'type' => 'list', 'data' => { 'id' => '1' } })) + + expect { client.list_page('conversations', per_page: 50) } + .to raise_error(APIError, /unexpected response shape.*'data' is not a list/m) + end + + it 'refuses a response carrying no data at all' do + stub_request(:get, "#{base}/conversations").with(query: hash_including({})) + .to_return(json({ 'type' => 'list', 'total_count' => 0 })) + + expect { client.list_page('conversations', per_page: 50) }.to raise_error(APIError, /'data' is not a list/) + end + + it 'serves an empty page as an empty page, zero being an answer' do + stub_request(:get, "#{base}/conversations").with(query: hash_including({})) + .to_return(json(list_body([], total: 0))) + + expect(client.list_page('conversations', per_page: 50)) + .to have_attributes(records: [], next_cursor: nil, total_count: 0) + end + + # nil rather than 0: zero is an answer, and this is the absence of one. + it 'reports no count when Intercom sends none' do + stub_request(:get, "#{base}/conversations").with(query: hash_including({})) + .to_return(json({ 'type' => 'list', 'data' => [] })) + + expect(client.list_page('conversations', per_page: 50).total_count).to be_nil + end + + it 'reads a page through the boot connection when asked to' do + stub_request(:get, "#{base}/ticket_types").with(query: hash_including({})) + .to_return(json(list_body([{ 'id' => '1' }]))) + + expect(client.list_page('ticket_types', per_page: 50, boot: true).records.size).to eq(1) + end + + it 'names the endpoint when the read fails' do + stub_request(:get, "#{base}/conversations").with(query: hash_including({})) + .to_return(json({ 'errors' => [{ 'code' => 'not_found' }] }, 404)) + + expect { client.list_page('conversations', per_page: 50) } + .to raise_error(APIError, /conversations: HTTP 404 not_found/) + end + end + + describe '#fetch_all' do + it 'reads the records under the key the endpoint uses' do + stub_request(:get, "#{base}/admins") + .to_return(json('type' => 'admin.list', 'admins' => [{ 'id' => '1' }, { 'id' => '2' }])) + + expect(client.fetch_all('admins', list_key: 'admins').size).to eq(2) + end + + # Intercom is not consistent about it: /admins and /teams use their own + # key, /ticket_types the `data` envelope every paginated listing uses. + it 'falls back to the data envelope' do + stub_request(:get, "#{base}/ticket_types").to_return(json('type' => 'list', 'data' => [{ 'id' => '1' }])) + + expect(client.fetch_all('ticket_types')).to eq([{ 'id' => '1' }]) + end + + it 'asks for no page: these endpoints answer whole' do + stub_request(:get, "#{base}/teams").to_return(json('teams' => [])) + + client.fetch_all('teams', list_key: 'teams') + + expect(WebMock).to have_requested(:get, "#{base}/teams").with(query: {}) + end + + # A reference collection read as empty is a state column with no values and + # an assignee shown as a raw id -- worse than a failure naming the shape. + it 'refuses a response holding neither key' do + stub_request(:get, "#{base}/admins").to_return(json('type' => 'admin.list', 'admins' => { 'id' => '1' })) + + expect { client.fetch_all('admins', list_key: 'admins') } + .to raise_error(APIError, /neither 'admins' nor 'data' is a list/) + end + + it 'reads an empty body as no record' do + stub_request(:get, "#{base}/admins").to_return(status: 200, body: '') + + expect(client.fetch_all('admins', list_key: 'admins')).to eq([]) + end + + # No pagination parameter in the specification is not a promise that a + # large workspace answers in one response, and a truncated reference + # collection would show an operator a state list missing its last states. + it 'follows a cursor if one is advertised anyway' do + stub_request(:get, "#{base}/tags").with(query: {}) + .to_return(json('data' => [{ 'id' => '1' }], + 'pages' => { 'next' => { 'starting_after' => 'c2' } })) + stub_request(:get, "#{base}/tags").with(query: { 'starting_after' => 'c2' }) + .to_return(json('data' => [{ 'id' => '2' }])) + + expect(client.fetch_all('tags').map { |tag| tag['id'] }).to eq(%w[1 2]) + end + + it 'stops at its page cap and says what it left out' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + stub_request(:get, "#{base}/tags").with(query: hash_including({})) + .to_return(json('data' => [{ 'id' => '1' }], + 'pages' => { 'next' => { 'starting_after' => 'c' } })) + + client.fetch_all('tags') + + expect(WebMock).to have_requested(:get, "#{base}/tags") + .with(query: hash_including({})).times(described_class::MAX_COLLECTED_PAGES) + expect(ForestAdminDatasourceIntercom.logger).to have_received(:warn).with(/Stopped reading tags/) + end + + it 'reads through the boot connection when asked to' do + stub_request(:get, "#{base}/ticket_types").to_return(json('data' => [])) + + expect(client.fetch_all('ticket_types', boot: true)).to eq([]) + end + + it 'names the endpoint when the read fails' do + stub_request(:get, "#{base}/admins").to_return(json({ 'errors' => [{ 'code' => 'forbidden' }] }, 403)) + + expect { client.fetch_all('admins', list_key: 'admins') } + .to raise_error(APIError, /admins: HTTP 403 forbidden/) + end + end + + describe '.bounded_per_page' do + # Intercom answers `invalid_per_page` past 150 instead of clamping, so a + # page size is bounded before it is sent or the list view breaks. + it 'caps a page size at what Intercom accepts' do + expect(described_class.bounded_per_page(200)).to eq(150) + end + + it 'leaves an acceptable size alone' do + expect(described_class.bounded_per_page(50)).to eq(50) + end + + it 'asks for one record rather than none, an empty page being no answer' do + expect([described_class.bounded_per_page(0), described_class.bounded_per_page(-5)]).to eq([1, 1]) + end + end + + describe 'the boot connection' do + # What is read while the datasource is being constructed waits far less + # than a request that already has a page on screen: the wait there is + # minutes of Rails boot the operator sits through. + it 'honours the configured boot timeouts' do + booted = described_class.new(Configuration.new(access_token: 's3cr3t', boot_open_timeout: 1, boot_timeout: 2)) + conn = booted.send(:boot_connection) + + expect(conn.options).to have_attributes(open_timeout: 1, timeout: 2) + end + + it 'keeps the patience of a regular request on the regular connection' do + expect(client.send(:connection).options).to have_attributes(open_timeout: 5, timeout: 30) + end + + it 'reads through it when asked to' do + stub_request(:get, "#{base}/me").to_return(json('type' => 'admin')) + + expect(client.me(boot: true)).to eq('type' => 'admin') + end + end + + describe 'pacing' do + let(:limiter) { instance_double(RateLimiter, acquire: nil, observe: nil) } + let(:paced) do + described_class.new(Configuration.new(access_token: 's3cr3t', retry_policy: retry_policy, + rate_limiter: limiter)) + end + + it 'asks the limiter for room, and feeds it the window back' do + stub_request(:get, "#{base}/me").to_return(json({ 'type' => 'admin' }, 200, + 'x-ratelimit-remaining' => '1666')) + + paced.me + + expect(limiter).to have_received(:acquire) + expect(limiter).to have_received(:observe).with(hash_including('x-ratelimit-remaining' => '1666')) + end + + # The throttle sits inside the retry, so a replay waits for the window + # like a first attempt rather than going out on a budget already spent. + it 'asks again for every replay, not once per call' do + stub_request(:get, "#{base}/me") + .to_return(json({}, 429)).then.to_return(json('type' => 'admin')) + + paced.me + + expect(limiter).to have_received(:acquire).twice + end + end + + describe '#inspect' do + it 'never prints the token its connections carry' do + expect(client.inspect).to include(base) + expect(client.inspect).not_to include('s3cr3t') + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/admin_spec.rb b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/admin_spec.rb new file mode 100644 index 000000000..a98cfd112 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/admin_spec.rb @@ -0,0 +1,61 @@ +module ForestAdminDatasourceIntercom + RSpec.describe Collections::Admin do + subject(:collection) { described_class.new(datasource) } + + let(:datasource) { Datasource.new(access_token: 's3cr3t', rate_limiter: nil) } + let(:base) { datasource.configuration.url } + + def filter + ForestAdminDatasourceToolkit::Components::Query::Filter.new + end + + def stub_admins(*admins) + stub_request(:get, "#{base}/admins") + .to_return(status: 200, body: { 'type' => 'admin.list', 'admins' => admins }.to_json, + headers: { 'Content-Type' => 'application/json' }) + end + + it 'is named IntercomAdmin' do + expect(collection.name).to eq('IntercomAdmin') + end + + it 'exposes the columns an ops lead reads before assigning anything' do + expect(collection.fields.keys) + .to eq(%w[id name email job_title away_mode_enabled away_mode_reassign has_inbox_seat team_ids]) + end + + it 'declares id as the primary key' do + expect(collection.fields['id']).to have_attributes(is_primary_key: true, column_type: 'String') + end + + # `/admins` puts its records under `admins` rather than under the `data` + # envelope the paginated listings use. + it 'reads the endpoint and flattens the teammate' do + stub_admins('type' => 'admin', 'id' => '1', 'name' => 'Alice', 'email' => 'alice@acme.test', + 'job_title' => 'Support', 'away_mode_enabled' => true, 'away_mode_reassign' => false, + 'has_inbox_seat' => true, 'team_ids' => [814_865]) + + expect(collection.list(nil, filter, nil)) + .to eq([{ 'id' => '1', 'name' => 'Alice', 'email' => 'alice@acme.test', 'job_title' => 'Support', + 'away_mode_enabled' => true, 'away_mode_reassign' => false, 'has_inbox_seat' => true, + 'team_ids' => %w[814865] }]) + end + + # Intercom types a team id as a number here and as a string on the team + # itself; a filter value from Forest always arrives as a string. + it 'stringifies the ids so both sides of the membership match' do + stub_admins('id' => 493_881, 'team_ids' => [814_865, 814_866]) + + row = collection.list(nil, filter, nil).first + + expect(row['id']).to eq('493881') + expect(row['team_ids']).to eq(%w[814865 814866]) + end + + it 'reads a teammate with no team as one with no team, not as one with a null' do + stub_admins('id' => '1', 'team_ids' => nil) + + expect(collection.list(nil, filter, nil).first['team_ids']).to eq([]) + end + end +end diff --git a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/conversation_spec.rb b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/conversation_spec.rb new file mode 100644 index 000000000..eb4a24ad2 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/conversation_spec.rb @@ -0,0 +1,480 @@ +module ForestAdminDatasourceIntercom + RSpec.describe Collections::Conversation do + subject(:collection) { datasource.get_collection('IntercomConversation') } + + let(:datasource) { Datasource.new(access_token: 's3cr3t', rate_limiter: nil) } + let(:base) { datasource.configuration.url } + let(:operators) { ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Operators } + + def filter(condition_tree: nil, page: nil, sort: nil) + ForestAdminDatasourceToolkit::Components::Query::Filter.new(condition_tree: condition_tree, page: page, + sort: sort) + end + + def leaf(field, operator, value = nil) + ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Nodes::ConditionTreeLeaf + .new(field, operator, value) + end + + def json(payload, status = 200) + { status: status, body: payload.to_json, headers: { 'Content-Type' => 'application/json' } } + end + + # Hand-written from the OpenAPI 2.16 spec, never captured from a workspace: + # a conversation body is personal data. + def conversation(id, overrides = {}) + { + 'type' => 'conversation', 'id' => id, 'title' => "Facture #{id}", 'state' => 'closed', + 'priority' => 'priority', 'open' => false, 'read' => true, + 'created_at' => 1_700_000_000, 'updated_at' => 1_700_003_600, + 'waiting_since' => nil, 'snoozed_until' => nil, + 'admin_assignee_id' => 493_881, 'team_assignee_id' => 814_865, + 'company' => { 'type' => 'company', 'id' => '696dd52099f73812610d9c7b', 'name' => 'Acme' }, + 'contacts' => { 'type' => 'contact.list', + 'contacts' => [{ 'type' => 'contact', 'id' => 'c1' }, { 'type' => 'contact', 'id' => 'c2' }] }, + 'tags' => { 'type' => 'tag.list', 'tags' => [{ 'id' => 't1', 'name' => 'billing' }] }, + 'ai_agent_participated' => true, + 'source' => { 'type' => 'conversation', 'id' => 's1', 'delivered_as' => 'customer_initiated', + 'subject' => 'Ma facture', 'body' => 'Bonjour, ou est ma facture ?', + 'author' => { 'type' => 'user', 'id' => 'c1', 'name' => 'Camille', + 'email' => 'camille@acme.test' }, + 'attachments' => [] }, + 'statistics' => { 'type' => 'conversation_statistics', 'first_close_at' => 1_700_002_000, + 'last_close_at' => 1_700_003_000, 'last_closed_by_id' => '493881', + 'first_contact_reply_at' => 1_700_000_050, 'last_contact_reply_at' => 1_700_001_000, + 'last_admin_reply_at' => 1_700_002_500, 'count_reopens' => 1, + 'count_conversation_parts' => 4 } + }.merge(overrides) + end + + def parts(*entries) + { 'conversation_parts' => { 'type' => 'conversation_part.list', 'conversation_parts' => entries } } + end + + def part(part_type, overrides = {}) + { 'type' => 'conversation_part', 'id' => 'p1', 'part_type' => part_type, 'body' => 'Je regarde.', + 'created_at' => 1_700_002_500, 'redacted' => false, 'attachments' => [], + 'author' => { 'type' => 'admin', 'id' => '493881', 'name' => 'Alice', + 'email' => 'alice@acme.test' } }.merge(overrides) + end + + # Intercom puts the records under `conversations`, not under the `data` + # envelope -- measured on `/tickets/search`, and the listings follow the same + # habit. + def stub_list(*records, next_cursor: nil, total: nil, query: hash_including({})) + body = { 'type' => 'conversation.list', 'conversations' => records, + 'total_count' => total || records.size, 'pages' => { 'type' => 'pages', 'page' => 1 } } + body['pages']['next'] = { 'starting_after' => next_cursor } if next_cursor + + stub_request(:get, "#{base}/conversations").with(query: query).to_return(json(body)) + end + + def stub_record(id, payload, status = 200) + stub_request(:get, "#{base}/conversations/#{id}").with(query: hash_including({})).to_return(json(payload, status)) + end + + def ids(rows) + rows.map { |row| row['id'] } + end + + describe 'schema' do + it 'is named IntercomConversation' do + expect(collection.name).to eq('IntercomConversation') + end + + # Intercom ignores a sort on this endpoint without a word and filters + # nothing on the listing, so a column advertising either would put in the + # interface what the read then refuses. + it 'declares every column unsortable and unfilterable, except the primary key' do + others = collection.fields.except('id') + + expect(others.values.map(&:is_sortable).uniq).to eq([false]) + expect(others.values.map(&:filter_operators).flatten.uniq).to be_empty + end + + # The record detail is `id equals X`, answered by the record endpoint + # rather than by a filter. + it 'answers the primary key with equal and in' do + expect(collection.fields['id'].filter_operators).to eq(%w[equal in]) + end + + it 'is countable, since total_count is exact' do + expect(collection.is_countable?).to be(true) + end + + # No aggregate endpoint, so no group-by may be offered. + it 'declares no column groupable' do + expect(collection.fields.values.map(&:is_groupable).uniq).to eq([false]) + end + + # This lot writes nothing: an editable column would offer a Save that + # reaches an `update` the collection does not implement. + it 'declares every column read-only' do + expect(collection.fields.values.map(&:is_read_only).uniq).to eq([true]) + end + end + + describe '#list' do + it 'reads the listing endpoint as plain text and pages by cursor' do + stub_list(conversation('1')) + + collection.list(nil, filter, nil) + + expect(WebMock).to have_requested(:get, "#{base}/conversations") + .with(query: hash_including('display_as' => 'plaintext')) + end + + it 'flattens the payload into the row the schema declares' do + stub_list(conversation('1')) + + row = collection.list(nil, filter, nil).first + + expect(row).to include('id' => '1', 'title' => 'Facture 1', 'state' => 'closed', 'open' => false, + 'company_id' => '696dd52099f73812610d9c7b', 'company_name' => 'Acme', + 'admin_assignee_id' => '493881', 'team_assignee_id' => '814865', + 'tag_names' => %w[billing], 'ai_agent_participated' => true) + end + + # Epoch seconds are what Intercom sends; a Date column and a date filter + # both read ISO8601, and UTC is where Intercom truncates. + it 'reads the dates as ISO8601 in UTC' do + row = (stub_list(conversation('1')) && collection.list(nil, filter, nil)).first + + expect(row['created_at']).to eq('2023-11-14T22:13:20Z') + end + + it 'flattens the lifecycle Intercom keeps in statistics' do + stub_list(conversation('1')) + + expect(collection.list(nil, filter, nil).first) + .to include('closed_at' => '2023-11-14T23:03:20Z', 'closed_by_id' => '493881', + 'last_admin_reply_at' => '2023-11-14T22:55:00Z', 'reopen_count' => 1, 'part_count' => 4) + end + + # A conversation Intercom has computed nothing for yet answers a null + # statistics; the columns then read as absent rather than as zero. + it 'reads a missing statistics block as absent, not as zero' do + stub_list(conversation('1', 'statistics' => nil)) + + expect(collection.list(nil, filter, nil).first) + .to include('closed_at' => nil, 'reopen_count' => nil) + end + + # A group conversation has several contacts: the row names how many rather + # than presenting one of them as the one. + it 'carries the contact ids and their count' do + stub_list(conversation('1')) + + expect(collection.list(nil, filter, nil).first) + .to include('contact_ids' => %w[c1 c2], 'contact_count' => 2) + end + + it 'narrows the row to the projection' do + stub_list(conversation('1')) + + expect(collection.list(nil, filter, %w[id state])).to eq([{ 'id' => '1', 'state' => 'closed' }]) + end + + it 'walks the cursor until the window is covered' do + first = { 'conversations' => [conversation('1'), conversation('2')], + 'pages' => { 'next' => { 'starting_after' => 'c2' } } } + stub_request(:get, "#{base}/conversations").with(query: hash_including('per_page' => '3')) + .to_return(json(first)) + stub_request(:get, "#{base}/conversations").with(query: hash_including('starting_after' => 'c2')) + .to_return(json('conversations' => [conversation('3')])) + + page = ForestAdminDatasourceToolkit::Components::Query::Page.new(offset: 2, limit: 1) + + expect(ids(collection.list(nil, filter(page: page), %w[id]))).to eq(%w[3]) + end + end + + describe '#list of one record' do + # What a record detail is. It goes to the record endpoint rather than to + # the listing, which is also what brings the parts along. + it 'reads id equals X through the record endpoint' do + stub_record('1', conversation('1')) + + expect(ids(collection.list(nil, filter(condition_tree: leaf('id', operators::EQUAL, '1')), %w[id]))) + .to eq(%w[1]) + end + + # A stale link, or a record outside the token's scope: no record, not a + # failed page. + it 'reads a 404 as no record' do + stub_record('gone', { 'errors' => [{ 'code' => 'not_found' }] }, 404) + + expect(collection.list(nil, filter(condition_tree: leaf('id', operators::EQUAL, 'gone')), %w[id])).to eq([]) + end + + it 'still raises on a failure that is not a missing record' do + stub_record('1', { 'errors' => [{ 'code' => 'forbidden' }] }, 403) + + expect { collection.list(nil, filter(condition_tree: leaf('id', operators::EQUAL, '1')), %w[id]) } + .to raise_error(APIError) + end + end + + describe '#list of several records by id' do + # What a pointing collection asks for when it reads related records in + # bulk. Intercom has no "read these records" endpoint, so it is one + # request per id -- and therefore bounded. + it 'reads each id through the record endpoint' do + %w[1 2].each do |id| + stub_record(id, conversation(id)) + end + + rows = collection.list(nil, filter(condition_tree: leaf('id', operators::IN, %w[1 2])), %w[id]) + + expect(ids(rows)).to eq(%w[1 2]) + end + + it 'reads the first of too many and says the result is truncated' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + asked = (1..(Collections::CursorCollection::MAX_ID_READS + 3)).map(&:to_s) + stub_request(:get, %r{/conversations/\d+}).to_return(json(conversation('1'))) + + rows = collection.list(nil, filter(condition_tree: leaf('id', operators::IN, asked)), %w[id]) + + expect(rows.size).to eq(Collections::CursorCollection::MAX_ID_READS) + expect(ForestAdminDatasourceIntercom.logger).to have_received(:warn).with(/read the first 25/) + end + end + + describe 'a filter it cannot honour' do + # Translating a Forest tree into Intercom's search DSL is the next lot. + # Until then a page that looks filtered without being it is the one answer + # this datasource must not give. + it 'refuses a condition on anything but the primary key' do + expect { collection.list(nil, filter(condition_tree: leaf('state', operators::EQUAL, 'open')), %w[id]) } + .to raise_error(UnsupportedOperatorError, /cannot answer a condition/) + end + + it 'refuses a free-text search' do + searched = ForestAdminDatasourceToolkit::Components::Query::Filter.new(search: 'facture') + + expect { collection.list(nil, searched, %w[id]) } + .to raise_error(UnsupportedOperatorError, /cannot answer a free-text search/) + end + + it 'says where the filtering will come from, so the message is actionable' do + expect { collection.list(nil, filter(condition_tree: leaf('state', operators::EQUAL, 'open')), %w[id]) } + .to raise_error(UnsupportedOperatorError, /search endpoint.*filter translation/m) + end + end + + describe 'a sort Intercom ignores' do + before { allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) } + + # Measured: a sort sent to this endpoint raises nothing and changes + # nothing, so an order the operator asked for and did not get can only be + # reported here. + it 'reports the order it did not get' do + stub_list(conversation('1')) + sort = ForestAdminDatasourceToolkit::Components::Query::Sort.new([{ field: 'created_at', ascending: false }]) + + collection.list(nil, filter(sort: sort), %w[id]) + + expect(ForestAdminDatasourceIntercom.logger).to have_received(:warn).with(/ignores a sort/) + end + + it 'stays quiet on the primary-key order the agent injects by default' do + stub_list(conversation('1')) + sort = ForestAdminDatasourceToolkit::Components::Query::Sort.new([{ field: 'id', ascending: true }]) + + collection.list(nil, filter(sort: sort), %w[id]) + + expect(ForestAdminDatasourceIntercom.logger).not_to have_received(:warn) + end + end + + describe '#aggregate' do + def aggregation(operation, field: nil, groups: []) + ForestAdminDatasourceToolkit::Components::Query::Aggregation.new(operation: operation, field: field, + groups: groups) + end + + # One request, and exact on the whole collection rather than on the page + # the walk happened to read. + it 'counts through total_count' do + stub_list(conversation('1'), total: 81_142, query: hash_including('per_page' => '1')) + + expect(collection.aggregate(nil, filter, aggregation('Count'))) + .to eq([{ 'group' => {}, 'value' => 81_142 }]) + end + + it 'counts the records an id lookup found' do + stub_record('1', conversation('1')) + + expect(collection.aggregate(nil, filter(condition_tree: leaf('id', operators::EQUAL, '1')), + aggregation('Count')).first['value']).to eq(1) + end + + # Grouping over the pages a walk collected would look exact while + # answering a fraction. + it 'refuses a group-by' do + expect { collection.aggregate(nil, filter, aggregation('Count', groups: [{ field: 'state' }])) } + .to raise_error(UnsupportedOperatorError, /can only be counted/) + end + + it 'refuses a sum' do + expect { collection.aggregate(nil, filter, aggregation('Sum', field: 'reopen_count')) } + .to raise_error(UnsupportedOperatorError, /can only be counted/) + end + + it 'refuses a condition it could not honour on the list either' do + expect do + collection.aggregate(nil, filter(condition_tree: leaf('state', operators::EQUAL, 'open')), + aggregation('Count')) + end.to raise_error(UnsupportedOperatorError, /cannot answer a condition/) + end + + # Counting the pages a walk collected would answer a fraction as if it + # were the whole, so a listing with no total_count is a listing this + # cannot count. + it 'refuses to count a listing Intercom answered without a total_count' do + stub_request(:get, "#{base}/conversations").with(query: hash_including({})) + .to_return(json('conversations' => [], + 'pages' => { 'type' => 'pages' })) + + expect { collection.aggregate(nil, filter, aggregation('Count')) } + .to raise_error(UnsupportedOperatorError, /without a total_count/) + end + end + + describe 'the contact identity' do + before do + stub_list(conversation('1')) + stub_request(:post, "#{base}/contacts/search") + .to_return(json('type' => 'list', + 'data' => [{ 'id' => 'c1', 'name' => 'Camille', 'email' => 'camille@acme.test' }])) + end + + # Denormalized rather than declared as a relation: the Contacts collection + # arrives in lot 4, and a relation whose target is missing is a schema the + # agent refuses to boot on. + it 'reads the identity of the page in one request and puts it on the row' do + row = collection.list(nil, filter, %w[id contact_name contact_email]).first + + expect(row).to include('contact_name' => 'Camille', 'contact_email' => 'camille@acme.test') + expect(WebMock).to have_requested(:post, "#{base}/contacts/search").once + end + + it 'asks for the contacts of the page by id' do + collection.list(nil, filter, %w[id contact_name]) + + expect(WebMock).to have_requested(:post, "#{base}/contacts/search") + .with(body: hash_including('query' => { 'field' => 'id', 'operator' => 'IN', 'value' => %w[c1] })) + end + + # A page that never asked for the identity must not pay for it. + it 'reads nothing when the projection does not name it' do + collection.list(nil, filter, %w[id state]) + + expect(WebMock).not_to have_requested(:post, "#{base}/contacts/search") + end + + # An identity that could not be read is not a page that could not be + # served: it costs the two columns. + it 'degrades to empty columns when the read fails' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + stub_request(:post, "#{base}/contacts/search").to_return(json({ 'errors' => [] }, 403)) + + row = collection.list(nil, filter, %w[id contact_name]).first + + expect(row['contact_name']).to be_nil + expect(ForestAdminDatasourceIntercom.logger).to have_received(:warn).with(/could not read the contacts/) + end + end + + describe 'the timeline' do + let(:conversation_with_parts) do + conversation('1').merge(parts(part('assignment', 'id' => 'p1', 'body' => nil, 'created_at' => 1_700_000_100), + part('comment', 'id' => 'p2', 'created_at' => 1_700_002_500))) + end + + # The message that opened the conversation lives in `source`, not in the + # parts: a timeline built from the parts alone loses what the customer + # actually asked. + it 'opens on the source message' do + stub_record('1', conversation_with_parts) + + timeline = collection.list(nil, filter(condition_tree: leaf('id', operators::EQUAL, '1')), + %w[id timeline]).first['timeline'] + + expect(timeline.first).to include('part_type' => 'conversation_started', + 'body' => 'Bonjour, ou est ma facture ?', + 'author_name' => 'Camille', 'created_at' => '2023-11-14T22:13:20Z') + end + + # An assignment, a note and a reply are not the same event; a thread that + # flattens them reads as a conversation that never happened that way. + it 'keeps the part type of every entry' do + stub_record('1', conversation_with_parts) + + timeline = collection.list(nil, filter(condition_tree: leaf('id', operators::EQUAL, '1')), + %w[timeline]).first['timeline'] + + expect(timeline.map { |entry| entry['part_type'] }).to eq(%w[conversation_started assignment comment]) + end + + it 'costs no request on a record read, the parts riding along with it' do + stub_record('1', conversation_with_parts) + + collection.list(nil, filter(condition_tree: leaf('id', operators::EQUAL, '1')), %w[timeline]) + + expect(WebMock).to have_requested(:get, "#{base}/conversations/1") + .with(query: hash_including({})).once + end + + # Intercom returns the parts only when retrieving one conversation, so a + # list view pays a request per row. + it 'reads the record when a listed row has no parts' do + stub_list(conversation('1')) + stub_record('1', conversation_with_parts) + + rows = collection.list(nil, filter, %w[id timeline]) + + expect(rows.first['timeline'].size).to eq(3) + end + + it 'reads nothing when the projection does not name it' do + stub_list(conversation('1')) + + collection.list(nil, filter, %w[id state]) + + expect(WebMock).not_to have_requested(:get, "#{base}/conversations/1").with(query: hash_including({})) + end + + # A conversation deleted between the page and the read of its timeline: + # the row keeps a nil timeline rather than failing the whole page. + it 'leaves the timeline unread when the record has gone' do + stub_list(conversation('1')) + stub_record('1', { 'errors' => [{ 'code' => 'not_found' }] }, 404) + + expect(collection.list(nil, filter, %w[id timeline]).first['timeline']).to be_nil + end + + it 'still raises when the timeline read fails for another reason' do + stub_list(conversation('1')) + stub_record('1', { 'errors' => [{ 'code' => 'forbidden' }] }, 403) + + expect { collection.list(nil, filter, %w[id timeline]) }.to raise_error(APIError) + end + + # Rows past the cap keep a nil timeline -- unknown -- rather than an empty + # list, which would read as "this conversation has no message". + it 'bounds the fan-out and says what it left unread' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + listed = (1..(described_class::MAX_TIMELINE_READS + 2)).map { |index| conversation(index.to_s) } + stub_list(*listed) + stub_request(:get, %r{/conversations/\d+}).to_return(json(conversation_with_parts)) + + rows = collection.list(nil, filter, %w[id timeline]) + + expect(rows.count { |row| row['timeline'].nil? }).to eq(2) + expect(ForestAdminDatasourceIntercom.logger).to have_received(:warn).with(/left the timeline of 2 row/) + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/fetch_all_collection_spec.rb b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/fetch_all_collection_spec.rb new file mode 100644 index 000000000..09b5cefc0 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/fetch_all_collection_spec.rb @@ -0,0 +1,252 @@ +module ForestAdminDatasourceIntercom + # The in-memory tier is exercised through Admin, a real collection carrying one + # column of each kind it has to handle: strings, booleans and a list. + RSpec.describe Collections::FetchAllCollection do + subject(:collection) { Collections::Admin.new(datasource) } + + let(:datasource) { Datasource.new(access_token: 's3cr3t', rate_limiter: nil) } + let(:base) { datasource.configuration.url } + let(:operators) { ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Operators } + + def leaf(field, operator, value = nil) + ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Nodes::ConditionTreeLeaf + .new(field, operator, value) + end + + def filter(condition_tree: nil, page: nil, sort: nil) + ForestAdminDatasourceToolkit::Components::Query::Filter.new(condition_tree: condition_tree, page: page, + sort: sort) + end + + def page(offset, limit) + ForestAdminDatasourceToolkit::Components::Query::Page.new(offset: offset, limit: limit) + end + + def sort(*clauses) + ForestAdminDatasourceToolkit::Components::Query::Sort.new(clauses) + end + + def aggregation(operation, field: nil, groups: []) + ForestAdminDatasourceToolkit::Components::Query::Aggregation.new(operation: operation, field: field, + groups: groups) + end + + def admin(id, overrides = {}) + { 'type' => 'admin', 'id' => id, 'name' => "Admin #{id}", 'email' => "#{id}@acme.test", + 'away_mode_enabled' => false, 'has_inbox_seat' => true, 'team_ids' => [] }.merge(overrides) + end + + def stub_admins(*admins) + stub_request(:get, "#{base}/admins") + .to_return(status: 200, body: { 'type' => 'admin.list', 'admins' => admins }.to_json, + headers: { 'Content-Type' => 'application/json' }) + end + + def ids(records) + records.map { |record| record['id'] } + end + + # An operator is evaluable in memory when `ConditionTreeLeaf#match` handles + # it natively or the toolkit can rewrite it into operators that it does. + def evaluable?(operator, column_type) + ForestAdminDatasourceToolkit::Components::Query::ConditionTree::ConditionTreeEquivalent + .equivalent_tree?(operator, described_class::IN_MEMORY_OPERATORS, column_type) + end + + # The count and the group are taken over every record Intercom holds, not + # over a page of them, which is what makes them exact. + it 'is countable' do + expect(collection.is_countable?).to be(true) + end + + describe 'columns' do + it 'declares a scalar column filterable, sortable and groupable' do + expect(collection.fields['name']) + .to have_attributes(is_sortable: true, is_groupable: true) + expect(collection.fields['name'].filter_operators).to include(operators::EQUAL) + end + + # This lot writes nothing: an editable column would offer a Save that + # reaches an `update` the collection does not implement. + it 'declares every column read-only' do + expect(collection.fields.values.map(&:is_read_only).uniq).to eq([true]) + end + + # A list has no in-memory counterpart for any of the three. + it 'declares a Json column neither filterable nor sortable' do + expect(collection.fields['team_ids']) + .to have_attributes(column_type: 'Json', is_sortable: false, is_groupable: false, filter_operators: []) + end + + # A filter the UI offers and the collection then answers by emptying the + # page is the failure this whole datasource is built to avoid. + it 'advertises only operators it can actually evaluate' do + advertised = collection.fields.flat_map do |_name, column| + column.filter_operators.map { |operator| [operator, column.column_type] } + end + + expect(advertised.reject { |operator, type| evaluable?(operator, type) }).to be_empty + end + end + + describe '#list' do + it 'reads every record of the endpoint and serializes it' do + stub_admins(admin('1'), admin('2')) + + expect(ids(collection.list(nil, filter, nil))).to eq(%w[1 2]) + end + + it 'narrows the record to the projection' do + stub_admins(admin('1')) + + expect(collection.list(nil, filter, %w[id email])).to eq([{ 'id' => '1', 'email' => '1@acme.test' }]) + end + + # Freshness over rate-limit thrift: nothing is kept from the previous list, + # so an operator sees the teammates the workspace has now. + it 'reads the endpoint again on the next list' do + stub_admins(admin('1')) + + 2.times { collection.list(nil, filter, nil) } + + expect(WebMock).to have_requested(:get, "#{base}/admins").twice + end + + it 'propagates a failure rather than answering with no record' do + stub_request(:get, "#{base}/admins").to_return(status: 500, body: '{}', + headers: { 'Content-Type' => 'application/json' }) + + expect { collection.list(nil, filter, nil) }.to raise_error(APIError) + end + end + + describe '#list with a filter' do + before { stub_admins(admin('1', 'name' => 'Alice'), admin('2', 'name' => 'Bob', 'has_inbox_seat' => false)) } + + def filtered(field, operator, value = nil) + ids(collection.list(nil, filter(condition_tree: leaf(field, operator, value)), nil)) + end + + it 'keeps the rows a string condition names' do + expect(filtered('name', operators::EQUAL, 'Alice')).to eq(%w[1]) + end + + it 'keeps the rows a boolean condition names' do + expect(filtered('has_inbox_seat', operators::EQUAL, false)).to eq(%w[2]) + end + + it 'answers an operator it advertises through an equivalence' do + expect(filtered('name', operators::I_CONTAINS, 'ali')).to eq(%w[1]) + end + + it 'answers nothing when nothing matches, rather than everything' do + expect(filtered('name', operators::EQUAL, 'Nobody')).to be_empty + end + + # `match` answers nil for an operator with no equivalence and `apply` reads + # that as "no match", so this would otherwise be an empty page an operator + # cannot tell from a real answer. The schema advertises no such operator; a + # scope or a segment can still send one. + it 'refuses a condition it cannot evaluate instead of emptying the page' do + expect { filtered('team_ids', operators::EQUAL, 'x') } + .to raise_error(UnsupportedOperatorError, /cannot filter 'team_ids' with 'equal'/) + end + + it 'refuses a condition on a column it does not carry' do + expect { filtered('unknown', operators::EQUAL, 'x') } + .to raise_error(UnsupportedOperatorError, /cannot filter 'unknown'/) + end + + it 'names a refusal after the operator, so the message says what to change' do + expect { filtered('name', operators::LESS_THAN, 'x') } + .to raise_error(UnsupportedOperatorError, /'less_than'/) + end + end + + describe '#list with a sort' do + before do + stub_admins(admin('2', 'name' => 'Bob'), admin('1', 'name' => 'Alice'), admin('3', 'name' => nil)) + end + + it 'orders on the column asked for' do + expect(ids(collection.list(nil, filter(sort: sort({ field: 'name', ascending: true })), nil))) + .to eq(%w[1 2 3]) + end + + # Nulls last ascending, first descending, the way a database orders them. + it 'puts a null first on a descending order' do + expect(ids(collection.list(nil, filter(sort: sort({ field: 'name', ascending: false })), nil))) + .to eq(%w[3 2 1]) + end + + it 'keeps the order Intercom returned for rows the sort cannot separate' do + rows = collection.list(nil, filter(sort: sort({ field: 'away_mode_enabled', ascending: true })), nil) + + expect(ids(rows)).to eq(%w[2 1 3]) + end + + it 'drops a clause naming a column it does not carry' do + rows = collection.list(nil, filter(sort: sort({ field: 'unknown', ascending: true })), nil) + + expect(ids(rows)).to eq(%w[2 1 3]) + end + end + + describe '#list with a page' do + before { stub_admins(admin('1'), admin('2'), admin('3')) } + + it 'cuts the window out of the records in hand' do + expect(ids(collection.list(nil, filter(page: page(1, 1)), nil))).to eq(%w[2]) + end + + it 'reads a page with no limit as every record past the offset' do + expect(ids(collection.list(nil, filter(page: page(1, 0)), nil))).to eq(%w[2 3]) + end + + it 'answers an offset past the end with no record' do + expect(collection.list(nil, filter(page: page(50, 10)), nil)).to be_empty + end + end + + describe '#aggregate' do + before { stub_admins(admin('1', 'name' => 'Alice'), admin('2', 'name' => 'Bob', 'has_inbox_seat' => false)) } + + it 'counts every record, exactly' do + expect(collection.aggregate(nil, filter, aggregation('Count'))) + .to eq([{ 'group' => {}, 'value' => 2 }]) + end + + it 'counts the rows a filter keeps' do + filtered = filter(condition_tree: leaf('has_inbox_seat', operators::EQUAL, true)) + + expect(collection.aggregate(nil, filtered, aggregation('Count')).first['value']).to eq(1) + end + + it 'groups on a column, which is exact for the same reason' do + rows = collection.aggregate(nil, filter, aggregation('Count', groups: [{ field: 'has_inbox_seat' }])) + + expect(rows.sum { |row| row['value'] }).to eq(2) + expect(rows.size).to eq(2) + end + end + + describe 'the hooks a collection has to implement' do + let(:incomplete) do + Class.new(described_class) do + def initialize(datasource) + super(datasource, 'Incomplete') + end + + def define_schema + add_column('id', 'String', is_primary_key: true) + end + end + end + + it 'says which one is missing rather than failing obscurely' do + expect { incomplete.new(datasource).list(nil, filter, nil) } + .to raise_error(NotImplementedError, /did not implement fetch_all/) + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/team_spec.rb b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/team_spec.rb new file mode 100644 index 000000000..91f52dce4 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/team_spec.rb @@ -0,0 +1,41 @@ +module ForestAdminDatasourceIntercom + RSpec.describe Collections::Team do + subject(:collection) { described_class.new(datasource) } + + let(:datasource) { Datasource.new(access_token: 's3cr3t', rate_limiter: nil) } + let(:base) { datasource.configuration.url } + + def filter + ForestAdminDatasourceToolkit::Components::Query::Filter.new + end + + def stub_teams(*teams) + stub_request(:get, "#{base}/teams") + .to_return(status: 200, body: { 'type' => 'team.list', 'teams' => teams }.to_json, + headers: { 'Content-Type' => 'application/json' }) + end + + it 'is named IntercomTeam' do + expect(collection.name).to eq('IntercomTeam') + end + + it 'exposes the team and its membership' do + expect(collection.fields.keys).to eq(%w[id name admin_ids]) + end + + # Intercom carries the membership on the team and on the admin both. Left as + # a plain list it stays readable on either side; declared as a relation it + # would give the schema two halves of a many-to-many with no join collection. + it 'keeps the membership a list rather than a relation' do + expect(collection.fields['admin_ids']) + .to have_attributes(type: 'Column', column_type: 'Json', filter_operators: []) + end + + it 'reads the endpoint under its own key and stringifies the ids' do + stub_teams('type' => 'team', 'id' => '814865', 'name' => 'Support', 'admin_ids' => [493_881]) + + expect(collection.list(nil, filter, nil)) + .to eq([{ 'id' => '814865', 'name' => 'Support', 'admin_ids' => %w[493881] }]) + end + end +end diff --git a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/ticket_spec.rb b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/ticket_spec.rb new file mode 100644 index 000000000..55c13c07a --- /dev/null +++ b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/ticket_spec.rb @@ -0,0 +1,307 @@ +module ForestAdminDatasourceIntercom + RSpec.describe Collections::Ticket do + subject(:collection) { described_class.new(datasource, attributes: attributes) } + + let(:datasource) { Datasource.new(access_token: 's3cr3t', rate_limiter: nil) } + # Not read off the datasource: that would build it, and boot the ticket-type + # introspection before the stub of it exists. + let(:base) { Configuration::REGION_HOSTS[:us] } + let(:operators) { ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Operators } + let(:attributes) { [attribute('_default_title_'), attribute('Due', column_type: 'Date')] } + + # `column_name` is what the schema publishes and `name` the key the payload + # uses; they differ when the workspace's own name cannot travel through a + # Forest query string. + def attribute(name, column_name: nil, column_type: 'String') + Schema::TicketAttributesIntrospector::Attribute.new(name: name, column_name: column_name || name, + column_type: column_type, data_type: 'string', + ids_by_ticket_type: { '1' => '9001' }) + end + + def json(payload, status = 200) + { status: status, body: payload.to_json, headers: { 'Content-Type' => 'application/json' } } + end + + def filter(condition_tree: nil, page: nil) + ForestAdminDatasourceToolkit::Components::Query::Filter.new(condition_tree: condition_tree, page: page) + end + + def leaf(field, operator, value = nil) + ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Nodes::ConditionTreeLeaf + .new(field, operator, value) + end + + # Hand-written from the shape measured on a real workspace: the state comes + # embedded, the company as a bare id, and the parts ride along. + def ticket(id, overrides = {}) + { 'type' => 'ticket', 'id' => id, 'ticket_id' => "1#{id}", 'category' => 'request', + 'open' => true, 'is_shared' => false, 'created_at' => 1_700_000_000, 'updated_at' => 1_700_003_600, + 'admin_assignee_id' => 493_881, 'team_assignee_id' => 0, + 'company_id' => '696dd52099f73812610d9c7b', + 'ticket_state' => { 'type' => 'ticket_state', 'id' => '19', 'category' => 'in_progress', + 'internal_label' => 'En cours Tech', 'external_label' => 'Investigation en cours' }, + 'previous_ticket_state_id' => '14', + 'ticket_type' => { 'type' => 'ticket_type', 'id' => '1', 'name' => 'Bug' }, + 'contacts' => { 'type' => 'contact.list', 'contacts' => [{ 'type' => 'contact', 'id' => 'c1' }] }, + 'ticket_attributes' => { '_default_title_' => 'Facture manquante' }, + 'ticket_parts' => { 'type' => 'ticket_part.list', 'total_count' => 0, 'ticket_parts' => [] } } + .merge(overrides) + end + + def parts(*entries, total: nil) + { 'ticket_parts' => { 'type' => 'ticket_part.list', 'total_count' => total || entries.size, + 'ticket_parts' => entries } } + end + + def state_change(to, from: 'in_progress', at: 1_700_002_000, by: 'Alice', part_type: nil) + { 'type' => 'ticket_part', 'id' => "s#{at}", 'part_type' => part_type || 'ticket_state_updated_by_admin', + 'ticket_state' => to, 'previous_ticket_state' => from, 'created_at' => at, + 'author' => { 'type' => 'admin', 'id' => '1', 'name' => by, 'email' => 'alice@acme.test' } } + end + + def comment(at:, by: 'Alice', type: 'admin', part_type: 'comment') + { 'type' => 'ticket_part', 'id' => "c#{at}", 'part_type' => part_type, 'body' => 'Je regarde.', + 'created_at' => at, 'author' => { 'type' => type, 'id' => '1', 'name' => by } } + end + + def stub_search(*records, total: nil, body: nil) + answer = { 'type' => 'ticket.list', 'tickets' => records, 'total_count' => total || records.size, + 'pages' => { 'type' => 'pages', 'page' => 1 } } + + request = stub_request(:post, "#{base}/tickets/search") + request = request.with(body: hash_including(body)) if body + request.to_return(json(answer)) + end + + def rows(projection = nil, **options) + collection.list(nil, filter(**options), projection) + end + + describe 'schema' do + it 'is named IntercomTicket' do + expect(collection.name).to eq('IntercomTicket') + end + + # The state travels embedded, so its labels cost nothing and the row does + # not depend on IntercomTicketState to be readable. + it 'flattens the embedded state into its labels' do + expect(collection.fields.keys) + .to include('state_id', 'state_category', 'state_label', 'state_external_label', 'previous_state_id') + end + + it 'carries the attributes of every ticket type in union' do + expect(collection.fields.keys).to include('_default_title_', 'Due') + expect(collection.fields['Due'].column_type).to eq('Date') + end + + # `/tickets/search` filters none of these and ignores a sort without + # saying so, so nothing but the primary key may advertise anything. + it 'declares every column unfilterable and unsortable, except the primary key' do + others = collection.fields.except('id') + + expect(others.values.map(&:filter_operators).flatten.uniq).to be_empty + expect(others.values.map(&:is_sortable).uniq).to eq([false]) + end + + # An attribute overwriting a native column would show the attribute where + # the operator expects the ticket field. + it 'skips an attribute whose name a native column already carries' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + + collection = described_class.new(datasource, attributes: [attribute('category')]) + + expect(collection.fields['category'].column_type).to eq('String') + expect(ForestAdminDatasourceIntercom.logger).to have_received(:warn).with(/skips the ticket attribute/) + end + end + + describe '#list' do + # There is no GET /tickets at all: even an unfiltered list view goes + # through the search endpoint with a predicate that matches everything. + it 'reads the search endpoint with a predicate matching every ticket' do + stub_search(ticket('1')) + + rows(%w[id]) + + expect(WebMock).to have_requested(:post, "#{base}/tickets/search") + .with(body: hash_including('query' => { 'field' => 'created_at', 'operator' => '>', 'value' => '0' })) + end + + # Not the 150 the API accepts: a page carries every ticket's whole + # timeline, and there is no way to ask Intercom for less. + it 'asks for far fewer tickets than the API would allow' do + stub_search(ticket('1')) + + rows(%w[id]) + + pagination = hash_including('per_page' => described_class::MAX_TICKETS_PER_PAGE) + + expect(WebMock).to have_requested(:post, "#{base}/tickets/search") + .with(body: hash_including('pagination' => pagination)) + end + + it 'reads the records under the tickets key' do + stub_search(ticket('1'), ticket('2')) + + expect(rows(%w[id]).map { |row| row['id'] }).to eq(%w[1 2]) + end + + it 'flattens the ticket into the row the schema declares' do + stub_search(ticket('1')) + + expect(rows.first) + .to include('id' => '1', 'ticket_id' => '11', 'category' => 'request', 'open' => true, + 'state_id' => '19', 'state_category' => 'in_progress', 'state_label' => 'En cours Tech', + 'previous_state_id' => '14', 'ticket_type_name' => 'Bug', + 'company_id' => '696dd52099f73812610d9c7b', 'admin_assignee_id' => '493881') + end + + # Intercom keys the values by attribute name, which is what lets one + # collection display the union -- and what stops it from filtering on them. + it 'reads an attribute value by its name' do + stub_search(ticket('1')) + + expect(rows.first['_default_title_']).to eq('Facture manquante') + end + + # Forest lists the fields of a request in a comma-separated query + # parameter, so a column name carrying one splits the projection into + # fields no collection has -- a 400 before the page is ever read. The + # introspector renames such an attribute; the value is still read under the + # name Intercom keys it by. + it 'reads a renamed attribute under the name the payload uses' do + renamed = attribute('ID de l\'objet (immo, facture)', column_name: "ID de l'objet (immo facture)") + collection = described_class.new(datasource, attributes: [renamed]) + stub_search(ticket('1', 'ticket_attributes' => { 'ID de l\'objet (immo, facture)' => 'immo_42' })) + + row = collection.list(nil, filter, nil).first + + expect(row["ID de l'objet (immo facture)"]).to eq('immo_42') + end + + # The invariant behind the rename, asserted on the whole schema rather than + # on one column. + it 'publishes no column name a Forest query string could not carry' do + collection = described_class.new(datasource, attributes: [attribute('Scope', column_name: 'Scope')]) + + expect(collection.fields.keys.grep(/[,:]/)).to be_empty + end + + it 'leaves an attribute of another ticket type absent rather than empty' do + stub_search(ticket('1')) + + expect(rows.first['Due']).to be_nil + end + + it 'reads a date attribute as ISO8601 like every other Intercom date' do + stub_search(ticket('1', 'ticket_attributes' => { 'Due' => 1_700_000_000 })) + + expect(rows.first['Due']).to eq('2023-11-14T22:13:20Z') + end + + # A record detail goes to its own endpoint, which is not the search one. + it 'reads one ticket through the record endpoint' do + stub_request(:get, "#{base}/tickets/1").to_return(json(ticket('1'))) + + expect(rows(%w[id], condition_tree: leaf('id', operators::EQUAL, '1')).map { |row| row['id'] }).to eq(%w[1]) + end + + it 'refuses a condition it cannot honour' do + expect { rows(%w[id], condition_tree: leaf('state_category', operators::EQUAL, 'resolved')) } + .to raise_error(UnsupportedOperatorError, /cannot answer a condition/) + end + end + + describe '#aggregate' do + it 'counts through the total_count of the search, exactly' do + stub_search(ticket('1'), total: 81_142) + aggregation = ForestAdminDatasourceToolkit::Components::Query::Aggregation.new(operation: 'Count') + + expect(collection.aggregate(nil, filter, aggregation)).to eq([{ 'group' => {}, 'value' => 81_142 }]) + end + end + + describe 'the derived columns' do + # A ticket has no statistics block -- measured on 81 142 tickets -- so the + # closure date exists nowhere but in the parts, which ride along anyway. + it 'reads the closure from the last transition into a resolved state' do + stub_search(ticket('1', **parts(state_change('in_progress', from: 'submitted', at: 1_700_001_000), + state_change('resolved', at: 1_700_002_000, by: 'Alice')))) + + expect(rows.first).to include('closed_at' => '2023-11-14T22:46:40Z', 'closed_by_name' => 'Alice') + end + + # This workspace runs workflows: a closure done by automation carries + # another variant of the same event, and matching the admin one in full + # would make it invisible. + it 'reads a closure whatever the variant of the state-change event' do + stub_search(ticket('1', **parts(state_change('resolved', at: 1_700_002_000, + part_type: 'ticket_state_updated_by_workflow')))) + + expect(rows.first['closed_at']).to eq('2023-11-14T22:46:40Z') + end + + # Measured: a part can record a transition to the state the ticket was + # already in, and that is not an event. + it 'ignores a transition that changed nothing' do + stub_search(ticket('1', **parts(state_change('resolved', from: 'resolved', at: 1_700_002_000)))) + + expect(rows.first['closed_at']).to be_nil + end + + it 'keeps the last closure of a ticket that was reopened' do + stub_search(ticket('1', **parts(state_change('resolved', at: 1_700_001_000), + state_change('in_progress', from: 'resolved', at: 1_700_001_500), + state_change('resolved', at: 1_700_002_000)))) + + expect(rows.first['closed_at']).to eq('2023-11-14T22:46:40Z') + end + + it 'names the last responder and which side they are on' do + stub_search(ticket('1', **parts(comment(at: 1_700_001_000, by: 'Alice'), + comment(at: 1_700_002_000, by: 'Camille', type: 'contact')))) + + expect(rows.first) + .to include('last_reply_at' => '2023-11-14T22:46:40Z', 'last_responder_name' => 'Camille', + 'last_responder_type' => 'contact') + end + + # An internal note is a touch, not an answer: it would name as last + # responder someone who never wrote to the person waiting. + it 'ignores an internal note' do + stub_search(ticket('1', **parts(comment(at: 1_700_001_000, by: 'Alice'), + comment(at: 1_700_002_000, by: 'Bob', part_type: 'note')))) + + expect(rows.first['last_responder_name']).to eq('Alice') + end + + it 'leaves both columns empty on a ticket nothing happened to' do + stub_search(ticket('1')) + + expect(rows.first).to include('closed_at' => nil, 'last_responder_name' => nil) + end + + # Intercom keeps the 500 most recent parts. A resolved ticket whose + # transition fell out of that window has an *unknown* closure date, not an + # absent one -- a Date column cannot say it, so the log does. + it 'reports a resolved ticket whose timeline was truncated' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + resolved = { 'ticket_state' => { 'id' => '20', 'category' => 'resolved', 'internal_label' => 'Resolu' } } + stub_search(ticket('1', **resolved, **parts(comment(at: 1_700_001_000), total: 500))) + + rows(%w[id closed_at]) + + expect(ForestAdminDatasourceIntercom.logger).to have_received(:warn).with(/truncated their timeline/) + end + + it 'stays quiet when the closure is simply absent from a complete timeline' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + stub_search(ticket('1', **parts(comment(at: 1_700_001_000)))) + + rows(%w[id closed_at]) + + expect(ForestAdminDatasourceIntercom.logger).not_to have_received(:warn) + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/ticket_state_spec.rb b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/ticket_state_spec.rb new file mode 100644 index 000000000..88ab24ded --- /dev/null +++ b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/ticket_state_spec.rb @@ -0,0 +1,38 @@ +module ForestAdminDatasourceIntercom + RSpec.describe Collections::TicketState do + subject(:collection) { described_class.new(datasource) } + + let(:datasource) { Datasource.new(access_token: 's3cr3t', rate_limiter: nil) } + let(:base) { datasource.configuration.url } + + def filter + ForestAdminDatasourceToolkit::Components::Query::Filter.new + end + + def stub_ticket_states(*states) + stub_request(:get, "#{base}/ticket_states") + .to_return(status: 200, body: { 'type' => 'list', 'data' => states }.to_json, + headers: { 'Content-Type' => 'application/json' }) + end + + it 'is named IntercomTicketState' do + expect(collection.name).to eq('IntercomTicketState') + end + + # Two labels rather than one: what the support team sees, and what the + # customer is shown. + it 'exposes both labels of a state' do + expect(collection.fields.keys).to eq(%w[id category internal_label external_label archived]) + end + + it 'reads the endpoint and serializes a state' do + stub_ticket_states('type' => 'ticket_state', 'id' => '3', 'category' => 'submitted', + 'internal_label' => 'Waiting on triage', 'external_label' => 'We are on it', + 'archived' => false) + + expect(collection.list(nil, filter, nil)) + .to eq([{ 'id' => '3', 'category' => 'submitted', 'internal_label' => 'Waiting on triage', + 'external_label' => 'We are on it', 'archived' => false }]) + end + end +end diff --git a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/ticket_type_spec.rb b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/ticket_type_spec.rb new file mode 100644 index 000000000..38d1f490d --- /dev/null +++ b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/ticket_type_spec.rb @@ -0,0 +1,45 @@ +module ForestAdminDatasourceIntercom + RSpec.describe Collections::TicketType do + subject(:collection) { described_class.new(datasource) } + + let(:datasource) { Datasource.new(access_token: 's3cr3t', rate_limiter: nil) } + let(:base) { datasource.configuration.url } + + def filter + ForestAdminDatasourceToolkit::Components::Query::Filter.new + end + + def stub_types(*types) + stub_request(:get, "#{base}/ticket_types") + .to_return(status: 200, body: { 'type' => 'list', 'data' => types }.to_json, + headers: { 'Content-Type' => 'application/json' }) + end + + it 'is named IntercomTicketType' do + expect(collection.name).to eq('IntercomTicketType') + end + + it 'exposes what makes a ticket type readable' do + expect(collection.fields.keys).to eq(%w[id name description category icon archived]) + end + + # This endpoint uses the `data` envelope, unlike /admins and /teams. + it 'reads the endpoint through the data envelope' do + stub_types('type' => 'ticket_type', 'id' => '1', 'name' => 'Bug', 'description' => 'A bug', + 'category' => 'request', 'icon' => '🐛', 'archived' => false) + + expect(collection.list(nil, filter, nil)) + .to eq([{ 'id' => '1', 'name' => 'Bug', 'description' => 'A bug', 'category' => 'request', + 'icon' => '🐛', 'archived' => false }]) + end + + # The attribute definitions nested here are what the ticket collection reads + # to build its columns -- an attribute of the same name carries a different + # id from one type to the next -- and they are meaningless as a column. + it 'leaves the nested attribute definitions out of the schema' do + stub_types('id' => '1', 'ticket_type_attributes' => { 'type' => 'list', 'data' => [{ 'id' => '9' }] }) + + expect(collection.list(nil, filter, nil).first.keys).not_to include('ticket_type_attributes') + end + end +end diff --git a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/configuration_spec.rb b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/configuration_spec.rb new file mode 100644 index 000000000..64b9079a1 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/configuration_spec.rb @@ -0,0 +1,98 @@ +module ForestAdminDatasourceIntercom + RSpec.describe Configuration do + subject(:configuration) { described_class.new(access_token: 's3cr3t') } + + it 'defaults to the US host, since that is where a workspace lands unasked' do + expect(configuration.url).to eq('https://api.intercom.io') + end + + it 'pins the API version the spike ran against' do + expect(configuration.api_version).to eq('2.16') + end + + it 'points at the regional host it is given' do + expect(described_class.new(access_token: 's3cr3t', region: :eu).url).to eq('https://api.eu.intercom.io') + end + + it 'takes the region as a string too' do + expect(described_class.new(access_token: 's3cr3t', region: 'AU').url).to eq('https://api.au.intercom.io') + end + + it 'lets an explicit base_url win over the region, for a proxy or a mock server' do + configured = described_class.new(access_token: 's3cr3t', region: :eu, base_url: 'https://intercom.test/api/') + + expect(configured.url).to eq('https://intercom.test/api') + end + + it 'reports the subpath a base_url is mounted under' do + configured = described_class.new(access_token: 's3cr3t', base_url: 'https://intercom.test/api') + + expect(configured.base_path).to eq('/api') + end + + it 'reports no subpath against the API itself' do + expect(configuration.base_path).to eq('') + end + + describe 'validation' do + it 'refuses a missing access token' do + expect { described_class.new(access_token: nil) } + .to raise_error(ConfigurationError, /missing required config: access_token/) + end + + it 'refuses a blank access token' do + expect { described_class.new(access_token: ' ') } + .to raise_error(ConfigurationError, /access_token/) + end + + it 'names the regions it knows when handed one it does not' do + expect { described_class.new(access_token: 's3cr3t', region: :moon) } + .to raise_error(ConfigurationError, /unknown region :moon.*:us, :eu, :au/m) + end + + # A relative base_url makes Faraday resolve paths against the working + # directory, which surfaces much later as a failure naming nothing. + it 'refuses a base_url that is not absolute' do + expect { described_class.new(access_token: 's3cr3t', base_url: 'api.intercom.io') } + .to raise_error(ConfigurationError, /must be an absolute http\(s\) url/) + end + + it 'refuses a base_url that is not a url at all' do + expect { described_class.new(access_token: 's3cr3t', base_url: 'http://[bad') } + .to raise_error(ConfigurationError, /not a valid url/) + end + + it 'refuses an empty api_version, which would let the workspace default decide' do + expect { described_class.new(access_token: 's3cr3t', api_version: '') } + .to raise_error(ConfigurationError, /api_version cannot be empty/) + end + end + + describe 'defaults' do + it 'paces requests and retries unless told otherwise' do + expect(configuration).to have_attributes(rate_limiter: an_instance_of(RateLimiter), + retry_policy: an_instance_of(RetryPolicy)) + end + + it 'is patient on a request and impatient on the boot' do + expect(configuration).to have_attributes(timeout: 30, open_timeout: 5, boot_timeout: 10, + boot_open_timeout: 3) + end + + it 'takes the pacing out of the stack when handed no limiter' do + expect(described_class.new(access_token: 's3cr3t', rate_limiter: nil).rate_limiter).to be_nil + end + end + + describe '#inspect' do + it 'never prints the bearer token' do + expect(configuration.inspect).to include('[FILTERED]') + expect(configuration.inspect).not_to include('s3cr3t') + end + + it 'still names the host and version, which is what one inspects it for' do + expect(configuration.inspect).to include('https://api.intercom.io', '2.16') + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/datasource_spec.rb b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/datasource_spec.rb new file mode 100644 index 000000000..c5387e059 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/datasource_spec.rb @@ -0,0 +1,59 @@ +module ForestAdminDatasourceIntercom + RSpec.describe Datasource do + subject(:datasource) { described_class.new(access_token: 's3cr3t') } + + it 'boots without reaching Intercom' do + expect { datasource }.not_to raise_error + end + + # The reference collections come first: they are what turns an assignee id + # into a teammate and a state id into a label. Conversations follow, Tickets + # next. + it 'publishes the collections of the lot' do + expect(datasource.collections.keys) + .to eq(%w[IntercomAdmin IntercomTeam IntercomTicketType IntercomTicketState IntercomConversation + IntercomTicket]) + end + + # The one read a boot performs: the attributes a workspace declares on its + # ticket types are columns of the Tickets collection, and a ticket payload + # carries the values of its own type only, so they cannot be discovered from + # the records. + it 'introspects the ticket-type attributes while registering, and reads nothing else' do + datasource + + expect(WebMock).to have_requested(:get, /ticket_types/).once + expect(WebMock).not_to have_requested(:get, /conversations|admins|teams/) + end + + # A token without that permission costs the attribute columns, never the + # agent. + it 'boots without the attribute columns when the introspection is refused' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + stub_request(:get, /ticket_types/).to_return(status: 403, body: '{}', + headers: { 'Content-Type' => 'application/json' }) + + expect(datasource.get_collection('IntercomTicket').fields.keys).not_to include('_default_title_') + end + + it 'configures a client from the options it is handed' do + stub_ticket_types(base: 'https://api.eu.intercom.io') + configured = described_class.new(access_token: 's3cr3t', region: :eu, rate_limiter: nil) + + expect(configured.configuration.url).to eq('https://api.eu.intercom.io') + expect(configured.client).to be_a(Client) + end + + it 'refuses to boot on a configuration it cannot use' do + expect { described_class.new(access_token: nil) }.to raise_error(ConfigurationError) + end + + it 'names the collections it holds when printed' do + expect(datasource.inspect).to include('IntercomAdmin', 'IntercomTicketState') + end + + it 'never prints the token the client carries' do + expect(datasource.inspect).not_to include('s3cr3t') + end + end +end diff --git a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/pagination/cursor_walker_spec.rb b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/pagination/cursor_walker_spec.rb new file mode 100644 index 000000000..70529cc32 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/pagination/cursor_walker_spec.rb @@ -0,0 +1,173 @@ +module ForestAdminDatasourceIntercom + module Pagination + RSpec.describe CursorWalker do + subject(:walker) { described_class.new } + + let(:asked) { [] } + + def record(id) + { 'id' => id } + end + + def page(records, next_cursor: nil) + Client::Page.new(records: records, next_cursor: next_cursor, total_count: nil) + end + + # A page source Intercom's own shape: each page advertises the cursor of + # the next one, and the last advertises nothing. + def source(*pages) + queue = pages.dup + + lambda do |per_page, cursor| + asked << [per_page, cursor] + queue.shift || page([]) + end + end + + def ids(records) + records.map { |r| r['id'] } + end + + it 'serves a window one page already covers' do + records = walker.walk(offset: 0, limit: 2, &source(page([record('a'), record('b')], next_cursor: 'c1'))) + + expect(ids(records)).to eq(%w[a b]) + end + + it 'asks only for the records the window still needs' do + walker.walk(offset: 0, limit: 3, &source(page([record('a'), record('b'), record('c')]))) + + expect(asked).to eq([[3, nil]]) + end + + it 'walks pages until the window is covered, then slices the offset out' do + pages = source(page([record('a'), record('b')], next_cursor: 'c1'), + page([record('c'), record('d')], next_cursor: 'c2')) + + records = walker.walk(offset: 2, limit: 2, &pages) + + expect(ids(records)).to eq(%w[c d]) + expect(asked).to eq([[4, nil], [2, 'c1']]) + end + + it 'follows the cursor each page advertises' do + walker.walk(offset: 0, limit: 4, &source(page([record('a')], next_cursor: 'c1'), + page([record('b')], next_cursor: 'c2'), + page([record('c')]))) + + expect(asked.map(&:last)).to eq([nil, 'c1', 'c2']) + end + + it 'stops where Intercom stops advertising a next page' do + records = walker.walk(offset: 0, limit: 10, &source(page([record('a')]))) + + expect(ids(records)).to eq(%w[a]) + expect(asked.size).to eq(1) + end + + it 'stops on an empty page' do + records = walker.walk(offset: 0, limit: 10, &source(page([], next_cursor: 'c1'))) + + expect(records).to be_empty + end + + # None of this happens against Intercom today, but a walk driven by a + # remote value stops on its own terms rather than on the caps only. + it 'stops on a cursor it has already followed' do + pages = source(page([record('a')], next_cursor: 'loop'), + page([record('b')], next_cursor: 'loop'), + page([record('c')], next_cursor: 'loop')) + + walker.walk(offset: 0, limit: 10, &pages) + + expect(asked.size).to eq(2) + end + + # Intercom documents duplicates on a dataset that moves between two + # paginated requests, and conversations move constantly. Two rows carrying + # one id is what a list view renders as two identical lines. + it 'drops a record a previous page already served' do + pages = source(page([record('a'), record('b')], next_cursor: 'c1'), + page([record('b'), record('c')])) + + records = walker.walk(offset: 0, limit: 10, &pages) + + expect(ids(records)).to eq(%w[a b c]) + end + + it 'keeps records carrying no id rather than deciding they are not records' do + anonymous = source(page([{ 'email' => 'a@b.test' }, { 'email' => 'c@d.test' }])) + records = walker.walk(offset: 0, limit: 10, &anonymous) + + expect(records.size).to eq(2) + end + + it 'returns nothing, and asks nothing, for a limit of zero' do + records = walker.walk(offset: 0, limit: 0, &source(page([record('a')]))) + + expect(records).to be_empty + expect(asked).to be_empty + end + + it 'reads an offset past the end as an empty window rather than an error' do + records = walker.walk(offset: 50, limit: 10, &source(page([record('a')]))) + + expect(records).to be_empty + end + + it 'treats a negative offset as the beginning' do + records = walker.walk(offset: -5, limit: 1, &source(page([record('a')]))) + + expect(ids(records)).to eq(%w[a]) + end + + describe 'a limit of nil, which asks for everything past the offset' do + it 'walks to the end' do + pages = source(page([record('a')], next_cursor: 'c1'), page([record('b')])) + + expect(ids(walker.walk(offset: 0, limit: nil, &pages))).to eq(%w[a b]) + end + + it 'asks for the largest page Intercom accepts' do + walker.walk(offset: 0, limit: nil, &source(page([record('a')]))) + + expect(asked.first.first).to eq(Client::MAX_PER_PAGE) + end + end + + describe 'caps' do + before { allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) } + + it 'stops after the page it is allowed, and says the result is truncated' do + capped = described_class.new(max_pages: 2) + pages = source(page([record('a')], next_cursor: 'c1'), + page([record('b')], next_cursor: 'c2'), + page([record('c')], next_cursor: 'c3')) + + capped.walk(offset: 0, limit: nil, &pages) + + expect(asked.size).to eq(2) + expect(ForestAdminDatasourceIntercom.logger).to have_received(:warn).with(/truncated/) + end + + it 'stops on the record budget, and never asks for more than it has left' do + capped = described_class.new(max_records: 3) + pages = source(page([record('a'), record('b')], next_cursor: 'c1'), + page([record('c'), record('d')], next_cursor: 'c2')) + + capped.walk(offset: 0, limit: nil, &pages) + + expect(asked).to eq([[3, nil], [1, 'c1']]) + end + + # A walk that covered the window it was given hands back exactly that, + # and has nothing to report -- unlike one a cap cut short. + it 'stays quiet when the window was covered' do + walker.walk(offset: 0, limit: 1, &source(page([record('a')], next_cursor: 'c1'))) + + expect(ForestAdminDatasourceIntercom.logger).not_to have_received(:warn) + end + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/rate_limiter_spec.rb b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/rate_limiter_spec.rb new file mode 100644 index 000000000..78c234aa6 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/rate_limiter_spec.rb @@ -0,0 +1,150 @@ +module ForestAdminDatasourceIntercom + RSpec.describe RateLimiter do + # Sleeping moves the clock, the way it does outside a spec: a window waited + # out is a window that has refilled by the time the next request asks. + subject(:limiter) { build_limiter } + + let(:time) { [1_000.0] } + let(:slept) { [] } + + def build_limiter(**options) + sleeper = lambda do |seconds| + slept << seconds + time[0] += seconds + end + + described_class.new(now: -> { time[0] }, sleeper: sleeper, **options) + end + + # What Intercom answers with: the limit of the 10-second window, what is + # left of it, and the epoch second it refills at. + def headers(remaining:, reset_in: 4, limit: 1667) + { 'x-ratelimit-limit' => limit.to_s, + 'x-ratelimit-remaining' => remaining.to_s, + 'x-ratelimit-reset' => (time[0] + reset_in).to_i.to_s } + end + + it 'lets the first request through: the budget is what that request discovers' do + limiter.acquire + + expect(slept).to be_empty + end + + it 'lets a request through while the window still has room' do + limiter.observe(headers(remaining: 5)) + limiter.acquire + + expect(slept).to be_empty + end + + it 'waits for the reset once the window is spent' do + limiter.observe(headers(remaining: 0, reset_in: 4)) + limiter.acquire + + expect(slept).to eq([4.0]) + end + + it 'lets requests through again once the reset has passed' do + limiter.observe(headers(remaining: 0, reset_in: -1)) + limiter.acquire + + expect(slept).to be_empty + end + + # Several requests can be in flight before any of them answers, and a + # `remaining` that only moves on a response lets all of them through on the + # same stale figure. + it 'counts its own requests down rather than trusting the last response' do + limiter.observe(headers(remaining: 2, reset_in: 3)) + 3.times { limiter.acquire } + + expect(slept).to eq([3.0]) + end + + it 'ignores a response from a window older than the one it knows' do + limiter.observe(headers(remaining: 0, reset_in: 5)) + limiter.observe(headers(remaining: 100, reset_in: -10)) + limiter.acquire + + expect(slept).to eq([5.0]) + end + + it 'adopts a new window whole, generous remaining included' do + limiter.observe(headers(remaining: 0, reset_in: 2)) + limiter.observe(headers(remaining: 50, reset_in: 12)) + limiter.acquire + + expect(slept).to be_empty + end + + # A response that left Intercom before the requests now in flight were made + # must not hand their budget back. + it 'keeps the smaller remaining inside one window' do + limiter.observe(headers(remaining: 2, reset_in: 3)) + limiter.acquire + limiter.observe(headers(remaining: 99, reset_in: 3)) + 2.times { limiter.acquire } + + expect(slept).to eq([3.0]) + end + + it 'reads the headers whatever their case' do + limiter.observe('X-RateLimit-Remaining' => '0', 'X-RateLimit-Reset' => (time[0] + 6).to_i.to_s) + limiter.acquire + + expect(slept).to eq([6.0]) + end + + it 'ignores a response carrying no rate-limit headers at all' do + limiter.observe('content-type' => 'application/json') + limiter.acquire + + expect(slept).to be_empty + end + + it 'ignores a remaining that is not a number' do + limiter.observe('x-ratelimit-remaining' => 'many', 'x-ratelimit-reset' => (time[0] + 3).to_i.to_s) + limiter.acquire + + expect(slept).to be_empty + end + + it 'sleeps a reset out when it fits within the bound it was given' do + capped = build_limiter(max_wait: 2.0) + capped.observe(headers(remaining: 0, reset_in: 1)) + capped.acquire + + expect(slept).to eq([1.0]) + end + + describe 'a reset further out than one window' do + # Intercom's reset is a timestamp from its clock. One that far out is the + # two clocks disagreeing, not a window emptying, so the request goes + # through and the log says why -- waiting an hour would read as a hang. + before do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + limiter.observe(headers(remaining: 0, reset_in: 3_600)) + end + + it 'lets the request through instead of waiting it out' do + limiter.acquire + + expect(slept).to be_empty + end + + it 'says so once, not once per request' do + 3.times { limiter.acquire } + + expect(ForestAdminDatasourceIntercom.logger) + .to have_received(:warn).once.with(/rate-limit window is spent.*limit 1667/m) + end + end + + it 'sleeps for real when handed no sleeper' do + real = described_class.new(now: -> { 999.96 }) + real.observe('x-ratelimit-remaining' => '0', 'x-ratelimit-reset' => '1000') + + expect { real.acquire }.not_to raise_error + end + end +end diff --git a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/retry_policy_spec.rb b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/retry_policy_spec.rb new file mode 100644 index 000000000..060565df4 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/retry_policy_spec.rb @@ -0,0 +1,49 @@ +module ForestAdminDatasourceIntercom + RSpec.describe RetryPolicy do + describe '#to_faraday_options' do + subject(:options) { described_class.new.to_faraday_options } + + it 'retries the statuses worth another attempt' do + expect(options[:retry_statuses]).to eq([429, 500, 502, 503, 504]) + end + + # A 502 on the way back from a POST Intercom did perform would be replayed + # into a second reply on the conversation. + it 'only replays the verbs that change nothing' do + expect(options[:methods]).to eq(%i[get head options]) + end + + it 'replays a 429 whatever the verb, Intercom having rejected it unprocessed' do + expect(options[:retry_if].call({ status: 429 }, nil)).to be(true) + end + + it 'leaves any other status to the methods list' do + expect(options[:retry_if].call({ status: 502 }, nil)).to be(false) + end + + # faraday-retry abandons outright when Retry-After exceeds max_interval, + # so the cap has to cover Intercom's whole 10-second window. + it 'waits out a full rate-limit window' do + expect(options[:max_interval]).to be > RateLimiter::WINDOW + end + + it 'absorbs a dropped connection, which faraday-retry does not by default' do + expect(options[:exceptions]).to include(Faraday::ConnectionFailed) + end + end + + describe '.boot' do + subject(:options) { described_class.boot.to_faraday_options } + + it 'retries once: a boot read is never revisited, and never worth a long wait' do + expect(options[:max]).to eq(1) + end + + # Below a rate-limit window on purpose: past the cap faraday-retry gives + # up at once, which is what keeps a 429 from holding the Rails boot. + it 'gives up rather than waiting a 429 out' do + expect(options[:max_interval]).to be < RateLimiter::WINDOW + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/schema/ticket_attributes_introspector_spec.rb b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/schema/ticket_attributes_introspector_spec.rb new file mode 100644 index 000000000..a6119eaea --- /dev/null +++ b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/schema/ticket_attributes_introspector_spec.rb @@ -0,0 +1,164 @@ +module ForestAdminDatasourceIntercom + module Schema + RSpec.describe TicketAttributesIntrospector do + subject(:introspector) { described_class.new(Client.new(configuration)) } + + let(:configuration) { Configuration.new(access_token: 's3cr3t', rate_limiter: nil) } + let(:base) { configuration.url } + + def json(payload, status = 200) + { status: status, body: payload.to_json, headers: { 'Content-Type' => 'application/json' } } + end + + def attribute(name, id, data_type: 'string', archived: false) + { 'id' => id, 'name' => name, 'data_type' => data_type, 'archived' => archived } + end + + def ticket_type(id, name, *attributes) + { 'type' => 'ticket_type', 'id' => id, 'name' => name, + 'ticket_type_attributes' => { 'type' => 'list', 'data' => attributes } } + end + + def stub_types(*types) + stub_request(:get, "#{base}/ticket_types").to_return(json('type' => 'list', 'data' => types)) + end + + it 'reads one entry per attribute name' do + stub_types(ticket_type('1', 'Bug', attribute('Severity', '9001')), + ticket_type('2', 'Task', attribute('Due', '9002', data_type: 'datetime'))) + + expect(introspector.attributes.map(&:name)).to contain_exactly('Severity', 'Due') + end + + # Measured: two ticket types share the names `_default_title_` and + # `_default_description_` while carrying different attribute ids. A union + # column has no single id to be filtered by, which is why these ship + # unfilterable -- and why the ids are kept, since that is what a filter + # per ticket type will need. + it 'keeps the id each ticket type gives the same attribute name' do + stub_types(ticket_type('1', 'Bug', attribute('_default_title_', '14162161')), + ticket_type('2', 'Task', attribute('_default_title_', '14162165'))) + + expect(introspector.attributes.map(&:ids_by_ticket_type)) + .to eq([{ '1' => '14162161', '2' => '14162165' }]) + end + + # Forest lists the fields of a request in a comma-separated query + # parameter: a comma in a column name splits the projection into fields no + # collection has, and the agent rejects the page with a 400 before reading + # anything. Measured on a real workspace, several attributes carry one. + it 'takes the commas out of a column name, keeping the name the payload uses' do + stub_types(ticket_type('1', 'Bug', attribute("ID de l'objet (immo, facture, user)", '9001'))) + + expect(introspector.attributes.first) + .to have_attributes(name: "ID de l'objet (immo, facture, user)", + column_name: "ID de l'objet (immo facture user)") + end + + # A colon is how Forest names a field through a relation. + it 'takes a colon out too' do + stub_types(ticket_type('1', 'Bug', attribute('Scope: mobile', '9001'))) + + expect(introspector.attributes.first.column_name).to eq('Scope mobile') + end + + # Intercom hands the names back HTML-escaped, which is an artefact of where + # they were typed rather than part of the name. + it 'unescapes what Intercom escaped' do + stub_types(ticket_type('1', 'Bug', attribute('Ce que j'ai vérifié & validé', '9001'))) + + expect(introspector.attributes.first.column_name).to eq("Ce que j'ai vérifié & validé") + end + + it 'leaves a name that needs nothing alone' do + stub_types(ticket_type('1', 'Bug', attribute('Severity', '9001'))) + + expect(introspector.attributes.first).to have_attributes(name: 'Severity', column_name: 'Severity') + end + + # Two different attributes landing on one column would otherwise share an + # entry, and the second's values would be read under the first's name -- + # wrong values rather than missing ones. + it 'leaves out a second attribute that reads as an existing column' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + stub_types(ticket_type('1', 'Bug', attribute('Scope, mobile', '9001'), attribute('Scope mobile', '9002'))) + + expect(introspector.attributes.map(&:name)).to eq(['Scope, mobile']) + expect(ForestAdminDatasourceIntercom.logger).to have_received(:warn).with(/is left out/) + end + + it 'leaves out an attribute whose name is nothing but separators' do + stub_types(ticket_type('1', 'Bug', attribute(' , : ', '9001'))) + + expect(introspector.attributes).to be_empty + end + + it 'maps the Intercom data types onto what Forest renders' do + stub_types(ticket_type('1', 'Bug', attribute('n', '1', data_type: 'integer'), + attribute('d', '2', data_type: 'decimal'), + attribute('b', '3', data_type: 'boolean'), + attribute('t', '4', data_type: 'datetime'), + attribute('l', '5', data_type: 'list'), + attribute('f', '6', data_type: 'files'))) + + expect(introspector.attributes.map(&:column_type)).to eq(%w[Number Number Boolean Date String Json]) + end + + # Showing the value Intercom sent beats hiding a column because its type + # is one this datasource has not met yet. + it 'reads an unknown data type as a string rather than dropping the column' do + stub_types(ticket_type('1', 'Bug', attribute('x', '1', data_type: 'quantum'))) + + expect(introspector.attributes.map(&:column_type)).to eq(%w[String]) + end + + it 'leaves out an archived attribute, which is not offered any more' do + stub_types(ticket_type('1', 'Bug', attribute('Gone', '1', archived: true), attribute('Here', '2'))) + + expect(introspector.attributes.map(&:name)).to eq(%w[Here]) + end + + it 'leaves out an attribute with no name to be a column of' do + stub_types(ticket_type('1', 'Bug', attribute('', '1'))) + + expect(introspector.attributes).to be_empty + end + + it 'reads a ticket type declaring no attribute as declaring none' do + stub_types({ 'id' => '1', 'name' => 'Bug' }) + + expect(introspector.attributes).to be_empty + end + + # It runs while Rails is starting, so it waits far less than a request + # that already has a page on screen. + it 'reads through the boot connection' do + slow = Configuration.new(access_token: 's3cr3t', rate_limiter: nil, boot_timeout: 2, timeout: 30) + client = Client.new(slow) + stub_types + + described_class.new(client).attributes + + expect(client.send(:boot_connection).options.timeout).to eq(2) + end + + it 'reads once and remembers, a schema being built once' do + stub_types(ticket_type('1', 'Bug', attribute('Severity', '9001'))) + + 2.times { introspector.attributes } + + expect(WebMock).to have_requested(:get, "#{base}/ticket_types").once + end + + # A token without the ticket-types permission costs the attribute columns, + # never the boot of the agent. + it 'degrades to no attribute when the read is refused' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + stub_request(:get, "#{base}/ticket_types").to_return(json({ 'errors' => [{ 'code' => 'forbidden' }] }, 403)) + + expect(introspector.attributes).to eq([]) + expect(ForestAdminDatasourceIntercom.logger).to have_received(:warn).with(/boots without its attribute/) + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/throttle_spec.rb b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/throttle_spec.rb new file mode 100644 index 000000000..a2209a2a7 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/throttle_spec.rb @@ -0,0 +1,40 @@ +module ForestAdminDatasourceIntercom + RSpec.describe Throttle do + let(:limiter) { instance_double(RateLimiter, acquire: nil, observe: nil) } + let(:connection) do + Faraday.new(url: 'https://api.intercom.test') do |f| + f.use described_class, limiter: limiter + end + end + + before do + stub_request(:get, 'https://api.intercom.test/me') + .to_return(status: 200, body: '{}', + headers: { 'Content-Type' => 'application/json', 'x-ratelimit-remaining' => '7' }) + end + + it 'asks for room before the request leaves' do + connection.get('me') + + expect(limiter).to have_received(:acquire) + end + + it 'hands the window back what the response says about it' do + connection.get('me') + + expect(limiter).to have_received(:observe).with(hash_including('x-ratelimit-remaining' => '7')) + end + + # The 429 carries the most useful reset of all, so the observation cannot be + # limited to the responses that succeeded. + it 'observes a rejected response too' do + stub_request(:get, 'https://api.intercom.test/me') + .to_return(status: 429, body: '{}', + headers: { 'Content-Type' => 'application/json', 'x-ratelimit-remaining' => '0' }) + + connection.get('me') + + expect(limiter).to have_received(:observe).with(hash_including('x-ratelimit-remaining' => '0')) + end + end +end diff --git a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom_spec.rb b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom_spec.rb new file mode 100644 index 000000000..f38f4adfe --- /dev/null +++ b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom_spec.rb @@ -0,0 +1,49 @@ +RSpec.describe ForestAdminDatasourceIntercom do + describe 'VERSION' do + # The release `sed` in .releaserc.js only matches `VERSION = "x.y.z"`, and a + # format it misses is a version that stays behind with no CI failure to say so. + it 'is a double-quoted semantic version' do + expect(described_class::VERSION).to match(/\A\d+\.\d+\.\d+\z/) + end + end + + describe '.logger' do + around do |example| + previous = described_class.logger + example.run + described_class.logger = previous + end + + it 'defaults to a logger named after the package' do + described_class.logger = nil + + expect(described_class.logger.progname).to eq('forest_admin_datasource_intercom') + end + + it 'takes the logger it is handed' do + logger = Logger.new(File::NULL) + described_class.logger = logger + + expect(described_class.logger).to be(logger) + end + end + + describe 'errors' do + it 'reports a filter Intercom cannot express as a validation error' do + expect(described_class::UnsupportedOperatorError.new('nope')) + .to be_a(ForestAdminDatasourceToolkit::Exceptions::ValidationError) + end + + it 'carries the status and parsed body of a failed call' do + error = described_class::APIError.new('boom', status: 429, body: { 'type' => 'error.list' }) + + expect(error).to have_attributes(message: 'boom', status: 429, body: { 'type' => 'error.list' }) + end + + it 'leaves the status and body unset when the call failed before answering' do + error = described_class::APIError.new('timeout') + + expect(error).to have_attributes(status: nil, body: nil) + end + end +end diff --git a/packages/forest_admin_datasource_intercom/spec/spec_helper.rb b/packages/forest_admin_datasource_intercom/spec/spec_helper.rb new file mode 100644 index 000000000..056ee440e --- /dev/null +++ b/packages/forest_admin_datasource_intercom/spec/spec_helper.rb @@ -0,0 +1,59 @@ +require 'simplecov' +# JSON output is consumed by the qlty CI coverage step; HTML is for local +# inspection. simplecov-html and simplecov_json_formatter are required only +# in Gemfile-test, so guard the require for local Gemfile runs. +begin + require 'simplecov_json_formatter' + require 'simplecov-html' + SimpleCov.formatters = [SimpleCov::Formatter::JSONFormatter, SimpleCov::Formatter::HTMLFormatter] +rescue LoadError + # Local Gemfile run without the CI formatters; default text output is fine. +end + +SimpleCov.start do + add_filter '/spec/' + enable_coverage :branch + minimum_coverage 90 +end + +SimpleCov.coverage_dir 'coverage' + +require 'webmock/rspec' +require 'forest_admin_datasource_customizer' +require 'forest_admin_datasource_intercom' + +# Every payload the specs feed in is hand-written from the Intercom OpenAPI 2.16 +# spec, never captured from a workspace: a conversation body is personal data, +# and a fixture is read by everyone who clones the repo. +WebMock.disable_net_connect!(allow_localhost: true) + +# A datasource introspects the ticket-type attributes while it registers its +# collections, so every spec building one issues that read. The base url is not +# taken from the datasource on purpose: reading it would build the datasource, +# and boot the very read this stubs. +module IntercomBootStubs + def stub_ticket_types(*types, base: ForestAdminDatasourceIntercom::Configuration::REGION_HOSTS[:us]) + stub_request(:get, "#{base}/ticket_types") + .to_return(status: 200, body: { 'type' => 'list', 'data' => types }.to_json, + headers: { 'Content-Type' => 'application/json' }) + end +end + +RSpec.configure do |config| + config.include IntercomBootStubs + config.expect_with :rspec do |c| + c.syntax = :expect + end + config.mock_with :rspec do |m| + m.verify_partial_doubles = true + end + config.disable_monkey_patching! + config.warnings = false + config.order = :random + Kernel.srand config.seed + + config.before do + WebMock.reset! + stub_ticket_types + end +end From 5a00d7194460626ad2d705027a96c85acbfd3bf5 Mon Sep 17 00:00:00 2001 From: Christophe Brun Date: Thu, 3 Sep 2026 18:03:17 +0200 Subject: [PATCH 2/6] feat(datasource intercom): search filters and per-endpoint operator tables (lot 2) (#384) * fix(intercom): page id lookups and report explicit sorts Two findings from the lot 1 review, both places where a cursor collection answers something other than what it was asked for. An `id in [...]` read returned every record it fetched whatever page was asked for, so page 1 and page 2 of a related-record list rendered the same rows. The window is now cut out of the ids before they are read, which also spares the requests the discarded records cost: Intercom reads them one request each. `default_pk_sort?` read a symbol-keyed `false` as an absent key, so an explicit `?sort=-id` was taken for the ascending default the agent injects, and the warning about Intercom silently ignoring a sort never fired. Refs PRD-1118 Co-Authored-By: Claude Opus 5 (1M context) --- .../README.md | 126 +++++- .../bin/probe_search_fields | 252 ++++++++++++ .../lib/forest_admin_datasource_intercom.rb | 1 + .../client.rb | 20 +- .../collections/conversation.rb | 10 + .../collections/cursor_collection.rb | 150 +++++-- .../collections/ticket.rb | 12 +- .../query/caller_zone.rb | 56 +++ .../query/condition_tree_translator.rb | 150 +++++++ .../query/day_bounds.rb | 69 ++++ .../query/filter_value.rb | 156 +++++++ .../query/operator_table.rb | 79 ++++ .../query/search_fields.rb | 155 +++++++ .../query/search_fields.yml | 388 ++++++++++++++++++ .../collections/conversation_spec.rb | 217 ++++++++-- .../collections/ticket_spec.rb | 77 +++- .../query/condition_tree_translator_spec.rb | 194 +++++++++ .../query/filter_value_spec.rb | 248 +++++++++++ .../query/operator_table_spec.rb | 110 +++++ .../query/search_fields_spec.rb | 152 +++++++ .../spec/probe_search_fields_spec.rb | 171 ++++++++ 21 files changed, 2696 insertions(+), 97 deletions(-) create mode 100755 packages/forest_admin_datasource_intercom/bin/probe_search_fields create mode 100644 packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/query/caller_zone.rb create mode 100644 packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/query/condition_tree_translator.rb create mode 100644 packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/query/day_bounds.rb create mode 100644 packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/query/filter_value.rb create mode 100644 packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/query/operator_table.rb create mode 100644 packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/query/search_fields.rb create mode 100644 packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/query/search_fields.yml create mode 100644 packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/query/condition_tree_translator_spec.rb create mode 100644 packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/query/filter_value_spec.rb create mode 100644 packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/query/operator_table_spec.rb create mode 100644 packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/query/search_fields_spec.rb create mode 100644 packages/forest_admin_datasource_intercom/spec/probe_search_fields_spec.rb diff --git a/packages/forest_admin_datasource_intercom/README.md b/packages/forest_admin_datasource_intercom/README.md index 3505508e0..38503e773 100644 --- a/packages/forest_admin_datasource_intercom/README.md +++ b/packages/forest_admin_datasource_intercom/README.md @@ -66,7 +66,7 @@ degrades to no attribute column, and a collection whose endpoint answers 403 fai | Collection | Endpoint | Paginated | Countable | | --- | --- | --- | --- | -| `IntercomConversation` | `GET /conversations`, `GET /conversations/{id}` | cursor | yes, exactly | +| `IntercomConversation` | `GET /conversations`, `POST /conversations/search`, `GET /conversations/{id}` | cursor | yes, exactly | | `IntercomTicket` | `POST /tickets/search`, `GET /tickets/{id}` | cursor | yes, exactly | | `IntercomAdmin` | `GET /admins` | read whole | yes, exactly | | `IntercomTeam` | `GET /teams` | read whole | yes, exactly | @@ -82,8 +82,9 @@ this lot, and the only ones a chart may group by. The cost is bandwidth, not cor **Cursor** — conversations and tickets. What is in hand is a page of something far larger, so nothing is filtered or sorted in memory. Three routes and no fourth: no condition walks the listing, -`id equals X` reads the record through its own endpoint, and **anything else is refused** with a -message naming the lot that will answer it. +`id equals X` reads the record through its own endpoint, and anything else is translated into +Intercom's search DSL and walked through the search endpoint. What the translation cannot express is +**refused by name** — see [Filtering](#filtering). ## What the API cannot do, and what this does about it @@ -97,6 +98,9 @@ arrive as a 400 carrying the text. - **Duplicates on a moving dataset.** Intercom documents that records modified between two paginated requests can be served twice; the walk deduplicates by id. The missed counterpart is inherent to cursor pagination and cannot be repaired — it is documented rather than papered over. +- **A search takes no sort at all.** Neither search endpoint accepts one, so **no column of + `IntercomConversation` or `IntercomTicket` is sortable** and an explicit order is reported in the + log. The only collections Intercom sorts are the ones read whole, in memory. - **A sort is accepted and ignored.** Measured: `sort` on these endpoints raises nothing and changes nothing. Since the lack of support is undetectable at runtime, no column is declared sortable and a requested order is reported in the log. The rows come back in the order the API imposes. @@ -114,6 +118,111 @@ arrive as a 400 carrying the text. `/admins` under `admins`, `/teams` under `teams`. A response carrying neither the expected key nor `data` is refused rather than read as an empty page. +## Filtering + +`POST /conversations/search` and `POST /tickets/search` answer the condition trees Forest sends, on +the fields Intercom really filters and with the operators each endpoint really validates. Anything +else is **refused with a message naming what to change** — a condition dropped on the way out comes +back as an unfiltered page that looks filtered, which is the one answer this datasource must not +give. A refusal costs no request: it is raised before anything leaves the process. + +### The table is measured, not documented + +The fields a search endpoint filters are not the fields its specification lists. Measured: +`/tickets/search` refuses `company_id` with `invalid_field` although every ticket carries one. So +the source of truth is a committed table — `lib/forest_admin_datasource_intercom/query/search_fields.yml` +— one row per column, each carrying its provenance: + +| `source` | What it means | +| --- | --- | +| `measured` | observed against a real workspace, by `bin/probe_search_fields` or during the spike | +| `spec` | read off Intercom's documentation, and therefore still a candidate | + +Every `filter_operators` a column publishes is **derived** from that table, so a column cannot +advertise a filter the translator would then refuse, and a column the table does not carry +advertises nothing at all. + +To measure a workspace of your own: + +```bash +INTERCOM_ACCESS_TOKEN=... bin/probe_search_fields --endpoint tickets --out measured.yml +``` + +It sends one search per (field, operator) cell, reads Intercom's refusal codes — `invalid_field` for +a field the endpoint does not filter, `data_invalid` for an operator it refuses on that field — and +prints what the committed table promises that Intercom refuses, plus what Intercom accepts that the +table does not know about. It writes evidence rather than rewriting the table, which carries the +prose a generated file would drop. + +### A date filter is day-granular, and the day is the UTC one + +Intercom truncates a date search to the day, at the **UTC** boundary — measured, and against its own +documentation, which promises the workspace's timezone. `> V` answers from the start of the day +*after* V; `< V` answers before the start of V's own day. + +Sent as they come, the two bounds an interval is rewritten into cancel each other out: `today` +reaches the datasource as `> 00:00` and `< 23:59` of one day, which Intercom reads as "from +tomorrow" *and* "before today" — no rows at all, to the most ordinary filter there is. So each bound +is moved to the boundary that makes Intercom answer the day the filter named. + +What follows from that: + +- a bound naming a time of day matches **from the start of that day, or through the end of it**. It + is the granularity the Intercom interface itself filters on; +- a caller in UTC gets exactly the day they asked for; +- a caller in another timezone gets the UTC days their window overlaps — up to a day wider at each + end — and the agent logs that once per filter; +- a date column publishes `>` and `<` only, and no equality. Everything an operator actually uses — + `before`, `after`, `today`, `yesterday`, `past`, `future`, the whole `previous_*` family — is + rewritten by the agent into a pair of those bounds. An equality on an instant is what stays out, + and a day-granular filter could not have honoured it anyway. + +### What is filterable + +| Collection | Filterable on | +| --- | --- | +| `IntercomConversation` | `id`, `state`, `priority`, `open`, `read`, `title`, `admin_assignee_id`, `team_assignee_id`, `source_type`, `source_subject`, `source_body`, `source_delivered_as`, `source_author_email`, `closed_by_id`, `reopen_count`, `part_count`, `ai_agent_participated`, and the dates `created_at`, `updated_at`, `waiting_since`, `snoozed_until`, `closed_at`, `first_closed_at`, `first_contact_reply_at`, `last_contact_reply_at`, `last_admin_reply_at` | +| `IntercomTicket` | `id`, `open`, `category`, `ticket_type_id`, `admin_assignee_id`, `team_assignee_id`, `created_at`, `updated_at` | + +**The primary key** is filterable like any other column, but a filter naming it *alone* is not +answered by a search: `id equals X` and `id in [...]` read the record endpoint directly, one request +per record. The search answers it only when something else is filtered alongside it — a permission +scope, a segment, or a second filter. + +**Free-text search** is answered on `IntercomConversation` only, through `~` on `source.body` — the +message that opened the conversation. Intercom matches it **per word, not as a substring**: searching +`fact` does not find `facture`. `IntercomTicket` exposes no text column this endpoint matches and +refuses a search by name. + +### What is not filterable, and why + +- **the columns a ticket derives from its parts** — `closed_at`, `closed_by_name`, `last_reply_at`, + `last_responder_name`, `last_responder_type`. They exist nowhere in Intercom; `/tickets/search` + filters none of them and ignores a sort on them without a word; +- **the account of a ticket** — `company_id`, refused by the endpoint itself with `invalid_field`; +- **the ticket attributes** — filtered as `ticket_attribute.{id}`, and the same attribute carries a + different id per ticket type, so a union column has no single id to translate to. See + [Tickets](#tickets); +- **the tag names, the company name and the contact identity of a conversation** — read from + somewhere the search endpoint does not filter, or filtered by an id the column does not hold; +- **absence** — `present`, `blank` and `missing` are derived by the agent from an equality and + rewritten into a comparison with an empty value. Intercom's search matches values and has no + operator for the lack of one, so the rewritten condition is refused rather than sent as a + comparison against the empty string; +- **group-by**, on either cursor collection: there is no aggregate endpoint, and grouping over the + pages a walk collected would look exact while answering a fraction. + +### The limits of a search, checked before the request leaves + +Intercom nests a search **two levels** deep and takes **fifteen conditions per group**. Past either +it answers a 400 whose body names neither the limit nor the part of the filter that reached it, so +both are checked here and refused with a message naming what to simplify. + +Fifteen is reached without trying: a scope, a segment and an operator's own filter add up, and a +condition naming several values arrives expanded into **one condition per value** — Intercom accepts +no membership operator on these fields. Branches carrying a single condition are unwrapped and spend +no level. + ## Conversations The row carries what a queue is read for: state, priority, assignee and team ids, the company, the @@ -160,16 +269,18 @@ Four things to know about them: ceiling the transition falls out of the window. That case is detected and logged, since a Date column cannot say "unknown". -Both columns are **display only**, and not temporarily: `/tickets/search` filters on neither and -ignores a sort, so neither advertises an operator. +Both are **display only**, and not temporarily: `/tickets/search` filters on neither and ignores a +sort, so neither advertises an operator. The attributes a workspace declares on its ticket types are introspected once at boot and published as the **union** of every type's, keyed by name the way the payload is. Filtering one is a different matter: Intercom filters an attribute by id (`ticket_attribute.{id}`), and the same name carries a different id from one ticket type to the next — measured, `_default_title_` is `14162161` on one type and `14162165` on another. A union column has no single id to translate to, so filtering on a -ticket attribute means one collection per ticket type. The ids are kept per type for the lot that -will need them. +ticket attribute means one collection per ticket type — more collections in the interface, and a +schema that changes shape whenever the customer adds a type. Until that trade is worth paying for, +the attributes stay display-only and advertise no operator. The ids are kept per type so the day the +answer changes costs no second boot round trip. ## Rate limits @@ -215,7 +326,6 @@ Everything else is read when a collection is listed, so an agent boots whatever | Lot | What it brings | | --- | --- | -| 2 | Filter translation into Intercom's search DSL, free-text search, per-endpoint operator tables, UTC date bounds | | 3 | Writes and business actions: reply, close, snooze, reopen, assign, tag, convert | | 4 | Contacts and companies, and the relations promoted from today's denormalized columns | | 5 | Notes, tags, segments | diff --git a/packages/forest_admin_datasource_intercom/bin/probe_search_fields b/packages/forest_admin_datasource_intercom/bin/probe_search_fields new file mode 100755 index 000000000..2ba05fabf --- /dev/null +++ b/packages/forest_admin_datasource_intercom/bin/probe_search_fields @@ -0,0 +1,252 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +# Enumerates what Intercom's search endpoints really filter, and with which +# operators, against a real workspace. The answer is what `search_fields.yml` +# holds and what the schema derives its filters from -- and it cannot be read +# off the documentation: measured during lot 1, `/tickets/search` refuses +# `company_id` with `invalid_field` although a ticket carries one. +# +# INTERCOM_ACCESS_TOKEN=... bin/probe_search_fields --endpoint tickets +# INTERCOM_ACCESS_TOKEN=... bin/probe_search_fields --region eu --out measured.yml +# +# It sends one search per (field, operator) cell, asking for a single record, +# and reads Intercom's refusal codes: `invalid_field` for a field the endpoint +# does not filter at all -- the rest of its row is then skipped -- `data_invalid` +# for an operator it refuses on that field, `invalid_value` for a value shape it +# refuses. Read-only: a search changes nothing, and no payload is printed or +# written, only field names, operators and error codes. +# +# What it writes with `--out` is *evidence*, deliberately not the table itself: +# `search_fields.yml` carries the prose that says why a column stays refused, +# and a generated file would drop it. Merge the measurements into the table by +# hand, which is also where deciding to expose a newly discovered field belongs. + +$LOAD_PATH.unshift(File.expand_path('../lib', __dir__)) + +require 'forest_admin_datasource_intercom' +require 'optparse' + +module ProbeSearchFields + SearchFields = ForestAdminDatasourceIntercom::Query::SearchFields + + # A cell is probed with the value shapes its field plausibly takes, most + # likely first: a wrong shape is refused with `data_invalid` just like an + # unsupported operator, so a single attempt would report an operator as + # refused when only the value was wrong. + VALUES = { + 'date' => ['1', 1, '2026-01-01'], + 'number' => [0, '0'], + 'boolean' => [true, 'true'], + 'string' => ['forest-probe', 0], + 'text' => ['forest-probe'] + }.freeze + + LIST_OPERATORS = %w[IN NIN].freeze + + # A candidate carries no declared type -- discovering it is the point -- so + # one is guessed from its name, and the fallbacks above cover a wrong guess. + def self.guess_type(field) + case field + when /(_at|_since|_until)\z/ then 'date' + when /\Acount_|_count\z|\Atime_to_|_time\z/ then 'number' + when /\A(open|read|is_|has_|.*_participated)/ then 'boolean' + else 'string' + end + end + + class Probe + def initialize(client, endpoint) + @client = client + @endpoint = endpoint + end + + # Every operator of the alphabet on every field of the row: what the table + # already declares, plus the candidates. Nothing is assumed from the + # declared operators -- an operator missing from the table is exactly what + # this is here to find. + def run(field, type) + results = {} + + SearchFields::KNOWN_OPERATORS.each do |operator| + outcome = probe(field, operator, type) + return { unfilterable: outcome } if outcome[:code] == 'invalid_field' + + results[operator] = outcome + end + + { operators: results } + end + + private + + def probe(field, operator, type) + last = nil + + VALUES.fetch(type, VALUES['string']).each do |value| + outcome = attempt(field, operator, operator_value(operator, value)) + # `invalid_field` is about the field, not about what was sent with it: + # another value shape cannot make a field the endpoint does not filter + # appear, so the row ends here rather than paying a request per shape. + return outcome if outcome[:ok] || outcome[:code] == 'invalid_field' + + last = outcome + end + + last + end + + def attempt(field, operator, value) + @client.search_page(@endpoint.path, query: { 'field' => field, 'operator' => operator, 'value' => value }, + per_page: 1, list_key: list_key) + { ok: true } + rescue ForestAdminDatasourceIntercom::APIError => e + { ok: false, code: error_code(e), status: e.status } + end + + def operator_value(operator, value) + LIST_OPERATORS.include?(operator) ? [value] : value + end + + # `/tickets/search` answers under `tickets` and `/conversations/search` + # under `conversations`; a body read under the wrong key raises before the + # outcome is classified. + def list_key = @endpoint.name + + def error_code(error) + errors = error.body.is_a?(Hash) ? error.body['errors'] : nil + first = Array(errors).first + + (first.is_a?(Hash) ? first['code'] : nil) || "http_#{error.status}" + end + end + + class Report + def initialize(endpoint) + @endpoint = endpoint + @rows = {} + end + + def record(field, column, result) + @rows[field] = { 'column' => column, 'result' => result } + end + + # What the run says about the committed table, which is the only reason to + # run it: an operator the table promises and Intercom refuses is a filter + # the interface offers and the read cannot honour. + def print_diff + @rows.each do |field, row| + column = row['column'] + result = row['result'] + + if result[:unfilterable] + declared = column ? " <- table declares it on column '#{column}'" : '' + puts " #{field.ljust(42)} NOT FILTERABLE (#{result[:unfilterable][:code]})#{declared}" + next + end + + print_operators(field, column, result[:operators]) + end + end + + def to_yaml_document + { 'endpoint' => @endpoint.name, 'path' => @endpoint.path, 'measured_at' => Time.now.utc.strftime('%Y-%m-%d'), + 'fields' => @rows.to_h { |field, row| [field, measured_row(row)] } }.to_yaml + end + + private + + def print_operators(field, column, outcomes) + supported = outcomes.select { |_, outcome| outcome[:ok] }.keys + declared = column ? Array(@endpoint.field(column)&.operators) : [] + puts " #{field.ljust(42)} #{supported.empty? ? "(no operator answered)" : supported.join(" ")}" + + report_drift(declared - supported, supported - declared) + end + + def report_drift(promised, discovered) + puts " ! the table promises #{promised.join(", ")} here, Intercom refuses it" unless promised.empty? + puts " + Intercom also accepts #{discovered.join(", ")}" unless discovered.empty? + end + + def measured_row(row) + result = row['result'] + return { 'filterable' => false, 'code' => result[:unfilterable][:code] } if result[:unfilterable] + + { 'filterable' => true, + 'operators' => result[:operators].select { |_, outcome| outcome[:ok] }.keys, + 'refused' => result[:operators].reject { |_, outcome| outcome[:ok] } + .transform_values { |outcome| outcome[:code] } } + end + end + + class CLI + def self.call(argv) + new(options(argv)).run + end + + def self.options(argv) + options = { endpoints: SearchFields.endpoints, region: 'us', token: ENV.fetch('INTERCOM_ACCESS_TOKEN', nil) } + + OptionParser.new do |parser| + parser.banner = 'Usage: INTERCOM_ACCESS_TOKEN=... bin/probe_search_fields [options]' + parser.on('--endpoint NAME', "one of #{SearchFields.endpoints.join(", ")}") { |v| options[:endpoints] = [v] } + parser.on('--region NAME', 'us (default), eu or au') { |v| options[:region] = v } + parser.on('--token TOKEN', 'defaults to $INTERCOM_ACCESS_TOKEN') { |v| options[:token] = v } + parser.on('--out PATH', 'write the measurements as YAML evidence') { |v| options[:out] = v } + end.parse!(argv) + + options + end + + def initialize(options) + @options = options + end + + def run + abort('No token: set INTERCOM_ACCESS_TOKEN or pass --token.') if @options[:token].to_s.empty? + + documents = @options[:endpoints].map { |name| probe_endpoint(SearchFields.fetch(name)) } + write(documents) if @options[:out] + 0 + end + + private + + def probe_endpoint(endpoint) + puts "\n#{endpoint.path} -- #{endpoint.fields.size} declared field(s), #{endpoint.candidates.size} candidate(s)" + probe = Probe.new(client, endpoint) + report = Report.new(endpoint) + + targets(endpoint).each { |field, column, type| report.record(field, column, probe.run(field, type)) } + report.print_diff + report.to_yaml_document + end + + # The declared fields first, with the type the table gives them; then the + # candidates, whose type is guessed. A candidate already covered by a + # declared field is not probed twice. + def targets(endpoint) + declared = endpoint.fields.values.map { |field| [field.field, field.column, field.type] } + candidates = (endpoint.candidates - declared.map(&:first)).map { |field| [field, nil, guess_type(field)] } + + declared + candidates + end + + def guess_type(field) = ProbeSearchFields.guess_type(field) + + def client + @client ||= ForestAdminDatasourceIntercom::Client.new( + ForestAdminDatasourceIntercom::Configuration.new(access_token: @options[:token], region: @options[:region]) + ) + end + + def write(documents) + File.write(@options[:out], documents.join("\n")) + puts "\nWritten to #{@options[:out]}. Merge the measurements into search_fields.yml by hand: " \ + 'the table carries the prose a generated file would drop.' + end + end +end + +exit(ProbeSearchFields::CLI.call(ARGV)) if $PROGRAM_NAME == __FILE__ diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom.rb index 43d8fe9c9..b90198801 100644 --- a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom.rb +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom.rb @@ -5,6 +5,7 @@ require 'set' require 'time' require 'uri' +require 'yaml' require 'zeitwerk' require 'faraday' require 'faraday/retry' diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/client.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/client.rb index e9b278e3a..ecfe093b3 100644 --- a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/client.rb +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/client.rb @@ -61,15 +61,16 @@ def list_page(path, per_page:, starting_after: nil, params: {}, list_key: 'data' must_succeed(path) { to_page(get(path, query, boot: boot).body, path, list_key) } end - # One page of a search endpoint. The query is written by the caller rather - # than translated from a Forest filter -- that translation is lot 2 -- so - # what goes on the wire is what the caller asked for. - def search_page(path, query:, per_page:, starting_after: nil, list_key: 'data') + # One page of a search endpoint. The query is the one the condition-tree + # translator wrote, and it travels in the body; `params` is what still + # belongs in the query string -- `display_as` above all, which is not part + # of the search payload. + def search_page(path, query:, per_page:, starting_after: nil, list_key: 'data', params: {}) pagination = { 'per_page' => self.class.bounded_per_page(per_page) } pagination['starting_after'] = starting_after unless blank?(starting_after) body = { 'query' => query, 'pagination' => pagination } - must_succeed(path) { to_page(post(path, body).body, path, list_key) } + must_succeed(path) { to_page(post(path, body, params: params).body, path, list_key) } end # One record from its own endpoint. Raises on a 404 like on any other @@ -123,8 +124,13 @@ def get(path, params = nil, boot: false) (boot ? boot_connection : connection).get(path, params) end - def post(path, body, boot: false) - (boot ? boot_connection : connection).post(path, body) + def post(path, body, params: {}, boot: false) + http = boot ? boot_connection : connection + + http.post(path) do |request| + request.params.update(params) unless params.nil? || params.empty? + request.body = body + end end # Intercom serves the version its workspace defaults to when the pin is not diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/conversation.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/conversation.rb index 9c0d13fc6..dbe8a53d0 100644 --- a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/conversation.rb +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/conversation.rb @@ -25,12 +25,22 @@ class Conversation < CursorCollection def initialize(datasource) super(datasource, 'IntercomConversation') + # The one collection of this datasource Intercom matches text on: `~` on + # `source.body`, which is the message that opened the conversation. + enable_search end protected def list_endpoint = 'conversations' def list_key = 'conversations' + def searchable = 'conversations' + def search_column = 'source_body' + + # Sent on the search too, where Intercom does not document it: the bodies + # are HTML written by end customers (R10), and a parameter it ignores costs + # a query string while the one it honours saves every filtered row from + # coming back as markup. def read_params = { 'display_as' => 'plaintext' } # The contact identity and the timeline, each read only when the projection diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/cursor_collection.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/cursor_collection.rb index eb8fa4721..17c7d3c39 100644 --- a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/cursor_collection.rb +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/cursor_collection.rb @@ -11,10 +11,10 @@ module Collections # * no condition at all -- a list view -- walks the listing endpoint; # * `id equals X` reads the record through its own endpoint, which is what a # record detail is; - # * anything else is **refused**. Translating a Forest condition tree into - # Intercom's search DSL is lot 2, and until it exists a filter that cannot - # be honoured has to say so: an unfiltered page served in answer to a - # filter is the one failure this datasource is built to avoid. + # * anything else is translated into Intercom's search DSL and walked + # through the search endpoint. What the translator will not express, it + # refuses by name: an unfiltered page served in answer to a filter is the + # one failure this datasource is built to avoid. # # Counting is the exception that costs nothing: `total_count` is exact on # every response, filter included, so the record counter is one request. @@ -35,10 +35,10 @@ def initialize(datasource, name) enable_count end - def list(_caller, filter, projection) + def list(caller, filter, projection) warn_ignored_sort(filter&.sort) - records = fetch_records(filter) + records = fetch_records(caller, filter) rows = records.map { |record| project(serialize(record), projection) } enrich(records, rows, projection) rows @@ -48,10 +48,10 @@ def list(_caller, filter, projection) # and grouping over the pages a walk happened to collect would look exact # while answering a fraction. Refused here rather than through the # contract's NotImplementedError, which reads as an oversight. - def aggregate(_caller, filter, aggregation, _limit = nil) + def aggregate(caller, filter, aggregation, _limit = nil) refuse_unsupported_aggregation!(aggregation) - [{ 'group' => {}, 'value' => count_records(filter) }] + [{ 'group' => {}, 'value' => count_records(caller, filter) }] end protected @@ -63,6 +63,19 @@ def record_endpoint = list_endpoint def list_key = 'data' def read_params = {} + # The row of the measured table this collection is filtered through: what + # its columns may advertise, and what the translator is allowed to write. + def searchable = raise(NotImplementedError, "#{self.class} did not implement searchable") + + def search_endpoint + @search_endpoint ||= Query::SearchFields.fetch(searchable) + end + + # The column a free-text search is answered on, for a collection whose + # endpoint has one. Nil elsewhere, and a search is then refused rather than + # answered by a page that ignored it. + def search_column = nil + # One Intercom entity flattened into a record matching the schema. def serialize(_entity) = raise(NotImplementedError, "#{self.class} did not implement serialize") @@ -78,20 +91,32 @@ def max_page_size = Client::MAX_PER_PAGE # One page of the collection. A listing for conversations, a search for # tickets -- Intercom exposes no `GET /tickets` at all -- so the endpoint # and its shape belong to the collection, while walking it does not. - def read_page(per_page:, cursor:) - client.list_page(list_endpoint, per_page: [per_page, max_page_size].min, - starting_after: cursor, params: read_params, list_key: list_key) + def read_page(per_page:, cursor:, query: nil) + size = [per_page, max_page_size].min + + if query.nil? + client.list_page(list_endpoint, per_page: size, starting_after: cursor, + params: read_params, list_key: list_key) + else + client.search_page(search_endpoint.path, query: query, per_page: size, starting_after: cursor, + params: read_params, list_key: list_key) + end end - # A column of this tier advertises no filter and no sort, because the - # collection can honour neither -- except on the primary key, which is - # answered by the record endpoint rather than by a filter. A schema that - # advertised more would put filters in the interface that the read then - # refuses. Read-only for the same reason, on the write side. + # A column advertises exactly the filters the search endpoint answers on + # it, taken from the measured table and derived by the operator table -- + # never written by hand here, so a column cannot offer a filter the + # translator would refuse. A column the table does not carry advertises + # none, which is how a refusal is spelled in a schema. + # + # The primary key is the exception, and it is not a filter: `id equals X` + # and `id in [...]` are answered by the record endpoint. + # + # No column is sortable: Intercom takes no sort on either search endpoint + # and ignores the one it is sent. Read-only, this lot writing nothing. def add_column(name, type, is_primary_key: false) - operators = is_primary_key ? [Operators::EQUAL, Operators::IN] : [] add_field(name, ColumnSchema.new(column_type: type, - filter_operators: operators, + filter_operators: column_operators(name, is_primary_key), is_primary_key: is_primary_key, is_read_only: true, is_sortable: false, @@ -104,17 +129,54 @@ def walker private - def fetch_records(filter) + def column_operators(name, is_primary_key) + return [Operators::EQUAL, Operators::IN] if is_primary_key + + field = search_endpoint.field(name) + + field ? Query::OperatorTable.forest_operators(field) : [] + end + + def fetch_records(caller, filter) ids = id_lookup(filter) - return records_by_ids(ids) if ids + # The window is cut out of the ids rather than out of the records they + # read: Intercom reads them one request each, so paging after the read + # would pay for a whole page to hand back a slice of it -- and page 2 of + # a set larger than the cap would come back empty, the records it names + # having been dropped by the truncation before the window was applied. + return records_by_ids(page_window(ids, filter)) if ids + + listed_records(filter, translate(caller, filter)) + end - refuse_filter!(filter) unless browsing?(filter) + # The Intercom query a filter comes down to, or nil for a list view, which + # walks the listing endpoint instead. The free-text search is folded into + # the condition tree rather than added to the translated query: written as + # one tree, it is checked against the nesting Intercom allows like every + # other condition, instead of adding a level nothing counted. + def translate(caller, filter) + tree = combined_tree(filter) - listed_records(filter) + Query::ConditionTreeTranslator.call(tree, endpoint: search_endpoint, collection: name, + timezone: timezone_for(caller)) end - def browsing?(filter) - filter.nil? || (filter.condition_tree.nil? && blank_search?(filter)) + def combined_tree(filter) + conditions = [filter&.condition_tree, search_condition(filter)].compact + return conditions.first if conditions.size < 2 + + ForestAdminDatasourceToolkit::Components::Query::ConditionTree::ConditionTreeFactory.intersect(conditions) + end + + # A free-text search reaches Intercom as a condition on the one column its + # endpoint matches text on -- per word, not as a substring, which the + # README says rather than the interface implying otherwise. + def search_condition(filter) + return nil if blank_search?(filter) + + refuse_search! if search_column.nil? + + Leaf.new(search_column, Operators::CONTAINS, filter.search.to_s.strip) end def blank_search?(filter) @@ -158,10 +220,12 @@ def records_by_ids(ids) end end - def listed_records(filter) + def listed_records(filter, query) offset, limit = translate_page(filter&.page) - walker.walk(offset: offset, limit: limit) { |per_page, cursor| read_page(per_page: per_page, cursor: cursor) } + walker.walk(offset: offset, limit: limit) do |per_page, cursor| + read_page(per_page: per_page, cursor: cursor, query: query) + end end # A filter with no page asks for every record it matched; the walker reads @@ -176,13 +240,11 @@ def translate_page(page) # Exact, and one request: `total_count` counts what the filter names, not # what a page happened to hold. An id lookup counts the records it found, # which is cheaper still. - def count_records(filter) + def count_records(caller, filter) ids = id_lookup(filter) return records_by_ids(ids).size if ids - refuse_filter!(filter) unless browsing?(filter) - - page = read_page(per_page: 1, cursor: nil) + page = read_page(per_page: 1, cursor: nil, query: translate(caller, filter)) return page.total_count if page.total_count raise UnsupportedOperatorError, @@ -200,18 +262,10 @@ def refuse_unsupported_aggregation!(aggregation) 'Chart it on a collection read whole, or wait for the bounded group-by of the reporting lot.' end - def refuse_filter!(filter) - detail = if filter&.condition_tree - 'a condition on this collection' - else - 'a free-text search' - end - + def refuse_search! raise UnsupportedOperatorError, - "#{name} cannot answer #{detail} yet: it reads Intercom's listing endpoint, which takes no filter. " \ - 'Server-side filtering goes through the search endpoint and arrives with the filter translation. ' \ - 'Until then, remove the condition, the scope or the segment carrying it rather than being served a ' \ - 'page that would look filtered without being it.' + "#{name} cannot answer a free-text search: #{search_endpoint.path} matches values field by field, " \ + 'and this collection exposes no text column it searches. Filter on a column instead of searching.' end # Intercom accepts a `sort` on these endpoints and ignores it without a @@ -230,9 +284,17 @@ def warn_ignored_sort(sort) end def default_pk_sort?(clauses) - clauses.size == 1 && - (clauses.first[:field] || clauses.first['field']).to_s == primary_key && - (clauses.first[:ascending] || clauses.first['ascending']) != false + return false unless clauses.size == 1 + + clause = clauses.first + return false unless (clause[:field] || clause['field']).to_s == primary_key + + # `key?` rather than `||`: a descending clause carries `false`, which an + # `||` fallback reads as "absent" -- so an explicit `?sort=-id` would be + # taken for the ascending default the agent injects, and the one order + # Intercom silently drops would go unreported. + ascending = clause.key?(:ascending) ? clause[:ascending] : clause['ascending'] + ascending != false end def warn_truncated_ids(asked) diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/ticket.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/ticket.rb index 69dcf6953..2569e4552 100644 --- a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/ticket.rb +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/ticket.rb @@ -38,16 +38,16 @@ def initialize(datasource, attributes: []) protected - def list_endpoint = 'tickets/search' def record_endpoint = 'tickets' def list_key = 'tickets' + def searchable = 'tickets' def max_page_size = MAX_TICKETS_PER_PAGE - # A search rather than a listing, which is the whole reason this hook - # exists. - def read_page(per_page:, cursor:) - client.search_page(list_endpoint, query: MATCH_EVERY_TICKET, list_key: list_key, - per_page: [per_page, max_page_size].min, starting_after: cursor) + # Intercom exposes no `GET /tickets`, so a list view searches too: with the + # filter it was given, or with the predicate that matches everything when + # it was given none. + def read_page(per_page:, cursor:, query: nil) + super(per_page: per_page, cursor: cursor, query: query || MATCH_EVERY_TICKET) end def enrich(records, rows, projection) diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/query/caller_zone.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/query/caller_zone.rb new file mode 100644 index 000000000..f06eac6f8 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/query/caller_zone.rb @@ -0,0 +1,56 @@ +module ForestAdminDatasourceIntercom + module Query + # The timezone a filter was written in, and what reading a value in it + # costs when the caller names one nothing knows. + # + # Split from FilterValue for the reason DayBounds was: that class knows + # what shape a field expects on the wire, and whether a value carries the + # caller's wall clock or the server's is a different question. + class CallerZone + # Kept stripped rather than only checked stripped: as it came, a + # `" Europe/Paris "` passes the blank guard and then fails the zone + # lookup, and a day boundary lands an offset away from where the filter + # meant it. + attr_reader :name + + def initialize(identifier) + stripped = identifier.to_s.strip + @name = stripped.empty? ? 'UTC' : stripped + end + + # A day with no time of day is the caller's day, not the server's: it is + # the timezone the filter was written in that says when that day starts. + def start_of_day(date) + read('the day boundary') { Time.zone.local(date.year, date.month, date.day).to_i } + end + + # A timestamp carrying no offset is the caller's wall clock. `FilterFactory` + # writes the bounds of a previous period as `%Y-%m-%d %H:%M:%S`, so a chart + # comparing to the previous month sent a midnight read in whatever timezone + # the process happened to run in -- and a midnight moved by any offset at + # all lands on another UTC day once truncated, which is a whole day of rows + # beside the ones asked for. + # + # A timestamp carrying an offset is untouched, which is every operator the + # toolkit rewrites into a pair of bounds: those come through as UTC ISO8601 + # and read the same in any zone. + def timestamp(value) + read('the timestamp') { Time.zone.parse(value).to_i } + end + + private + + # Falling back to UTC silently would move a boundary by the offset, which + # is the failure a timezone is read for in the first place. + def read(what, &block) + Time.use_zone(@name, &block) + rescue ArgumentError + ForestAdminDatasourceIntercom.logger.warn( + "[forest_admin_datasource_intercom] unknown timezone #{@name.inspect}, reading #{what} of a date " \ + 'filter in UTC instead.' + ) + Time.use_zone('UTC', &block) + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/query/condition_tree_translator.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/query/condition_tree_translator.rb new file mode 100644 index 000000000..3418a7951 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/query/condition_tree_translator.rb @@ -0,0 +1,150 @@ +module ForestAdminDatasourceIntercom + module Query + # Turns a Forest condition tree into the query `POST /conversations/search` + # and `POST /tickets/search` take: + # + # leaf -> { 'field' => ..., 'operator' => ..., 'value' => ... } + # branch -> { 'operator' => 'AND' | 'OR', 'value' => [...] } + # + # What it will not translate, it refuses. A condition dropped on the way to + # Intercom comes back as a page of unfiltered records that looks filtered, + # and every refusal below therefore names the field, the operator, or the + # thing to change -- an operator reads that message and nothing else. + # + # Which field the endpoint filters, and with which operator, is not decided + # here: it is `search_fields.yml`, measured against a real workspace. This + # walks the tree and formats what the table allows. + class ConditionTreeTranslator + Branch = ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Nodes::ConditionTreeBranch + Leaf = ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Nodes::ConditionTreeLeaf + + AGGREGATORS = { 'and' => 'AND', 'or' => 'OR' }.freeze + + # Intercom nests a search two levels deep and takes fifteen conditions per + # group. Both are checked here rather than left to the API: over either, + # Intercom answers a 400 whose body names neither the limit nor the part of + # the filter that reached it, and an operator reading it has no way of + # knowing that their segment plus their scope plus their own filter is what + # went over. + MAX_DEPTH = 2 + MAX_GROUP_SIZE = 15 + + def self.call(condition_tree, endpoint:, collection:, timezone: nil) + return nil if condition_tree.nil? + + new(endpoint: endpoint, collection: collection, timezone: timezone).translate(condition_tree) + end + + def initialize(endpoint:, collection:, timezone: nil) + @endpoint = endpoint + @collection = collection + @value = FilterValue.new(collection: collection, timezone: timezone) + end + + def translate(node, depth = 1) + case node + when Branch then translate_branch(node, depth) + when Leaf then translate_leaf(node) + else raise UnsupportedOperatorError, "#{@collection} cannot read #{node.class} as a condition." + end + end + + private + + def translate_branch(branch, depth) + conditions = Array(branch.conditions) + refuse_empty_branch!(branch) if conditions.empty? + + # Read before the unwrap below, so a branch is refused on the aggregator + # it carries rather than on how many conditions it holds. + operator = aggregator(branch) + + # A branch holding one condition needs no group of its own. The agent + # builds a tree one branch at a time -- a scope, then a segment, then the + # operator's own filter -- and the nesting Intercom allows is shallow + # enough that a wrapper around nothing is a level worth not spending. + return translate(conditions.first, depth) if conditions.size == 1 + + refuse_too_deep!(depth) if depth > MAX_DEPTH + refuse_too_wide!(branch, conditions.size) if conditions.size > MAX_GROUP_SIZE + + { 'operator' => operator, 'value' => conditions.map { |condition| translate(condition, depth + 1) } } + end + + # What reaches this depth is a group inside a group inside a group. The + # message names the shape rather than a number, since the tree an operator + # can act on is the segment and the scope they wrote, not the one the agent + # assembled out of them. + def refuse_too_deep!(depth) + raise UnsupportedOperatorError, + "#{@collection} cannot answer this filter: Intercom nests a search #{MAX_DEPTH} levels deep and this " \ + "one reaches #{depth}. A group inside a group inside a group is one level too many -- flatten the " \ + 'segment, the scope or the filter carrying the innermost one.' + end + + # Fifteen is reached without trying: a scope, a segment and a filter add up, + # and a condition naming several values is expanded into one condition per + # value on the way here, Intercom taking no membership operator on these + # fields. + def refuse_too_wide!(branch, size) + raise UnsupportedOperatorError, + "#{@collection} cannot answer this filter: Intercom takes #{MAX_GROUP_SIZE} conditions per group and " \ + "this #{branch.aggregator} carries #{size}. A filter naming several values counts one condition per " \ + 'value here, so narrowing the list, the segment or the scope is what brings it back under the limit.' + end + + def aggregator(branch) + AGGREGATORS[branch.aggregator.to_s.downcase] || + raise(UnsupportedOperatorError, + "#{@collection} cannot read #{branch.aggregator.inspect} as a condition tree aggregator; " \ + "expected 'And' or 'Or'.") + end + + def translate_leaf(leaf) + field = @endpoint.field(leaf.field.to_s) || refuse_unfilterable!(leaf.field.to_s) + spelling = OperatorTable.intercom_operator(field, leaf.operator) || refuse_operator!(leaf, field) + + { 'field' => field.field, 'operator' => spelling, 'value' => @value.call(leaf, field, spelling) } + end + + # A column the endpoint does not filter, and the reason it does not, taken + # from the table when it carries one: those reasons are the difference + # between "no" and a message an operator can do something with. + def refuse_unfilterable!(column) + raise UnsupportedOperatorError, "#{@collection} cannot filter #{column.inspect}: #{unfilterable_reason(column)}" + end + + def unfilterable_reason(column) + return relation_reason(column) if column.include?(':') + + refusal = @endpoint.refusal(column) + return refusal.reason if refusal + + "#{@endpoint.path} takes no filter on it. Filter on one of: #{@endpoint.filterable_columns.join(", ")}." + end + + # A relation reaches the translator as `relation:field`. None of the + # collections this endpoint serves declares one yet, so the condition can + # only come from a scope or a segment written against a schema this + # datasource does not have. + def relation_reason(column) + "#{@collection} declares no relation, so #{column.inspect} names a field it cannot reach. Filter on one " \ + "of its own columns: #{@endpoint.filterable_columns.join(", ")}." + end + + def refuse_operator!(leaf, field) + supported = OperatorTable.forest_operators(field) + + raise UnsupportedOperatorError, + "#{@collection} cannot filter #{leaf.field.inspect} with #{leaf.operator.inspect}: " \ + "#{@endpoint.path} answers #{supported.join(", ")} on #{field.field.inspect} and nothing else." + end + + def refuse_empty_branch!(branch) + raise UnsupportedOperatorError, + "#{@collection} was given a #{branch.aggregator} branch carrying no condition, which names no record " \ + 'and no filter.' + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/query/day_bounds.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/query/day_bounds.rb new file mode 100644 index 000000000..c5df547db --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/query/day_bounds.rb @@ -0,0 +1,69 @@ +module ForestAdminDatasourceIntercom + module Query + # Where a date bound lands once Intercom's day truncation is accounted for. + # + # Intercom truncates a date search to the day, at the **UTC** boundary -- + # measured, and it contradicts the documentation, which promises the + # workspace's timezone. `> V` answers from the start of the day *after* V, + # and `< V` answers before the start of V's own day. + # + # Sent as they come, the two bounds of an interval cancel each other out: + # `today` reaches here as `> 00:00` and `< 23:59` of one day, which Intercom + # reads as "from tomorrow" and "before today" -- an empty answer to the most + # ordinary filter there is. So each bound is moved to the day boundary that + # makes Intercom answer the day the filter named: + # + # `>` V -> the day before V's day, so the answer starts at V's day; + # `<` V -> the day after V's day, so the answer runs through V's day, + # except when V already sits on a boundary, where V's day is + # exactly what was asked to be left out. + # + # The window is therefore day-granular: a bound naming a time of day matches + # from the start of that day, or through the end of it. That is the + # granularity the Intercom interface filters on, and the README says so. + # + # Split from FilterValue, which knows how to read a date out of whatever a + # condition carries but has no business knowing that Intercom answers a + # different question from the one it was asked. + class DayBounds + UTC_DAY = 86_400 + + def initialize(collection:, timezone:) + @collection = collection + @timezone = timezone + end + + def call(seconds, spelling) + day = seconds - (seconds % UTC_DAY) + report_utc_day(seconds, day) + + return day - UTC_DAY if spelling == '>' + + seconds == day ? seconds : day + UTC_DAY + end + + private + + # A window written in another timezone is answered on the UTC days it + # overlaps, which is up to a day wider at each end. Reported once per + # filter rather than per bound: what an operator needs to know is that the + # day boundary is not theirs, not how many bounds crossed it. + def report_utc_day(seconds, day) + return if @reported || same_day_locally?(seconds, day) + + @reported = true + ForestAdminDatasourceIntercom.logger.warn( + "[forest_admin_datasource_intercom] #{@collection} was filtered on a date written in #{@timezone}, and " \ + 'Intercom truncates a date search to the UTC day whatever the workspace timezone says. The rows come ' \ + 'back for the UTC days the window overlaps, which is up to a day wider at each end.' + ) + end + + def same_day_locally?(seconds, day) + Time.use_zone(@timezone) { Time.zone.at(seconds).to_date } == Time.at(day).utc.to_date + rescue ArgumentError + true + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/query/filter_value.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/query/filter_value.rb new file mode 100644 index 000000000..c14be8fc3 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/query/filter_value.rb @@ -0,0 +1,156 @@ +module ForestAdminDatasourceIntercom + module Query + # How the value of a Forest condition reaches Intercom's search DSL, and + # every way it can fail to. Split from the translator, which knows the shape + # of the tree but not what a field expects on the wire. + # + # Intercom types its filter values: a date is epoch seconds, a number is a + # number, a flag is a boolean. A value of the wrong shape is refused with + # `data_invalid`, the same code an unsupported operator returns, so the + # conversion belongs here rather than in a rescue reading error codes. + class FilterValue + # What the frontend sends for a Dateonly column, and what a segment written + # in Ruby carries: a day with no time of day, which is midnight in the + # timezone of whoever wrote the filter rather than in the server's. + DATE_ONLY = /\A\d{4}-\d{2}-\d{2}\z/ + + INTEGER = /\A-?\d+\z/ + + def initialize(collection:, timezone: nil) + @collection = collection + @zone = CallerZone.new(timezone) + @day_bounds = DayBounds.new(collection: collection, timezone: @zone.name) + end + + def call(leaf, field, spelling) + return list(leaf, field) if OperatorTable.list_operator?(spelling) + + refuse_absence!(leaf, field) if blank?(leaf.value) + + value = scalar(leaf.value, leaf, field) + field.type == 'date' ? @day_bounds.call(value, spelling) : value + end + + private + + # Dropping the blanks would answer a different question: `not_in [nil, + # 'open']` was asked to exclude the records carrying neither and would come + # back including them. An empty list is as bad the other way round, being a + # filter that matches everything. + def list(leaf, field) + values = Array(leaf.value) + refuse_empty_list!(leaf) if values.empty? + refuse_absence!(leaf, field) if values.any? { |value| blank?(value) } + + values.map { |value| scalar(value, leaf, field) } + end + + def scalar(value, leaf, field) + case field.type + when 'date' then epoch(value, leaf) + when 'number' then number(value, leaf) + when 'boolean' then boolean(value, leaf) + else value.to_s + end + end + + # Intercom stores and compares its dates as epoch seconds. What arrives + # here is an ISO8601 string most of the time -- the frontend sends one, and + # so does every interval operator the toolkit rewrites into a pair of + # bounds -- but a scope or a segment written in Ruby carries a Time or a + # Date, and neither has a timezone of its own. + def epoch(value, leaf) + case value + when DateTime then value.to_time.to_i + when Date then @zone.start_of_day(value) + when Time then value.to_i + when Numeric then seconds(value, leaf) + when String then parse(value, leaf) + else refuse_value!(leaf, value, 'a date') + end + end + + # An Infinity or a NaN -- a cast that overflowed above this datasource -- + # makes `Integer()` raise a FloatDomainError naming a float where the + # operator asked for a date, and a Complex a RangeError, which is the + # same class one level up. Refused like every other value the field + # cannot take, for the reason `number` refuses them too. + def seconds(value, leaf) + Integer(value) + rescue RangeError, TypeError + refuse_value!(leaf, value, 'a date') + end + + # `Time.parse` is called for what it refuses, not for what it returns: + # it raises on a string naming no date, where `Time.zone.parse` answers + # today -- a filter on `last tuesday` coming back as a filter on today is + # the silent wrong answer this datasource exists not to give. + def parse(value, leaf) + return @zone.start_of_day(Date.parse(value)) if DATE_ONLY.match?(value) + + Time.parse(value) + @zone.timestamp(value) + rescue ArgumentError, TypeError + refuse_value!(leaf, value, 'a date') + end + + # The agent casts every Number column with `to_f`, so an integer field + # would be filtered with `42.0` -- a form none of its values carry. A float + # with nothing after the point travels as the integer it is. + # + # A cast that overflowed to Infinity, or a NaN, is refused rather than + # passed on: the JSON encoder raises on both, a step later, as a 500 naming + # nothing the operator can act on. + def number(value, leaf) + case value + when Integer then value + when Float then finite(value, leaf) + when String then value.match?(INTEGER) ? value.to_i : finite(Float(value, exception: false), leaf) + else refuse_value!(leaf, value, 'a number') + end + end + + def finite(value, leaf) + refuse_value!(leaf, value, 'a number') unless value.is_a?(Float) && value.finite? + + value == value.to_i ? value.to_i : value + end + + def boolean(value, leaf) + case value + when true, false then value + when 'true' then true + when 'false' then false + else refuse_value!(leaf, value, 'a true or a false') + end + end + + def blank?(value) = value.nil? || value.to_s.empty? + + # `present`, `blank` and `missing` are derived by the agent from an + # equality, above this datasource, and rewritten into a comparison with an + # empty value. Intercom's search matches values and has no spelling for the + # absence of one, so the rewritten condition is refused here rather than + # sent as a comparison against the empty string -- which Intercom would + # answer as if it were a value of its own. + def refuse_absence!(leaf, field) + raise UnsupportedOperatorError, + "#{@collection} cannot filter #{leaf.field.inspect} for absence: Intercom's search matches values " \ + "and has no operator for the lack of one, so a #{leaf.operator} condition on " \ + "#{field.field.inspect} cannot be translated. Filter on a value instead." + end + + def refuse_empty_list!(leaf) + raise UnsupportedOperatorError, + "#{@collection} was asked to filter #{leaf.field.inspect} with #{leaf.operator} and an empty list, " \ + 'which names no record and no filter. Pass at least one value.' + end + + def refuse_value!(leaf, value, expected) + raise UnsupportedOperatorError, + "#{@collection} cannot filter #{leaf.field.inspect} with #{value.inspect}: Intercom expects " \ + "#{expected} on this field." + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/query/operator_table.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/query/operator_table.rb new file mode 100644 index 000000000..e08f14f74 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/query/operator_table.rb @@ -0,0 +1,79 @@ +module ForestAdminDatasourceIntercom + module Query + # How a Forest operator is spelled in Intercom's search DSL, per kind of + # field. Two things read this table and they must never disagree: the schema, + # which publishes a column's `filter_operators`, and the translator, which + # writes the filter. Both go through `forest_operators` and + # `intercom_operator`, so a column cannot advertise a filter the translator + # would then refuse. + # + # What an endpoint really accepts on a given field is not here -- that is + # `search_fields.yml`, measured. This is only the spelling, and the set is + # narrowed by the table before anything is published. + module OperatorTable + Operators = ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Operators + + EQUALITY = { Operators::EQUAL => '=', Operators::NOT_EQUAL => '!=', + Operators::IN => 'IN', Operators::NOT_IN => 'NIN' }.freeze + + # `contains` and `i_contains` both land on `~`: Intercom documents one + # substring operator and no case semantics for it, and the frontend sends + # either spelling depending on the column. Declaring one alone would leave + # the other refused at read time on a field Intercom does filter. + # + # `not_i_contains` is deliberately absent although Intercom would answer it + # the same way as `not_contains`: the toolkit's own `Rules` does not allow + # it on a String column, so publishing it would put a filter in the + # interface that the agent rejects before this datasource is ever reached + # -- the failure PRD-989 describes. + SUBSTRING = { Operators::CONTAINS => '~', Operators::I_CONTAINS => '~', + Operators::NOT_CONTAINS => '!~', + Operators::STARTS_WITH => '^', Operators::ENDS_WITH => '$' }.freeze + + BOUNDS = { Operators::GREATER_THAN => '>', Operators::LESS_THAN => '<' }.freeze + + # A date field carries the two bounds and nothing else, even where the + # endpoint accepts `=`, `!=`, `>=` and `<=` -- measured, it does on both + # search endpoints. The reason is on the agent's side: declaring `equal` on + # a Date column makes the toolkit republish `in`, which its own validator + # then refuses (PRD-989), so the interface would offer a date filter + # answered by a 400 having nothing to do with Intercom. + # + # Nothing is lost that an operator can see. From the two bounds the toolkit + # derives `before`, `after`, `today`, `yesterday`, `past`, `future` and the + # whole `previous_*` family -- twenty operators, all rewritten into a pair + # of bounds before they reach here. What stays out is an equality on an + # instant, which day-granular filtering could not honour anyway. + MAPS = { + 'string' => EQUALITY, + 'boolean' => EQUALITY, + 'number' => EQUALITY.merge(BOUNDS), + 'text' => EQUALITY.merge(SUBSTRING), + 'date' => BOUNDS + }.freeze + + # The operators that take a list rather than a value, on the wire. + LIST_OPERATORS = %w[IN NIN].freeze + + class << self + def types = MAPS.keys + + # What the column publishes: the Forest operators whose Intercom spelling + # this endpoint accepts on this field, and no other. + def forest_operators(field) + MAPS.fetch(field.type).select { |_, spelling| field.operators.include?(spelling) }.keys + end + + # nil when the endpoint does not accept that operator on that field, + # which the translator turns into a refusal naming what it does accept. + def intercom_operator(field, forest_operator) + spelling = MAPS.fetch(field.type)[forest_operator] + + spelling if spelling && field.operators.include?(spelling) + end + + def list_operator?(spelling) = LIST_OPERATORS.include?(spelling) + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/query/search_fields.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/query/search_fields.rb new file mode 100644 index 000000000..554b6d175 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/query/search_fields.rb @@ -0,0 +1,155 @@ +module ForestAdminDatasourceIntercom + module Query + # Reads `search_fields.yml`, the table of what Intercom's search endpoints + # filter, and hands it to the schema and to the translator as objects rather + # than as nested hashes. + # + # The table is data rather than code for one reason: `bin/probe_search_fields` + # rewrites it from a real workspace. Anything derived from it -- which + # columns are filterable, with which Forest operators, and what an operator + # is told about the ones that are not -- therefore follows a measurement + # instead of a hand-written list that drifts from the endpoint. + # + # Every row is validated on load. The file ships with the gem and is written + # by a script, so a typo in an operator, or a column filed as both filterable + # and refused, is a defect of this package: it fails at boot rather than + # producing a schema nobody can explain. + module SearchFields + PATH = File.expand_path('search_fields.yml', __dir__) + + # The operators Intercom's search DSL spells, and nothing else: `=`, `!=`, + # `>`, `<`, `>=`, `<=`, the substring pair `~` / `!~`, the anchors `^` / + # `$`, and the membership pair. Which of them an endpoint honours on a + # given field is the table's business; this is only the alphabet. + KNOWN_OPERATORS = ['=', '!=', '>', '<', '>=', '<=', '~', '!~', '^', '$', 'IN', 'NIN'].freeze + # Read off the operator table rather than listed again here: a type with + # no spelling of its own would pass this validation and raise when the + # schema asked what to publish on it. + KNOWN_TYPES = OperatorTable.types + KNOWN_SOURCES = %w[measured spec].freeze + + # `source` says where a row comes from, and `measured?` is what the boot + # report and the README section read: a row taken from the documentation is + # a candidate the probe has not confirmed. + Field = Struct.new(:column, :field, :type, :operators, :source, keyword_init: true) do + def measured? = source == 'measured' + end + + # A column that stays unfilterable, and why. The reason travels into the + # refusal the operator reads, so it names what to filter on instead + # wherever there is something to name. + Refusal = Struct.new(:column, :reason, :source, keyword_init: true) do + def measured? = source == 'measured' + end + + Endpoint = Struct.new(:name, :path, :measured_at, :fields, :refused, :candidates, :ticket_attributes, + keyword_init: true) do + # Whether the probe has run against a real workspace for this endpoint. + # False means every `spec` row is still a candidate. + def measured? = !measured_at.nil? + + def field(column) = fields[column] + def refusal(column) = refused[column] + def filterable_columns = fields.keys + def unmeasured_fields = fields.values.reject(&:measured?) + end + + class << self + def fetch(name) + table[name.to_s] || + raise(ConfigurationError, "Unknown Intercom search endpoint #{name.inspect}; " \ + "the table declares #{table.keys.join(", ")}.") + end + + def endpoints = table.keys + + def table + @table ||= build(YAML.safe_load_file(PATH)) + end + + # Public so a spec can feed it a table of its own: this validation is the + # reason the file can be rewritten by a script without the package + # trusting whatever comes back. + def build(raw) + raw.fetch('endpoints').to_h { |name, definition| [name, endpoint(name, definition)] }.freeze + end + + private + + def endpoint(name, definition) + Endpoint.new( + name: name, + path: definition.fetch('path'), + measured_at: definition['measured_at'], + fields: fields(name, definition['fields']), + refused: refusals(name, definition['refused']), + candidates: Array(definition['candidates']).freeze, + ticket_attributes: definition['ticket_attributes'] + ).freeze + end + + def fields(endpoint, declared) + (declared || {}).to_h do |column, row| + field = Field.new(column: column, field: row.fetch('field'), type: row.fetch('type'), + operators: Array(row['operators']).freeze, source: row.fetch('source')).freeze + validate_field!(endpoint, field) + + [column, field] + end.freeze + end + + def refusals(endpoint, declared) + (declared || {}).to_h do |column, row| + refusal = Refusal.new(column: column, reason: squish(row.fetch('reason')), + source: row.fetch('source')).freeze + validate_source!(endpoint, column, refusal) + + [column, refusal] + end.freeze + end + + def validate_field!(endpoint, field) + validate_source!(endpoint, field.column, field) + validate_type!(endpoint, field) + validate_operators!(endpoint, field) + end + + def validate_type!(endpoint, field) + return if KNOWN_TYPES.include?(field.type) + + malformed!(endpoint, field.column, "type #{field.type.inspect} is not one of #{KNOWN_TYPES.join(", ")}") + end + + # An empty operator list is how a table stops short of saying anything: + # it would publish a filterable column no operator can reach. A column + # Intercom does not filter belongs in the refused table, where it comes + # with the reason an operator reads. + def validate_operators!(endpoint, field) + if field.operators.empty? + malformed!(endpoint, field.column, + 'it declares no operator; a column Intercom cannot filter belongs in the refused table') + end + + unknown = field.operators - KNOWN_OPERATORS + return if unknown.empty? + + malformed!(endpoint, field.column, "Intercom's search DSL has no operator #{unknown.join(", ")}") + end + + def validate_source!(endpoint, column, row) + return if KNOWN_SOURCES.include?(row.source) + + malformed!(endpoint, column, "source #{row.source.inspect} is neither #{KNOWN_SOURCES.join(" nor ")}") + end + + def malformed!(endpoint, column, detail) + raise ConfigurationError, "#{File.basename(PATH)} is malformed at #{endpoint}.#{column}: #{detail}." + end + + # A YAML folded block keeps the newlines the file needs to stay readable; + # the reason travels into a one-line message. + def squish(text) = text.to_s.split.join(' ') + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/query/search_fields.yml b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/query/search_fields.yml new file mode 100644 index 000000000..d2bb0ad2b --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/query/search_fields.yml @@ -0,0 +1,388 @@ +# The table of (endpoint x field x operator): what Intercom's search endpoints +# really filter, and with which operators. It is the single source of truth of +# this datasource's filtering -- the schema derives every column's +# filter_operators from it, so no collection can advertise a filter the +# translator would then refuse. +# +# `source` is the provenance of a row, and it is not decoration: +# +# measured -- observed against a real workspace, by `bin/probe_search_fields` +# or during a spike, and recorded here with the date; +# spec -- read off Intercom's documentation and nothing else. +# +# The distinction exists because the two disagree. Measured during lot 1: +# `company_id` is documented on a ticket and refused by `/tickets/search` with +# `invalid_field`. A `spec` row is therefore a candidate, never a promise, and +# the probe is what turns it into one. Run it against the customer's workspace +# and commit what it writes. +# +# `type` decides how a column's Forest operators are derived from the Intercom +# ones, and how a value reaches the wire: +# +# string -- an exact-match field +# text -- a field Intercom also matches per word (`~`) +# date -- epoch seconds on the wire, truncated to the UTC day on read +# boolean, number +version: 1 + +endpoints: + conversations: + path: conversations/search + # Nothing has been probed yet: fill this in with the probe's output. + measured_at: null + fields: + # The primary key, and the one column whose Forest operators are not + # derived from this table: every collection publishes `equal` and `in` on + # its key whatever is written here, the toolkit refusing a collection + # whose key carries neither. It has to be declared all the same, or the + # schema advertises a filter the translator has no row to write -- which + # is what a record detail becomes the moment a permission scope turns + # `id equals X` into an `and` the record endpoint cannot answer alone. + # + # A bare `id equals X` still reads the record endpoint, one request + # instead of a search. This row is for the compound case. + id: + field: id + type: string + operators: ['=', 'IN'] + source: spec + state: + field: state + type: string + operators: ['=', '!='] + source: spec + open: + field: open + type: boolean + operators: ['='] + source: spec + read: + field: read + type: boolean + operators: ['='] + source: spec + priority: + field: priority + type: string + operators: ['=', '!='] + source: spec + title: + field: title + type: text + operators: ['=', '!=', '~', '!~', '^', '$'] + source: spec + admin_assignee_id: + field: admin_assignee_id + type: string + operators: ['=', '!='] + source: spec + team_assignee_id: + field: team_assignee_id + type: string + operators: ['=', '!='] + source: spec + source_type: + field: source.type + type: string + operators: ['=', '!='] + source: spec + source_delivered_as: + field: source.delivered_as + type: string + operators: ['=', '!='] + source: spec + source_subject: + field: source.subject + type: text + operators: ['=', '!=', '~', '!~', '^', '$'] + source: spec + # The full-text search of the collection: Intercom matches `~` per word, + # not as a substring, which the README states rather than the interface + # implying otherwise. + source_body: + field: source.body + type: text + operators: ['~', '!~'] + source: spec + source_author_email: + field: source.author.email + type: text + operators: ['=', '!=', '~', '!~', '^', '$'] + source: spec + # The date operators are measured, and they are not the same on every + # endpoint: `/contacts/search` refuses `>=`, `<=` and `!=` where this one + # accepts them (25 August 2026, API 2.16). Whatever this table allows, a + # Date column publishes the two bounds alone -- see the translator. + created_at: + field: created_at + type: date + operators: ['>', '<', '>=', '<=', '=', '!='] + source: measured + updated_at: + field: updated_at + type: date + operators: ['>', '<', '>=', '<=', '=', '!='] + source: measured + waiting_since: + field: waiting_since + type: date + operators: ['>', '<', '>=', '<=', '=', '!='] + source: spec + snoozed_until: + field: snoozed_until + type: date + operators: ['>', '<', '>=', '<=', '=', '!='] + source: spec + closed_at: + field: statistics.last_close_at + type: date + operators: ['>', '<', '>=', '<=', '=', '!='] + source: spec + first_closed_at: + field: statistics.first_close_at + type: date + operators: ['>', '<', '>=', '<=', '=', '!='] + source: spec + closed_by_id: + field: statistics.last_closed_by_id + type: string + operators: ['=', '!='] + source: spec + first_contact_reply_at: + field: statistics.first_contact_reply_at + type: date + operators: ['>', '<', '>=', '<=', '=', '!='] + source: spec + last_contact_reply_at: + field: statistics.last_contact_reply_at + type: date + operators: ['>', '<', '>=', '<=', '=', '!='] + source: spec + last_admin_reply_at: + field: statistics.last_admin_reply_at + type: date + operators: ['>', '<', '>=', '<=', '=', '!='] + source: spec + reopen_count: + field: statistics.count_reopens + type: number + operators: ['=', '!=', '>', '<'] + source: spec + part_count: + field: statistics.count_conversation_parts + type: number + operators: ['=', '!=', '>', '<'] + source: spec + ai_agent_participated: + field: ai_agent_participated + type: boolean + operators: ['='] + source: spec + # Columns this collection publishes and will not filter, with the reason an + # operator reads when they try. A column absent from both tables is refused + # by name too -- these are the ones whose refusal is permanent and has an + # explanation worth giving. + refused: + contact_ids: + reason: >- + Intercom does match a conversation against one of its contact ids, but + the column holds the list and is therefore typed Json, whose filter + values the agent's own validator requires to be Json too -- an id + would be rejected before this datasource saw it. Filtering by contact + arrives with the Contacts collection and a relation. + source: measured + tag_names: + reason: >- + Intercom filters conversations by tag id, and this column holds the + tag names. Filtering it would mean resolving a name to an id per + request, and a name the workspace renamed would silently match + nothing. + source: spec + company_name: + reason: >- + The conversation carries its company as an object read from the + payload; the search endpoint filters no company field. + source: spec + company_id: + reason: >- + The search endpoint filters no company field. Measured on + `/tickets/search`, which refuses `company_id` with `invalid_field` + although a ticket carries one; the same is assumed here until probed. + source: spec + timeline: + reason: >- + Built by the agent from the parts of a conversation, which the search + endpoint does not read. + source: measured + contact_name: + reason: >- + Read from the Contacts endpoint, not from the conversation. Filtering + on it arrives with the Contacts collection. + source: spec + contact_email: + reason: >- + Read from the Contacts endpoint, not from the conversation. Filter on + `source_author_email` instead, which is on the conversation itself. + source: spec + contact_count: + reason: Counted by the agent from the contacts the payload carries. + source: measured + # What `bin/probe_search_fields` enumerates on top of the fields above: + # names the documentation mentions, or that an ops team would plausibly + # search on. A candidate that turns out to be filterable becomes a field + # above -- with a column to expose it on, or with none, in which case it is + # a column the next lot may add. + candidates: + - source.id + - source.author.id + - source.author.type + - source.author.name + - statistics.time_to_assignment + - statistics.time_to_admin_reply + - statistics.time_to_first_close + - statistics.median_time_to_reply + - statistics.first_assignment_at + - statistics.first_admin_reply_at + - statistics.last_assignment_at + - statistics.count_assignments + - conversation_rating.score + - conversation_rating.remark + - conversation_rating.contact_id + - ai_agent.resolution_state + - ai_agent.last_answer_type + - ai_agent.rating + - channel_initiated + - tag_ids + - teammate_ids + - company_id + - topics + + tickets: + path: tickets/search + measured_at: null + fields: + # The primary key, declared here for the same reason as on conversations: + # the schema publishes `equal` and `in` on it whatever this table says, + # so a row has to exist for the translator to write. `GET /tickets/{id}` + # answers the key on its own; the search answers it in an `and`. + id: + field: id + type: string + operators: ['=', 'IN'] + source: spec + open: + field: open + type: boolean + operators: ['='] + source: spec + category: + field: category + type: string + operators: ['=', '!='] + source: spec + ticket_type_id: + field: ticket_type_id + type: string + operators: ['=', '!='] + source: spec + admin_assignee_id: + field: admin_assignee_id + type: string + operators: ['=', '!='] + source: spec + team_assignee_id: + field: team_assignee_id + type: string + operators: ['=', '!='] + source: spec + created_at: + field: created_at + type: date + operators: ['>', '<', '>=', '<=', '=', '!='] + source: measured + updated_at: + field: updated_at + type: date + operators: ['>', '<', '>=', '<=', '=', '!='] + source: measured + refused: + contact_ids: + reason: >- + Intercom does match a ticket against one of its contact ids, but + the column holds the list and is therefore typed Json, whose filter + values the agent's own validator requires to be Json too -- an id + would be rejected before this datasource saw it. Filtering by contact + arrives with the Contacts collection and a relation. + source: measured + company_id: + reason: >- + Measured during lot 1: `/tickets/search` refuses `company_id` with + `invalid_field`, although a ticket carries one. Filtering tickets by + account is not something this endpoint does. + source: measured + closed_at: + reason: >- + Derived by the agent from the parts of the ticket. The search endpoint + filters nothing of the sort, and ignores a sort on it without a word. + source: measured + closed_by_name: + reason: Derived by the agent from the parts of the ticket. + source: measured + last_reply_at: + reason: Derived by the agent from the parts of the ticket. + source: measured + last_responder_name: + reason: Derived by the agent from the parts of the ticket. + source: measured + last_responder_type: + reason: Derived by the agent from the parts of the ticket. + source: measured + state_label: + reason: >- + Read off the state object the ticket embeds. Whether the endpoint + filters a ticket state at all, and under which name, is one of the + probe's questions. + source: spec + state_external_label: + reason: Read off the state object the ticket embeds. + source: spec + ticket_type_name: + reason: >- + Read off the type object the ticket embeds. Filter on + `ticket_type_id`, which the endpoint does take. + source: spec + part_count: + reason: Counted by the agent from the parts the payload carries. + source: measured + # R7, and it is a product decision rather than an implementation choice: a + # ticket attribute is filtered as `ticket_attribute.{id}`, and a same-named + # attribute carries a different id per ticket type (measured: + # `_default_title_` is 14162161 on one type and 14162165 on another). One + # union column cannot know which id a row's type uses, so filtering it + # server-side would take one collection per ticket type -- more collections + # in the interface, and a schema that changes shape when the customer adds a + # type. Until the customer says that trade is worth it, the attributes stay + # display-only, as lot 1 published them. + ticket_attributes: + filterable: false + reason: >- + An Intercom ticket attribute is filtered through an id that differs from + one ticket type to the next, so a column showing the attribute of every + type at once cannot say which id to filter on. Filter on the ticket type + and a native field instead. + source: measured + + candidates: + - ticket_id + - state + - ticket_state.category + - ticket_state.id + - state_id + - is_shared + - previous_state_id + - contact_id + - source.author.email + - source.subject + - source.body + - title + - description diff --git a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/conversation_spec.rb b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/conversation_spec.rb index eb4a24ad2..3b562221b 100644 --- a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/conversation_spec.rb +++ b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/conversation_spec.rb @@ -16,6 +16,11 @@ def leaf(field, operator, value = nil) .new(field, operator, value) end + def branch(aggregator, *conditions) + ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Nodes::ConditionTreeBranch + .new(aggregator, conditions) + end + def json(payload, status = 200) { status: status, body: payload.to_json, headers: { 'Content-Type' => 'application/json' } } end @@ -69,6 +74,14 @@ def stub_list(*records, next_cursor: nil, total: nil, query: hash_including({})) stub_request(:get, "#{base}/conversations").with(query: query).to_return(json(body)) end + def stub_search(*records, total: nil, next_cursor: nil) + body = { 'type' => 'conversation.list', 'conversations' => records, + 'total_count' => total || records.size, 'pages' => { 'type' => 'pages', 'page' => 1 } } + body['pages']['next'] = { 'starting_after' => next_cursor } if next_cursor + + stub_request(:post, "#{base}/conversations/search").with(query: hash_including({})).to_return(json(body)) + end + def stub_record(id, payload, status = 200) stub_request(:get, "#{base}/conversations/#{id}").with(query: hash_including({})).to_return(json(payload, status)) end @@ -82,14 +95,32 @@ def ids(rows) expect(collection.name).to eq('IntercomConversation') end - # Intercom ignores a sort on this endpoint without a word and filters - # nothing on the listing, so a column advertising either would put in the - # interface what the read then refuses. - it 'declares every column unsortable and unfilterable, except the primary key' do - others = collection.fields.except('id') + # Neither search endpoint takes a sort, and Intercom ignores the one it is + # sent without a word, so no column of this tier may advertise one. + it 'declares every column unsortable' do + expect(collection.fields.values.map(&:is_sortable).uniq).to eq([false]) + end + + # Derived from the measured table, never written by hand: a column + # advertises exactly what the search endpoint answers on it. + it 'advertises the filters the search endpoint answers, and only those' do + expect(collection.fields['state'].filter_operators).to eq(%w[equal not_equal]) + expect(collection.fields['created_at'].filter_operators).to eq(%w[greater_than less_than]) + expect(collection.fields['source_body'].filter_operators).to eq(%w[contains i_contains not_contains]) + end + + # A column the table does not carry advertises nothing, which is how a + # refusal is spelled in a schema: the tag names, the company, the timeline + # and the contact identity are all read from somewhere the endpoint does + # not filter. + it 'advertises no filter on a column the endpoint does not filter' do + %w[tag_names company_name contact_email timeline contact_ids].each do |column| + expect(collection.fields[column].filter_operators).to be_empty, "#{column} advertises a filter" + end + end - expect(others.values.map(&:is_sortable).uniq).to eq([false]) - expect(others.values.map(&:filter_operators).flatten.uniq).to be_empty + it 'is searchable, Intercom matching text on the body of the first message' do + expect(collection.is_searchable?).to be(true) end # The record detail is `id equals X`, answered by the record endpoint @@ -213,6 +244,33 @@ def ids(rows) expect { collection.list(nil, filter(condition_tree: leaf('id', operators::EQUAL, '1')), %w[id]) } .to raise_error(APIError) end + + # A permission scope turns the record detail into `id equals X and `, which the record endpoint cannot answer: the ids name a wider + # set than the scope does, and reading them alone would serve a record the + # scope excludes. It goes to the search, where the key is a field like any + # other -- and the whole condition travels, or none of it does. + it 'reads the key through the search once a scope is filtered alongside it' do + search = stub_search(conversation('1')) + tree = branch('And', leaf('id', operators::EQUAL, '1'), leaf('state', operators::EQUAL, 'closed')) + + expect(ids(collection.list(nil, filter(condition_tree: tree), %w[id]))).to eq(%w[1]) + expect(search).to have_been_requested + end + + it 'sends the scope and the key as the one query, neither dropped' do + stub_search(conversation('1')) + tree = branch('And', leaf('id', operators::IN, %w[1 2]), leaf('open', operators::EQUAL, false)) + + collection.list(nil, filter(condition_tree: tree), %w[id]) + + expected = { 'operator' => 'AND', + 'value' => [{ 'field' => 'id', 'operator' => 'IN', 'value' => %w[1 2] }, + { 'field' => 'open', 'operator' => '=', 'value' => false }] } + + expect(a_request(:post, "#{base}/conversations/search") + .with(query: hash_including({}), body: hash_including('query' => expected))).to have_been_made + end end describe '#list of several records by id' do @@ -229,6 +287,19 @@ def ids(rows) expect(ids(rows)).to eq(%w[1 2]) end + # A pointing collection pages through the records it named, and every page + # must name different ones: cut out of the records instead of out of the + # ids, the window would render the same rows on page 1 and page 2 -- and + # a page past the cap would come back empty, its ids having been dropped + # by the truncation before the window was applied. + it 'reads only the ids the page window names' do + stub_record('2', conversation('2')) + page = ForestAdminDatasourceToolkit::Components::Query::Page.new(offset: 1, limit: 1) + tree = leaf('id', operators::IN, %w[1 2 3]) + + expect(ids(collection.list(nil, filter(condition_tree: tree, page: page), %w[id]))).to eq(%w[2]) + end + it 'reads the first of too many and says the result is truncated' do allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) asked = (1..(Collections::CursorCollection::MAX_ID_READS + 3)).map(&:to_s) @@ -241,25 +312,101 @@ def ids(rows) end end - describe 'a filter it cannot honour' do - # Translating a Forest tree into Intercom's search DSL is the next lot. - # Until then a page that looks filtered without being it is the one answer - # this datasource must not give. - it 'refuses a condition on anything but the primary key' do - expect { collection.list(nil, filter(condition_tree: leaf('state', operators::EQUAL, 'open')), %w[id]) } - .to raise_error(UnsupportedOperatorError, /cannot answer a condition/) + describe 'a filter Intercom answers' do + # A condition switches the read from the listing to the search endpoint, + # which is the only one that takes a filter. + it 'searches instead of listing, with the query the translator wrote' do + search = stub_search + tree = leaf('state', operators::EQUAL, 'open') + + collection.list(nil, filter(condition_tree: tree), %w[id]) + + expect(search.with(body: hash_including('query' => { 'field' => 'state', 'operator' => '=', + 'value' => 'open' }))).to have_been_made end - it 'refuses a free-text search' do - searched = ForestAdminDatasourceToolkit::Components::Query::Filter.new(search: 'facture') + # The bodies are HTML written by end customers (R10), and a filtered read + # must not come back as markup where an unfiltered one comes back as text. + it 'asks the search for plain text too' do + search = stub_search - expect { collection.list(nil, searched, %w[id]) } - .to raise_error(UnsupportedOperatorError, /cannot answer a free-text search/) + collection.list(nil, filter(condition_tree: leaf('open', operators::EQUAL, true)), %w[id]) + + expect(search.with(query: hash_including('display_as' => 'plaintext'))).to have_been_made end - it 'says where the filtering will come from, so the message is actionable' do - expect { collection.list(nil, filter(condition_tree: leaf('state', operators::EQUAL, 'open')), %w[id]) } - .to raise_error(UnsupportedOperatorError, /search endpoint.*filter translation/m) + it 'walks the search cursor for the window a list view asked for' do + stub_request(:post, "#{base}/conversations/search") + .with(query: hash_including({})) + .to_return(json({ 'conversations' => [conversation('1'), conversation('2')], + 'pages' => { 'next' => { 'starting_after' => 'c2' } } })) + stub_request(:post, "#{base}/conversations/search") + .with(query: hash_including({}), + body: hash_including('pagination' => hash_including('starting_after' => 'c2'))) + .to_return(json('conversations' => [conversation('3')])) + page = ForestAdminDatasourceToolkit::Components::Query::Page.new(offset: 2, limit: 1) + + rows = collection.list(nil, filter(condition_tree: leaf('open', operators::EQUAL, true), page: page), %w[id]) + + expect(ids(rows)).to eq(%w[3]) + end + + # Per word rather than as a substring, which the README says out loud. + it 'answers a free-text search on the body of the message that opened the conversation' do + search = stub_search + searched = ForestAdminDatasourceToolkit::Components::Query::Filter.new(search: ' facture ') + + collection.list(nil, searched, %w[id]) + + expect(search.with(body: hash_including('query' => { 'field' => 'source.body', 'operator' => '~', + 'value' => 'facture' }))).to have_been_made + end + + # Written as one tree rather than added to the translated query: the + # nesting Intercom allows is then checked over the whole of it. + it 'ands a free-text search with the condition it came with' do + search = stub_search + searched = ForestAdminDatasourceToolkit::Components::Query::Filter.new( + search: 'facture', condition_tree: leaf('open', operators::EQUAL, true) + ) + + collection.list(nil, searched, %w[id]) + + expect(search.with { |request| JSON.parse(request.body)['query']['operator'] == 'AND' }).to have_been_made + end + + # The date bounds Intercom answers are the ones the day rule moved, which + # is what makes an interval answer the day it names. + it 'sends a date bound on the UTC day boundary that answers the day asked for' do + search = stub_search + tree = leaf('created_at', operators::GREATER_THAN, '2026-09-01T08:30:00Z') + + collection.list(nil, filter(condition_tree: tree), %w[id]) + + expect(search.with(body: hash_including('query' => hash_including('value' => Time.utc(2026, 8, + 31).to_i)))) + .to have_been_made + end + end + + describe 'a filter it cannot honour' do + # A condition dropped on the way to Intercom comes back as an unfiltered + # page that looks filtered, which is the one answer this datasource must + # not give. + it 'refuses a condition on a column the endpoint does not filter' do + expect { collection.list(nil, filter(condition_tree: leaf('tag_names', operators::EQUAL, 'billing')), %w[id]) } + .to raise_error(UnsupportedOperatorError, /cannot filter "tag_names"/) + end + + it 'refuses an operator the endpoint does not answer on that column' do + expect { collection.list(nil, filter(condition_tree: leaf('state', operators::CONTAINS, 'op')), %w[id]) } + .to raise_error(UnsupportedOperatorError, /answers equal, not_equal on "state"/) + end + + it 'makes no request at all when it refuses' do + expect { collection.list(nil, filter(condition_tree: leaf('tag_names', operators::EQUAL, 'x')), %w[id]) } + .to raise_error(UnsupportedOperatorError) + expect(a_request(:post, "#{base}/conversations/search")).not_to have_been_made end end @@ -278,6 +425,17 @@ def ids(rows) expect(ForestAdminDatasourceIntercom.logger).to have_received(:warn).with(/ignores a sort/) end + # `?sort=-id` is an order the operator asked for, not the ascending default + # the agent injects when a request names none. + it 'reports an explicit descending order on the primary key' do + stub_list(conversation('1')) + sort = ForestAdminDatasourceToolkit::Components::Query::Sort.new([{ field: 'id', ascending: false }]) + + collection.list(nil, filter(sort: sort), %w[id]) + + expect(ForestAdminDatasourceIntercom.logger).to have_received(:warn).with(/ignores a sort/) + end + it 'stays quiet on the primary-key order the agent injects by default' do stub_list(conversation('1')) sort = ForestAdminDatasourceToolkit::Components::Query::Sort.new([{ field: 'id', ascending: true }]) @@ -322,11 +480,22 @@ def aggregation(operation, field: nil, groups: []) .to raise_error(UnsupportedOperatorError, /can only be counted/) end - it 'refuses a condition it could not honour on the list either' do + # `total_count` is exact on a search too, so a filtered count is one + # request over the whole filtered set rather than over a page of it. + it 'counts a filtered collection through the search, in one request' do + stub_search(total: 1_234) + + value = collection.aggregate(nil, filter(condition_tree: leaf('state', operators::EQUAL, 'open')), + aggregation('Count')).first['value'] + + expect(value).to eq(1_234) + end + + it 'refuses to count what it refuses to list' do expect do - collection.aggregate(nil, filter(condition_tree: leaf('state', operators::EQUAL, 'open')), + collection.aggregate(nil, filter(condition_tree: leaf('tag_names', operators::EQUAL, 'billing')), aggregation('Count')) - end.to raise_error(UnsupportedOperatorError, /cannot answer a condition/) + end.to raise_error(UnsupportedOperatorError, /cannot filter "tag_names"/) end # Counting the pages a walk collected would answer a fraction as if it diff --git a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/ticket_spec.rb b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/ticket_spec.rb index 55c13c07a..bf1b8fddb 100644 --- a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/ticket_spec.rb +++ b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/ticket_spec.rb @@ -94,13 +94,42 @@ def rows(projection = nil, **options) expect(collection.fields['Due'].column_type).to eq('Date') end - # `/tickets/search` filters none of these and ignores a sort without - # saying so, so nothing but the primary key may advertise anything. - it 'declares every column unfilterable and unsortable, except the primary key' do - others = collection.fields.except('id') + # `/tickets/search` ignores a sort without saying so, on every column. + it 'declares every column unsortable' do + expect(collection.fields.values.map(&:is_sortable).uniq).to eq([false]) + end + + it 'advertises the filters the search endpoint answers, and only those' do + expect(collection.fields['category'].filter_operators).to eq(%w[equal not_equal]) + expect(collection.fields['created_at'].filter_operators).to eq(%w[greater_than less_than]) + end - expect(others.values.map(&:filter_operators).flatten.uniq).to be_empty - expect(others.values.map(&:is_sortable).uniq).to eq([false]) + # Measured during lot 1: `/tickets/search` refuses `company_id` with + # `invalid_field` although a ticket carries one. Filtering tickets by + # account is not something this endpoint does. + it 'advertises no filter on the account, which the endpoint refuses' do + expect(collection.fields['company_id'].filter_operators).to be_empty + end + + # Derived by the agent from the parts of the ticket. A column advertising a + # filter the read cannot honour is what this lot exists to prevent. + it 'advertises no filter on the columns derived from the parts' do + %w[closed_at closed_by_name last_reply_at last_responder_name last_responder_type].each do |column| + expect(collection.fields[column].filter_operators).to be_empty, "#{column} advertises a filter" + end + end + + # R7: an attribute is filtered through an id that differs from one ticket + # type to the next, so the union column cannot say which id to use. + it 'advertises no filter on a ticket attribute while the arbitration stands' do + expect(collection.fields['Due'].filter_operators).to be_empty + expect(collection.fields['_default_title_'].filter_operators).to be_empty + end + + # Intercom matches text field by field, and this endpoint exposes none + # this collection carries. + it 'is not searchable' do + expect(collection).not_to be_is_searchable end # An attribute overwriting a native column would show the attribute where @@ -206,13 +235,45 @@ def rows(projection = nil, **options) expect(rows(%w[id], condition_tree: leaf('id', operators::EQUAL, '1')).map { |row| row['id'] }).to eq(%w[1]) end - it 'refuses a condition it cannot honour' do + # The search carries the filter it was given, in place of the predicate + # that matches everything. + it 'searches with the query the translator wrote' do + search = stub_search(ticket('1'), body: { 'query' => { 'field' => 'category', 'operator' => '=', + 'value' => 'request' } }) + + rows(%w[id], condition_tree: leaf('category', operators::EQUAL, 'request')) + + expect(search).to have_been_made + end + + it 'refuses a condition on a column the endpoint does not filter, by name' do expect { rows(%w[id], condition_tree: leaf('state_category', operators::EQUAL, 'resolved')) } - .to raise_error(UnsupportedOperatorError, /cannot answer a condition/) + .to raise_error(UnsupportedOperatorError, /cannot filter "state_category"/) + end + + # The account, measured as refused by the endpoint itself. + it 'refuses a condition on the account with the measurement as its reason' do + expect { rows(%w[id], condition_tree: leaf('company_id', operators::EQUAL, '696dd')) } + .to raise_error(UnsupportedOperatorError, /invalid_field/) + end + + it 'refuses a free-text search, having no text column the endpoint matches' do + searched = ForestAdminDatasourceToolkit::Components::Query::Filter.new(search: 'facture') + + expect { collection.list(nil, searched, %w[id]) } + .to raise_error(UnsupportedOperatorError, /cannot answer a free-text search/) end end describe '#aggregate' do + it 'counts a filtered collection through the total_count of its search' do + stub_search(total: 12, body: { 'query' => { 'field' => 'open', 'operator' => '=', 'value' => true } }) + aggregation = ForestAdminDatasourceToolkit::Components::Query::Aggregation.new(operation: 'Count') + + expect(collection.aggregate(nil, filter(condition_tree: leaf('open', operators::EQUAL, true)), + aggregation).first['value']).to eq(12) + end + it 'counts through the total_count of the search, exactly' do stub_search(ticket('1'), total: 81_142) aggregation = ForestAdminDatasourceToolkit::Components::Query::Aggregation.new(operation: 'Count') diff --git a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/query/condition_tree_translator_spec.rb b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/query/condition_tree_translator_spec.rb new file mode 100644 index 000000000..b252c3867 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/query/condition_tree_translator_spec.rb @@ -0,0 +1,194 @@ +module ForestAdminDatasourceIntercom + RSpec.describe Query::ConditionTreeTranslator do + let(:operators) { ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Operators } + let(:nodes) { ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Nodes } + let(:endpoint) { Query::SearchFields.fetch('conversations') } + + def translate(tree, timezone: 'UTC') + described_class.call(tree, endpoint: endpoint, collection: 'IntercomConversation', timezone: timezone) + end + + def leaf(field, operator, value = nil) + nodes::ConditionTreeLeaf.new(field, operator, value) + end + + def branch(aggregator, *conditions) + nodes::ConditionTreeBranch.new(aggregator, conditions) + end + + describe 'a leaf' do + it 'writes the Intercom field, operator and value the endpoint takes' do + expect(translate(leaf('state', operators::EQUAL, 'open'))) + .to eq({ 'field' => 'state', 'operator' => '=', 'value' => 'open' }) + end + + # The column is the operator's name for it; the field is Intercom's. They + # are not the same on a statistic, which the column flattens onto the row. + it 'filters a flattened statistic through the field Intercom nests it in' do + expect(translate(leaf('closed_at', operators::LESS_THAN, '2026-09-01T00:00:00Z'))['field']) + .to eq('statistics.last_close_at') + end + + it 'answers nil for no condition at all, which is a list view' do + expect(translate(nil)).to be_nil + end + end + + # A bare `id equals X` never reaches here -- the collection reads the record + # endpoint instead, one request rather than a search. What does reach here + # is the key nested in an `and`: a permission scope, a segment, or a second + # filter alongside it, which the record endpoint cannot answer on its own. + describe 'the primary key, once something else is filtered alongside it' do + it 'writes the key as the search field it is' do + tree = branch('And', leaf('id', operators::EQUAL, '42'), leaf('state', operators::EQUAL, 'open')) + + expect(translate(tree)).to eq({ 'operator' => 'AND', + 'value' => [{ 'field' => 'id', 'operator' => '=', 'value' => '42' }, + { 'field' => 'state', 'operator' => '=', 'value' => 'open' }] }) + end + + it 'writes a membership on the key as the list Intercom takes' do + tree = branch('And', leaf('id', operators::IN, %w[1 2]), leaf('open', operators::EQUAL, true)) + + expect(translate(tree)['value'].first) + .to eq({ 'field' => 'id', 'operator' => 'IN', 'value' => %w[1 2] }) + end + end + + describe 'a branch' do + it 'groups its conditions under the aggregator Intercom spells in capitals' do + tree = branch('Or', leaf('state', operators::EQUAL, 'open'), leaf('state', operators::EQUAL, 'snoozed')) + + expect(translate(tree)).to eq({ 'operator' => 'OR', + 'value' => [{ 'field' => 'state', 'operator' => '=', 'value' => 'open' }, + { 'field' => 'state', 'operator' => '=', 'value' => 'snoozed' }] }) + end + + # The agent builds a tree one branch at a time -- a scope, then a segment, + # then the operator's filter -- and Intercom allows two levels of nesting. + # A wrapper around a single condition is a level worth not spending. + it 'unwraps a branch carrying one condition rather than spending a level on it' do + tree = branch('And', branch('And', leaf('open', operators::EQUAL, true))) + + expect(translate(tree)).to eq({ 'field' => 'open', 'operator' => '=', 'value' => true }) + end + + it 'nests a group inside a group' do + tree = branch('And', leaf('open', operators::EQUAL, true), + branch('Or', leaf('state', operators::EQUAL, 'open'), + leaf('state', operators::EQUAL, 'snoozed'))) + + expect(translate(tree)['value'].last['operator']).to eq('OR') + end + + it 'refuses an aggregator that is neither and nor or' do + expect { translate(branch('Xor', leaf('open', operators::EQUAL, true))) } + .to raise_error(UnsupportedOperatorError, /cannot read "Xor" as a condition tree aggregator/) + end + + # A branch with nothing in it names no record and no filter, so sending it + # would answer a filtered question with the whole collection. + it 'refuses a branch carrying no condition' do + expect { translate(branch('And')) } + .to raise_error(UnsupportedOperatorError, /carrying no condition/) + end + + it 'refuses a node that is neither a leaf nor a branch' do + expect { translate(Object.new) } + .to raise_error(UnsupportedOperatorError, /cannot read Object as a condition/) + end + end + + # The headline failure this lot had to avoid: Intercom truncates a date + # search to the UTC day, so the pair of bounds the toolkit rewrites `today` + # into reads as "from tomorrow" and "before today" -- an empty answer to the + # most ordinary filter there is -- unless each bound is moved to the day + # boundary that makes Intercom answer the day the filter named. + describe 'the two bounds an interval is rewritten into' do + it 'asks for the day the interval names rather than cancelling out' do + tree = branch('And', leaf('created_at', operators::GREATER_THAN, '2026-09-01T00:00:00Z'), + leaf('created_at', operators::LESS_THAN, '2026-09-01T23:59:59Z')) + + expect(translate(tree)['value'].map { |bound| Time.at(bound['value']).utc.iso8601 }) + .to eq(['2026-08-31T00:00:00Z', '2026-09-02T00:00:00Z']) + end + end + + # Over either limit Intercom answers a 400 whose body names neither the + # limit nor the part of the filter that reached it. + describe 'the limits of the search DSL, checked before the request leaves' do + def leaves(count) + Array.new(count) { |index| leaf('state', operators::EQUAL, "state-#{index}") } + end + + it 'takes a group nested one level inside another' do + tree = branch('And', leaf('open', operators::EQUAL, true), + branch('Or', *leaves(2))) + + expect(translate(tree)['value'].last['operator']).to eq('OR') + end + + it 'refuses a group inside a group inside a group, naming the shape' do + tree = branch('And', leaf('open', operators::EQUAL, true), + branch('Or', leaf('read', operators::EQUAL, true), + branch('And', *leaves(2)))) + + expect { translate(tree) } + .to raise_error(UnsupportedOperatorError, /nests a search 2 levels deep and this one reaches 3/) + end + + # A branch carrying a single condition is unwrapped, so it spends no level: + # the agent wraps a scope and a segment one branch at a time. + it 'does not spend a level on the branches the agent wraps around one condition' do + tree = branch('And', branch('And', branch('And', leaf('open', operators::EQUAL, true)))) + + expect(translate(tree)).to eq({ 'field' => 'open', 'operator' => '=', 'value' => true }) + end + + it 'takes a group of fifteen conditions' do + expect(translate(branch('Or', *leaves(15)))['value'].size).to eq(15) + end + + # A scope, a segment and a filter add up; and a condition naming several + # values arrives expanded into one condition per value, Intercom taking no + # membership operator on these fields. + it 'refuses a group of sixteen, and says what brings it back under' do + expect { translate(branch('Or', *leaves(16))) } + .to raise_error(UnsupportedOperatorError, /takes 15 conditions per group and this Or carries 16/) + end + end + + describe 'what it will not translate' do + # The whole point of the lot: a condition dropped on the way to Intercom + # comes back as an unfiltered page that looks filtered. + it 'refuses a column the endpoint does not filter, and names what it does' do + expect { translate(leaf('contact_name', operators::EQUAL, 'Camille')) } + .to raise_error(UnsupportedOperatorError, /cannot filter "contact_name".*Contacts endpoint/m) + end + + it 'refuses a column nothing declares, listing the ones it takes' do + expect { translate(leaf('nope', operators::EQUAL, 'x')) } + .to raise_error(UnsupportedOperatorError, /takes no filter on it. Filter on one of: id, state, open/) + end + + # None of these collections declares a relation yet, so a `relation:field` + # can only come from a scope or a segment written against another schema. + it 'refuses a condition on a relation by name' do + expect { translate(leaf('contact:email', operators::EQUAL, 'camille@acme.test')) } + .to raise_error(UnsupportedOperatorError, /declares no relation/) + end + + it 'refuses an operator the endpoint does not answer on that field' do + expect { translate(leaf('state', operators::CONTAINS, 'op')) } + .to raise_error(UnsupportedOperatorError, /answers equal, not_equal on "state" and nothing else/) + end + + # Published on a date column by the toolkit and refused by this endpoint: + # the datasource cannot express an equality on a day it truncates. + it 'refuses an equality on a date, which is not one of the two bounds' do + expect { translate(leaf('created_at', operators::EQUAL, '2026-09-01T00:00:00Z')) } + .to raise_error(UnsupportedOperatorError, /answers greater_than, less_than/) + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/query/filter_value_spec.rb b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/query/filter_value_spec.rb new file mode 100644 index 000000000..13ee73781 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/query/filter_value_spec.rb @@ -0,0 +1,248 @@ +module ForestAdminDatasourceIntercom + RSpec.describe Query::FilterValue do + subject(:formatter) { described_class.new(collection: 'IntercomConversation', timezone: timezone) } + + let(:timezone) { 'Europe/Paris' } + let(:operators) { ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Operators } + let(:nodes) { ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Nodes } + + def field(type, operators = ['=']) + Query::SearchFields::Field.new(column: 'c', field: 'c', type: type, operators: operators, source: 'spec') + end + + def call(type, value, spelling: '=', operator: nil) + leaf = nodes::ConditionTreeLeaf.new('c', operator || operators::EQUAL, value) + + formatter.call(leaf, field(type), spelling) + end + + # Intercom truncates a date search to the UTC day -- measured, and against + # its own documentation, which promises the workspace timezone. `>` answers + # from the start of the day after the value, `<` before the start of the + # day of the value. Sent as they come, the two bounds of an interval cancel + # each other out and `today` answers nothing. + describe 'a date, and the UTC day Intercom truncates it to' do + def bound(value, spelling) + Time.at(call('date', value, spelling: spelling, + operator: spelling == '>' ? operators::GREATER_THAN : operators::LESS_THAN)) + .utc.iso8601 + end + + it 'moves a lower bound back a day, so Intercom answers from the day it names' do + expect(bound('2026-09-01T08:30:00Z', '>')).to eq('2026-08-31T00:00:00Z') + end + + it 'moves an upper bound forward a day, so Intercom answers through the day it names' do + expect(bound('2026-09-01T08:30:00Z', '<')).to eq('2026-09-02T00:00:00Z') + end + + # An upper bound already sitting on a day boundary names the day to leave + # out, which is what Intercom answers on its own. + it 'leaves an upper bound already on a UTC day boundary where it is' do + expect(bound('2026-09-01T00:00:00Z', '<')).to eq('2026-09-01T00:00:00Z') + end + + # The pair the toolkit rewrites `today` into, from a caller in UTC: what + # comes back is that day and nothing else. + it 'answers the day itself for the two bounds of an interval' do + expect(bound('2026-09-01T00:00:00Z', '>')).to eq('2026-08-31T00:00:00Z') + expect(bound('2026-09-01T23:59:59Z', '<')).to eq('2026-09-02T00:00:00Z') + end + + it 'keeps the offset an ISO8601 timestamp carries' do + expect(bound('2026-09-01T10:30:00+02:00', '<')).to eq('2026-09-02T00:00:00Z') + end + + # `FilterFactory` writes the bounds of a previous period with strftime + # and no offset at all, so a chart comparing to the previous month sends + # `2026-08-01 00:00:00` meaning the caller's midnight. Read in the + # process timezone it is a different instant, and a midnight moved by any + # offset lands on another UTC day once truncated -- a whole day of rows + # beside the ones the chart named. + it 'reads a timestamp carrying no offset in the timezone of the caller' do + # Paris midnight on 1 September is 2026-08-31T22:00:00Z, whose UTC day + # is the 31st, so `>` must answer from the 31st and sit on the 30th. + expect(bound('2026-09-01 00:00:00', '>')).to eq('2026-08-30T00:00:00Z') + end + + it 'reads a whole previous-period window in the timezone of the caller' do + expect(bound('2026-08-01 00:00:00', '>')).to eq('2026-07-30T00:00:00Z') + expect(bound('2026-09-01 00:00:00', '<')).to eq('2026-09-01T00:00:00Z') + end + + # A day with no time of day is the caller's day: it is the timezone the + # filter was written in that says when that day starts. + it 'reads a bare date as midnight in the timezone of the caller' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + + expect(bound('2026-09-01', '>')).to eq('2026-08-30T00:00:00Z') + end + + it 'reads a Ruby Date, a Time, a DateTime and epoch seconds the same way' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + + expect(bound(Date.new(2026, 9, 1), '>')).to eq('2026-08-30T00:00:00Z') + expect(bound(Time.utc(2026, 9, 1, 8, 30), '<')).to eq('2026-09-02T00:00:00Z') + expect(bound(DateTime.new(2026, 9, 1, 8, 30, 0), '<')).to eq('2026-09-02T00:00:00Z') + expect(bound(1_788_251_400, '<')).to eq('2026-09-02T00:00:00Z') + end + + # A window written in another timezone is answered on the UTC days it + # overlaps, and an operator has no way of guessing that from the rows. + it 'reports the UTC day boundary once, when it is not the day of the caller' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + + bound('2026-08-31T22:00:00Z', '>') + bound('2026-08-31T23:00:00Z', '>') + + expect(ForestAdminDatasourceIntercom.logger).to have_received(:warn).once.with(/truncates a date search/) + end + + it 'stays quiet when the window and the UTC day are the same day anyway' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + + bound('2026-09-01T08:30:00Z', '>') + + expect(ForestAdminDatasourceIntercom.logger).not_to have_received(:warn) + end + + context 'when the caller names a timezone nothing knows' do + let(:timezone) { 'Middle-Earth/Shire' } + + # Falling back to UTC silently would move a day boundary by the offset, + # which is the failure a timezone is read for in the first place. + it 'reads the day boundary in UTC and says so' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + + expect(bound('2026-09-01', '>')).to eq('2026-08-31T00:00:00Z') + expect(ForestAdminDatasourceIntercom.logger).to have_received(:warn).with(/unknown timezone/) + end + + it 'reads a timestamp carrying no offset in UTC and says so' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + + expect(bound('2026-09-01 00:00:00', '>')).to eq('2026-08-31T00:00:00Z') + expect(ForestAdminDatasourceIntercom.logger).to have_received(:warn).with(/unknown timezone/) + end + end + + context 'when the caller names no timezone at all' do + let(:timezone) { ' ' } + + it 'reads the day boundary in UTC' do + expect(bound('2026-09-01', '>')).to eq('2026-08-31T00:00:00Z') + end + end + + it 'refuses a string that is not a date' do + expect { call('date', 'last tuesday', spelling: '>') } + .to raise_error(UnsupportedOperatorError, /Intercom expects a date on this field/) + end + + it 'refuses a value that is not a date at all' do + expect { call('date', { 'day' => 1 }, spelling: '>') } + .to raise_error(UnsupportedOperatorError, /Intercom expects a date on this field/) + end + + # Reading either as epoch seconds raises a FloatDomainError, which would + # leave the read with an error naming a float where the operator asked + # for a date. The number branch already refuses them; a date is no + # different. + it 'refuses a cast that overflowed to Infinity, and a NaN' do + expect { call('date', Float::INFINITY, spelling: '>') } + .to raise_error(UnsupportedOperatorError, /Intercom expects a date on this field/) + expect { call('date', Float::NAN, spelling: '>') } + .to raise_error(UnsupportedOperatorError, /Intercom expects a date on this field/) + end + + it 'reads a finite number as the epoch seconds Intercom stores' do + expect(bound(1_767_225_600, '>')).to eq('2025-12-31T00:00:00Z') + end + end + + describe 'a number' do + # The agent casts every Number column with `to_f`, so an integer field + # would otherwise be filtered with `42.0`, a form none of its values carry. + it 'sends a whole float as the integer it is' do + expect(call('number', 42.0)).to eq(42) + end + + it 'keeps a decimal, and an integer, as they are' do + expect(call('number', 42.5)).to eq(42.5) + expect(call('number', 42)).to eq(42) + end + + it 'reads an integer written as a string' do + expect(call('number', '42')).to eq(42) + end + + it 'reads a decimal written as a string' do + expect(call('number', '42.5')).to eq(42.5) + end + + # `to_i` raises on both, and so does the JSON encoder a step later, as a + # 500 naming nothing the operator can act on. + it 'refuses a cast that overflowed and a value that is not a number' do + expect { call('number', Float::INFINITY) }.to raise_error(UnsupportedOperatorError, /expects a number/) + expect { call('number', 'many') }.to raise_error(UnsupportedOperatorError, /expects a number/) + expect { call('number', []) }.to raise_error(UnsupportedOperatorError, /expects a number/) + end + end + + describe 'a boolean' do + it 'sends a flag as a boolean, whichever way the filter spelled it' do + expect(call('boolean', true)).to be(true) + expect(call('boolean', false)).to be(false) + expect(call('boolean', 'true')).to be(true) + expect(call('boolean', 'false')).to be(false) + end + + it 'refuses anything else, rather than reading it as truthy' do + expect { call('boolean', 'yes') }.to raise_error(UnsupportedOperatorError, /expects a true or a false/) + end + end + + describe 'a list' do + it 'formats every value of an IN the way the field takes it' do + expect(call('number', ['4', 5.0], spelling: 'IN', operator: operators::IN)).to eq([4, 5]) + end + + # A filter matching everything is not what an empty list was asked for. + it 'refuses an empty list' do + expect { call('string', [], spelling: 'IN', operator: operators::IN) } + .to raise_error(UnsupportedOperatorError, /empty list/) + end + + # `not_in [nil, 'open']` was asked to exclude the records carrying neither + # and would come back including the blank ones. + it 'refuses a list holding a blank, rather than dropping it' do + expect { call('string', [nil, 'open'], spelling: 'NIN', operator: operators::NOT_IN) } + .to raise_error(UnsupportedOperatorError, /cannot filter "c" for absence/) + end + end + + describe 'a condition on the absence of a value' do + # `present`, `blank` and `missing` are derived from an equality above this + # datasource and rewritten into a comparison with an empty value. Intercom + # would answer it as if the empty string were a value of its own. + it 'refuses the rewritten comparison and says to filter on a value' do + expect { call('string', nil) } + .to raise_error(UnsupportedOperatorError, /matches values and has no operator for the lack of one/) + end + + it 'refuses an empty string the same way' do + expect { call('string', '') }.to raise_error(UnsupportedOperatorError, /for absence/) + end + end + + describe 'a string' do + it 'sends it as it came' do + expect(call('string', 'open')).to eq('open') + end + + it 'sends a text the same way' do + expect(call('text', 'facture')).to eq('facture') + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/query/operator_table_spec.rb b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/query/operator_table_spec.rb new file mode 100644 index 000000000..42a998071 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/query/operator_table_spec.rb @@ -0,0 +1,110 @@ +module ForestAdminDatasourceIntercom + RSpec.describe Query::OperatorTable do + let(:operators) { ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Operators } + let(:rules) { ForestAdminDatasourceToolkit::Validations::Rules } + let(:equivalent) { ForestAdminDatasourceToolkit::Components::Query::ConditionTree::ConditionTreeEquivalent } + + def field(type, operators) + Query::SearchFields::Field.new(column: 'c', field: 'c', type: type, operators: operators, source: 'spec') + end + + describe '.forest_operators' do + it 'publishes only what the endpoint accepts on that field' do + expect(described_class.forest_operators(field('string', ['=']))).to eq([operators::EQUAL]) + expect(described_class.forest_operators(field('string', ['=', '!=']))) + .to eq([operators::EQUAL, operators::NOT_EQUAL]) + end + + # Intercom documents one substring operator and no case semantics for it, + # and the frontend sends either spelling depending on the column. + it 'reads both spellings of contains onto the one operator Intercom has' do + published = described_class.forest_operators(field('text', ['~'])) + + expect(published).to eq([operators::CONTAINS, operators::I_CONTAINS]) + end + + # The point of the whole table: a date column carries the two bounds even + # where the endpoint accepts more, because declaring an equality on a Date + # makes the toolkit republish `in`, which its own validator refuses. + it 'keeps a date column to the two bounds whatever the endpoint accepts' do + published = described_class.forest_operators(field('date', ['>', '<', '>=', '<=', '=', '!='])) + + expect(published).to eq([operators::GREATER_THAN, operators::LESS_THAN]) + end + + it 'publishes nothing for a field the endpoint answers no known operator on' do + expect(described_class.forest_operators(field('string', ['~']))).to be_empty + end + end + + describe '.intercom_operator' do + it 'spells a Forest operator the way the search DSL does' do + expect(described_class.intercom_operator(field('number', ['>']), operators::GREATER_THAN)).to eq('>') + end + + it 'answers nil for an operator the endpoint does not accept on that field' do + expect(described_class.intercom_operator(field('string', ['=']), operators::NOT_EQUAL)).to be_nil + end + + it 'answers nil for an operator no field of that type carries' do + expect(described_class.intercom_operator(field('boolean', ['=']), operators::CONTAINS)).to be_nil + end + end + + describe 'the operators every published column ends up with' do + # The invariant PRD-989 says nothing checks: everything the agent publishes + # from what a column declares must be an operator its own validator allows. + # A column advertising a filter the agent then rejects is a 400 in the + # interface for a reason that has nothing to do with Intercom. + it 'is a set the toolkit validator allows, for every column of every endpoint' do + column_types = { 'string' => 'String', 'text' => 'String', 'date' => 'Date', + 'boolean' => 'Boolean', 'number' => 'Number' } + + Query::SearchFields.endpoints.each do |name| + Query::SearchFields.fetch(name).fields.each_value do |searchable| + declared = described_class.forest_operators(searchable) + column_type = column_types.fetch(searchable.type) + published = operators.all.select { |o| equivalent.equivalent_tree?(o, declared, column_type) } + + expect(published - rules.get_allowed_operators_for_column_type(column_type)) + .to be_empty, "#{name}.#{searchable.column} publishes an operator Rules refuses" + end + end + end + + # Walked from the schema rather than from the table, which is the only way + # to see the primary key: `add_column` writes its operators by hand -- the + # toolkit refuses a collection whose key carries neither `equal` nor `in` + # -- so the loop above, reading the table, could never reach the one + # column no row of the table is derived from. A column the schema + # publishes and the table cannot spell is a filter the translator refuses + # at read time, which is what a record detail hits the moment a scope + # nests `id equals X` in an `and`. + it 'is a set the table can express, for every column the datasource publishes' do + cursor_collections.each do |collection| + endpoint = collection.send(:search_endpoint) + + collection.fields.each do |column, schema| + published = Array(schema.respond_to?(:filter_operators) ? schema.filter_operators : nil) + next if published.empty? + + searchable = endpoint.field(column) + expect(searchable).not_to be_nil, + "#{collection.name}.#{column} publishes #{published.join(", ")} and " \ + "#{endpoint.path} carries no row to translate it" + expect(published - described_class.forest_operators(searchable)) + .to be_empty, "#{collection.name}.#{column} publishes an operator the translator would refuse" + end + end + end + end + + # Only the two collections a search endpoint backs: the reference ones are + # read whole and filtered in memory, and publish no operator from a table. + def cursor_collections + datasource = Datasource.new(access_token: 's3cr3t', rate_limiter: nil) + + datasource.collections.each_value.grep(Collections::CursorCollection) + end + end +end diff --git a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/query/search_fields_spec.rb b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/query/search_fields_spec.rb new file mode 100644 index 000000000..b203c5545 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/query/search_fields_spec.rb @@ -0,0 +1,152 @@ +module ForestAdminDatasourceIntercom + RSpec.describe Query::SearchFields do + def table(fields: {}, refused: {}, path: 'tickets/search', measured_at: nil, candidates: []) + described_class.build( + 'endpoints' => { 'tickets' => { 'path' => path, 'measured_at' => measured_at, 'fields' => fields, + 'refused' => refused, 'candidates' => candidates } } + )['tickets'] + end + + def field_row(overrides = {}) + { 'field' => 'created_at', 'type' => 'date', 'operators' => ['>', '<'], 'source' => 'spec' }.merge(overrides) + end + + describe 'the committed table' do + # The schema derives its filters from this file, so a malformed row is a + # boot failure rather than a column nobody can explain. Reading it here is + # what turns that guarantee into a test. + it 'reads the two search endpoints' do + expect(described_class.endpoints).to eq(%w[conversations tickets]) + end + + it 'names the path each endpoint is searched through' do + expect(described_class.fetch('conversations').path).to eq('conversations/search') + expect(described_class.fetch('tickets').path).to eq('tickets/search') + end + + # The one thing lot 1 measured about the field lists: `/tickets/search` + # refuses `company_id`, which the specification lists on a ticket. + it 'refuses company_id on tickets, with the measurement as its reason' do + refusal = described_class.fetch('tickets').refusal('company_id') + + expect(refusal.reason).to include('invalid_field') + expect(refusal).to be_measured + end + + # The columns the agent derives from the parts of a ticket: Intercom + # filters none of them, and lot 1 published them without an operator. + it 'refuses every column derived from the parts of a ticket' do + refused = described_class.fetch('tickets').refused.keys + + expect(refused).to include('closed_at', 'closed_by_name', 'last_reply_at', 'last_responder_name', + 'last_responder_type') + end + + it 'keeps the ticket attributes unfilterable while the arbitration stands' do + expect(described_class.fetch('tickets').ticket_attributes['filterable']).to be(false) + end + + # Until the probe runs against the customer's workspace, the date rows are + # the only ones a measurement backs. + it 'reports which rows nothing has measured yet' do + conversations = described_class.fetch('conversations') + + expect(conversations).not_to be_measured + expect(conversations.unmeasured_fields.map(&:column)).not_to include('created_at', 'updated_at') + end + + it 'declares no column both filterable and refused' do + described_class.endpoints.each do |name| + endpoint = described_class.fetch(name) + + expect(endpoint.filterable_columns & endpoint.refused.keys).to be_empty + end + end + + # A candidate is what the probe enumerates on top of the table; one that + # is already declared would be probed twice and read as a discovery. + it 'names no candidate already declared as a field' do + described_class.endpoints.each do |name| + endpoint = described_class.fetch(name) + + expect(endpoint.candidates & endpoint.fields.values.map(&:field)).to be_empty + end + end + end + + # The README is where an operator reads what they may filter on before the + # interface shows it to them, so it is checked against the table rather than + # left to drift from it. + describe 'the README section the table feeds' do + let(:filterable) do + File.read(File.expand_path('../../../README.md', __dir__), encoding: 'UTF-8')[ + /### What is filterable\n(.*?)\n### /m, 1 + ] + end + + it 'lists exactly the columns each endpoint filters' do + { 'IntercomConversation' => 'conversations', 'IntercomTicket' => 'tickets' }.each do |collection, endpoint| + row = filterable.lines.find { |line| line.start_with?("| `#{collection}` |") } + listed = row.to_s.scan(/`([a-z_]+)`/).flatten + + expect(listed).to match_array(described_class.fetch(endpoint).filterable_columns) + end + end + end + + describe 'a table that cannot be trusted' do + it 'refuses an operator Intercom has no spelling for' do + expect { table(fields: { 'created_at' => field_row('operators' => ['~=']) }) } + .to raise_error(ConfigurationError, /has no operator ~=/) + end + + it 'refuses a type nothing knows how to send' do + expect { table(fields: { 'created_at' => field_row('type' => 'timestamp') }) } + .to raise_error(ConfigurationError, /type "timestamp" is not one of/) + end + + # An empty list would publish a filterable column no operator can reach. + it 'refuses a field declaring no operator, and says where it belongs' do + expect { table(fields: { 'created_at' => field_row('operators' => []) }) } + .to raise_error(ConfigurationError, /belongs in the refused table/) + end + + it 'refuses a provenance that is neither measured nor read off the documentation' do + expect { table(fields: { 'created_at' => field_row('source' => 'guessed') }) } + .to raise_error(ConfigurationError, /source "guessed" is neither measured nor spec/) + end + + it 'refuses a refusal with no provenance of its own' do + expect { table(refused: { 'company_id' => { 'reason' => 'no', 'source' => 'hearsay' } }) } + .to raise_error(ConfigurationError, /tickets.company_id/) + end + + it 'names the endpoint and the column it choked on' do + expect { table(fields: { 'created_at' => field_row('type' => 'timestamp') }) } + .to raise_error(ConfigurationError, /search_fields\.yml is malformed at tickets\.created_at/) + end + end + + describe 'what it hands to the schema' do + it 'carries the Intercom field a column is filtered through' do + expect(described_class.fetch('conversations').field('closed_at').field).to eq('statistics.last_close_at') + end + + it 'reads a refusal reason as one line, whatever the YAML wrapping' do + reason = table(refused: { 'company_id' => { 'reason' => "one\ntwo\n", 'source' => 'spec' } }) + .refusal('company_id').reason + + expect(reason).to eq('one two') + end + + it 'refuses an endpoint nothing declares, rather than filtering nothing' do + expect { described_class.fetch('contacts') } + .to raise_error(ConfigurationError, /Unknown Intercom search endpoint "contacts"/) + end + + it 'is measured once the probe has stamped a date on it' do + expect(table(measured_at: '2026-09-01')).to be_measured + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/spec/probe_search_fields_spec.rb b/packages/forest_admin_datasource_intercom/spec/probe_search_fields_spec.rb new file mode 100644 index 000000000..405d2f85f --- /dev/null +++ b/packages/forest_admin_datasource_intercom/spec/probe_search_fields_spec.rb @@ -0,0 +1,171 @@ +require 'tmpdir' + +load File.expand_path('../bin/probe_search_fields', __dir__) + +module ForestAdminDatasourceIntercom + RSpec.describe ProbeSearchFields do + let(:base) { Configuration::REGION_HOSTS[:us] } + let(:endpoint) { Query::SearchFields.fetch('tickets') } + let(:client) { Client.new(Configuration.new(access_token: 's3cr3t', rate_limiter: nil)) } + + def json(payload, status = 200) + { status: status, body: payload.to_json, headers: { 'Content-Type' => 'application/json' } } + end + + # Every probe is a search asking for one record; what a stub answers is + # therefore an empty page or one of Intercom's refusal codes. + def stub_search(code: nil, operator: nil, value: nil) + body = if code + { 'type' => 'error.list', 'errors' => [{ 'code' => code, 'message' => 'nope' }] } + else + { 'type' => 'list', 'tickets' => [], 'total_count' => 0 } + end + + stub_request(:post, "#{base}/tickets/search").with { |request| matches?(request, operator, value) } + .to_return(json(body, code ? 400 : 200)) + end + + def matches?(request, operator, value) + query = JSON.parse(request.body)['query'] + + (operator.nil? || query['operator'] == operator) && (value.nil? || query['value'] == value) + end + + describe 'the type guessed for a candidate field' do + # A candidate is a name and nothing else -- discovering what it is is the + # point -- so the value shape sent with it is guessed from that name. + it 'reads a timestamp, a count and a flag off the name' do + expect(described_class.guess_type('statistics.last_close_at')).to eq('date') + expect(described_class.guess_type('count_reopens')).to eq('number') + expect(described_class.guess_type('open')).to eq('boolean') + expect(described_class.guess_type('state')).to eq('string') + end + end + + describe described_class::Probe do + subject(:probe) { described_class.new(client, endpoint) } + + it 'keeps the operators the endpoint answers and drops the ones it refuses' do + stub_search(code: 'data_invalid') + stub_search(operator: '=') + stub_search(operator: '!=') + + result = probe.run('category', 'string') + + expect(result[:operators].select { |_, outcome| outcome[:ok] }.keys).to eq(['=', '!=']) + end + + # A field the endpoint does not filter at all is worth one request, not + # twelve: `invalid_field` ends the row. + it 'stops at the first invalid_field and reports the field unfilterable' do + stub_search(code: 'invalid_field') + + result = probe.run('company_id', 'string') + + expect(result[:unfilterable][:code]).to eq('invalid_field') + expect(a_request(:post, "#{base}/tickets/search")).to have_been_made.once + end + + # A wrong value shape is refused with the same code as an unsupported + # operator, so a single attempt would report a filter Intercom does answer + # as refused. + it 'retries a refused cell with the other value shapes its type takes' do + stub_search(code: 'data_invalid') + stub_search(operator: '>', value: '2026-01-01') + + result = probe.run('created_at', 'date') + + expect(result[:operators]['>'][:ok]).to be(true) + expect(result[:operators]['<'][:ok]).to be(false) + end + + it 'names the failure by its HTTP status when Intercom sends no error code' do + stub_request(:post, "#{base}/tickets/search").to_return(json({ 'nope' => true }, 500)) + + result = probe.run('category', 'string') + + expect(result[:operators]['='][:code]).to eq('http_500') + end + end + + describe described_class::Report do + subject(:report) { described_class.new(endpoint) } + + def run_probe(field, type) + ProbeSearchFields::Probe.new(client, endpoint).run(field, type) + end + + # The only reason to run the probe: an operator the table promises and + # Intercom refuses is a filter the interface offers and the read cannot + # honour. + it 'reports an operator the table promises and Intercom refuses' do + stub_search(code: 'data_invalid') + stub_search(operator: '=') + report.record('category', 'category', run_probe('category', 'string')) + + expect { report.print_diff }.to output(/! the table promises != here, Intercom refuses it/).to_stdout + end + + it 'reports an operator the table does not know about yet' do + stub_search(code: 'data_invalid') + ['=', '!=', '~'].each { |operator| stub_search(operator: operator) } + report.record('category', 'category', run_probe('category', 'string')) + + expect { report.print_diff }.to output(/\+ Intercom also accepts ~/).to_stdout + end + + it 'names the column a field the endpoint refuses is declared on' do + stub_search(code: 'invalid_field') + report.record('category', 'category', run_probe('category', 'string')) + + expect { report.print_diff }.to output(/NOT FILTERABLE \(invalid_field\).*column 'category'/).to_stdout + end + + it 'writes the measurement as evidence, refusal codes included' do + stub_search(code: 'data_invalid') + stub_search(operator: '=') + report.record('category', 'category', run_probe('category', 'string')) + + written = YAML.safe_load(report.to_yaml_document) + + expect(written['fields']['category']['operators']).to eq(['=']) + expect(written['fields']['category']['refused']['!=']).to eq('data_invalid') + end + + it 'writes a field the endpoint refuses as unfilterable' do + stub_search(code: 'invalid_field') + report.record('company_id', nil, run_probe('company_id', 'string')) + + written = YAML.safe_load(report.to_yaml_document) + + expect(written['fields']['company_id']).to eq({ 'filterable' => false, 'code' => 'invalid_field' }) + end + end + + describe described_class::CLI do + # A probe with no token would abort on the first request with an Intercom + # 401, which reads as a workspace problem rather than as a missing option. + it 'refuses to start without a token' do + expect { described_class.call(['--endpoint', 'tickets']) } + .to raise_error(SystemExit).and output(/No token/).to_stderr + end + + it 'refuses an endpoint the table does not declare' do + expect { described_class.call(['--endpoint', 'contacts', '--token', 's3cr3t']) } + .to raise_error(ConfigurationError, /Unknown Intercom search endpoint/) + end + + it 'probes every declared endpoint and writes the evidence where it was asked to' do + out = File.join(Dir.tmpdir, 'intercom-probe.yml') + stub_search(code: 'invalid_field') + stub_request(:post, "#{base}/conversations/search") + .to_return(json({ 'type' => 'error.list', 'errors' => [{ 'code' => 'invalid_field' }] }, 400)) + + expect { described_class.call(['--token', 's3cr3t', '--out', out]) }.to output(/NOT FILTERABLE/).to_stdout + expect(YAML.safe_load_file(out)['endpoint']).to eq('conversations') + ensure + FileUtils.rm_f(out) + end + end + end +end From 33987828d32e113586bd0bbc23c3694f83f88e86 Mon Sep 17 00:00:00 2001 From: Christophe Brun Date: Mon, 7 Sep 2026 14:56:08 +0200 Subject: [PATCH 3/6] feat(datasource intercom): relations of the reference tier (lot 2.5) (#385) * feat(datasource): intercom relations of the reference tier Declares the relations between the collections this datasource already serves, and answers them (lot 2.5, PRD-1145). Publishing a many-to-one is not free: the agent marks it filterable as soon as any column of its target is, so the interface offers `assignee:name` the moment the relation exists. The traversal is therefore the bulk of this change, not the declarations. * eight many-to-one relations towards Admin, Team, TicketState and TicketType, whose targets all shipped with lot 1; * IntercomTeamMembership, synthesized from GET /teams: Intercom carries the membership on both sides and exposes no resource for the pair, while a many-to-many needs a collection to travel through. Read-only, Intercom writing none; * admin_names and team_names replace the arrays of ids on Team and Admin, read only when a projection asks for them; * one label per object on a ticket: state_category and state_external_label are dropped, both being a hop away on the state relation and neither having ever been filterable; * a projection through a relation reads its target once per page; a condition through one is resolved by the target, exactly, and becomes a condition on the foreign key. Over fifteen matching ids, or on a key the endpoint does not filter, it is refused by name rather than approximated. Co-Authored-By: Claude Opus 5 (1M context) --- .../README.md | 111 ++++++- .../collections/admin.rb | 41 ++- .../collections/base_collection.rb | 48 ++- .../collections/conversation.rb | 34 ++- .../collections/cursor_collection.rb | 79 ++++- .../collections/fetch_all_collection.rb | 48 ++- .../collections/relations.rb | 266 +++++++++++++++++ .../collections/team.rb | 40 ++- .../collections/team_membership.rb | 61 ++++ .../collections/ticket.rb | 37 ++- .../collections/ticket/serializer.rb | 4 +- .../datasource.rb | 4 + .../query/condition_tree_translator.rb | 24 +- .../collections/admin_spec.rb | 86 +++++- .../collections/conversation_spec.rb | 58 +++- .../collections/fetch_all_collection_spec.rb | 33 +- .../collections/team_membership_spec.rb | 206 +++++++++++++ .../collections/team_spec.rb | 167 +++++++++-- .../collections/ticket_spec.rb | 282 +++++++++++++++++- .../datasource_spec.rb | 7 +- .../query/condition_tree_translator_spec.rb | 16 +- .../utils/collection.rb | 21 +- .../utils/collection_spec.rb | 25 +- 23 files changed, 1589 insertions(+), 109 deletions(-) create mode 100644 packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/relations.rb create mode 100644 packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/team_membership.rb create mode 100644 packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/team_membership_spec.rb diff --git a/packages/forest_admin_datasource_intercom/README.md b/packages/forest_admin_datasource_intercom/README.md index 38503e773..43ce826e8 100644 --- a/packages/forest_admin_datasource_intercom/README.md +++ b/packages/forest_admin_datasource_intercom/README.md @@ -60,7 +60,15 @@ beats not running. A read-only token is enough, and is what to recommend for this lot. A permission the token lacks costs **columns or a collection, never the boot of the agent**: the ticket-type introspection -degrades to no attribute column, and a collection whose endpoint answers 403 fails its own page. +degrades to no attribute column, a collection whose endpoint answers 403 fails its own page, and a +token that cannot read `/admins` or `/teams` leaves the `admin_names` / `team_names` column empty +rather than failing the page it is on. + +A **relation is the exception**, and it is worth knowing before scoping a token: resolving one reads +the target endpoint, and that read is not guarded the way the names above are. A token denied +`/admins` fails any page projecting `admin_assignee:name`, and fails the related list behind +`IntercomTeam#admins` — the failure lands on the collection being read, not on the one that was +denied. Scope the token to the endpoints in the table below, or to none of them. ## Collections @@ -70,12 +78,13 @@ degrades to no attribute column, and a collection whose endpoint answers 403 fai | `IntercomTicket` | `POST /tickets/search`, `GET /tickets/{id}` | cursor | yes, exactly | | `IntercomAdmin` | `GET /admins` | read whole | yes, exactly | | `IntercomTeam` | `GET /teams` | read whole | yes, exactly | +| `IntercomTeamMembership` | `GET /teams` | read whole | yes, exactly | | `IntercomTicketType` | `GET /ticket_types` | read whole | yes, exactly | | `IntercomTicketState` | `GET /ticket_states` | read whole | yes, exactly | Two tiers, and they behave differently on purpose. -**Read whole** — admins, teams, ticket types, ticket states. Their endpoints answer in one response, +**Read whole** — admins, teams, team memberships, ticket types, ticket states. Their endpoints answer in one response, so filtering, sorting, paging and counting them in memory is *exact*: the records in hand are every record Intercom holds. These are the only collections that can be filtered, sorted and grouped in this lot, and the only ones a chart may group by. The cost is bandwidth, not correctness. @@ -86,6 +95,66 @@ nothing is filtered or sorted in memory. Three routes and no fourth: no conditio Intercom's search DSL and walked through the search endpoint. What the translation cannot express is **refused by name** — see [Filtering](#filtering). +## Relations + +Intercom joins nothing: a ticket carries an assignee id, and the teammate behind it is a second read +of a second endpoint. What makes eight relations affordable is that every collection on the far end +is read whole in one request — so a relation resolves for a **whole page at the price of one read**, +never one read per row. The price is per target *collection*, not per relation: a ticket's `state` +and `previous_state` are one read of `/ticket_states`, over the ids both of them name. + +*Exactly*, with one bound worth naming: "read whole" is what the endpoint answers, and `fetch_all` +stops after [`MAX_COLLECTED_PAGES`](lib/forest_admin_datasource_intercom/client.rb) pages if Intercom +paginates one of these on its own — it logs when it does. A workspace whose `/admins` or `/teams` +runs past that cap resolves the relations pointing at the records it dropped as empty. The figure is +sized for reference collections, which is what every target here is. + +| Collection | Relation | Target | Filterable through | +| --- | --- | --- | --- | +| `IntercomConversation` | `admin_assignee`, `closed_by` | `IntercomAdmin` | yes | +| `IntercomConversation` | `team_assignee` | `IntercomTeam` | yes | +| `IntercomTicket` | `admin_assignee` | `IntercomAdmin` | yes | +| `IntercomTicket` | `team_assignee` | `IntercomTeam` | yes | +| `IntercomTicket` | `ticket_type` | `IntercomTicketType` | yes | +| `IntercomTicket` | `state`, `previous_state` | `IntercomTicketState` | **no** — read and navigate only | +| `IntercomTeam` | `admins` | `IntercomAdmin` | no (many-to-many) | +| `IntercomAdmin` | `teams` | `IntercomTeam` | no (many-to-many) | +| `IntercomTeamMembership` | `team`, `admin` | `IntercomTeam`, `IntercomAdmin` | yes | + +Every one of them is **read-only**: this lot writes nothing, and Intercom exposes no endpoint that +writes a team membership at all. + +**`IntercomTeamMembership` exists because Intercom's does not.** The workspace carries the +membership on the team (`admin_ids`) and on the teammate (`team_ids`) both and exposes no resource +for the pair, while a many-to-many needs a collection to travel through. It is synthesized from +`GET /teams`, one record per pair, keyed `teamId:adminId`. Without it, both sides read as an array of +ids nobody can click. + +Two consequences of travelling through it are worth knowing. A **related list of teammates is +ordered by the membership, not by the teammate**: the agent hands the through collection the columns +of the collection the relation reaches, so an order on `name` or `email` cannot be resolved there and +is logged rather than silently dropped. And a `admin_ids` entry naming a teammate `/admins` does not +answer -- one who left, one outside the token's reach -- **drops out of the related list** instead of +appearing as an empty row. + +Alongside it, a team names its teammates (`admin_names`) and a teammate its teams (`team_names`) on +the row itself, so a list view reads without a join. **Those replace the arrays of ids** the first +lots published: one readable form plus a relation to navigate, rather than two ways to read one fact. +They are read only when a projection asks for them, and a token that cannot read the other side +costs the column and nothing else — never the page, and never the relation. + +The same rule settled the ticket labels: `state_label` and `ticket_type_name` stay on the row, +`state_category` and `state_external_label` are gone — they are a hop away, on the `state` relation, +and neither was ever filterable, so no segment, scope or saved filter could rest on them. + +A relation reads its target **undecorated**, so a permission scope or a segment defined on the target +does not narrow what a relation resolves — the same way a native datasource joins a table without +applying the scopes of the collection mapped to it. + +One semantic worth stating plainly: **a row whose foreign key is null matches no relation filter**, +the way a join drops it, negated filters included. A ticket with no assignee is not "assigned to +someone other than Marie". + ## What the API cannot do, and what this does about it Where Forest asks for something Intercom has no equivalent for, this datasource **refuses with a @@ -200,6 +269,9 @@ refuses a search by name. `last_responder_name`, `last_responder_type`. They exist nowhere in Intercom; `/tickets/search` filters none of them and ignores a sort on them without a word; - **the account of a ticket** — `company_id`, refused by the endpoint itself with `invalid_field`; +- **the state of a ticket** — the measured table carries no filter on a state id, so `state_id`, + `previous_state_id` and the `state` relation are read and navigated rather than filtered. Whether + the endpoint filters one at all is one of the probe's open questions; - **the ticket attributes** — filtered as `ticket_attribute.{id}`, and the same attribute carries a different id per ticket type, so a union column has no single id to translate to. See [Tickets](#tickets); @@ -212,6 +284,39 @@ refuses a search by name. - **group-by**, on either cursor collection: there is no aggregate endpoint, and grouping over the pages a walk collected would look exact while answering a fraction. +### Through a relation + +A relation is published filterable as soon as *any* column of its target is — the agent decides that, +not this datasource — so the interface offers `admin_assignee:name` the moment the relation exists. +What Intercom is really filtered on is the foreign key: the **target says which of its records +match**, over every record it holds rather than over a page, and the ids it names become the +condition the search carries. + +That is exact, and it has three visible edges: + +- Intercom takes no membership operator on these fields, so several matches become **one equality per + match**, inside an `OR` — which counts against the fifteen conditions a group allows. A relation + condition matching more records than that is refused by name rather than sent and answered with a + 400 naming neither the limit nor the filter that hit it. That `OR` is **inlined into a parent that + aggregates the same way**, so it costs no level of nesting where it does not have to: the two + levels Intercom allows are spent on the filter that was written, not on the expansion of a + relation. Where inlining it would take the parent past fifteen conditions it stays nested, width + being the scarcer of the two. +- A condition the target matched **no record** with names no row, and the DSL cannot say so: the + search is skipped entirely rather than sent as a filter that would come back with everything. +- A relation whose foreign key the endpoint does not filter — the ticket `state` — is refused with a + message saying which of the two it is: the relation is there to be read and navigated. Whether + `/tickets/search` filters a state id at all is one of the probe's open questions; the answer lands + in the table, not in an assumption. + +On the collections read whole the same condition costs nothing: they filter in memory, so the ids go +in as a plain membership and none of the DSL's limits apply. + +A **many-to-many is published unfilterable** — `admins` and `teams` — and a condition written on one +anyway, in a scope or a segment, is refused before it reaches this datasource: the agent's own +validator answers a 400 naming the field and its type. Filter on a column of the collection next +door instead. + ### The limits of a search, checked before the request leaves Intercom nests a search **two levels** deep and takes **fifteen conditions per group**. Past either @@ -327,7 +432,7 @@ Everything else is read when a collection is listed, so an agent boots whatever | Lot | What it brings | | --- | --- | | 3 | Writes and business actions: reply, close, snooze, reopen, assign, tag, convert | -| 4 | Contacts and companies, and the relations promoted from today's denormalized columns | +| 4 | Contacts and companies, and the relations towards them promoted from today's denormalized columns | | 5 | Notes, tags, segments | | 6 | Bounded group-by and the reporting export | diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/admin.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/admin.rb index 45704ea82..9cba4a2d3 100644 --- a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/admin.rb +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/admin.rb @@ -25,11 +25,37 @@ def serialize(admin) 'away_mode_enabled' => attrs['away_mode_enabled'], 'away_mode_reassign' => attrs['away_mode_reassign'], 'has_inbox_seat' => attrs['has_inbox_seat'], - 'team_ids' => Array(attrs['team_ids']).map { |id| stringify_id(id) } } + # Not a column: it is what the names below are read from, and what the + # membership collection turns into the relation. + 'team_ids' => Array(attrs['team_ids']).map { |id| stringify_id(id) }, + 'team_names' => nil } + end + + # The teams the teammate belongs to, by name, and only when a projection + # asked for them: one read of `/teams` for the whole page. A token that + # cannot read them costs the column and nothing else. + def enrich(records, rows, projection) + return unless column_asked?(projection, 'team_names') + + names = team_names + records.each_with_index do |record, index| + rows[index]['team_names'] = Array(record['team_ids']).filter_map { |id| names[id] } + end end private + def team_names + client.fetch_all('teams', list_key: 'teams') + .to_h { |team| [stringify_id(team['id']), team['name']] } + rescue APIError => e + ForestAdminDatasourceIntercom.logger.warn( + "[forest_admin_datasource_intercom] #{name} could not read the teams of the workspace (HTTP " \ + "#{e.status || "-"}); the names are left empty. The relation to IntercomTeam is unaffected." + ) + {} + end + def define_schema add_column('id', 'String', is_primary_key: true) add_column('name', 'String') @@ -41,11 +67,14 @@ def define_schema add_column('away_mode_enabled', 'Boolean') add_column('away_mode_reassign', 'Boolean') add_column('has_inbox_seat', 'Boolean') - # A list, so neither filterable nor sortable. It stays a plain column - # rather than a relation: Intercom carries the membership on the admin - # and on the team both, so declaring it twice would give the schema two - # sides of a many-to-many with no join collection to hold it. - add_column('team_ids', 'Json') + # A list, so neither filterable nor sortable -- as the array of ids it + # replaces was. It reads the teammate without a join; the relation below + # is what navigates it, through the membership collection that gives the + # many-to-many the join Intercom does not expose. + add_column('team_names', 'Json') + add_many_to_many('teams', foreign_collection: 'IntercomTeam', + through_collection: 'IntercomTeamMembership', + origin_key: 'admin_id', foreign_key: 'team_id') end end end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/base_collection.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/base_collection.rb index 9c4731f27..30d11d7e7 100644 --- a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/base_collection.rb +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/base_collection.rb @@ -4,11 +4,15 @@ module Collections # record is narrowed to the projection asked for, and how a window is cut # out of records already in hand. # - # Read-only for now. The writes and the business actions arrive with lot 3, - # and the relations with lot 4, once Contacts and Companies exist -- a - # relation whose target collection is missing is a schema the agent refuses - # to boot on. + # Read-only for now: the writes and the business actions arrive with lot 3. + # The relations towards Contacts and Companies wait for lot 4, those two + # collections not existing yet -- a relation whose target collection is + # missing is a schema the agent refuses to boot on. The relations between the + # collections this datasource already serves are declared and answered here, + # through `Relations`. class BaseCollection < ForestAdminDatasourceToolkit::Collection + include Relations + ColumnSchema = ForestAdminDatasourceToolkit::Schema::ColumnSchema Operators = ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Operators Equivalent = ForestAdminDatasourceToolkit::Components::Query::ConditionTree::ConditionTreeEquivalent @@ -27,14 +31,34 @@ def client def define_schema = raise(NotImplementedError, "#{self.class} did not implement define_schema") - # A record narrowed to what was asked for. A projection naming a field the - # record does not carry yields nil rather than nothing at all: the agent - # asked for a column, and an absent key would read as a record missing it. + # A record narrowed to the columns that were asked for. A projection naming + # a field the record does not carry yields nil rather than nothing at all: + # the agent asked for a column, and an absent key would read as a record + # missing it. + # + # A path through a relation is not a column and is skipped here: it is + # answered by `embed_relations`, which nests a whole row under the relation + # name once the page is in hand. + # + # No projection at all asks for every column -- which is not the same as + # every key a serialized record happens to carry: a couple of them hold + # what a column is read *from*, the ids behind a membership for one, and + # publishing those would show the operator the plumbing. def project(record, projection) - fields = Array(projection) - return record if fields.empty? + asked = Array(projection).map(&:to_s) + return record.slice(*column_names) if asked.empty? + + asked.reject { |field| field.include?(':') }.to_h { |field| [field, record[field]] } + end + + # Whether a projection asks for a column. No projection at all asks for + # every declared column, which is how `project` reads it -- an enrichment + # guarded on the column being named would leave nil the very column the + # projection publishes. + def column_asked?(projection, column) + asked = Array(projection).map(&:to_s) - fields.to_h { |field| [field, record[field]] } + asked.empty? || asked.include?(column) end # The window a list view asked for, cut out of records already in hand. @@ -54,6 +78,10 @@ def page_window(records, filter) records[offset, limit] || [] end + def column_names + @column_names ||= fields.select { |_, field| field.is_a?(ColumnSchema) }.keys + end + # The timezone in-memory date comparisons are evaluated in. The caller's, # since that is whose "today" the filter was written against. def timezone_for(caller) diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/conversation.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/conversation.rb index dbe8a53d0..380caeec9 100644 --- a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/conversation.rb +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/conversation.rb @@ -10,7 +10,7 @@ module Collections # customers, and rendering third-party HTML inside Forest is neither safe nor # useful (R10). # Long by line count only: most of it declares the columns, one call each. - class Conversation < CursorCollection + class Conversation < CursorCollection # rubocop:disable Metrics/ClassLength include ContactIdentity include Conversation::Serializer include Conversation::Timeline @@ -69,18 +69,38 @@ def define_schema add_column('updated_at', 'Date') add_column('waiting_since', 'Date') add_column('snoozed_until', 'Date') - add_column('admin_assignee_id', 'String') - add_column('team_assignee_id', 'String') - # The conversation carries its company as a whole object, so the account - # name is free here -- unlike on a ticket, which carries the id alone. - add_column('company_id', 'String') - add_column('company_name', 'String') + define_assignment_columns define_contact_columns define_source_columns define_statistics_columns add_column('tag_names', 'Json') add_column('ai_agent_participated', 'Boolean') add_column('timeline', 'Json') + define_relations + end + + # Who the conversation sits with, and who closed it. All three targets are + # read whole in one request, and `/conversations/search` takes a filter on + # each of the three keys -- so these relations can be read, navigated and + # filtered through alike. + # + # No relation towards the company: a conversation carries its account as a + # whole object, so the name is already on the row, and the Companies + # collection arrives with lot 4. + def define_relations + add_many_to_one('admin_assignee', foreign_collection: 'IntercomAdmin', foreign_key: 'admin_assignee_id') + add_many_to_one('team_assignee', foreign_collection: 'IntercomTeam', foreign_key: 'team_assignee_id') + add_many_to_one('closed_by', foreign_collection: 'IntercomAdmin', foreign_key: 'closed_by_id') + end + + # Who the conversation sits with, and which account it belongs to. The + # conversation carries its company as a whole object, so the account name + # is free here -- unlike on a ticket, which carries the id alone. + def define_assignment_columns + add_column('admin_assignee_id', 'String') + add_column('team_assignee_id', 'String') + add_column('company_id', 'String') + add_column('company_name', 'String') end # The contact identity is denormalized onto the row rather than declared as diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/cursor_collection.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/cursor_collection.rb index 17c7d3c39..62545008f 100644 --- a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/cursor_collection.rb +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/cursor_collection.rb @@ -16,6 +16,10 @@ module Collections # refuses by name: an unfiltered page served in answer to a filter is the # one failure this datasource is built to avoid. # + # A condition through a relation is resolved before any of that: the target + # collection is asked which of its records match, and what reaches Intercom + # is a condition on the foreign key. See `Relations`. + # # Counting is the exception that costs nothing: `total_count` is exact on # every response, filter included, so the record counter is one request. # Long by line count only: half of it is the refusals, and a refusal that @@ -39,8 +43,14 @@ def list(caller, filter, projection) warn_ignored_sort(filter&.sort) records = fetch_records(caller, filter) - rows = records.map { |record| project(serialize(record), projection) } + # Serialized whole and projected afterwards rather than the other way + # round: a projection reaching through a relation names no foreign key, + # and the key is where the relation is read from. + serialized = records.map { |record| serialize(record) } + rows = serialized.map { |record| project(record, projection) } + enrich(records, rows, projection) + embed_relations(caller, serialized, rows, projection) rows end @@ -146,7 +156,13 @@ def fetch_records(caller, filter) # having been dropped by the truncation before the window was applied. return records_by_ids(page_window(ids, filter)) if ids - listed_records(filter, translate(caller, filter)) + query = translate(caller, filter) + # A condition through a relation the target matched no record with names + # no row, and Intercom's DSL cannot say so: the read is skipped rather + # than sent as a filter that would come back with everything. + return [] if query == NOTHING + + listed_records(filter, query) end # The Intercom query a filter comes down to, or nil for a list view, which @@ -155,12 +171,64 @@ def fetch_records(caller, filter) # one tree, it is checked against the nesting Intercom allows like every # other condition, instead of adding a level nothing counted. def translate(caller, filter) - tree = combined_tree(filter) + tree = rewrite_relation_conditions(caller, combined_tree(filter)) do |key, ids, leaf| + relation_group(key, ids, leaf) + end + return NOTHING if tree == NOTHING Query::ConditionTreeTranslator.call(tree, endpoint: search_endpoint, collection: name, timezone: timezone_for(caller)) end + # The ids the target matched, written as the filter Intercom does take on + # the foreign key: a group of equalities, its DSL offering no membership + # operator on these fields. + # + # That group counts against the fifteen conditions Intercom allows, so a + # relation condition matching more records than that is refused rather than + # sent -- and refused here, where the message can name the relation the + # operator filtered on rather than the key it resolved to. + # + # It also costs a level of nesting, which is why it is built as the group + # `Relations` can tell from one the operator wrote: inlined into a parent + # that aggregates the same way, it costs none. + def relation_group(key, ids, leaf) + refuse_fan_out!(leaf, key, ids) if ids.size > Query::ConditionTreeTranslator::MAX_GROUP_SIZE + return Leaf.new(key, Operators::EQUAL, ids.first) if ids.size == 1 + + RelationBranch.new('Or', ids.map { |id| Leaf.new(key, Operators::EQUAL, id) }, leaf.field) + end + + # Inlining a relation group into its parent trades a level of nesting for + # width, and Intercom bounds both. Past the conditions a group may hold, + # the nested form is the one it answers. + def absorb_relation_group?(size) + size <= Query::ConditionTreeTranslator::MAX_GROUP_SIZE + end + + # A relation this endpoint filters nothing through. It is navigable all the + # same -- the read costs nothing, the target being read whole -- and saying + # which of the two it is, is the whole point of the message. + def check_relation_filterable!(leaf, relation) + key = relation.foreign_key + refuse_unfilterable_key!(leaf, key) if search_endpoint.field(key).nil? + end + + def refuse_unfilterable_key!(leaf, key) + raise UnsupportedOperatorError, + "#{name} cannot filter #{leaf.field.inspect}: the relation resolves to #{key.inspect}, on " \ + "which #{search_endpoint.path} takes no filter. The relation is there to be read and " \ + "navigated; filter on one of: #{search_endpoint.filterable_columns.join(", ")}." + end + + def refuse_fan_out!(leaf, key, ids) + raise UnsupportedOperatorError, + "#{name} cannot filter #{leaf.field.inspect}: it names #{ids.size} records, " \ + "#{search_endpoint.path} answers #{key.inspect} one value at a time, and Intercom takes " \ + "#{Query::ConditionTreeTranslator::MAX_GROUP_SIZE} conditions per group. Narrow the condition " \ + "on the relation, or filter on #{key.inspect} itself." + end + def combined_tree(filter) conditions = [filter&.condition_tree, search_condition(filter)].compact return conditions.first if conditions.size < 2 @@ -244,7 +312,10 @@ def count_records(caller, filter) ids = id_lookup(filter) return records_by_ids(ids).size if ids - page = read_page(per_page: 1, cursor: nil, query: translate(caller, filter)) + query = translate(caller, filter) + return 0 if query == NOTHING + + page = read_page(per_page: 1, cursor: nil, query: query) return page.total_count if page.total_count raise UnsupportedOperatorError, diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/fetch_all_collection.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/fetch_all_collection.rb index ceeefe5f5..2cb9d8b30 100644 --- a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/fetch_all_collection.rb +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/fetch_all_collection.rb @@ -17,7 +17,7 @@ module Collections # Each read re-reads the endpoint, so an operator sees what Intercom holds # now rather than what it held when the process booted. One request per list # against a 10 000-a-minute budget is not a figure any list view approaches. - class FetchAllCollection < BaseCollection + class FetchAllCollection < BaseCollection # rubocop:disable Metrics/ClassLength # The filters a column may advertise, per column type. Restricted to what # the toolkit can evaluate in memory, since the in-memory pass is the only # pass there is here: an operator with no equivalence makes `match` answer @@ -57,8 +57,12 @@ def initialize(datasource, name) def list(caller, filter, projection) records = sort_in_memory(filtered_records(caller, filter), filter&.sort) + window = page_window(records, filter) + rows = window.map { |record| project(record, projection) } - page_window(records, filter).map { |record| project(record, projection) } + enrich(window, rows, projection) + embed_relations(caller, window, rows, projection) + rows end # Exact, like the filter and the sort above it, which is why these columns @@ -90,6 +94,12 @@ def add_column(name, type, is_primary_key: false) is_groupable: type != 'Json')) end + # Hook for what a row needs beyond the endpoint this collection reads -- + # a name held on the other side of a membership, say. Called with the rows + # of the page only, and with the projection, so a column nobody asked for + # costs no request. + def enrich(_records, _rows, _projection); end + # Every record of the collection, straight from its endpoint. def fetch_all = raise(NotImplementedError, "#{self.class} did not implement fetch_all") @@ -105,6 +115,13 @@ def filtered_records(caller, filter) tree = filter&.condition_tree return records if tree.nil? + # A condition through a relation becomes a membership on the foreign key: + # this tier filters in memory, where a list of ids costs no more than one + # id and none of Intercom's group limits apply -- they bound its search + # DSL, which nothing here goes through. + tree = rewrite_relation_conditions(caller, tree) { |key, ids, _| Leaf.new(key, Operators::IN, ids) } + return [] if tree == NOTHING + refuse_unevaluable!(tree) tree.apply(records, self, timezone_for(caller)) end @@ -156,19 +173,36 @@ def sort_in_memory(records, sort) # A sort clause naming a field this collection does not carry is dropped: # ordering by a column that is not there would compare nil to nil on every - # row and leave the order to the tie-break. + # row and leave the order to the tie-break. Reported when it happens, + # rather than dropped in silence -- an order asked for and not honoured is + # reported everywhere else in this datasource, and there is one route that + # reaches this tier with a clause it cannot resolve: a related list through + # a many-to-many is served by the collection it travels through, and + # `Filter#nest` prefixes the condition tree without prefixing the sort, so + # the membership is handed the columns of the collection it reaches. def sort_clauses(sort) - Array(sort).filter_map do |clause| - field = clause[:field] || clause['field'] - next unless fields.key?(field) + known, unknown = Array(sort).partition { |clause| fields.key?(sort_field(clause)) } + warn_unsortable(unknown) unless unknown.empty? + known.map do |clause| # `key?` rather than `||`: a descending clause carries `false`, which an # `||` fallback would read as "absent" and turn back into ascending. ascending = clause.key?(:ascending) ? clause[:ascending] : clause['ascending'] - [field, ascending != false] + [sort_field(clause), ascending != false] end end + def sort_field(clause) = clause[:field] || clause['field'] + + def warn_unsortable(clauses) + ForestAdminDatasourceIntercom.logger.warn( + "[forest_admin_datasource_intercom] #{name} was asked to sort on " \ + "#{clauses.map { |clause| sort_field(clause).inspect }.join(", ")}, which it does not carry; the rows " \ + 'come back in the order Intercom returned them. A related list through a many-to-many is ordered by the ' \ + 'columns of the collection it travels through, not by those of the collection it reaches.' + ) + end + def compare_clauses(left, right, clauses) clauses.each do |field, ascending| comparison = compare_values(left[field], right[field]) diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/relations.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/relations.rb new file mode 100644 index 000000000..bcb7dc03b --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/relations.rb @@ -0,0 +1,266 @@ +module ForestAdminDatasourceIntercom + module Collections + # How this datasource declares a relation, and how it answers one. + # + # Intercom joins nothing: a ticket carries an assignee id, and the teammate + # behind it is a second read of a second endpoint. What makes that affordable + # -- and exact -- is that every collection on the far end of these relations + # is read whole in one request, the tier `FetchAllCollection` serves. + # + # Declaring a relation obliges both halves below, and the second is not + # optional: a many-to-one is published filterable as soon as *any* column of + # its target is (`GeneratorField#build_many_to_one_schema`), so the interface + # offers `assignee:name` the moment the relation exists. Left unanswered, + # that filter reaches a translator refusing every traversing field -- the + # interface offering a filter the datasource then refuses, which is the one + # thing this package is built not to do. + # + # * a projection through a relation (`assignee:name`) is answered by reading + # the target once per page and nesting its row under the relation name; + # * a filter through a relation is answered by asking the target which of its + # records match, and filtering Intercom on the ids it names. + # + # The second is exact rather than approximate -- the target answers over + # every record it holds, not over a page -- with one semantic worth stating + # plainly: a row whose foreign key is null matches no relation filter, the + # way a join drops it, negated filters included. A ticket with no assignee is + # not "assigned to someone other than Marie". + module Relations # rubocop:disable Metrics/ModuleLength + ManyToOneSchema = ForestAdminDatasourceToolkit::Schema::Relations::ManyToOneSchema + ManyToManySchema = ForestAdminDatasourceToolkit::Schema::Relations::ManyToManySchema + Filter = ForestAdminDatasourceToolkit::Components::Query::Filter + Projection = ForestAdminDatasourceToolkit::Components::Query::Projection + Operators = ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Operators + Branch = ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Nodes::ConditionTreeBranch + Leaf = ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Nodes::ConditionTreeLeaf + + # What a relation condition comes to when the target matched no record: no + # row can satisfy it. It is a value rather than an empty condition because + # the two tiers spell "match nothing" differently -- in memory an `in []` + # says it, while Intercom's search DSL has no way to. + NOTHING = :matches_nothing + + # The group a relation condition expands into, told apart from a group the + # operator wrote. Two things hang on the difference: this one may be + # inlined into a parent aggregating the same way -- the level it adds is + # one nobody budgeted for -- and a tier refusing it can name the relation + # rather than a shape the operator never wrote. + class RelationBranch < Branch + attr_reader :relation_field + + def initialize(aggregator, conditions, relation_field) + @relation_field = relation_field + super(aggregator, conditions) + end + end + + protected + + # Every relation of this package points at a collection keyed by `id` and + # is read-only: nothing in this lot writes, and an editable relation would + # offer an association the collection cannot perform. + def add_many_to_one(name, foreign_collection:, foreign_key:) + add_field(name, ManyToOneSchema.new(foreign_collection: foreign_collection, + foreign_key: foreign_key, + foreign_key_target: 'id', + is_read_only: true)) + end + + def add_many_to_many(name, foreign_collection:, through_collection:, origin_key:, foreign_key:) + add_field(name, ManyToManySchema.new(foreign_collection: foreign_collection, + through_collection: through_collection, + origin_key: origin_key, origin_key_target: 'id', + foreign_key: foreign_key, foreign_key_target: 'id', + is_read_only: true)) + end + + # The rows of one page, with the relations the projection named nested onto + # them. `records` are the same rows before projection: that is where the + # foreign keys are read, since a projection naming `assignee:name` does not + # have to name `admin_assignee_id`. + # + # One request per target *collection* per page -- never one per row, and + # never twice for two relations that point at the same collection: a + # ticket's `state` and `previous_state` are one read of `/ticket_states`, + # over the ids both of them name. + def embed_relations(caller, records, rows, projection) + asked = many_to_one_asked(projection) + return if asked.empty? + + indexed = indexed_targets(caller, records, asked) + + asked.each do |name, relation, wanted| + rows_of_target = indexed[relation.foreign_collection] + + # Nil rather than absent when an id names no record: a teammate who + # left the workspace reads as no teammate, not as a broken row. Sliced + # back to what this relation asked for, the read having been widened to + # the union of what every relation on that collection did. + records.each_with_index do |record, index| + target_row = rows_of_target[record[relation.foreign_key]] + rows[index][name] = target_row&.slice(*wanted) + end + end + end + + # The tree this tier can filter on, every `relation:field` leaf traded for + # a condition on the foreign key. The block is handed that key and the ids + # the target matched, and answers the node the tier wants: an `in` where + # the filtering is done in memory, a group of equalities where Intercom's + # DSL takes no membership operator. + # + # An `and` carrying a leaf that matches nothing matches nothing itself; an + # `or` drops that leaf and keeps its siblings. + def rewrite_relation_conditions(caller, node, &builder) + case node + when Branch then rewrite_branch(caller, node, &builder) + when Leaf then relation_leaf?(node) ? rewrite_relation_leaf(caller, node, &builder) : node + else node + end + end + + private + + def relation_leaf?(node) + node.is_a?(Leaf) && node.field.to_s.include?(':') + end + + def relations_asked(projection) + Projection.new(Array(projection).map(&:to_s)).relations + end + + # The many-to-one relations the projection named, each with the columns it + # asked of its target. The target's own key travels with them whether or + # not it was asked for: it is what the rows are indexed by here, and what + # makes the nested row a link rather than a label in the interface. + def many_to_one_asked(projection) + relations_asked(projection).filter_map do |name, sub_projection| + relation = fields[name] + next unless relation.is_a?(ManyToOneSchema) + + [name, relation, Array(sub_projection).map(&:to_s).union([relation.foreign_key_target])] + end + end + + def indexed_targets(caller, records, asked) + asked.group_by { |_, relation, _| relation.foreign_collection } + .transform_values { |group| target_rows(caller, records, group) } + end + + # One read per target collection, over the ids every relation pointing at + # it names and the union of the columns they asked for. + def target_rows(caller, records, group) + relation = group.first[1] + ids = group.flat_map { |_, rel, _| records.filter_map { |record| record[rel.foreign_key] } }.uniq + return {} if ids.empty? + + target = relation.foreign_key_target + wanted = Projection.new(group.flat_map { |_, _, columns| columns }.uniq) + filter = Filter.new(condition_tree: Leaf.new(target, Operators::IN, ids)) + + foreign_collection(relation).list(caller, filter, wanted).to_h { |row| [row[target], row] } + end + + def rewrite_branch(caller, branch, &builder) + rewritten = Array(branch.conditions).map { |node| rewrite_relation_conditions(caller, node, &builder) } + + if branch.aggregator.to_s.casecmp('or').zero? + kept = rewritten.reject { |node| node == NOTHING } + kept.empty? ? NOTHING : Branch.new(branch.aggregator, absorb_groups(branch.aggregator, kept)) + elsif rewritten.include?(NOTHING) + NOTHING + else + Branch.new(branch.aggregator, absorb_groups(branch.aggregator, rewritten)) + end + end + + # A relation group aggregating the way its parent does is inlined into it: + # `or(x, or(a, b))` is `or(x, a, b)` and one level shallower. The level it + # saves is one nothing budgeted for -- what a tier measured its nesting + # limit against is the tree the operator wrote, not the equalities a + # relation expands into afterwards. + # + # Only a relation group is inlined. A group the operator wrote is what + # their filter means, and flattening it would spend on one group the + # conditions two groups were holding. + def absorb_groups(aggregator, nodes) + absorbed = nodes.flat_map do |node| + inlinable_group?(node, aggregator) ? Array(node.conditions) : [node] + end + + absorb_relation_group?(absorbed.size) ? absorbed : nodes + end + + def inlinable_group?(node, aggregator) + node.is_a?(RelationBranch) && node.aggregator.to_s.casecmp(aggregator.to_s).zero? + end + + # Hook for a tier that bounds how many conditions a group may hold: past + # that, the nested form is the one that fits, and the level it costs is + # spent rather than the width. + def absorb_relation_group?(_size) = true + + def rewrite_relation_leaf(caller, leaf) + name, path = leaf.field.to_s.split(':', 2) + relation = fields[name] + refuse_unknown_relation!(leaf, name) unless relation.is_a?(ManyToOneSchema) + refuse_two_hops!(leaf) if path.include?(':') + + check_relation_filterable!(leaf, relation) + + ids = matching_ids(caller, relation, Leaf.new(path, leaf.operator, leaf.value)) + return NOTHING if ids.empty? + + yield(relation.foreign_key, ids, leaf) + end + + # Hook for a tier that cannot filter on every foreign key it declares a + # relation on. Answered before the target is read: a refusal that spends a + # request first costs exactly what it refuses to do. + def check_relation_filterable!(_leaf, _relation); end + + # Which records of the target the condition names, asked of the target + # itself: it owns what its columns can be filtered with, and it answers + # over every record Intercom holds rather than over a page of them. + def matching_ids(caller, relation, leaf) + target = relation.foreign_key_target + + foreign_collection(relation) + .list(caller, Filter.new(condition_tree: leaf), Projection.new([target])) + .filter_map { |row| row[target] } + .uniq + end + + # The target as the datasource holds it, undecorated -- so a permission + # scope or a segment defined on the target does not narrow what a relation + # resolves. That is how a native datasource behaves too: it joins the table + # without applying the scopes of the collection mapped to it. + def foreign_collection(relation) + datasource.get_collection(relation.foreign_collection) + end + + # Either a name this collection carries no relation under -- a condition + # from a scope or a segment written against another schema -- or a + # many-to-many, which is published unfilterable: resolving one would mean + # reading the collection it travels through once per value. + def refuse_unknown_relation!(leaf, relation_name) + filterable = fields.select { |_, field| field.is_a?(ManyToOneSchema) }.keys + + raise UnsupportedOperatorError, + "#{name} cannot filter #{leaf.field.inspect}: #{relation_name.inspect} is not a relation it filters " \ + "through. #{filterable.empty? ? "It has none." : "Those it does: #{filterable.join(", ")}."} Filter " \ + 'on a column of the collection next door instead.' + end + + # Two hops would mean resolving a relation of a relation, one read per + # level, and nothing in this datasource publishes a filter that deep -- a + # many-to-many is published unfilterable. It is refused by name rather than + # half-answered. + def refuse_two_hops!(leaf) + raise UnsupportedOperatorError, + "#{name} cannot filter #{leaf.field.inspect}: it reaches through two relations, and this datasource " \ + 'resolves one. Filter on a column of the collection next door instead.' + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/team.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/team.rb index 2286bfa46..a0e111bcf 100644 --- a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/team.rb +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/team.rb @@ -19,17 +19,49 @@ def serialize(team) { 'id' => stringify_id(attrs['id']), 'name' => attrs['name'], - 'admin_ids' => Array(attrs['admin_ids']).map { |id| stringify_id(id) } } + # Not a column: it is what the names below are read from, and what the + # membership collection turns into the relation. Intercom types these + # as numbers here and as strings on the admin itself, so they are + # stringified for both sides to carry the same id. + 'admin_ids' => Array(attrs['admin_ids']).map { |id| stringify_id(id) }, + 'admin_names' => nil } + end + + # The teammates of the team, by name, and only when a projection asked for + # them: one read of `/admins` for the whole page, never one per team. A + # token that cannot read the teammates costs the column and nothing else. + def enrich(records, rows, projection) + return unless column_asked?(projection, 'admin_names') + + names = admin_names + records.each_with_index do |record, index| + rows[index]['admin_names'] = Array(record['admin_ids']).filter_map { |id| names[id] } + end end private + def admin_names + client.fetch_all('admins', list_key: 'admins') + .to_h { |admin| [stringify_id(admin['id']), admin['name']] } + rescue APIError => e + ForestAdminDatasourceIntercom.logger.warn( + "[forest_admin_datasource_intercom] #{name} could not read the teammates of the workspace (HTTP " \ + "#{e.status || "-"}); the names are left empty. The relation to IntercomAdmin is unaffected." + ) + {} + end + def define_schema add_column('id', 'String', is_primary_key: true) add_column('name', 'String') - # Intercom types these as numbers here and as strings on the admin - # itself; they are stringified so both sides carry the same id. - add_column('admin_ids', 'Json') + # A list, so neither filterable nor sortable -- as the array of ids it + # replaces was. It reads the team without a join; the relation below is + # what navigates it, and filtering happens on the Admins collection. + add_column('admin_names', 'Json') + add_many_to_many('admins', foreign_collection: 'IntercomAdmin', + through_collection: 'IntercomTeamMembership', + origin_key: 'team_id', foreign_key: 'admin_id') end end end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/team_membership.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/team_membership.rb new file mode 100644 index 000000000..9a1d662b0 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/team_membership.rb @@ -0,0 +1,61 @@ +module ForestAdminDatasourceIntercom + module Collections + # The membership of the inbox teams: one record per team-and-teammate pair. + # + # It exists because Intercom's does not. The workspace carries the membership + # on the team (`admin_ids`) and on the teammate (`team_ids`) both, and offers + # no resource for the pair -- while a many-to-many needs a collection to + # travel through, whose two many-to-one relations are what + # `Utils::Collection.get_through_target` looks for. Declared here, Teams and + # Admins navigate to each other in both directions; left undeclared, both + # sides read as an array of ids. + # + # Read from `/teams` alone. The admin side names the same pairs, so reading + # it too would spend a request confirming what the first answer already said. + # Read-only, like the relations it carries: Intercom exposes no endpoint that + # writes a membership, and an editable relation would offer an association + # that could only fail. + class TeamMembership < FetchAllCollection + def initialize(datasource) + super(datasource, 'IntercomTeamMembership') + end + + protected + + def fetch_all + client.fetch_all('teams', list_key: 'teams').flat_map { |team| pairs_of(team) } + end + + # Keyed by both ids rather than by a counter: a related list is read over + # two requests, and a key that changed between them would move the rows + # under the operator. + def serialize(pair) + { 'id' => "#{pair["team_id"]}:#{pair["admin_id"]}", + 'team_id' => pair['team_id'], + 'admin_id' => pair['admin_id'] } + end + + private + + def pairs_of(team) + return [] unless team.is_a?(Hash) + + team_id = stringify_id(team['id']) + return [] if team_id.nil? + + Array(team['admin_ids']).filter_map do |admin_id| + id = stringify_id(admin_id) + { 'team_id' => team_id, 'admin_id' => id } unless id.nil? + end + end + + def define_schema + add_column('id', 'String', is_primary_key: true) + add_column('team_id', 'String') + add_column('admin_id', 'String') + add_many_to_one('team', foreign_collection: 'IntercomTeam', foreign_key: 'team_id') + add_many_to_one('admin', foreign_collection: 'IntercomAdmin', foreign_key: 'admin_id') + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/ticket.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/ticket.rb index 2569e4552..f071cd37c 100644 --- a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/ticket.rb +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/ticket.rb @@ -85,17 +85,23 @@ def define_schema define_contact_columns define_derived_columns add_column('part_count', 'Number') + # Before the attribute columns rather than after: a workspace attribute + # whose name lands on a relation is then skipped with a warning, the way + # one landing on a column already is. Declared after, it would collide + # and take the boot with it. + define_relations register_attribute_columns end - # The state arrives embedded as a whole object, so its labels cost nothing. - # `IntercomTicketState` remains a collection of its own -- it is the list of - # what a state can be -- but a row does not depend on it to be readable. + # The state arrives embedded as a whole object, so its label costs nothing: + # a queue reads without a join. One label and not three -- the category and + # the customer-facing label are read through the `state` relation, which is + # where every field of a state lives. Neither of the two was ever + # filterable, so nothing that could be saved in a segment or a scope + # depended on them. def define_state_columns add_column('state_id', 'String') - add_column('state_category', 'String') add_column('state_label', 'String') - add_column('state_external_label', 'String') add_column('previous_state_id', 'String') end @@ -104,6 +110,23 @@ def define_type_columns add_column('ticket_type_name', 'String') end + # The four reference collections a ticket points at. Every target is read + # whole in one request, so a relation resolves for a page at the price of a + # single read. + # + # Only two of them can be filtered *through*: `/tickets/search` takes a + # filter on `admin_assignee_id`, `team_assignee_id` and `ticket_type_id`, + # and none on a state id -- which the refusal names when a filter reaches + # for it, rather than letting the interface offer what the endpoint drops. + def define_relations + add_many_to_one('admin_assignee', foreign_collection: 'IntercomAdmin', foreign_key: 'admin_assignee_id') + add_many_to_one('team_assignee', foreign_collection: 'IntercomTeam', foreign_key: 'team_assignee_id') + add_many_to_one('state', foreign_collection: 'IntercomTicketState', foreign_key: 'state_id') + add_many_to_one('previous_state', foreign_collection: 'IntercomTicketState', + foreign_key: 'previous_state_id') + add_many_to_one('ticket_type', foreign_collection: 'IntercomTicketType', foreign_key: 'ticket_type_id') + end + # The attribute columns of every ticket type, in union. Read at boot by # `TicketAttributesIntrospector`, which is also where a workspace's own # name is turned into one a Forest query string can carry. An attribute @@ -118,8 +141,8 @@ def collides?(attribute) ForestAdminDatasourceIntercom.logger.warn( "[forest_admin_datasource_intercom] #{name} skips the ticket attribute #{attribute.name.inspect}: a " \ - "native column already carries the name #{attribute.column_name.inspect}, and overwriting it would show " \ - 'the attribute where the operator expects the ticket field.' + "native column or relation already carries the name #{attribute.column_name.inspect}, and overwriting " \ + 'it would show the attribute where the operator expects the ticket field.' ) true end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/ticket/serializer.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/ticket/serializer.rb index 3fb91030a..d547fefb0 100644 --- a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/ticket/serializer.rb +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/ticket/serializer.rb @@ -37,10 +37,10 @@ def native(attrs) def state_of(attrs) state = attrs['ticket_state'].is_a?(Hash) ? attrs['ticket_state'] : {} + # `internal_label` is what the support team reads. The category and the + # customer-facing label are a hop away, on the `state` relation. { 'state_id' => stringify_id(state['id']), - 'state_category' => state['category'], 'state_label' => state['internal_label'], - 'state_external_label' => state['external_label'], 'previous_state_id' => stringify_id(attrs['previous_ticket_state_id']) } end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/datasource.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/datasource.rb index f9ccf242a..c2a4c2bf0 100644 --- a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/datasource.rb +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/datasource.rb @@ -30,6 +30,10 @@ def inspect def register_collections add_collection(Collections::Admin.new(self)) add_collection(Collections::Team.new(self)) + # The join Intercom does not expose: without it the membership of a team is + # an array of ids on either side, since a many-to-many needs a collection + # to travel through. + add_collection(Collections::TeamMembership.new(self)) add_collection(Collections::TicketType.new(self)) add_collection(Collections::TicketState.new(self)) add_collection(Collections::Conversation.new(self)) diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/query/condition_tree_translator.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/query/condition_tree_translator.rb index 3418a7951..aa33b085d 100644 --- a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/query/condition_tree_translator.rb +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/query/condition_tree_translator.rb @@ -65,7 +65,7 @@ def translate_branch(branch, depth) # enough that a wrapper around nothing is a level worth not spending. return translate(conditions.first, depth) if conditions.size == 1 - refuse_too_deep!(depth) if depth > MAX_DEPTH + refuse_too_deep!(branch, depth) if depth > MAX_DEPTH refuse_too_wide!(branch, conditions.size) if conditions.size > MAX_GROUP_SIZE { 'operator' => operator, 'value' => conditions.map { |condition| translate(condition, depth + 1) } } @@ -75,11 +75,27 @@ def translate_branch(branch, depth) # message names the shape rather than a number, since the tree an operator # can act on is the segment and the scope they wrote, not the one the agent # assembled out of them. - def refuse_too_deep!(depth) + def refuse_too_deep!(branch, depth) raise UnsupportedOperatorError, "#{@collection} cannot answer this filter: Intercom nests a search #{MAX_DEPTH} levels deep and this " \ - "one reaches #{depth}. A group inside a group inside a group is one level too many -- flatten the " \ - 'segment, the scope or the filter carrying the innermost one.' + "one reaches #{depth}. #{deepening_cause(branch)}" + end + + # A group the operator wrote is theirs to flatten. A group a relation + # expanded into is not: it is the several records the relation matched, + # written one equality each because Intercom takes no membership operator + # on a foreign key -- and telling them to flatten a nesting they never + # wrote is a refusal they cannot act on. + def deepening_cause(branch) + field = branch.respond_to?(:relation_field) ? branch.relation_field : nil + + if field.nil? + 'A group inside a group inside a group is one level too many -- flatten the segment, the scope or the ' \ + 'filter carrying the innermost one.' + else + "The innermost group is what #{field.inspect} expanded into, one equality per record it matched: narrow " \ + 'that condition until it names a single record, or lift it out of the groups nesting it.' + end end # Fifteen is reached without trying: a scope, a segment and a filter add up, diff --git a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/admin_spec.rb b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/admin_spec.rb index a98cfd112..9ef97e218 100644 --- a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/admin_spec.rb +++ b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/admin_spec.rb @@ -9,10 +9,16 @@ def filter ForestAdminDatasourceToolkit::Components::Query::Filter.new end + def json(payload, status = 200) + { status: status, body: payload.to_json, headers: { 'Content-Type' => 'application/json' } } + end + def stub_admins(*admins) - stub_request(:get, "#{base}/admins") - .to_return(status: 200, body: { 'type' => 'admin.list', 'admins' => admins }.to_json, - headers: { 'Content-Type' => 'application/json' }) + stub_request(:get, "#{base}/admins").to_return(json('type' => 'admin.list', 'admins' => admins)) + end + + def stub_teams(*teams) + stub_request(:get, "#{base}/teams").to_return(json('type' => 'team.list', 'teams' => teams)) end it 'is named IntercomAdmin' do @@ -21,7 +27,7 @@ def stub_admins(*admins) it 'exposes the columns an ops lead reads before assigning anything' do expect(collection.fields.keys) - .to eq(%w[id name email job_title away_mode_enabled away_mode_reassign has_inbox_seat team_ids]) + .to eq(%w[id name email job_title away_mode_enabled away_mode_reassign has_inbox_seat team_names teams]) end it 'declares id as the primary key' do @@ -34,28 +40,80 @@ def stub_admins(*admins) stub_admins('type' => 'admin', 'id' => '1', 'name' => 'Alice', 'email' => 'alice@acme.test', 'job_title' => 'Support', 'away_mode_enabled' => true, 'away_mode_reassign' => false, 'has_inbox_seat' => true, 'team_ids' => [814_865]) + stub_teams('id' => '814865', 'name' => 'Support') expect(collection.list(nil, filter, nil)) .to eq([{ 'id' => '1', 'name' => 'Alice', 'email' => 'alice@acme.test', 'job_title' => 'Support', 'away_mode_enabled' => true, 'away_mode_reassign' => false, 'has_inbox_seat' => true, - 'team_ids' => %w[814865] }]) + 'team_names' => ['Support'] }]) + end + + # A projection naming nothing asks for every declared column, which is how + # `project` reads it -- so the enrichment has to read it that way too, or the + # one column the projection publishes comes back nil. + it 'fills the derived names when the projection names no column at all' do + stub_admins('id' => '1', 'name' => 'Alice', 'team_ids' => [814_865]) + stub_teams('id' => '814865', 'name' => 'Support') + + expect(collection.list(nil, filter, nil).first['team_names']).to eq(['Support']) end # Intercom types a team id as a number here and as a string on the team - # itself; a filter value from Forest always arrives as a string. + # itself; a filter value from Forest always arrives as a string. Both sides + # of the membership therefore carry the same id, without which the relation + # would resolve to nothing rather than to an error. it 'stringifies the ids so both sides of the membership match' do - stub_admins('id' => 493_881, 'team_ids' => [814_865, 814_866]) - - row = collection.list(nil, filter, nil).first + stub_admins('id' => 493_881, 'team_ids' => [814_865]) + stub_teams('id' => '814865', 'name' => 'Support') - expect(row['id']).to eq('493881') - expect(row['team_ids']).to eq(%w[814865 814866]) + expect(collection.list(nil, filter, %w[id team_names]).first) + .to eq({ 'id' => '493881', 'team_names' => ['Support'] }) end - it 'reads a teammate with no team as one with no team, not as one with a null' do - stub_admins('id' => '1', 'team_ids' => nil) + describe 'the teams of a teammate' do + it 'names them for the whole page in one read, rather than one read per row' do + stub_admins({ 'id' => '1', 'team_ids' => [814_865] }, { 'id' => '2', 'team_ids' => [814_865, 814_866] }) + stub_teams({ 'id' => '814865', 'name' => 'Support' }, { 'id' => '814866', 'name' => 'Billing' }) + + expect(collection.list(nil, filter, %w[id team_names])) + .to eq([{ 'id' => '1', 'team_names' => ['Support'] }, + { 'id' => '2', 'team_names' => %w[Support Billing] }]) + expect(WebMock).to have_requested(:get, "#{base}/teams").once + end + + # A column nobody asked for costs no request. + it 'reads nothing when no projection asks for the names' do + stub_admins('id' => '1', 'team_ids' => [814_865]) + + collection.list(nil, filter, %w[id name]) + + expect(WebMock).not_to have_requested(:get, "#{base}/teams") + end + + # A missing permission costs the column, never the page -- and never the + # relation, which reads the teams from the other side. + it 'leaves the names empty when the teams cannot be read' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + stub_admins('id' => '1', 'team_ids' => [814_865]) + stub_request(:get, "#{base}/teams").to_return(json({ 'type' => 'error.list' }, 403)) + + expect(collection.list(nil, filter, %w[id team_names]).first) + .to eq({ 'id' => '1', 'team_names' => [] }) + end + + it 'reads a teammate with no team as one with no team, not as one with a null' do + stub_admins('id' => '1', 'team_ids' => nil) + stub_teams('id' => '814865', 'name' => 'Support') + + expect(collection.list(nil, filter, %w[team_names]).first['team_names']).to eq([]) + end - expect(collection.list(nil, filter, nil).first['team_ids']).to eq([]) + it 'is a many-to-many through the membership collection, read-only' do + expect(collection.fields['teams']) + .to have_attributes(type: 'ManyToMany', foreign_collection: 'IntercomTeam', + through_collection: 'IntercomTeamMembership', + origin_key: 'admin_id', foreign_key: 'team_id', is_read_only: true) + end end end end diff --git a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/conversation_spec.rb b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/conversation_spec.rb index 3b562221b..a6cfe8db5 100644 --- a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/conversation_spec.rb +++ b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/conversation_spec.rb @@ -25,6 +25,12 @@ def json(payload, status = 200) { status: status, body: payload.to_json, headers: { 'Content-Type' => 'application/json' } } end + # The columns alone: a relation carries neither operators nor an order, and + # what it may be filtered through is asserted on its own below. + def columns + collection.fields.select { |_, field| field.type == 'Column' } + end + # Hand-written from the OpenAPI 2.16 spec, never captured from a workspace: # a conversation body is personal data. def conversation(id, overrides = {}) @@ -98,7 +104,7 @@ def ids(rows) # Neither search endpoint takes a sort, and Intercom ignores the one it is # sent without a word, so no column of this tier may advertise one. it 'declares every column unsortable' do - expect(collection.fields.values.map(&:is_sortable).uniq).to eq([false]) + expect(columns.values.map(&:is_sortable).uniq).to eq([false]) end # Derived from the measured table, never written by hand: a column @@ -135,7 +141,7 @@ def ids(rows) # No aggregate endpoint, so no group-by may be offered. it 'declares no column groupable' do - expect(collection.fields.values.map(&:is_groupable).uniq).to eq([false]) + expect(columns.values.map(&:is_groupable).uniq).to eq([false]) end # This lot writes nothing: an editable column would offer a Save that @@ -145,6 +151,54 @@ def ids(rows) end end + # All three targets are read whole in one request, and + # `/conversations/search` takes a filter on each of the three keys -- so + # these relations can be read, navigated and filtered through alike, unlike + # the state of a ticket. + describe 'the relations to the reference collections' do + it 'points every id at the collection that reads it' do + expect(collection.fields['admin_assignee']) + .to have_attributes(type: 'ManyToOne', foreign_collection: 'IntercomAdmin', + foreign_key: 'admin_assignee_id', foreign_key_target: 'id', is_read_only: true) + expect(collection.fields['team_assignee']) + .to have_attributes(foreign_collection: 'IntercomTeam', foreign_key: 'team_assignee_id') + expect(collection.fields['closed_by']) + .to have_attributes(foreign_collection: 'IntercomAdmin', foreign_key: 'closed_by_id') + end + + # The account is on the payload as a whole object, so the name is already a + # column; the Companies collection arrives with lot 4, and a relation whose + # target is missing is a schema the agent refuses to boot on. + it 'declares no relation towards the account' do + expect(collection.fields.keys).not_to include('company') + end + + it 'nests the teammate who closed it, reading the teammates once for the page' do + stub_list(conversation('1'), conversation('2')) + stub_request(:get, "#{base}/admins") + .to_return(json('type' => 'admin.list', 'admins' => [{ 'id' => '493881', 'name' => 'Alice' }])) + + rows = collection.list(nil, filter, ['id', 'closed_by:name']) + + expect(rows).to eq([{ 'id' => '1', 'closed_by' => { 'name' => 'Alice', 'id' => '493881' } }, + { 'id' => '2', 'closed_by' => { 'name' => 'Alice', 'id' => '493881' } }]) + expect(WebMock).to have_requested(:get, "#{base}/admins").once + end + + it 'filters on the foreign key the target resolved to' do + stub_request(:get, "#{base}/admins") + .to_return(json('type' => 'admin.list', 'admins' => [{ 'id' => '493881', 'name' => 'Alice' }])) + stub_search(conversation('1')) + + collection.list(nil, filter(condition_tree: leaf('admin_assignee:name', operators::EQUAL, 'Alice')), %w[id]) + + expect(WebMock).to have_requested(:post, "#{base}/conversations/search") + .with(query: hash_including({}), + body: hash_including('query' => { 'field' => 'admin_assignee_id', 'operator' => '=', + 'value' => '493881' })) + end + end + describe '#list' do it 'reads the listing endpoint as plain text and pages by cursor' do stub_list(conversation('1')) diff --git a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/fetch_all_collection_spec.rb b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/fetch_all_collection_spec.rb index 09b5cefc0..94d03933c 100644 --- a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/fetch_all_collection_spec.rb +++ b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/fetch_all_collection_spec.rb @@ -53,6 +53,14 @@ def evaluable?(operator, column_type) .equivalent_tree?(operator, described_class::IN_MEMORY_OPERATORS, column_type) end + # A projection naming nothing asks for every declared column, `team_names` + # among them, so a list with no projection reads the teams as well. + before do + stub_request(:get, "#{base}/teams") + .to_return(status: 200, body: { 'type' => 'team.list', 'teams' => [] }.to_json, + headers: { 'Content-Type' => 'application/json' }) + end + # The count and the group are taken over every record Intercom holds, not # over a page of them, which is what makes them exact. it 'is countable' do @@ -74,14 +82,14 @@ def evaluable?(operator, column_type) # A list has no in-memory counterpart for any of the three. it 'declares a Json column neither filterable nor sortable' do - expect(collection.fields['team_ids']) + expect(collection.fields['team_names']) .to have_attributes(column_type: 'Json', is_sortable: false, is_groupable: false, filter_operators: []) end # A filter the UI offers and the collection then answers by emptying the # page is the failure this whole datasource is built to avoid. it 'advertises only operators it can actually evaluate' do - advertised = collection.fields.flat_map do |_name, column| + advertised = collection.fields.select { |_, field| field.type == 'Column' }.flat_map do |_name, column| column.filter_operators.map { |operator| [operator, column.column_type] } end @@ -185,10 +193,29 @@ def filtered(field, operator, value = nil) expect(ids(rows)).to eq(%w[2 1 3]) end - it 'drops a clause naming a column it does not carry' do + # An order asked for and not honoured is reported everywhere else in this + # datasource, and a related list through a many-to-many reaches this tier + # with the columns of the collection it *reaches* rather than the one it + # travels through -- `Filter#nest` prefixes the condition tree and not the + # sort. Dropped in silence, that is the one order nothing would report. + it 'drops a clause naming a column it does not carry, and says so' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + rows = collection.list(nil, filter(sort: sort({ field: 'unknown', ascending: true })), nil) expect(ids(rows)).to eq(%w[2 1 3]) + expect(ForestAdminDatasourceIntercom.logger) + .to have_received(:warn).with(/sort on "unknown", which it does not carry/) + end + + it 'keeps the clauses it does carry alongside the one it drops' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + + rows = collection.list( + nil, filter(sort: sort({ field: 'unknown', ascending: true }, { field: 'name', ascending: true })), nil + ) + + expect(ids(rows)).to eq(%w[1 2 3]) end end diff --git a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/team_membership_spec.rb b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/team_membership_spec.rb new file mode 100644 index 000000000..656301c2a --- /dev/null +++ b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/team_membership_spec.rb @@ -0,0 +1,206 @@ +module ForestAdminDatasourceIntercom + RSpec.describe Collections::TeamMembership do + subject(:collection) { datasource.get_collection('IntercomTeamMembership') } + + let(:datasource) { Datasource.new(access_token: 's3cr3t', rate_limiter: nil) } + let(:base) { datasource.configuration.url } + let(:operators) { ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Operators } + + def filter(condition_tree: nil) + ForestAdminDatasourceToolkit::Components::Query::Filter.new(condition_tree: condition_tree) + end + + def leaf(field, operator, value = nil) + ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Nodes::ConditionTreeLeaf + .new(field, operator, value) + end + + def branch(aggregator, *conditions) + ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Nodes::ConditionTreeBranch + .new(aggregator, conditions) + end + + def json(payload, status = 200) + { status: status, body: payload.to_json, headers: { 'Content-Type' => 'application/json' } } + end + + def stub_teams(*teams) + stub_request(:get, "#{base}/teams").to_return(json('type' => 'team.list', 'teams' => teams)) + end + + def stub_admins(*admins) + stub_request(:get, "#{base}/admins").to_return(json('type' => 'admin.list', 'admins' => admins)) + end + + it 'is named IntercomTeamMembership' do + expect(collection.name).to eq('IntercomTeamMembership') + end + + it 'exposes the pair and a relation to either side of it' do + expect(collection.fields.keys).to eq(%w[id team_id admin_id team admin]) + end + + # One record per pair, keyed by both ids: a related list is read over two + # requests, and a key that changed in between would move the rows under the + # operator. + it 'reads one record per pair from the team side alone' do + stub_teams({ 'id' => '814865', 'name' => 'Support', 'admin_ids' => [493_881, 493_882] }, + { 'id' => '814866', 'name' => 'Billing', 'admin_ids' => [493_882] }) + + expect(collection.list(nil, filter, nil)) + .to eq([{ 'id' => '814865:493881', 'team_id' => '814865', 'admin_id' => '493881' }, + { 'id' => '814865:493882', 'team_id' => '814865', 'admin_id' => '493882' }, + { 'id' => '814866:493882', 'team_id' => '814866', 'admin_id' => '493882' }]) + expect(WebMock).not_to have_requested(:get, "#{base}/admins") + end + + it 'holds no record for a team nobody belongs to' do + stub_teams({ 'id' => '814865', 'admin_ids' => [] }, { 'id' => '814866', 'admin_ids' => nil }) + + expect(collection.list(nil, filter, nil)).to be_empty + end + + # Intercom types the id as a number inside `admin_ids` and as a string on the + # teammate itself: the pair carries the form the other side answers by, or + # the relation resolves to nothing. + it 'stringifies both ids' do + stub_teams('id' => 814_865, 'admin_ids' => [493_881]) + + expect(collection.list(nil, filter, nil).first) + .to eq({ 'id' => '814865:493881', 'team_id' => '814865', 'admin_id' => '493881' }) + end + + describe 'a projection through a relation' do + it 'nests the teammate under the relation, reading them once for the page' do + stub_teams('id' => '814865', 'admin_ids' => [493_881, 493_882]) + stub_admins({ 'id' => '493881', 'name' => 'Alice' }, { 'id' => '493882', 'name' => 'Bruno' }) + + expect(collection.list(nil, filter, ['id', 'admin:name'])) + .to eq([{ 'id' => '814865:493881', 'admin' => { 'name' => 'Alice', 'id' => '493881' } }, + { 'id' => '814865:493882', 'admin' => { 'name' => 'Bruno', 'id' => '493882' } }]) + expect(WebMock).to have_requested(:get, "#{base}/admins").once + end + + # A teammate who left the workspace is still named by the membership until + # Intercom drops the pair. The row reads as having no teammate rather than + # as a broken record. + it 'nests nothing when the id names no record' do + stub_teams('id' => '814865', 'admin_ids' => [493_881]) + stub_admins('id' => '493882', 'name' => 'Bruno') + + expect(collection.list(nil, filter, ['id', 'admin:name']).first) + .to eq({ 'id' => '814865:493881', 'admin' => nil }) + end + + it 'reads no relation the projection did not name' do + stub_teams('id' => '814865', 'admin_ids' => [493_881]) + + collection.list(nil, filter, %w[id team_id]) + + expect(WebMock).not_to have_requested(:get, "#{base}/admins") + end + end + + # This tier filters in memory, over every record Intercom holds, so a + # condition through a relation becomes a plain membership on the foreign key + # -- none of the limits of Intercom's search DSL apply here, nothing going + # through it. + describe 'a condition through a relation' do + it 'keeps the pairs whose teammate the target matched' do + stub_teams('id' => '814865', 'admin_ids' => [493_881, 493_882]) + stub_admins({ 'id' => '493881', 'name' => 'Alice' }, { 'id' => '493882', 'name' => 'Bruno' }) + + rows = collection.list(nil, filter(condition_tree: leaf('admin:name', operators::EQUAL, 'Alice')), %w[id]) + + expect(rows).to eq([{ 'id' => '814865:493881' }]) + end + + it 'answers nothing when the target matched no record, rather than everything' do + stub_teams('id' => '814865', 'admin_ids' => [493_881]) + stub_admins('id' => '493881', 'name' => 'Alice') + + rows = collection.list(nil, filter(condition_tree: leaf('admin:name', operators::EQUAL, 'Zoe')), %w[id]) + + expect(rows).to be_empty + end + + it 'counts the pairs a relation condition keeps, exactly' do + stub_teams('id' => '814865', 'admin_ids' => [493_881, 493_882]) + stub_admins({ 'id' => '493881', 'name' => 'Alice' }, { 'id' => '493882', 'name' => 'Bruno' }) + aggregation = ForestAdminDatasourceToolkit::Components::Query::Aggregation.new(operation: 'Count') + + rows = collection.aggregate(nil, filter(condition_tree: leaf('admin:name', operators::EQUAL, 'Alice')), + aggregation) + + expect(rows).to eq([{ 'group' => {}, 'value' => 1 }]) + end + + # The shape the agent really builds: a scope, then a segment, then the + # operator's own filter, one branch at a time. + it 'keeps a relation condition standing next to a condition on its own column' do + stub_teams({ 'id' => '814865', 'admin_ids' => [493_881] }, { 'id' => '814866', 'admin_ids' => [493_881] }) + stub_admins('id' => '493881', 'name' => 'Alice') + tree = branch('And', leaf('team_id', operators::EQUAL, '814866'), + leaf('admin:name', operators::EQUAL, 'Alice')) + + expect(collection.list(nil, filter(condition_tree: tree), %w[id])) + .to eq([{ 'id' => '814866:493881' }]) + end + + # An `and` carrying a condition nothing can satisfy matches nothing itself. + it 'answers nothing when a relation condition inside an and matches nothing' do + stub_teams('id' => '814865', 'admin_ids' => [493_881]) + stub_admins('id' => '493881', 'name' => 'Alice') + tree = branch('And', leaf('team_id', operators::EQUAL, '814865'), + leaf('admin:name', operators::EQUAL, 'Zoe')) + + expect(collection.list(nil, filter(condition_tree: tree), %w[id])).to be_empty + end + + # An `or` drops it and keeps its siblings: what the others name is still + # named. + it 'keeps the siblings of a relation condition inside an or' do + stub_teams({ 'id' => '814865', 'admin_ids' => [493_881] }, { 'id' => '814866', 'admin_ids' => [493_882] }) + stub_admins({ 'id' => '493881', 'name' => 'Alice' }, { 'id' => '493882', 'name' => 'Bruno' }) + tree = branch('Or', leaf('admin:name', operators::EQUAL, 'Zoe'), + leaf('team_id', operators::EQUAL, '814866')) + + expect(collection.list(nil, filter(condition_tree: tree), %w[id])) + .to eq([{ 'id' => '814866:493882' }]) + end + + it 'answers nothing when every branch of an or matches nothing' do + stub_teams('id' => '814865', 'admin_ids' => [493_881]) + stub_admins('id' => '493881', 'name' => 'Alice') + tree = branch('Or', leaf('admin:name', operators::EQUAL, 'Zoe'), + leaf('admin:name', operators::EQUAL, 'Yann')) + + expect(collection.list(nil, filter(condition_tree: tree), %w[id])).to be_empty + end + + # The operators a relation condition may carry are the target's own: it is + # the one that evaluates them, and the one whose refusal is worth reading. + it 'refuses an operator the target cannot evaluate, naming it' do + stub_teams('id' => '814865', 'admin_ids' => [493_881]) + stub_admins('id' => '493881', 'name' => 'Alice') + + expect { collection.list(nil, filter(condition_tree: leaf('admin:team_names', operators::EQUAL, 'x')), nil) } + .to raise_error(UnsupportedOperatorError, /IntercomAdmin cannot filter 'team_names'/) + end + + it 'refuses a relation it does not declare, naming the ones it has' do + stub_teams('id' => '814865', 'admin_ids' => [493_881]) + + expect { collection.list(nil, filter(condition_tree: leaf('owner:name', operators::EQUAL, 'x')), nil) } + .to raise_error(UnsupportedOperatorError, /"owner" is not a relation it filters through.*team, admin/m) + end + + it 'refuses a path reaching through two relations' do + stub_teams('id' => '814865', 'admin_ids' => [493_881]) + + expect { collection.list(nil, filter(condition_tree: leaf('admin:teams:name', operators::EQUAL, 'x')), nil) } + .to raise_error(UnsupportedOperatorError, /reaches through two relations/) + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/team_spec.rb b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/team_spec.rb index 91f52dce4..54c01b75a 100644 --- a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/team_spec.rb +++ b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/team_spec.rb @@ -1,41 +1,172 @@ module ForestAdminDatasourceIntercom RSpec.describe Collections::Team do - subject(:collection) { described_class.new(datasource) } + subject(:collection) { datasource.get_collection('IntercomTeam') } let(:datasource) { Datasource.new(access_token: 's3cr3t', rate_limiter: nil) } let(:base) { datasource.configuration.url } - def filter - ForestAdminDatasourceToolkit::Components::Query::Filter.new + def filter(condition_tree: nil) + ForestAdminDatasourceToolkit::Components::Query::Filter.new(condition_tree: condition_tree) + end + + def admins_named(name) + operators = ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Operators + ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Nodes::ConditionTreeLeaf + .new('admins:name', operators::EQUAL, name) + end + + def json(payload, status = 200) + { status: status, body: payload.to_json, headers: { 'Content-Type' => 'application/json' } } end def stub_teams(*teams) - stub_request(:get, "#{base}/teams") - .to_return(status: 200, body: { 'type' => 'team.list', 'teams' => teams }.to_json, - headers: { 'Content-Type' => 'application/json' }) + stub_request(:get, "#{base}/teams").to_return(json('type' => 'team.list', 'teams' => teams)) end - it 'is named IntercomTeam' do - expect(collection.name).to eq('IntercomTeam') + def stub_admins(*admins) + stub_request(:get, "#{base}/admins").to_return(json('type' => 'admin.list', 'admins' => admins)) end - it 'exposes the team and its membership' do - expect(collection.fields.keys).to eq(%w[id name admin_ids]) + it 'is named IntercomTeam' do + expect(collection.name).to eq('IntercomTeam') end - # Intercom carries the membership on the team and on the admin both. Left as - # a plain list it stays readable on either side; declared as a relation it - # would give the schema two halves of a many-to-many with no join collection. - it 'keeps the membership a list rather than a relation' do - expect(collection.fields['admin_ids']) - .to have_attributes(type: 'Column', column_type: 'Json', filter_operators: []) + it 'exposes the team, its teammates by name, and the relation to them' do + expect(collection.fields.keys).to eq(%w[id name admin_names admins]) end it 'reads the endpoint under its own key and stringifies the ids' do stub_teams('type' => 'team', 'id' => '814865', 'name' => 'Support', 'admin_ids' => [493_881]) - expect(collection.list(nil, filter, nil)) - .to eq([{ 'id' => '814865', 'name' => 'Support', 'admin_ids' => %w[493881] }]) + expect(collection.list(nil, filter, %w[id name])).to eq([{ 'id' => '814865', 'name' => 'Support' }]) + end + + describe 'the teammates of a team' do + it 'names them for the whole page in one read, rather than one read per row' do + stub_teams({ 'id' => '814865', 'name' => 'Support', 'admin_ids' => [493_881, 493_882] }, + { 'id' => '814866', 'name' => 'Billing', 'admin_ids' => [493_882] }) + stub_admins({ 'id' => '493881', 'name' => 'Alice' }, { 'id' => '493882', 'name' => 'Bruno' }) + + expect(collection.list(nil, filter, %w[id admin_names])) + .to eq([{ 'id' => '814865', 'admin_names' => %w[Alice Bruno] }, + { 'id' => '814866', 'admin_names' => %w[Bruno] }]) + expect(WebMock).to have_requested(:get, "#{base}/admins").once + end + + it 'names them when the projection names no column at all' do + stub_teams('id' => '814865', 'name' => 'Support', 'admin_ids' => [493_881]) + stub_admins('id' => '493881', 'name' => 'Alice') + + expect(collection.list(nil, filter, nil).first['admin_names']).to eq(['Alice']) + end + + it 'reads nothing when no projection asks for the names' do + stub_teams('id' => '814865', 'name' => 'Support', 'admin_ids' => [493_881]) + + collection.list(nil, filter, %w[id name]) + + expect(WebMock).not_to have_requested(:get, "#{base}/admins") + end + + # A missing permission costs the column, never the page -- and never the + # relation, which reads the teammates from the other side. + it 'leaves the names empty when the teammates cannot be read' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + stub_teams('id' => '814865', 'admin_ids' => [493_881]) + stub_request(:get, "#{base}/admins").to_return(json({ 'type' => 'error.list' }, 403)) + + expect(collection.list(nil, filter, %w[id admin_names]).first) + .to eq({ 'id' => '814865', 'admin_names' => [] }) + end + end + + # Intercom carries the membership on the team and on the teammate both and + # exposes no resource for the pair, so the many-to-many travels through the + # membership collection this datasource synthesizes. + describe 'the relation to the teammates' do + it 'is a many-to-many through the membership collection' do + expect(collection.fields['admins']) + .to have_attributes(type: 'ManyToMany', foreign_collection: 'IntercomAdmin', + through_collection: 'IntercomTeamMembership', + origin_key: 'team_id', foreign_key: 'admin_id') + end + + # Intercom exposes no endpoint that writes a membership, and an editable + # relation would offer an association that could only fail. + it 'is read-only' do + expect(collection.fields['admins'].is_read_only).to be(true) + end + + # What the toolkit looks for on the collection a many-to-many travels + # through. Without the two, a related list falls back to a filter on the + # foreign collection that nothing there can answer. + it 'is reachable from both ends of the membership' do + utils = ForestAdminDatasourceToolkit::Utils::Collection + + expect(utils.get_through_target(collection, 'admins')).to eq('admin') + expect(utils.get_through_origin(collection, 'admins')).to eq('team') + end + + # A many-to-many is published unfilterable, so nothing the interface offers + # reaches here -- a scope or a segment still can, and it is refused rather + # than resolved by reading the membership once per value. + it 'refuses a condition through the membership' do + stub_teams('id' => '814865', 'admin_ids' => [493_881]) + + expect { collection.list(nil, filter(condition_tree: admins_named('Alice')), nil) } + .to raise_error(UnsupportedOperatorError, /"admins" is not a relation it filters through. It has none/) + end + + # The path a related list really takes: through the membership, whose own + # relation towards the teammate is what carries the rows back. + it 'lists the teammates of one team, through the membership' do + stub_teams({ 'id' => '814865', 'admin_ids' => [493_881] }, { 'id' => '814866', 'admin_ids' => [493_882] }) + stub_admins({ 'id' => '493881', 'name' => 'Alice' }, { 'id' => '493882', 'name' => 'Bruno' }) + query = ForestAdminDatasourceToolkit::Components::Query + + rows = ForestAdminDatasourceToolkit::Utils::Collection.list_relation( + collection, %w[814865], 'admins', nil, query::Filter.new, query::Projection.new(%w[id name]) + ) + + expect(rows).to eq([{ 'id' => '493881', 'name' => 'Alice' }]) + end + + # `admin_ids` names a teammate `/admins` does not answer -- one who left + # the workspace, one outside the token's reach, one past the page cap of a + # large workspace. The membership nests a nil under the relation, which is + # right on a row an operator reads; unwrapped into a related list it would + # be a row holding nothing, and the serializer reads a row by key. + it 'drops a teammate the workspace no longer answers, rather than listing an empty row' do + stub_teams('id' => '814865', 'admin_ids' => [493_881, 999_999]) + stub_admins('id' => '493881', 'name' => 'Alice') + query = ForestAdminDatasourceToolkit::Components::Query + + rows = ForestAdminDatasourceToolkit::Utils::Collection.list_relation( + collection, %w[814865], 'admins', nil, query::Filter.new, query::Projection.new(%w[id name]) + ) + + expect(rows).to eq([{ 'id' => '493881', 'name' => 'Alice' }]) + end + + # The membership is handed the columns of the collection the relation + # reaches, `Filter#nest` prefixing the condition tree and not the sort. It + # cannot order on them, and says so rather than answering an order it did + # not honour. + it 'reports the order it cannot honour on a related list' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + stub_teams('id' => '814865', 'admin_ids' => [493_881, 493_882]) + stub_admins({ 'id' => '493881', 'name' => 'Zoe' }, { 'id' => '493882', 'name' => 'Alice' }) + query = ForestAdminDatasourceToolkit::Components::Query + + ForestAdminDatasourceToolkit::Utils::Collection.list_relation( + collection, %w[814865], 'admins', nil, + query::Filter.new(sort: query::Sort.new([{ field: 'name', ascending: true }])), + query::Projection.new(%w[id name]) + ) + + expect(ForestAdminDatasourceIntercom.logger) + .to have_received(:warn).with(/sort on "name", which it does not carry/) + end end end end diff --git a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/ticket_spec.rb b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/ticket_spec.rb index bf1b8fddb..3e5530c25 100644 --- a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/ticket_spec.rb +++ b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/ticket_spec.rb @@ -31,6 +31,11 @@ def leaf(field, operator, value = nil) .new(field, operator, value) end + def branch(aggregator, *conditions) + ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Nodes::ConditionTreeBranch + .new(aggregator, conditions) + end + # Hand-written from the shape measured on a real workspace: the state comes # embedded, the company as a bare id, and the parts ride along. def ticket(id, overrides = {}) @@ -64,6 +69,14 @@ def comment(at:, by: 'Alice', type: 'admin', part_type: 'comment') 'created_at' => at, 'author' => { 'type' => type, 'id' => '1', 'name' => by } } end + def stub_admins(*admins) + stub_request(:get, "#{base}/admins").to_return(json('type' => 'admin.list', 'admins' => admins)) + end + + def stub_ticket_states(*states) + stub_request(:get, "#{base}/ticket_states").to_return(json('type' => 'list', 'data' => states)) + end + def stub_search(*records, total: nil, body: nil) answer = { 'type' => 'ticket.list', 'tickets' => records, 'total_count' => total || records.size, 'pages' => { 'type' => 'pages', 'page' => 1 } } @@ -77,16 +90,24 @@ def rows(projection = nil, **options) collection.list(nil, filter(**options), projection) end + # The columns alone: a relation carries neither operators nor an order. + def columns + collection.fields.select { |_, field| field.type == 'Column' } + end + describe 'schema' do it 'is named IntercomTicket' do expect(collection.name).to eq('IntercomTicket') end - # The state travels embedded, so its labels cost nothing and the row does - # not depend on IntercomTicketState to be readable. - it 'flattens the embedded state into its labels' do - expect(collection.fields.keys) - .to include('state_id', 'state_category', 'state_label', 'state_external_label', 'previous_state_id') + # The state travels embedded, so its label costs nothing and the row does + # not depend on IntercomTicketState to be readable. One label and not + # three: the category and the customer-facing label are a hop away, on the + # relation, and neither was ever filterable -- so no segment or scope could + # rest on them. + it 'flattens the embedded state into one label, the rest being a hop away' do + expect(collection.fields.keys).to include('state_id', 'state_label', 'previous_state_id') + expect(collection.fields.keys).not_to include('state_category', 'state_external_label') end it 'carries the attributes of every ticket type in union' do @@ -96,7 +117,7 @@ def rows(projection = nil, **options) # `/tickets/search` ignores a sort without saying so, on every column. it 'declares every column unsortable' do - expect(collection.fields.values.map(&:is_sortable).uniq).to eq([false]) + expect(columns.values.map(&:is_sortable).uniq).to eq([false]) end it 'advertises the filters the search endpoint answers, and only those' do @@ -180,7 +201,7 @@ def rows(projection = nil, **options) expect(rows.first) .to include('id' => '1', 'ticket_id' => '11', 'category' => 'request', 'open' => true, - 'state_id' => '19', 'state_category' => 'in_progress', 'state_label' => 'En cours Tech', + 'state_id' => '19', 'state_label' => 'En cours Tech', 'previous_state_id' => '14', 'ticket_type_name' => 'Bug', 'company_id' => '696dd52099f73812610d9c7b', 'admin_assignee_id' => '493881') end @@ -265,6 +286,253 @@ def rows(projection = nil, **options) end end + # Every target is read whole in one request, so a relation resolves for a + # whole page at the price of a single read -- which is what makes eight of + # them affordable at all. + describe 'the relations to the reference collections' do + it 'points every id at the collection that reads it' do + expect(collection.fields['admin_assignee']) + .to have_attributes(type: 'ManyToOne', foreign_collection: 'IntercomAdmin', + foreign_key: 'admin_assignee_id', foreign_key_target: 'id', is_read_only: true) + expect(collection.fields['team_assignee']) + .to have_attributes(foreign_collection: 'IntercomTeam', foreign_key: 'team_assignee_id') + expect(collection.fields['state']) + .to have_attributes(foreign_collection: 'IntercomTicketState', foreign_key: 'state_id') + expect(collection.fields['previous_state']) + .to have_attributes(foreign_collection: 'IntercomTicketState', foreign_key: 'previous_state_id') + expect(collection.fields['ticket_type']) + .to have_attributes(foreign_collection: 'IntercomTicketType', foreign_key: 'ticket_type_id') + end + + # Declared before the attribute columns, so a workspace attribute whose + # name lands on a relation is skipped with a warning the way one landing on + # a column is. Declared after, it would collide and take the boot with it. + it 'keeps the relation when a ticket attribute carries its name' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + + collection = described_class.new(datasource, attributes: [attribute('state')]) + + expect(collection.fields['state'].type).to eq('ManyToOne') + end + + it 'nests the assignee under the relation, reading the teammates once for the page' do + stub_search(ticket('1'), ticket('2')) + stub_admins('id' => '493881', 'name' => 'Alice', 'email' => 'alice@acme.test') + + rows = collection.list(nil, filter, ['id', 'admin_assignee:name']) + + expect(rows).to eq([{ 'id' => '1', 'admin_assignee' => { 'name' => 'Alice', 'id' => '493881' } }, + { 'id' => '2', 'admin_assignee' => { 'name' => 'Alice', 'id' => '493881' } }]) + expect(WebMock).to have_requested(:get, "#{base}/admins").once + end + + # A teammate who left the workspace: the row reads as unassigned rather + # than as a page that could not be served. + it 'nests nothing when the id names no teammate' do + stub_search(ticket('1')) + stub_admins('id' => '493882', 'name' => 'Bruno') + + expect(collection.list(nil, filter, ['id', 'admin_assignee:name']).first) + .to eq({ 'id' => '1', 'admin_assignee' => nil }) + end + + it 'reads no relation the projection did not name' do + stub_search(ticket('1')) + + collection.list(nil, filter, %w[id state_label]) + + expect(WebMock).not_to have_requested(:get, "#{base}/admins") + end + + # The price of a relation is one read per target *collection*, not one per + # relation: `state` and `previous_state` name the same endpoint, and it is + # read once, over the ids both of them point at. + it 'reads a collection two relations point at once, not once per relation' do + stub_search(ticket('1')) + stub_ticket_states({ 'id' => '19', 'internal_label' => 'En cours Tech' }, + { 'id' => '14', 'internal_label' => 'Recu' }) + + rows = collection.list(nil, filter, ['id', 'state:internal_label', 'previous_state:internal_label']) + + expect(rows.first).to eq({ 'id' => '1', + 'state' => { 'internal_label' => 'En cours Tech', 'id' => '19' }, + 'previous_state' => { 'internal_label' => 'Recu', 'id' => '14' } }) + expect(WebMock).to have_requested(:get, "#{base}/ticket_states").once + end + + # One read for two relations means one projection for two, and each of them + # still gets the columns it asked for rather than the union. + it 'nests under each relation only the columns that relation asked for' do + stub_search(ticket('1')) + stub_ticket_states({ 'id' => '19', 'category' => 'in_progress', 'internal_label' => 'En cours Tech' }, + { 'id' => '14', 'category' => 'submitted', 'internal_label' => 'Recu' }) + + rows = collection.list(nil, filter, ['id', 'state:category', 'previous_state:internal_label']) + + expect(rows.first).to eq({ 'id' => '1', + 'state' => { 'category' => 'in_progress', 'id' => '19' }, + 'previous_state' => { 'internal_label' => 'Recu', 'id' => '14' } }) + end + end + + # A relation is published filterable as soon as any column of its target is, + # so the interface offers `admin_assignee:name` the moment the relation + # exists. What Intercom is really filtered on is the foreign key: the target + # says which of its records match -- over every record it holds, not over a + # page -- and the ids it names are what the search carries. + describe 'a condition through a relation' do + it 'filters on the foreign key the target resolved to' do + stub_admins('id' => '493881', 'name' => 'Alice') + stub_search(ticket('1')) + + rows(%w[id], condition_tree: leaf('admin_assignee:name', operators::EQUAL, 'Alice')) + + expect(WebMock).to have_requested(:post, "#{base}/tickets/search") + .with(body: hash_including('query' => { 'field' => 'admin_assignee_id', 'operator' => '=', + 'value' => '493881' })) + end + + # Intercom takes no membership operator on these fields, so several matches + # become several conditions -- which is also why the group has a ceiling. + it 'writes a group of equalities when the target matched several records' do + stub_admins({ 'id' => '493881', 'name' => 'Alice' }, { 'id' => '493882', 'name' => 'Alice' }) + stub_search(ticket('1')) + + rows(%w[id], condition_tree: leaf('admin_assignee:name', operators::EQUAL, 'Alice')) + + expect(WebMock).to have_requested(:post, "#{base}/tickets/search") + .with(body: hash_including('query' => { + 'operator' => 'OR', + 'value' => [{ 'field' => 'admin_assignee_id', 'operator' => '=', + 'value' => '493881' }, + { 'field' => 'admin_assignee_id', 'operator' => '=', + 'value' => '493882' }] + })) + end + + # No row can satisfy it, and Intercom's DSL has no way of saying so: the + # search is skipped rather than sent as a filter that would come back with + # every ticket. + it 'answers nothing, and reads nothing, when the target matched no record' do + stub_admins('id' => '493881', 'name' => 'Alice') + + expect(rows(%w[id], condition_tree: leaf('admin_assignee:name', operators::EQUAL, 'Zoe'))).to be_empty + expect(WebMock).not_to have_requested(:post, "#{base}/tickets/search") + end + + it 'counts none of them either, without a request' do + stub_admins('id' => '493881', 'name' => 'Alice') + aggregation = ForestAdminDatasourceToolkit::Components::Query::Aggregation.new(operation: 'Count') + + counted = collection.aggregate(nil, filter(condition_tree: leaf('admin_assignee:name', + operators::EQUAL, 'Zoe')), aggregation) + + expect(counted).to eq([{ 'group' => {}, 'value' => 0 }]) + expect(WebMock).not_to have_requested(:post, "#{base}/tickets/search") + end + + # A relation group nested inside the tree the agent assembled: a scope, a + # segment and the operator's own filter, and the group counts as one of + # them. + it 'nests the group inside the condition it was written next to' do + stub_admins({ 'id' => '493881', 'name' => 'Alice' }, { 'id' => '493882', 'name' => 'Alice' }) + stub_search(ticket('1')) + tree = branch('And', leaf('category', operators::EQUAL, 'request'), + leaf('admin_assignee:name', operators::EQUAL, 'Alice')) + + collection.list(nil, filter(condition_tree: tree), %w[id]) + + expect(WebMock).to have_requested(:post, "#{base}/tickets/search") + .with(body: hash_including('query' => { + 'operator' => 'AND', + 'value' => [{ 'field' => 'category', 'operator' => '=', + 'value' => 'request' }, + { 'operator' => 'OR', + 'value' => [{ 'field' => 'admin_assignee_id', + 'operator' => '=', 'value' => '493881' }, + { 'field' => 'admin_assignee_id', + 'operator' => '=', 'value' => '493882' }] }] + })) + end + + # Intercom nests a search two levels deep, and the group a relation expands + # into is a level nobody wrote. Inlined into a parent aggregating the same + # way, it costs none -- without which a scope plus a "match any" filter + # carrying one relation condition would be refused for a nesting the + # operator cannot find in their own filter. + it 'inlines the group into a parent that aggregates the same way' do + stub_admins({ 'id' => '493881', 'name' => 'Alice' }, { 'id' => '493882', 'name' => 'Alice' }) + stub_search(ticket('1')) + tree = branch('And', leaf('category', operators::EQUAL, 'request'), + branch('Or', leaf('open', operators::EQUAL, true), + leaf('admin_assignee:name', operators::EQUAL, 'Alice'))) + + collection.list(nil, filter(condition_tree: tree), %w[id]) + + expect(WebMock).to have_requested(:post, "#{base}/tickets/search") + .with(body: hash_including('query' => { + 'operator' => 'AND', + 'value' => [{ 'field' => 'category', 'operator' => '=', + 'value' => 'request' }, + { 'operator' => 'OR', + 'value' => [{ 'field' => 'open', 'operator' => '=', + 'value' => true }, + { 'field' => 'admin_assignee_id', + 'operator' => '=', 'value' => '493881' }, + { 'field' => 'admin_assignee_id', + 'operator' => '=', 'value' => '493882' }] }] + })) + end + + # Inlining trades a level of nesting for width, and Intercom bounds both. + # Past fifteen conditions the nested form is the one that fits, so the + # group stays where it was rather than emptying the budget it was spared. + it 'leaves the group nested when inlining it would pass fifteen conditions' do + stub_admins(*(1..3).map { |index| { 'id' => index.to_s, 'name' => 'Alice' } }) + stub_search(ticket('1')) + others = (1..14).map { |index| leaf('category', operators::NOT_EQUAL, "c#{index}") } + tree = branch('Or', *others, leaf('admin_assignee:name', operators::EQUAL, 'Alice')) + + collection.list(nil, filter(condition_tree: tree), %w[id]) + + expect(WebMock).to(have_requested(:post, "#{base}/tickets/search").with do |request| + query = JSON.parse(request.body)['query'] + query['value'].size == 15 && query['value'].last['operator'] == 'OR' && + query['value'].last['value'].size == 3 + end) + end + + it 'reads nothing when a relation condition inside an and matches nothing' do + stub_admins('id' => '493881', 'name' => 'Alice') + tree = branch('And', leaf('category', operators::EQUAL, 'request'), + leaf('admin_assignee:name', operators::EQUAL, 'Zoe')) + + expect(collection.list(nil, filter(condition_tree: tree), %w[id])).to be_empty + expect(WebMock).not_to have_requested(:post, "#{base}/tickets/search") + end + + # Fifteen conditions per group is Intercom's limit, and a relation reaches + # it without trying. Refused by name rather than sent and answered with a + # 400 naming neither the limit nor the filter that hit it. + it 'refuses a relation condition matching more records than a group holds' do + stub_admins(*(1..16).map { |index| { 'id' => index.to_s, 'name' => 'Alice' } }) + + expect { rows(%w[id], condition_tree: leaf('admin_assignee:name', operators::EQUAL, 'Alice')) } + .to raise_error(UnsupportedOperatorError, /names 16 records.*15 conditions per group/m) + end + + # `/tickets/search` filters no state id -- the table carries none -- so the + # state relation is there to be read and navigated, and the message says + # which of the two it is rather than naming a column the operator never + # wrote. Refused before the target is read, a refusal that spends a request + # costing exactly what it refuses. + it 'refuses a condition through a relation the endpoint filters nothing on' do + expect { rows(%w[id], condition_tree: leaf('state:category', operators::EQUAL, 'in_progress')) } + .to raise_error(UnsupportedOperatorError, %r{resolves to "state_id", on which tickets/search takes no}) + expect(WebMock).not_to have_requested(:get, /ticket_states/) + end + end + describe '#aggregate' do it 'counts a filtered collection through the total_count of its search' do stub_search(total: 12, body: { 'query' => { 'field' => 'open', 'operator' => '=', 'value' => true } }) diff --git a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/datasource_spec.rb b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/datasource_spec.rb index c5387e059..91ad2f84f 100644 --- a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/datasource_spec.rb +++ b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/datasource_spec.rb @@ -8,11 +8,12 @@ module ForestAdminDatasourceIntercom # The reference collections come first: they are what turns an assignee id # into a teammate and a state id into a label. Conversations follow, Tickets - # next. + # next. The membership sits with the two collections it joins: a many-to-many + # needs a collection to travel through, and Intercom exposes none. it 'publishes the collections of the lot' do expect(datasource.collections.keys) - .to eq(%w[IntercomAdmin IntercomTeam IntercomTicketType IntercomTicketState IntercomConversation - IntercomTicket]) + .to eq(%w[IntercomAdmin IntercomTeam IntercomTeamMembership IntercomTicketType IntercomTicketState + IntercomConversation IntercomTicket]) end # The one read a boot performs: the attributes a workspace declares on its diff --git a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/query/condition_tree_translator_spec.rb b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/query/condition_tree_translator_spec.rb index b252c3867..797efa12a 100644 --- a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/query/condition_tree_translator_spec.rb +++ b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/query/condition_tree_translator_spec.rb @@ -134,7 +134,21 @@ def leaves(count) branch('And', *leaves(2)))) expect { translate(tree) } - .to raise_error(UnsupportedOperatorError, /nests a search 2 levels deep and this one reaches 3/) + .to raise_error(UnsupportedOperatorError, + /nests a search 2 levels deep and this one reaches 3.*flatten the segment/m) + end + + # The innermost group is sometimes not one the operator wrote: a relation + # expands into one equality per record it matched, and telling them to + # flatten a nesting they never wrote is a refusal they cannot act on. + it 'names the relation when the innermost group is what one expanded into' do + relation_group = Collections::Relations::RelationBranch.new('Or', leaves(2), 'admin_assignee:name') + tree = branch('Or', leaf('open', operators::EQUAL, true), + branch('And', leaf('read', operators::EQUAL, true), relation_group)) + + expect { translate(tree) } + .to raise_error(UnsupportedOperatorError, + /"admin_assignee:name" expanded into, one equality per record it matched/) end # A branch carrying a single condition is unwrapped, so it spends no level: diff --git a/packages/forest_admin_datasource_toolkit/lib/forest_admin_datasource_toolkit/utils/collection.rb b/packages/forest_admin_datasource_toolkit/lib/forest_admin_datasource_toolkit/utils/collection.rb index 537cdf97c..916d9884c 100644 --- a/packages/forest_admin_datasource_toolkit/lib/forest_admin_datasource_toolkit/utils/collection.rb +++ b/packages/forest_admin_datasource_toolkit/lib/forest_admin_datasource_toolkit/utils/collection.rb @@ -56,10 +56,15 @@ def self.other_inverse?(field, relation_field) field.origin_key == relation_field.foreign_key end + # ValidationError rather than a bare ForestException, for the same reason as + # below: every one of these three names a field the request asked for -- a + # filter, a sort, a scope written against another schema, a condition + # reaching through a relation that is not one this can traverse. That is a + # 400 the caller can read and fix, not a 500 saying the agent broke. def self.get_field_schema(collection, field_name) fields = collection.schema[:fields] unless field_name.include?(':') - raise ForestException, "Column not found #{collection.name}.#{field_name}" unless fields.key?(field_name) + raise ValidationError, "Column not found #{collection.name}.#{field_name}" unless fields.key?(field_name) return fields[field_name] end @@ -67,10 +72,11 @@ def self.get_field_schema(collection, field_name) association_name = field_name.split(':')[0] relation_schema = fields[association_name] - raise ForestException, "Relation not found #{collection.name}.#{association_name}" unless relation_schema + raise ValidationError, "Relation not found #{collection.name}.#{association_name}" unless relation_schema if relation_schema.type != 'ManyToOne' && relation_schema.type != 'OneToOne' - raise ForestException, "Unexpected field type #{relation_schema.type}: #{collection.name}.#{association_name}" + raise ValidationError, + "Unexpected field type #{relation_schema.type}: #{collection.name}.#{association_name}" end get_field_schema( @@ -161,7 +167,14 @@ def self.list_relation(collection, primary_key_values, relation_name, caller, fo projection.nest(prefix: foreign_relation) ) - return records.map { |r| r[foreign_relation] } + # Compacted: a through row whose target the foreign collection no + # longer answers -- a record deleted, outside the caller's reach, or + # dropped by a datasource that reads its targets in bounded pages -- + # carries a nil where a record was expected, and every consumer of a + # related list down to the JSON:API serializer reads a row by key. + # A join whose other side is gone yields no row, it does not yield an + # empty one. + return records.filter_map { |r| r[foreign_relation] } end end diff --git a/packages/forest_admin_datasource_toolkit/spec/lib/forest_admin_datasource_toolkit/utils/collection_spec.rb b/packages/forest_admin_datasource_toolkit/spec/lib/forest_admin_datasource_toolkit/utils/collection_spec.rb index fb7d88535..ce0036c25 100644 --- a/packages/forest_admin_datasource_toolkit/spec/lib/forest_admin_datasource_toolkit/utils/collection_spec.rb +++ b/packages/forest_admin_datasource_toolkit/spec/lib/forest_admin_datasource_toolkit/utils/collection_spec.rb @@ -221,7 +221,7 @@ module Utils expect do described_class.get_field_schema(collection_person, 'foo') - end.to raise_error(ForestException, 'Column not found Person.foo') + end.to raise_error(ValidationError, 'Column not found Person.foo') end it 'get_field_schema should work with simple column' do @@ -233,14 +233,14 @@ module Utils expect do described_class.get_field_schema(collection_person, 'unknown:foo') - end.to raise_error(ForestException, 'Relation not found Person.unknown') + end.to raise_error(ValidationError, 'Relation not found Person.unknown') end it 'get_field_schema should throw with invalid relation type' do expect do described_class.get_field_schema(collection_book, 'myBookPersons:bookId') - end.to raise_error(ForestException, 'Unexpected field type OneToMany: Book.myBookPersons') + end.to raise_error(ValidationError, 'Unexpected field type OneToMany: Book.myBookPersons') end it 'get_field_schema should work with relation column' do @@ -311,6 +311,25 @@ module Utils ForestAdminDatasourceToolkit::Components::Query::Projection.new)).to eq([1]) end + # A through row whose target the foreign collection no longer answers -- + # deleted, outside the caller's reach, or dropped by a datasource that + # reads its targets in bounded pages -- carries a nil where a record was + # expected, and every consumer down to the JSON:API serializer reads a + # row by key. A join whose other side is gone yields no row. + it 'list_relation should drop a through row whose target resolved to nothing' do + book_person_class = Struct.new(:bookId, :personId, :myPerson, :myBook) + stub_const('BookPerson', book_person_class) + allow(collection_book_person).to receive(:list).and_return( + [ + BookPerson.new(1, 1, 1, 1), + BookPerson.new(1, 2, nil, 1) + ] + ) + + expect(described_class.list_relation(collection_book, [1], 'myPersons', caller, ForestAdminDatasourceToolkit::Components::Query::Filter.new, + ForestAdminDatasourceToolkit::Components::Query::Projection.new)).to eq([1]) + end + it 'aggregate_relation should work with one to many relation' do allow(collection_book_person).to receive(:aggregate).and_return(1) From 313b031e31754a3099069a4428af74cdd2f6243b Mon Sep 17 00:00:00 2001 From: Christophe Brun Date: Tue, 8 Sep 2026 16:34:44 +0200 Subject: [PATCH 4/6] feat(datasource intercom): contacts, companies and the promotion of the relations (lot 4) (#386) * feat(intercom): contacts, companies and contact relations Publishes the two collections phase 2 is walked through, and turns the denormalized contact columns of lot 1 into a relation now that their target exists. Contacts read through the one endpoint of the API that sorts, so the measured table gained a sortable flag and the cursor tier a server-side order; the match-all predicate Tickets already sent is what routes a list view asking for an order through the search. Companies read through the third pagination tier, by offset, which is the one place the window a list view asks for maps onto what Intercom takes. In exchange it is looked up rather than searched, and anything past the two published lookups is refused by name. Contact and company custom attributes are typed from GET /data_attributes and carry api_writable for the lot that writes, though every column here is read-only. Resolving a relation condition is now bounded on the cursor tier: the target is read one record past what a group may hold, rather than walking a whole collection to refuse the fan-out afterwards. contact_email and contact_ids give way to contact_id and the contact relation, one readable label plus navigation, which is the rule lot 2.5 set for the ticket labels. No saved view can rest on them: both were already refused server-side. Co-Authored-By: Claude Opus 5 (1M context) --- .../README.md | 215 ++++++-- .../client.rb | 73 ++- .../collections/company.rb | 85 ++++ .../collections/company/serializer.rb | 44 ++ .../collections/contact.rb | 201 ++++++++ .../collections/contact/serializer.rb | 106 ++++ .../collections/contact_identity.rb | 41 +- .../collections/conversation.rb | 28 +- .../collections/conversation/timeline.rb | 45 +- .../collections/cursor_collection.rb | 136 +++-- .../collections/custom_attributes.rb | 69 +++ .../collections/offset_collection.rb | 303 +++++++++++ .../collections/relations.rb | 31 +- .../collections/ticket.rb | 71 +-- .../collections/ticket/serializer.rb | 33 +- .../collections/timeline.rb | 53 ++ .../datasource.rb | 22 +- .../query/search_fields.rb | 26 +- .../query/search_fields.yml | 281 +++++++++- .../schema/data_attributes_introspector.rb | 119 +++++ .../collections/company_spec.rb | 374 ++++++++++++++ .../collections/contact_spec.rb | 481 ++++++++++++++++++ .../collections/conversation_spec.rb | 52 +- .../collections/ticket_spec.rb | 103 +++- .../datasource_spec.rb | 36 +- .../query/search_fields_spec.rb | 38 +- .../data_attributes_introspector_spec.rb | 136 +++++ .../spec/probe_search_fields_spec.rb | 4 +- .../spec/spec_helper.rb | 19 +- 29 files changed, 2946 insertions(+), 279 deletions(-) create mode 100644 packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/company.rb create mode 100644 packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/company/serializer.rb create mode 100644 packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/contact.rb create mode 100644 packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/contact/serializer.rb create mode 100644 packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/custom_attributes.rb create mode 100644 packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/offset_collection.rb create mode 100644 packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/timeline.rb create mode 100644 packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/schema/data_attributes_introspector.rb create mode 100644 packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/company_spec.rb create mode 100644 packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/contact_spec.rb create mode 100644 packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/schema/data_attributes_introspector_spec.rb diff --git a/packages/forest_admin_datasource_intercom/README.md b/packages/forest_admin_datasource_intercom/README.md index 43ce826e8..7ca2838d0 100644 --- a/packages/forest_admin_datasource_intercom/README.md +++ b/packages/forest_admin_datasource_intercom/README.md @@ -59,10 +59,11 @@ beats not running. ### Token permissions A read-only token is enough, and is what to recommend for this lot. A permission the token lacks -costs **columns or a collection, never the boot of the agent**: the ticket-type introspection -degrades to no attribute column, a collection whose endpoint answers 403 fails its own page, and a -token that cannot read `/admins` or `/teams` leaves the `admin_names` / `team_names` column empty -rather than failing the page it is on. +costs **columns or a collection, never the boot of the agent**: the three boot-time introspections +each degrade to no attribute column, a collection whose endpoint answers 403 fails its own page, and +a token that cannot read `/admins` or `/teams` leaves the `admin_names` / `team_names` column empty +rather than failing the page it is on. A token denied contacts or companies costs those two +collections and the `contact_name` column, and leaves everything else standing. A **relation is the exception**, and it is worth knowing before scoping a token: resolving one reads the target endpoint, and that read is not guarded the way the names above are. A token denied @@ -81,19 +82,29 @@ denied. Scope the token to the endpoints in the table below, or to none of them. | `IntercomTeamMembership` | `GET /teams` | read whole | yes, exactly | | `IntercomTicketType` | `GET /ticket_types` | read whole | yes, exactly | | `IntercomTicketState` | `GET /ticket_states` | read whole | yes, exactly | +| `IntercomContact` | `GET /contacts`, `POST /contacts/search`, `GET /companies/{id}/contacts` | cursor | yes, exactly | +| `IntercomCompany` | `POST /companies/list`, `GET /companies?...`, `GET /companies/{id}` | **offset** | yes, exactly | -Two tiers, and they behave differently on purpose. +Three tiers, and they behave differently on purpose. **Read whole** — admins, teams, team memberships, ticket types, ticket states. Their endpoints answer in one response, so filtering, sorting, paging and counting them in memory is *exact*: the records in hand are every record Intercom holds. These are the only collections that can be filtered, sorted and grouped in this lot, and the only ones a chart may group by. The cost is bandwidth, not correctness. -**Cursor** — conversations and tickets. What is in hand is a page of something far larger, so -nothing is filtered or sorted in memory. Three routes and no fourth: no condition walks the listing, -`id equals X` reads the record through its own endpoint, and anything else is translated into -Intercom's search DSL and walked through the search endpoint. What the translation cannot express is -**refused by name** — see [Filtering](#filtering). +**Cursor** — conversations, tickets and contacts. What is in hand is a page of something far larger, +so nothing is filtered or sorted in memory. Three routes and no fourth: no condition walks the +listing, `id equals X` reads the record through its own endpoint, and anything else is translated +into Intercom's search DSL and walked through the search endpoint. What the translation cannot +express is **refused by name** — see [Filtering](#filtering). Contacts add two routes of their own, +both described under [Contacts](#contacts). + +**Offset** — companies, and nothing else. `POST /companies/list` takes a **page number**, which is +what a list view asks for: page 7 is one request rather than six pages walked to reach it, with no +cap and no truncation warning. It is the one place the [first limitation +below](#what-the-api-cannot-do-and-what-this-does-about-it) does not apply. What it pays for that is +filtering — there is no company search endpoint at all, so what a filter may say is a handful of +exact lookups and nothing else. See [Companies](#companies). ## Relations @@ -120,6 +131,12 @@ sized for reference collections, which is what every target here is. | `IntercomTeam` | `admins` | `IntercomAdmin` | no (many-to-many) | | `IntercomAdmin` | `teams` | `IntercomTeam` | no (many-to-many) | | `IntercomTeamMembership` | `team`, `admin` | `IntercomTeam`, `IntercomAdmin` | yes | +| `IntercomConversation` | `contact` | `IntercomContact` | yes | +| `IntercomTicket` | `contact` | `IntercomContact` | **spec, unprobed** — see below | +| `IntercomContact` | `owner` | `IntercomAdmin` | yes | +| `IntercomContact` | `company` | `IntercomCompany` | **no** — read and navigate only | +| `IntercomContact` | `conversations`, `tickets` | `IntercomConversation`, `IntercomTicket` | no (one-to-many) | +| `IntercomCompany` | `contacts` | `IntercomContact` | no (one-to-many) | Every one of them is **read-only**: this lot writes nothing, and Intercom exposes no endpoint that writes a team membership at all. @@ -143,6 +160,27 @@ lots published: one readable form plus a relation to navigate, rather than two w They are read only when a projection asks for them, and a token that cannot read the other side costs the column and nothing else — never the page, and never the relation. +**The 360 degrees is those last four rows.** From a ticket or a conversation, `contact` reaches the +person who wrote in; from them, `conversations` and `tickets` list everything they ever opened, and +`company` reaches their account, whose `contacts` lists their colleagues. Each of those lists is one +request: `/conversations/search` matches a conversation against one of its contact ids, and +`GET /companies/{id}/contacts` answers the contacts of an account — which is the one relation +`/contacts/search` could not have resolved, filtering no company field. + +**A conversation has several contacts, and the relation names the first of them** — the same one +`contact_name` and `contact_count` describe, so the column and the relation cannot disagree. The +others are a hop away: open that contact and read their conversations. The alternative, a +many-to-many through a join collection, would have been the honest cardinality at the price of three +collections of plumbing in the interface; naming the first contact and counting them is what lot 1 +already published, and lot 4 promotes it rather than replacing it. + +Two of these carry a caveat worth reading before scoping a token or writing a segment. The **ticket +side is a `spec` row the probe has not confirmed**: whether `/tickets/search` filters on +`contact_ids` at all is unmeasured, and if it does not, the relation stays navigable and the filter +moves to the refusal table — exactly what happened to the ticket `state`. And **the company +traversal is refused by name**: `/contacts/search` filters no company field, so `company:name` is +answered with a message saying to filter from the company side instead. + The same rule settled the ticket labels: `state_label` and `ticket_type_name` stay on the row, `state_category` and `state_external_label` are gone — they are a hop away, on the `state` relation, and neither was ever filterable, so no segment, scope or saved filter could rest on them. @@ -161,18 +199,21 @@ Where Forest asks for something Intercom has no equivalent for, this datasource message naming the reason** rather than answering something that looks right and is not. Those arrive as a 400 carrying the text. -- **No offset pagination.** Intercom hands out the page after a cursor and documents that jumping to - page N is unsupported, so reaching page 20 costs 20 sequential requests. The walk is capped at 50 - pages / 7 500 records and every truncation is logged, naming the window it stopped in. +- **No offset pagination, except on companies.** Intercom hands out the page after a cursor and + documents that jumping to page N is unsupported, so reaching page 20 costs 20 sequential requests. + The walk is capped at 50 pages / 7 500 records and every truncation is logged, naming the window + it stopped in. `POST /companies/list` is the exception and takes a page number, which is why + companies escape the walker and its caps entirely. - **Duplicates on a moving dataset.** Intercom documents that records modified between two paginated requests can be served twice; the walk deduplicates by id. The missed counterpart is inherent to cursor pagination and cannot be repaired — it is documented rather than papered over. -- **A search takes no sort at all.** Neither search endpoint accepts one, so **no column of - `IntercomConversation` or `IntercomTicket` is sortable** and an explicit order is reported in the - log. The only collections Intercom sorts are the ones read whole, in memory. -- **A sort is accepted and ignored.** Measured: `sort` on these endpoints raises nothing and changes - nothing. Since the lack of support is undetectable at runtime, no column is declared sortable and - a requested order is reported in the log. The rows come back in the order the API imposes. +- **One endpoint sorts, and it is `/contacts/search`.** Everything else comes back in the order the + API imposes: `POST /companies/list` has no order parameter at all, and the other two search + endpoints **accept a `sort` and ignore it** — measured, it raises nothing and changes nothing. + Since that is undetectable at runtime, no column of `IntercomConversation`, `IntercomTicket` or + `IntercomCompany` is declared sortable and a requested order is reported in the log. The + collections read whole sort in memory, exactly, and Contacts sort server-side on the columns the + measured table declares — see [Contacts](#contacts). - **No aggregate endpoint.** Counting is free and exact — `total_count` counts what the query names, not what a page held — so the record counter is one request. Anything beyond a count is refused on the cursor collections: grouping over the pages a walk collected would look exact while answering @@ -250,8 +291,10 @@ What follows from that: | Collection | Filterable on | | --- | --- | -| `IntercomConversation` | `id`, `state`, `priority`, `open`, `read`, `title`, `admin_assignee_id`, `team_assignee_id`, `source_type`, `source_subject`, `source_body`, `source_delivered_as`, `source_author_email`, `closed_by_id`, `reopen_count`, `part_count`, `ai_agent_participated`, and the dates `created_at`, `updated_at`, `waiting_since`, `snoozed_until`, `closed_at`, `first_closed_at`, `first_contact_reply_at`, `last_contact_reply_at`, `last_admin_reply_at` | -| `IntercomTicket` | `id`, `open`, `category`, `ticket_type_id`, `admin_assignee_id`, `team_assignee_id`, `created_at`, `updated_at` | +| `IntercomConversation` | `id`, `state`, `priority`, `open`, `read`, `title`, `admin_assignee_id`, `team_assignee_id`, `source_type`, `source_subject`, `source_body`, `source_delivered_as`, `source_author_email`, `closed_by_id`, `reopen_count`, `part_count`, `ai_agent_participated`, `contact_id`, and the dates `created_at`, `updated_at`, `waiting_since`, `snoozed_until`, `closed_at`, `first_closed_at`, `first_contact_reply_at`, `last_contact_reply_at`, `last_admin_reply_at` | +| `IntercomTicket` | `id`, `open`, `category`, `ticket_type_id`, `admin_assignee_id`, `team_assignee_id`, `contact_id`, `created_at`, `updated_at` | +| `IntercomContact` | `id`, `role`, `name`, `email`, `email_domain`, `phone`, `external_id`, `owner_id`, `unsubscribed_from_emails`, `has_hard_bounced`, `marked_email_as_spam`, `language_override`, `browser`, `browser_language`, `os`, `location_country`, `location_region`, `location_city`, and the dates `created_at`, `updated_at`, `signed_up_at`, `last_seen_at`, `last_contacted_at`, `last_replied_at`, `last_email_opened_at`, `last_email_clicked_at` | +| `IntercomCompany` | `id`, `company_id`, `name` — four lookups and no search endpoint, see [Companies](#companies) | **The primary key** is filterable like any other column, but a filter naming it *alone* is not answered by a search: `id equals X` and `id in [...]` read the record endpoint directly, one request @@ -265,6 +308,19 @@ refuses a search by name. ### What is not filterable, and why +- **every column of a company but two.** There is no `/companies/search`: Intercom looks a company + up by `name`, by `company_id`, by `tag_id` or by `segment_id`, one exact value at a time, and the + first two are the ones that name a column of the collection. Everything else — the industry, the + plan, the monthly spend — is refused by name. Filtering by tag or by segment belongs with the lot + that adds those collections; +- **a contact's `company_id`, `company_count`, `avatar` and `session_count`** — the endpoint filters + none of them. Reach the contacts of an account from the account instead, through its `contacts` + relation, which is one request; +- **the custom attributes of a contact or a company.** They are filtered as + `custom_attributes.{name}`, by name — the ambiguity that keeps ticket attributes display-only does + not arise here — but which operators Intercom answers on each data type has not been measured, and + this package publishes no filter it has not seen work. They ship typed and display-only, and the + probe is what turns that around; - **the columns a ticket derives from its parts** — `closed_at`, `closed_by_name`, `last_reply_at`, `last_responder_name`, `last_responder_type`. They exist nowhere in Intercom; `/tickets/search` filters none of them and ignores a sort on them without a word; @@ -292,8 +348,14 @@ What Intercom is really filtered on is the foreign key: the **target says which match**, over every record it holds rather than over a page, and the ids it names become the condition the search carries. -That is exact, and it has three visible edges: +That is exact, and it has four visible edges: +- **The target is read one record past what a group may hold, and no further.** Against a collection + read whole that costs nothing — every record is in hand — but Contacts are a page of something + far larger, and resolving `contact:email contains "@"` over a whole workspace to then refuse the + fan-out it comes to would spend a full cursor walk on a filter that was never going to be + answered. So the read is bounded, and the refusal says "more than fifteen" rather than a count it + deliberately did not go and measure. - Intercom takes no membership operator on these fields, so several matches become **one equality per match**, inside an `OR` — which counts against the fifteen conditions a group allows. A relation condition matching more records than that is refused by name rather than sent and answered with a @@ -304,10 +366,10 @@ That is exact, and it has three visible edges: being the scarcer of the two. - A condition the target matched **no record** with names no row, and the DSL cannot say so: the search is skipped entirely rather than sent as a filter that would come back with everything. -- A relation whose foreign key the endpoint does not filter — the ticket `state` — is refused with a - message saying which of the two it is: the relation is there to be read and navigated. Whether - `/tickets/search` filters a state id at all is one of the probe's open questions; the answer lands - in the table, not in an assumption. +- A relation whose foreign key the endpoint does not filter — the ticket `state`, a contact's + `company` — is refused with a message saying which of the two it is: the relation is there to be + read and navigated. Whether `/tickets/search` filters a state id or a contact id at all is one of + the probe's open questions; the answer lands in the table, not in an assumption. On the collections read whole the same condition costs nothing: they filter in memory, so the ids go in as a plain membership and none of the DSL's limits apply. @@ -348,8 +410,13 @@ Intercom returns the parts **only when retrieving a single conversation**, so: A conversation is capped at its **500 most recent parts**; a very long thread is therefore partial, and says so nowhere but here. -Contact name and e-mail are denormalized onto the row by **one bulk read per page**, not one per -row, and only when the projection names them. A failure there costs those two columns, not the page. +**The internal notes of the team are in the thread**, next to what the customer was told. That is +what a thread is on Intercom, and publishing half of it would be the more surprising answer — but it +is worth knowing before opening the collection to a role that should not read them. + +The contact's name is denormalized onto the row by **one bulk read per page**, not one per row, and +only when the projection names it. A failure there costs that column, not the page. The e-mail is a +hop away, on the `contact` relation. ## Tickets @@ -377,6 +444,15 @@ Four things to know about them: Both are **display only**, and not temporarily: `/tickets/search` filters on neither and ignores a sort, so neither advertises an operator. +**The thread is published too, and it is free here.** The same `timeline` column a conversation +carries — who said what, when, and through which kind of event, internal notes and state changes +included — built from the parts the response already holds. No request per row and no cap: where a +conversation read from a listing carries no parts at all and leaves the column `nil` for *unknown*, +a ticket always carries them, so an empty list means an empty thread. Both reads ask Intercom for +`display_as=plaintext`: the bodies are HTML written by end customers, and rendering third-party +markup inside Forest is neither safe nor useful. The 500-part ceiling applies here as well, which is +the same truncation that can hide a closure date. + The attributes a workspace declares on its ticket types are introspected once at boot and published as the **union** of every type's, keyed by name the way the payload is. Filtering one is a different matter: Intercom filters an attribute by id (`ticket_attribute.{id}`), and the same name carries a @@ -387,6 +463,64 @@ schema that changes shape whenever the customer adds a type. Until that trade is the attributes stay display-only and advertise no operator. The ids are kept per type so the day the answer changes costs no second boot round trip. +## Contacts + +The people who write in, users and leads alike. Cursor-paginated like conversations and tickets, +with two routes of its own and one thing no other collection has. + +**Intercom sorts this one.** `POST /contacts/search` is the only endpoint of the whole API that +takes a `sort` and applies it, so these are the only sortable columns of the datasource: `name`, +`email`, `created_at`, `updated_at`, `signed_up_at`, `last_seen_at`, `last_contacted_at`, +`last_replied_at`. The set is deliberately narrower than the documentation implies — nothing has +been measured, and a sort Intercom refuses is a list view that fails rather than one that comes back +unordered. A sort on any other column, or on two columns at once, is reported in the log and the +rows come back in the API's order: Intercom takes a single `{ field, order }`, and honouring the +first clause of two would order the page by something nobody asked for. + +An order is also what routes a plain list view through the search endpoint, the listing sorting +nothing: the read then carries the predicate matching everything that Tickets already send. + +**Its date operators are narrower than the other two endpoints'** — measured, 25 August 2026: +`/contacts/search` refuses `>=`, `<=` and `!=` on a date where `/conversations/search` and +`/tickets/search` take them. Nothing is lost that an operator can see, a Date column publishing the +two bounds alone everywhere in this datasource, but it is why the operator table is per endpoint. + +**A set of ids is read in one request** — `id IN [...]`, which this endpoint answers and no other +does — a hundred at a time, rather than one request per record. It is what makes a related list of +contacts affordable. + +**`company_id equals X` reads `GET /companies/{id}/contacts`.** The search filters no company field, +so without that route the contacts of an account would be a refusal rather than a list. It is a bare +equality only: an `and` also carrying a permission scope names a narrower set than the account does, +and answering it with the account alone would serve contacts the scope excludes. + +**A merged contact reads as gone, not as an error.** Intercom drops it from the listing and from the +search, and the record lives on under the id it was merged into. A row pointing at the old id comes +back empty rather than failing the page. + +The custom attributes a workspace declares on its contacts are introspected once at boot from +`GET /data_attributes?model=contact`, typed from `data_type`, and published display-only. + +## Companies + +The accounts contacts belong to, and the collection that behaves least like the others. + +**Paginated by offset**, which is the tier above. **Looked up, not searched**: `name` and +`company_id` — the identifier the customer's own system gave the account, not Intercom's — are the +two filters it publishes, each an exact equality, and anything else is refused by name. A record is +read through `GET /companies/{id}`, and a set of ids one request each, capped at 25 with the +truncation logged. + +`GET /companies/scroll` exists and is **deliberately rejected**: one open scroll per application, +expiring after a minute, cannot serve two operators looking at a list at the same time. + +A contact carries its accounts as a list of ids and nothing else, so **projecting `company:name` on +a contact list costs one request per distinct account on the page**. Reading the account from the +contact's record page, or listing contacts from the account, both cost one. + +Custom attributes are introspected at boot the same way, from `GET /data_attributes?model=company`, +and published display-only for the same reason. + ## Rate limits Intercom meters the app and, above it, the whole workspace — 25 000 requests a minute shared with @@ -420,10 +554,19 @@ The body of a conversation is raw personal data, and this datasource is built on ## Boot-time introspection -Constructing the datasource performs exactly **one** read: `GET /ticket_types`, for the attribute -columns of `IntercomTicket`. It runs on the boot connection — short timeouts, one quick retry — so a -slow Intercom cannot turn a Rails boot into minutes the operator sits through, and it degrades to no -attribute column rather than to a failed boot. +Constructing the datasource performs exactly **three** reads, and they are all of the same kind: +`GET /ticket_types` for the attribute columns of `IntercomTicket`, and +`GET /data_attributes?model=contact` and `?model=company` for those of `IntercomContact` and +`IntercomCompany`. A payload carries the values of the attributes that record happens to have been +given, never their definitions, which is why they cannot be discovered from the records. + +All three run on the boot connection — short timeouts, one quick retry — so a slow Intercom cannot +turn a Rails boot into minutes the operator sits through, and each degrades to no attribute column +rather than to a failed boot. + +`api_writable` is read alongside each attribute and kept, although every column of this lot is +published read-only: it is what tells an attribute the API may write from one Intercom fills in +itself, and reading it again later would be a second boot-time round trip. Everything else is read when a collection is listed, so an agent boots whatever Intercom is doing. @@ -431,11 +574,15 @@ Everything else is read when a collection is listed, so an agent boots whatever | Lot | What it brings | | --- | --- | -| 3 | Writes and business actions: reply, close, snooze, reopen, assign, tag, convert | -| 4 | Contacts and companies, and the relations towards them promoted from today's denormalized columns | +| 3 | Writes and business actions on tickets and conversations: reply, close, snooze, reopen, assign, tag, convert | +| 4b | Writes on contacts and companies: create, update, archive, block, merge, attach and detach | | 5 | Notes, tags, segments | | 6 | Bounded group-by and the reporting export | +Two questions this lot leaves in the table rather than in an assumption, both for +`bin/probe_search_fields` to answer against the customer's workspace: whether `/tickets/search` +filters on `contact_ids`, and which operators `/contacts/search` answers on a custom attribute. + ## Development ```bash diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/client.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/client.rb index ecfe093b3..46b9008d3 100644 --- a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/client.rb +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/client.rb @@ -24,7 +24,9 @@ class Client # rubocop:disable Metrics/ClassLength # callers never have to know how the absence is spelled on the wire. # `total_count` is exact, filter included, which is what makes Forest's # record counter and its "number of" charts one request each. - Page = Struct.new(:records, :next_cursor, :total_count, keyword_init: true) + # `total_pages` is filled by the one endpoint that paginates by offset and + # nil everywhere else: a cursor page has no notion of how many there are. + Page = Struct.new(:records, :next_cursor, :total_count, :total_pages, keyword_init: true) def initialize(configuration) @configuration = configuration @@ -65,14 +67,44 @@ def list_page(path, per_page:, starting_after: nil, params: {}, list_key: 'data' # translator wrote, and it travels in the body; `params` is what still # belongs in the query string -- `display_as` above all, which is not part # of the search payload. - def search_page(path, query:, per_page:, starting_after: nil, list_key: 'data', params: {}) + # `sort` is honoured by `/contacts/search` alone. The other two search + # endpoints accept one and ignore it without a word -- measured -- so the + # collections that read them never send one, and the parameter is here for + # the one that does. `{ field:, ascending: }`, translated to Intercom's own + # spelling on the way out. + def search_page(path, query:, per_page:, starting_after: nil, list_key: 'data', params: {}, sort: nil) pagination = { 'per_page' => self.class.bounded_per_page(per_page) } pagination['starting_after'] = starting_after unless blank?(starting_after) body = { 'query' => query, 'pagination' => pagination } + body['sort'] = sort_clause(sort) if sort must_succeed(path) { to_page(post(path, body, params: params).body, path, list_key) } end + # One page of an endpoint that paginates by **offset** rather than by + # cursor. `POST /companies/list` is the only one, and it is what lets the + # Companies collection answer page 7 of a list view with one request + # instead of walking six pages to reach it. Intercom counts pages from 1. + def offset_page(path, page:, per_page:, params: {}, list_key: 'data') + query = params.merge('page' => [page.to_i, 1].max, + 'per_page' => self.class.bounded_per_page(per_page)) + + must_succeed(path) { to_page(post(path, {}, params: query).body, path, list_key) } + end + + # An exact lookup, and the shape surprise that comes with it: `GET + # /companies?name=` answers the company itself where `?tag_id=` answers a + # list. A record is read here as a page of one, so a caller writes one route + # rather than testing the envelope. + def lookup_page(path, params:, list_key: 'data') + must_succeed(path) do + body = get(path, params).body + next Page.new(records: [body], next_cursor: nil, total_count: 1) if single_record?(body, list_key) + + to_page(body, path, list_key) + end + end + # One record from its own endpoint. Raises on a 404 like on any other # failure: what a missing record means -- a stale link, a record outside the # token's scope, a deletion -- is the caller's to decide, not the client's. @@ -98,8 +130,11 @@ def fetch_record(path, id, params: {}, boot: false) # parameter in the specification is not a promise that a large workspace # answers in one response, and a truncated reference collection would show # an operator a state list missing its last states. - def fetch_all(path, list_key: 'data', boot: false) - must_succeed(path) { collect_pages(path, list_key: list_key, boot: boot) } + # `params` is what narrows the endpoint rather than what pages it: + # `/data_attributes` answers the attributes of contacts and those of + # companies under `?model=`, and both are read whole. + def fetch_all(path, list_key: 'data', params: {}, boot: false) + must_succeed(path) { collect_pages(path, list_key: list_key, params: params, boot: boot) } end # The page size Intercom accepts, whatever was asked for. @@ -149,13 +184,28 @@ def verify_pinned_version(response) ) end - def collect_pages(path, list_key:, boot:) + # Intercom spells an order `{ "field": "...", "order": "descending" }`, + # and answers `data_invalid` on anything else -- so the clause is written + # here rather than by the caller, whose vocabulary is Forest's. + def sort_clause(sort) + { 'field' => sort[:field].to_s, 'order' => sort[:ascending] == false ? 'descending' : 'ascending' } + end + + # A record rather than a listing: no list under either key, and an id where + # a record carries one. An empty listing is not one of these -- it answers + # `data` as an empty array, which is a page of nothing rather than a record. + def single_record?(body, list_key) + body.is_a?(Hash) && !body[list_key].is_a?(Array) && !body['data'].is_a?(Array) && body.key?('id') + end + + def collect_pages(path, list_key:, params:, boot:) records = [] cursor = nil pages = 0 loop do - body = get(path, cursor.nil? ? nil : { 'starting_after' => cursor }, boot: boot).body + query = cursor.nil? ? params : params.merge('starting_after' => cursor) + body = get(path, query.empty? ? nil : query, boot: boot).body records.concat(extract_entities(body, path, list_key)) pages += 1 cursor = next_cursor(body, path) @@ -199,7 +249,8 @@ def log_collection_cap(path, pages, collected) def to_page(body, operation, list_key) Page.new(records: extract_entities(body, operation, list_key), next_cursor: next_cursor(body, operation), - total_count: extract_count(body)) + total_count: extract_count(body), + total_pages: extract_total_pages(body)) end # Absent on the last page, which is how the walk knows it is done. An older @@ -236,6 +287,14 @@ def extract_count(body) count.is_a?(Numeric) ? count.to_i : nil end + # How many pages the offset tier has to read through, when the endpoint + # counts them. nil on a cursor page, which counts nothing. + def extract_total_pages(body) + pages = body['pages'] if body.is_a?(Hash) + total = pages['total_pages'] if pages.is_a?(Hash) + total.is_a?(Numeric) ? total.to_i : nil + end + def refuse_body_shape(operation, detail) raise APIError.new("Intercom API call failed: #{operation}: unexpected response shape, #{detail}", status: nil) end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/company.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/company.rb new file mode 100644 index 000000000..5377cd4dc --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/company.rb @@ -0,0 +1,85 @@ +module ForestAdminDatasourceIntercom + module Collections + # The accounts the contacts belong to. + # + # The one collection Intercom paginates by offset, and the one place R1 -- + # a window the API cannot express -- does not apply: `POST /companies/list` + # takes a page number, which is what a list view asks for. See + # `OffsetCollection`. + # + # In exchange it is the least filterable collection of the datasource. + # There is no `/companies/search`, and what `GET /companies` answers is four + # exact lookups: by `name`, by `company_id` -- the workspace's own + # identifier, not Intercom's -- by `tag_id` and by `segment_id`. Two of them + # are published as filters here, the two that name a column of this + # collection; a tag and a segment are collections of their own and arrive + # with lot 5, which is where filtering by them belongs. + # + # `GET /companies/scroll` is deliberately rejected rather than used: one + # open scroll per application, expiring after a minute, cannot serve two + # operators looking at a list at the same time. + class Company < OffsetCollection + include Company::Serializer + include CustomAttributes + + # The column each lookup is written on, and the query parameter Intercom + # answers it under. They happen to share a name; keeping the mapping + # explicit is what lets a column be renamed without silently dropping the + # lookup. + LOOKUPS = { 'name' => 'name', 'company_id' => 'company_id' }.freeze + + def initialize(datasource, attributes: []) + @attributes = attributes + super(datasource, 'IntercomCompany') + end + + protected + + def list_path = 'companies/list' + def record_endpoint = 'companies' + def lookups = LOOKUPS + + private + + def define_schema + add_column('id', 'String', is_primary_key: true) + # Intercom's id and the workspace's own are two different things, and an + # ops team knows the second one: it is what their billing system calls + # the account. + add_column('company_id', 'String') + add_column('name', 'String') + define_profile_columns + define_activity_columns + # Before the attribute columns, so an attribute whose name lands on the + # relation is skipped with a warning rather than taking the boot with + # it. + add_one_to_many('contacts', foreign_collection: 'IntercomContact', origin_key: 'company_id') + register_attribute_columns + end + + def define_profile_columns + add_column('plan_name', 'String') + add_column('size', 'Number') + add_column('industry', 'String') + add_column('website', 'String') + add_column('monthly_spend', 'Number') + end + + def define_activity_columns + add_column('user_count', 'Number') + add_column('session_count', 'Number') + add_column('created_at', 'Date') + add_column('updated_at', 'Date') + add_column('last_request_at', 'Date') + # When the account was created in the customer's own system, which is + # not when Intercom heard about it. + add_column('remote_created_at', 'Date') + end + + # Typed from `GET /data_attributes?model=company`, and unfilterable for + # the same reason as everything else here: this collection is looked up, + # not searched. + def attribute_kind = 'company' + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/company/serializer.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/company/serializer.rb new file mode 100644 index 000000000..c449d1070 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/company/serializer.rb @@ -0,0 +1,44 @@ +module ForestAdminDatasourceIntercom + module Collections + class Company < OffsetCollection + # One Intercom company flattened into the row the schema declares. + module Serializer + protected + + def serialize(company) + attrs = company.is_a?(Hash) ? company : {} + plan = attrs['plan'].is_a?(Hash) ? attrs['plan'] : {} + + identity(attrs).merge( + 'plan_name' => plan['name'], + 'user_count' => attrs['user_count'], + 'session_count' => attrs['session_count'] + ).merge(dates_of(attrs)).merge(attribute_values(attrs['custom_attributes'])) + end + + private + + def identity(attrs) + { + 'id' => stringify_id(attrs['id']), + 'company_id' => stringify_id(attrs['company_id']), + 'name' => attrs['name'], + 'size' => attrs['size'], + 'industry' => attrs['industry'], + 'website' => attrs['website'], + 'monthly_spend' => attrs['monthly_spend'] + } + end + + def dates_of(attrs) + { + 'created_at' => stamp(attrs['created_at']), + 'updated_at' => stamp(attrs['updated_at']), + 'last_request_at' => stamp(attrs['last_request_at']), + 'remote_created_at' => stamp(attrs['remote_created_at']) + } + end + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/contact.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/contact.rb new file mode 100644 index 000000000..871591733 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/contact.rb @@ -0,0 +1,201 @@ +module ForestAdminDatasourceIntercom + module Collections + # The people who write to the workspace: users and leads alike. + # + # Cursor-paginated like conversations and tickets, with one thing no other + # collection of this datasource has -- **Intercom sorts it**. + # `POST /contacts/search` is the only endpoint of the whole API that takes a + # `sort` and applies it, so this is the only collection whose columns are + # published sortable, and the measured table is what says which ones. + # + # Two routes of its own, on top of the three the tier already has: + # + # * a set of ids is read in one request -- `id IN [...]`, which this + # endpoint answers and no other does -- rather than one request per id; + # * `company_id equals X` reads `GET /companies/{id}/contacts`, which is + # what serves the contacts of an account. `/contacts/search` filters no + # company field, so without this route the one relation an ops team walks + # the most would be a refusal. + # + # A contact merged into another **disappears** from the search and from the + # listing: a row whose contact was merged reads as gone rather than as an + # error, which is what a merge means -- the record still exists, under the + # id it was merged into. + # Long by line count only: most of it declares the columns, one call each. + class Contact < CursorCollection # rubocop:disable Metrics/ClassLength + include Contact::Serializer + include CustomAttributes + + # `/contacts/search` demands a query, so a read with no condition of its + # own -- a list view asking for an order -- sends the least noisy + # predicate that matches everything. Every contact has a creation date, + # and a bound at the epoch keeps whatever the day-granular truncation does + # to it harmless. The same predicate `Ticket` sends, for the same reason. + MATCH_EVERY_CONTACT = { 'field' => 'created_at', 'operator' => '>', 'value' => '0' }.freeze + + # How many ids one bulk read carries, and how many a single `id in [...]` + # may name. Both are far above what a page asks for; they keep a scope or + # a customizer naming thousands of ids from turning one list view into a + # rate limit. + IDS_PER_READ = 100 + MAX_IDS_READ = 300 + + def initialize(datasource, attributes: []) + @attributes = attributes + super(datasource, 'IntercomContact') + # Answered on the e-mail address, which is what an ops team types when + # they are looking for someone. Per word, not as a substring -- see the + # README. + enable_search + end + + protected + + def list_endpoint = 'contacts' + def searchable = 'contacts' + def search_column = 'email' + def match_all_query = MATCH_EVERY_CONTACT + + private + + def define_schema + add_column('id', 'String', is_primary_key: true) + define_identity_columns + define_date_columns + define_reachability_columns + define_device_columns + # Before the attribute columns rather than after: a workspace attribute + # whose name lands on a relation is then skipped with a warning, the way + # one landing on a column already is. Declared after, it would collide + # and take the boot with it. + define_relations + register_attribute_columns + end + + def define_identity_columns + add_column('role', 'String') + add_column('name', 'String') + add_column('email', 'String') + add_column('email_domain', 'String') + add_column('phone', 'String') + add_column('external_id', 'String') + add_column('avatar', 'String') + add_column('owner_id', 'String') + add_column('session_count', 'Number') + # The first of the accounts the contact belongs to, and how many there + # are: the same reading a conversation gives its contacts, and the + # foreign key the `company` relation is built on. + add_column('company_id', 'String') + add_column('company_count', 'Number') + end + + def define_date_columns + add_column('created_at', 'Date') + add_column('updated_at', 'Date') + add_column('signed_up_at', 'Date') + add_column('last_seen_at', 'Date') + add_column('last_contacted_at', 'Date') + add_column('last_replied_at', 'Date') + add_column('last_email_opened_at', 'Date') + add_column('last_email_clicked_at', 'Date') + end + + def define_reachability_columns + add_column('unsubscribed_from_emails', 'Boolean') + add_column('has_hard_bounced', 'Boolean') + add_column('marked_email_as_spam', 'Boolean') + end + + def define_device_columns + add_column('language_override', 'String') + add_column('browser', 'String') + add_column('browser_language', 'String') + add_column('os', 'String') + add_column('location_country', 'String') + add_column('location_region', 'String') + add_column('location_city', 'String') + end + + # Typed from `GET /data_attributes?model=contact` rather than guessed from + # a payload, and published unfilterable: which operators Intercom answers + # on `custom_attributes.{name}` has not been measured, and this package + # offers no filter it has not seen work. `api_writable` travels on the + # introspected attribute for lot 4b, not on the column -- everything here + # is read-only. + def attribute_kind = 'contact' + + # The owner is a teammate, read whole in one request, and + # `/contacts/search` filters on the key -- so that relation is readable, + # navigable and filterable alike. + # + # The company is none of those last two: the endpoint filters no company + # field, so the traversal is refused by name (see `check_relation_filterable!` + # on the tier). The two lists are the 360 degrees this lot exists for. + def define_relations + add_many_to_one('owner', foreign_collection: 'IntercomAdmin', foreign_key: 'owner_id') + add_many_to_one('company', foreign_collection: 'IntercomCompany', foreign_key: 'company_id') + add_one_to_many('conversations', foreign_collection: 'IntercomConversation', origin_key: 'contact_id') + add_one_to_many('tickets', foreign_collection: 'IntercomTicket', origin_key: 'contact_id') + end + + # The contacts of an account, which `/contacts/search` cannot answer and + # `GET /companies/{id}/contacts` can. Anything else goes the usual way. + def fetch_records(caller, filter, sort = nil) + company = company_lookup(filter) + return super unless company + + warn_ignored_sort(Array(filter&.sort)) if sort + offset, limit = translate_page(filter&.page) + + walker.walk(offset: offset, limit: limit) do |per_page, cursor| + read_company_page(company, per_page: per_page, cursor: cursor) + end + end + + def count_records(caller, filter) + company = company_lookup(filter) + return super unless company + + exact_count(read_company_page(company, per_page: 1, cursor: nil)) + end + + # A bare equality and nothing else: an `and` also carrying a scope names a + # narrower set than the account does, and answering it with the account + # alone would serve contacts the scope excludes. + def company_lookup(filter) + tree = filter&.condition_tree + return nil unless tree.is_a?(Leaf) && tree.field.to_s == 'company_id' + return nil unless tree.operator == Operators::EQUAL && blank_search?(filter) + + tree.value&.to_s + end + + def read_company_page(company, per_page:, cursor:) + client.list_page("companies/#{Faraday::Utils.escape(company)}/contacts", + per_page: [per_page, max_page_size].min, starting_after: cursor) + end + + # One request per hundred ids instead of one per id: this endpoint answers + # `id IN [...]`, which is what lot 1 already reads the contacts of a page + # through. A contact that was merged away is simply absent from the + # answer -- the row reads as gone, not as a failure. + def records_by_ids(ids) + wanted = ids.first(MAX_IDS_READ) + warn_truncated_ids(ids.size) if ids.size > wanted.size + + wanted.each_slice(IDS_PER_READ).flat_map do |chunk| + client.search_page(search_endpoint.path, per_page: chunk.size, + query: { 'field' => 'id', 'operator' => 'IN', + 'value' => chunk }).records + end + end + + def warn_truncated_ids(asked) + ForestAdminDatasourceIntercom.logger.warn( + "[forest_admin_datasource_intercom] #{name} was asked for #{asked} records by id and read the first " \ + "#{MAX_IDS_READ}: Intercom reads them #{IDS_PER_READ} at a time. The result is truncated." + ) + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/contact/serializer.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/contact/serializer.rb new file mode 100644 index 000000000..509351034 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/contact/serializer.rb @@ -0,0 +1,106 @@ +module ForestAdminDatasourceIntercom + module Collections + class Contact < CursorCollection + # One Intercom contact flattened into the row the schema declares. + # Nothing here reads a sub-resource: every value comes from the payload + # the search already returned. + module Serializer + protected + + def serialize(contact) + attrs = contact.is_a?(Hash) ? contact : {} + + native(attrs) + .merge(account_of(attrs['companies'])) + .merge(location_of(attrs['location'])) + .merge(attribute_values(attrs['custom_attributes'])) + end + + private + + def native(attrs) + identity(attrs).merge(dates_of(attrs)).merge(flags(attrs)).merge(device_of(attrs)) + end + + def identity(attrs) + { + 'id' => stringify_id(attrs['id']), + 'role' => attrs['role'], + 'name' => attrs['name'], + 'email' => attrs['email'], + # Derived rather than read, and filterable all the same: the search + # endpoint carries a field of its own for it, which is what turns + # "everyone at this customer" into a filter instead of a wildcard. + 'email_domain' => domain_of(attrs['email']), + 'phone' => attrs['phone'], + 'external_id' => attrs['external_id'], + 'avatar' => attrs['avatar'], + 'owner_id' => stringify_id(attrs['owner_id']), + 'session_count' => attrs['session_count'] + } + end + + def dates_of(attrs) + { + 'created_at' => stamp(attrs['created_at']), + 'updated_at' => stamp(attrs['updated_at']), + 'signed_up_at' => stamp(attrs['signed_up_at']), + 'last_seen_at' => stamp(attrs['last_seen_at']), + 'last_contacted_at' => stamp(attrs['last_contacted_at']), + 'last_replied_at' => stamp(attrs['last_replied_at']), + 'last_email_opened_at' => stamp(attrs['last_email_opened_at']), + 'last_email_clicked_at' => stamp(attrs['last_email_clicked_at']) + } + end + + def flags(attrs) + { + 'unsubscribed_from_emails' => attrs['unsubscribed_from_emails'], + 'has_hard_bounced' => attrs['has_hard_bounced'], + 'marked_email_as_spam' => attrs['marked_email_as_spam'] + } + end + + def device_of(attrs) + { + 'language_override' => attrs['language_override'], + 'browser' => attrs['browser'], + 'browser_language' => attrs['browser_language'], + 'os' => attrs['os'] + } + end + + # A contact belongs to several accounts, and the row names the first of + # them and counts them -- the same reading a conversation gives its + # contacts. The whole list is a hop away, on the `company` relation. + def account_of(companies) + list = nested_list(companies, 'data') + first = list.first.is_a?(Hash) ? list.first : {} + + { 'company_id' => stringify_id(first['id']), 'company_count' => account_count(companies, list) } + end + + # Intercom caps the accounts it nests on a contact and says how many + # there really are, so the count is read rather than measured on the + # list -- a contact belonging to twelve accounts must not read as + # belonging to the ten the payload had room for. + def account_count(companies, list) + declared = companies['total_count'] if companies.is_a?(Hash) + + declared.is_a?(Numeric) ? declared : list.size + end + + def location_of(location) + attrs = location.is_a?(Hash) ? location : {} + + { 'location_country' => attrs['country'], 'location_region' => attrs['region'], + 'location_city' => attrs['city'] } + end + + def domain_of(email) + email.to_s[/@(.+)\z/, 1] + end + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/contact_identity.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/contact_identity.rb index 2bc4fab54..c190f459d 100644 --- a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/contact_identity.rb +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/contact_identity.rb @@ -1,17 +1,22 @@ module ForestAdminDatasourceIntercom module Collections - # The contact of a conversation or of a ticket, denormalized onto the row. + # The contact of a conversation or of a ticket, on the row itself. # # Intercom nests only the ids -- `{"type": "contact.list", "contacts": - # [{"id": "..."}]}` -- so a name and an e-mail cost a read. That read is done - # once per page, for every row at once, and never per row: a page of 25 rows - # is one request, not 25. + # [{"id": "..."}]}` -- so a name costs a read. That read is done once per + # page, for every row at once, and never per row: a page of 25 rows is one + # request, not 25. # - # It stays a pair of columns rather than a relation because the Contacts - # collection arrives in lot 4, and a relation whose target collection is - # missing is a schema the agent refuses to boot on. + # **Three columns, where lot 1 published four.** Now that the Contacts + # collection exists, the identity is a relation, and the rule lot 2.5 set + # for the ticket labels applies here too: one readable label on the row plus + # the relation to navigate, rather than two ways to read one fact. + # `contact_email` is gone -- it is one hop away, on `contact:email` -- and + # `contact_ids` gave way to `contact_id`, which is a foreign key rather than + # a Json blob no filter could reach. The list of every contact of a group + # conversation is the `contacts` relation. module ContactIdentity - COLUMNS = %w[contact_name contact_email].freeze + COLUMNS = %w[contact_name].freeze # How many ids one `id in [...]` read carries. A page holds fewer than this # in practice; the chunk keeps the request bounded if it ever does not. @@ -20,22 +25,23 @@ module ContactIdentity private def define_contact_columns - add_column('contact_ids', 'Json') + add_column('contact_id', 'String') add_column('contact_count', 'Number') add_column('contact_name', 'String') - add_column('contact_email', 'String') end # A group conversation, or a ticket opened for several people, has more # than one contact: the row names the first and counts them, rather than - # presenting one of several as the one. + # presenting one of several as the one. The `contact` relation resolves + # that same first contact, so the column and the relation cannot disagree; + # the others are reached through the contact's own conversations. def contact_columns_for(attrs) ids = nested_list(attrs['contacts'], 'contacts').filter_map { |contact| stringify_id(contact['id']) } - { 'contact_ids' => ids, 'contact_count' => ids.size, + { 'contact_id' => ids.first, 'contact_count' => ids.size, # Filled by the bulk read below, and left nil when the projection did - # not ask for them. - 'contact_name' => nil, 'contact_email' => nil } + # not ask for it. + 'contact_name' => nil } end def first_contact_id(record) @@ -50,12 +56,11 @@ def embed_contact_identity(records, rows, projection) records.each_with_index do |record, index| identity = identities[first_contact_id(record)] || {} rows[index]['contact_name'] = identity['name'] if rows[index].key?('contact_name') - rows[index]['contact_email'] = identity['email'] if rows[index].key?('contact_email') end end - # A failure costs the two columns and nothing else: an identity that could - # not be read is not a page that could not be served. + # A failure costs the column and nothing else: an identity that could not + # be read is not a page that could not be served. def contact_identities(records) ids = records.filter_map { |record| first_contact_id(record) }.uniq return {} if ids.empty? @@ -69,7 +74,7 @@ def contact_identities(records) rescue APIError => e ForestAdminDatasourceIntercom.logger.warn( "[forest_admin_datasource_intercom] #{name} could not read the contacts of this page (HTTP " \ - "#{e.status || "-"}); the name and e-mail columns are left empty for it." + "#{e.status || "-"}); the name column is left empty for it." ) {} end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/conversation.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/conversation.rb index 380caeec9..09c8b390a 100644 --- a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/conversation.rb +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/conversation.rb @@ -13,6 +13,8 @@ module Collections class Conversation < CursorCollection # rubocop:disable Metrics/ClassLength include ContactIdentity include Conversation::Serializer + # The shared thread, then the hooks this collection puts over it. + include Collections::Timeline include Conversation::Timeline # How many conversations of one page may have their timeline read. The @@ -85,12 +87,21 @@ def define_schema # filtered through alike. # # No relation towards the company: a conversation carries its account as a - # whole object, so the name is already on the row, and the Companies - # collection arrives with lot 4. + # whole object, so the name is already on the row, and Intercom filters no + # company field on this endpoint -- the relation would be navigable and + # not filterable, where the column is already readable. + # + # The contact is a relation as of lot 4, and a filterable one: the + # endpoint matches a conversation against one of its contact ids + # (measured), so `contact:email` resolves against `/contacts/search` and + # is rewritten onto the key. A group conversation names its first contact + # here, like the column does; every contact of it is a hop away, through + # that contact's own conversations. def define_relations add_many_to_one('admin_assignee', foreign_collection: 'IntercomAdmin', foreign_key: 'admin_assignee_id') add_many_to_one('team_assignee', foreign_collection: 'IntercomTeam', foreign_key: 'team_assignee_id') add_many_to_one('closed_by', foreign_collection: 'IntercomAdmin', foreign_key: 'closed_by_id') + add_many_to_one('contact', foreign_collection: 'IntercomContact', foreign_key: 'contact_id') end # Who the conversation sits with, and which account it belongs to. The @@ -103,19 +114,6 @@ def define_assignment_columns add_column('company_name', 'String') end - # The contact identity is denormalized onto the row rather than declared as - # a relation: the Contacts collection arrives in lot 4, and a relation whose - # target collection is missing is a schema the agent refuses to boot on. - # - # A group conversation has several contacts; the row carries the first and - # says how many there are, rather than pretending there is one. - def define_contact_columns - add_column('contact_ids', 'Json') - add_column('contact_count', 'Number') - add_column('contact_name', 'String') - add_column('contact_email', 'String') - end - # The message that opened the conversation lives in `source`, not in the # parts. A timeline built from the parts alone loses it, which is the one # message nobody opens a conversation without wanting to read. diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/conversation/timeline.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/conversation/timeline.rb index 9b757f6c8..bb7fe62cd 100644 --- a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/conversation/timeline.rb +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/conversation/timeline.rb @@ -1,18 +1,12 @@ module ForestAdminDatasourceIntercom module Collections class Conversation < CursorCollection - # The thread of a conversation, as a structured list the record view can - # render: who said what, when, and through which kind of event. + # What a conversation adds to the shared thread: where its parts live, and + # the entry that opens it. # - # Two things this exists to get right. The opening message lives in - # `source`, not in the parts -- a timeline built from the parts alone opens - # on the first reply and loses what the customer actually asked. And - # `part_type` is kept on every entry: an assignment, a note and a reply are - # not the same event, and a thread that flattens them reads as a - # conversation that never happened the way it did. - # - # Intercom caps a conversation at its 500 most recent parts; the entry - # count is therefore what is in hand, not necessarily what exists. + # The opening message lives in `source`, not in the parts -- a timeline + # built from the parts alone opens on the first reply and loses what the + # customer actually asked. module Timeline # The pseudo type of the opening entry. Not an Intercom part type: it is # the source, and calling it `comment` would make it indistinguishable @@ -21,13 +15,6 @@ module Timeline private - def build_timeline(conversation) - attrs = conversation.is_a?(Hash) ? conversation : {} - entries = [source_entry(attrs)].compact - - entries + (parts_of(attrs) || []).map { |part| part_entry(part) } - end - # nil rather than an empty list when the payload carries no parts at all: # a listing response has none, and reading that as "this conversation is # empty" is exactly the answer that looks complete without being it. @@ -39,7 +26,7 @@ def parts_of(conversation) parts.is_a?(Array) ? parts : nil end - def source_entry(attrs) + def opening_entry(attrs) source = attrs['source'] return nil unless source.is_a?(Hash) @@ -47,26 +34,6 @@ def source_entry(attrs) body: source['body'], attachments: source['attachments']) .merge('id' => stringify_id(source['id'])) end - - def part_entry(part) - attrs = part.is_a?(Hash) ? part : {} - - entry(part_type: attrs['part_type'], created_at: attrs['created_at'], author: attrs['author'], - body: attrs['body'], attachments: attrs['attachments']) - .merge('id' => stringify_id(attrs['id']), 'redacted' => attrs['redacted']) - end - - def entry(part_type:, created_at:, author:, body:, attachments:) - writer = author.is_a?(Hash) ? author : {} - - { 'part_type' => part_type, - 'created_at' => stamp(created_at), - 'author_type' => writer['type'], - 'author_name' => writer['name'], - 'author_email' => writer['email'], - 'body' => body, - 'attachment_count' => Array(attachments).size } - end end end end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/cursor_collection.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/cursor_collection.rb index 62545008f..cb7ec67d9 100644 --- a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/cursor_collection.rb +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/cursor_collection.rb @@ -40,9 +40,7 @@ def initialize(datasource, name) end def list(caller, filter, projection) - warn_ignored_sort(filter&.sort) - - records = fetch_records(caller, filter) + records = fetch_records(caller, filter, server_sort(filter)) # Serialized whole and projected afterwards rather than the other way # round: a projection reaching through a relation names no foreign key, # and the key is where the relation is read from. @@ -101,15 +99,20 @@ def max_page_size = Client::MAX_PER_PAGE # One page of the collection. A listing for conversations, a search for # tickets -- Intercom exposes no `GET /tickets` at all -- so the endpoint # and its shape belong to the collection, while walking it does not. - def read_page(per_page:, cursor:, query: nil) + def read_page(per_page:, cursor:, query: nil, sort: nil) size = [per_page, max_page_size].min + # The listing endpoint neither filters nor sorts, and a collection whose + # records are only reachable through the search has no listing at all. + # Either way what routes the read to the search is a query, so the one + # that matches everything stands in for the condition there is none of. + query ||= match_all_query if sort || searchable_only? if query.nil? client.list_page(list_endpoint, per_page: size, starting_after: cursor, params: read_params, list_key: list_key) else client.search_page(search_endpoint.path, query: query, per_page: size, starting_after: cursor, - params: read_params, list_key: list_key) + params: read_params, list_key: list_key, sort: sort) end end @@ -122,17 +125,43 @@ def read_page(per_page:, cursor:, query: nil) # The primary key is the exception, and it is not a filter: `id equals X` # and `id in [...]` are answered by the record endpoint. # - # No column is sortable: Intercom takes no sort on either search endpoint - # and ignores the one it is sent. Read-only, this lot writing nothing. + # A column is sortable only where the measured table says the endpoint + # sorts, which is `/contacts/search` and nowhere else: the other two + # accept a `sort` and ignore it without a word, so a sortable column there + # would promise an order that never happens. def add_column(name, type, is_primary_key: false) add_field(name, ColumnSchema.new(column_type: type, filter_operators: column_operators(name, is_primary_key), is_primary_key: is_primary_key, is_read_only: true, - is_sortable: false, + is_sortable: sortable_column?(name), is_groupable: false)) end + def sortable_column?(name) + search_endpoint.field(name)&.sortable? == true + end + + # The Intercom query standing in for the condition a read has none of. + # Two collections need one and for different reasons: Intercom exposes no + # `GET /tickets` at all, and an order is applied by the search endpoint + # alone. Nil where the listing endpoint answers both, which is where a + # sort is reported as ignored rather than dropped. + def match_all_query = nil + + # Whether every read of this collection goes through the search, listing + # or not. + def searchable_only? = false + + # Bounded, unlike the tier read whole: one more than a group may hold is + # all it takes to know the fan-out will not fit, and reading further would + # walk a whole collection to refuse it afterwards. + def match_page + ForestAdminDatasourceToolkit::Components::Query::Page.new( + offset: 0, limit: Query::ConditionTreeTranslator::MAX_GROUP_SIZE + 1 + ) + end + def walker @walker ||= Pagination::CursorWalker.new end @@ -147,7 +176,7 @@ def column_operators(name, is_primary_key) field ? Query::OperatorTable.forest_operators(field) : [] end - def fetch_records(caller, filter) + def fetch_records(caller, filter, sort = nil) ids = id_lookup(filter) # The window is cut out of the ids rather than out of the records they # read: Intercom reads them one request each, so paging after the read @@ -162,7 +191,7 @@ def fetch_records(caller, filter) # than sent as a filter that would come back with everything. return [] if query == NOTHING - listed_records(filter, query) + listed_records(filter, query, sort) end # The Intercom query a filter comes down to, or nil for a list view, which @@ -221,9 +250,14 @@ def refuse_unfilterable_key!(leaf, key) "navigated; filter on one of: #{search_endpoint.filterable_columns.join(", ")}." end - def refuse_fan_out!(leaf, key, ids) + def refuse_fan_out!(leaf, key, _ids) + # "more than", never a count: the target is read one record past what a + # group may hold, so what is known is that it does not fit -- printing + # 16 where a workspace holds three thousand would read as a number the + # operator could go and narrow by one. raise UnsupportedOperatorError, - "#{name} cannot filter #{leaf.field.inspect}: it names #{ids.size} records, " \ + "#{name} cannot filter #{leaf.field.inspect}: it names more than " \ + "#{Query::ConditionTreeTranslator::MAX_GROUP_SIZE} records, " \ "#{search_endpoint.path} answers #{key.inspect} one value at a time, and Intercom takes " \ "#{Query::ConditionTreeTranslator::MAX_GROUP_SIZE} conditions per group. Narrow the condition " \ "on the relation, or filter on #{key.inspect} itself." @@ -288,11 +322,11 @@ def records_by_ids(ids) end end - def listed_records(filter, query) + def listed_records(filter, query, sort = nil) offset, limit = translate_page(filter&.page) walker.walk(offset: offset, limit: limit) do |per_page, cursor| - read_page(per_page: per_page, cursor: cursor, query: query) + read_page(per_page: per_page, cursor: cursor, query: query, sort: sort) end end @@ -315,7 +349,13 @@ def count_records(caller, filter) query = translate(caller, filter) return 0 if query == NOTHING - page = read_page(per_page: 1, cursor: nil, query: query) + exact_count(read_page(per_page: 1, cursor: nil, query: query)) + end + + # The count Intercom answered, or nothing at all. Counting the pages a + # walk collected would answer a fraction of the collection as if it were + # the whole of it, which is the one thing this tier does not do. + def exact_count(page) return page.total_count if page.total_count raise UnsupportedOperatorError, @@ -339,33 +379,65 @@ def refuse_search! 'and this collection exposes no text column it searches. Filter on a column instead of searching.' end - # Intercom accepts a `sort` on these endpoints and ignores it without a - # word -- measured -- so an order the operator asked for and did not get - # has to be reported here or nowhere. The ascending primary-key sort the - # agent injects when a request names none is not one of those. - def warn_ignored_sort(sort) - clauses = Array(sort) - return if clauses.empty? || default_pk_sort?(clauses) + # The order Intercom will really apply, written the way the client sends + # it -- or nil, and the order the operator asked for is then reported + # rather than dropped in silence. + # + # The ascending primary-key sort the agent injects when a request names + # none is neither honoured nor reported: it is not an order anybody asked + # for, and `/contacts/search` does not sort on an id anyway. + def server_sort(filter) + clauses = Array(filter&.sort) + return nil if clauses.empty? || default_pk_sort?(clauses) + + honoured = honourable_sort(clauses) + warn_ignored_sort(clauses) if honoured.nil? + honoured + end + # One clause, on a column the measured table says the endpoint sorts. + # Intercom takes a single `{ field, order }` and nothing composite, so a + # second clause is not half-honoured: honouring the first alone would + # order the page by something the operator did not ask for. + def honourable_sort(clauses) + return nil unless clauses.size == 1 + + clause = clauses.first + field = search_endpoint.field(sort_field(clause).to_s) + return nil unless field&.sortable? + + { field: field.field, ascending: ascending?(clause) } + end + + # Intercom accepts a `sort` on the other two endpoints and ignores it + # without a word -- measured -- so an order asked for and not applied has + # to be reported here or nowhere. + def warn_ignored_sort(clauses) ForestAdminDatasourceIntercom.logger.warn( "[forest_admin_datasource_intercom] #{name} was asked to sort on " \ - "#{clauses.map { |clause| clause[:field] || clause["field"] }.join(", ")}, and Intercom ignores a sort on " \ - 'this endpoint without reporting it. The rows come back in the order the API imposes.' + "#{clauses.map { |clause| sort_field(clause) }.join(", ")}, which Intercom does not sort this " \ + 'collection on -- and it ignores a sort it refuses without reporting it. The rows come back in the ' \ + 'order the API imposes.' ) end + def sort_field(clause) = clause[:field] || clause['field'] + + # `key?` rather than `||`: a descending clause carries `false`, which an + # `||` fallback reads as "absent" -- so an explicit `?sort=-id` would be + # taken for the ascending default the agent injects, and the one order + # Intercom silently drops would go unreported. + def ascending?(clause) + clause.key?(:ascending) ? clause[:ascending] : clause['ascending'] + end + def default_pk_sort?(clauses) return false unless clauses.size == 1 clause = clauses.first - return false unless (clause[:field] || clause['field']).to_s == primary_key - - # `key?` rather than `||`: a descending clause carries `false`, which an - # `||` fallback reads as "absent" -- so an explicit `?sort=-id` would be - # taken for the ascending default the agent injects, and the one order - # Intercom silently drops would go unreported. - ascending = clause.key?(:ascending) ? clause[:ascending] : clause['ascending'] - ascending != false + return false unless sort_field(clause).to_s == primary_key + + ascending?(clause) != false end def warn_truncated_ids(asked) diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/custom_attributes.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/custom_attributes.rb new file mode 100644 index 000000000..8437079d6 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/custom_attributes.rb @@ -0,0 +1,69 @@ +module ForestAdminDatasourceIntercom + module Collections + # The columns a workspace's own attributes become, and the two things that + # go wrong if they are published as they come. + # + # **A name can land on a column the collection already carries.** Measured + # on a real workspace: a custom contact attribute named `id`. Adding it + # would raise on the second declaration -- the toolkit refuses a field twice + # -- and, if it did not, the serializer would write the attribute where the + # operator expects the record's key. It is skipped, and the log names which + # one, since the fix is on Intercom's side. + # + # Relations count as taken names, which is why every collection here + # declares them **before** registering these columns. + # + # **A date arrives as epoch seconds**, like every other Intercom date, and a + # Date column that receives an integer renders as one. + module CustomAttributes + private + + def register_attribute_columns + @attribute_columns = @attributes.reject { |attribute| collides?(attribute) } + @attribute_columns.each { |attribute| add_column(attribute.column_name, attribute.column_type) } + end + + def attribute_columns = @attribute_columns || [] + + # What the log calls these, which is the workspace's own vocabulary: a + # ticket attribute is declared per ticket type, a contact attribute per + # model. + def attribute_kind = raise(NotImplementedError, "#{self.class} did not implement attribute_kind") + + def collides?(attribute) + return false unless fields.key?(attribute.column_name) + + ForestAdminDatasourceIntercom.logger.warn( + "[forest_admin_datasource_intercom] #{name} skips the #{attribute_kind} attribute " \ + "#{attribute.name.inspect}: a native column or relation already carries the name " \ + "#{attribute.column_name.inspect}, and overwriting it would show the attribute where the operator " \ + 'expects the record field. Rename it in Intercom to publish it.' + ) + true + end + + # The value of each published attribute, read under the name the workspace + # gave it and written under the column name the schema publishes -- the + # two differ whenever the first could not travel through a Forest query + # string. + # + # Nil rather than absent for an attribute the record does not carry: the + # column exists on every row, and an absent key would read as a record + # missing it. + def attribute_values(values) + held = values.is_a?(Hash) ? values : {} + + attribute_columns.to_h do |attribute| + [attribute.column_name, coerce_attribute(held[attribute.name], attribute)] + end + end + + def coerce_attribute(value, attribute) + return nil if value.nil? + return stamp(value) if attribute.column_type == 'Date' && value.is_a?(Numeric) + + value + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/offset_collection.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/offset_collection.rb new file mode 100644 index 000000000..3c11ddd7e --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/offset_collection.rb @@ -0,0 +1,303 @@ +module ForestAdminDatasourceIntercom + module Collections + # The third pagination tier, and the only one that maps onto what Forest + # asks for without translating anything: Intercom paginates + # `POST /companies/list` by **offset**, so page 7 of a list view is one + # request rather than six pages walked to reach it. No cursor walker, no + # cap, and no truncation warning. + # + # What it pays for that is filtering. There is no search endpoint for + # companies at all -- `GET /companies/scroll` exists and is deliberately + # rejected, one open scroll per app expiring in a minute cannot serve + # concurrent list views -- so what a filter may say is four exact lookups + # and nothing else. Everything past them is **refused by name**, the rule + # the cursor tier already set: a page served in answer to a filter it + # ignored is the one failure this datasource is built to avoid. + # + # In memory it does nothing: no filter, no sort, no group. What is in hand + # is a page of something larger, exactly like the cursor tier, and the same + # reasoning applies. + # Long by line count only: half of it is the refusals, and a refusal that + # does not say what to do instead is one an operator cannot act on. + class OffsetCollection < BaseCollection # rubocop:disable Metrics/ClassLength + Aggregation = ForestAdminDatasourceToolkit::Components::Query::Aggregation + + # How many records an `id in [...]` read may fetch. One request per id -- + # Intercom has no "read these records" endpoint here either -- so the + # fan-out is bounded rather than turned into a rate limit halfway through + # a page. + MAX_ID_READS = 25 + + # What one page holds when the read names no window: a relation resolving + # its target, a segment, a customizer. A list view always names one. + UNBOUNDED_PAGE_SIZE = Client::MAX_PER_PAGE + + # And how many such pages are read before the answer is cut short. The + # figure only ever applies to a read with no window of its own. + MAX_COLLECTED_PAGES = 10 + + def initialize(datasource, name) + super + enable_count + end + + def list(caller, filter, projection) + warn_ignored_sort(filter&.sort) + + records = fetch_records(filter) + serialized = records.map { |record| serialize(record) } + rows = serialized.map { |record| project(record, projection) } + + embed_relations(caller, serialized, rows, projection) + rows + end + + # Count only, and never a group: `total_count` is exact on every listing, + # while grouping the page in hand would answer a fraction as if it were + # the whole. + def aggregate(_caller, filter, aggregation, _limit = nil) + refuse_unsupported_aggregation!(aggregation) + + [{ 'group' => {}, 'value' => count_records(filter) }] + end + + protected + + # The endpoint that lists the collection by offset, the one that reads a + # record, and the lookups Intercom answers on the listing path. + def list_path = raise(NotImplementedError, "#{self.class} did not implement list_path") + def record_endpoint = raise(NotImplementedError, "#{self.class} did not implement record_endpoint") + def lookup_path = record_endpoint + def lookups = {} + + def serialize(_entity) = raise(NotImplementedError, "#{self.class} did not implement serialize") + + # A column advertises a filter only where Intercom looks the collection up + # by it, so a column cannot offer a filter this tier would then refuse. + # The primary key is the exception and it is not a filter: `id equals X` + # and `id in [...]` are answered by the record endpoint. + # + # Nothing is sortable: the listing takes no order and ordering a page in + # hand would order a fraction of the collection. + def add_column(name, type, is_primary_key: false) + add_field(name, ColumnSchema.new(column_type: type, + filter_operators: column_operators(name, is_primary_key), + is_primary_key: is_primary_key, + is_read_only: true, + is_sortable: false, + is_groupable: false)) + end + + private + + def column_operators(name, is_primary_key) + return [Operators::EQUAL, Operators::IN] if is_primary_key + + lookups.key?(name) ? [Operators::EQUAL] : [] + end + + def fetch_records(filter) + ids = id_lookup(filter) + return records_by_ids(page_window(ids, filter)) if ids + + lookup = lookup_condition(filter) + return page_window(looked_up_records(lookup), filter) if lookup + + refuse_condition!(filter.condition_tree) unless filter&.condition_tree.nil? + + listed_records(filter) + end + + def count_records(filter) + ids = id_lookup(filter) + return records_by_ids(ids).size if ids + + lookup = lookup_condition(filter) + return looked_up_records(lookup).size if lookup + + refuse_condition!(filter.condition_tree) unless filter&.condition_tree.nil? + + exact_count(read_offset_page(page: 1, per_page: 1)) + end + + # The window a list view asked for, read as the page Intercom counts from + # 1. An offset that does not fall on a page boundary is served by reading + # the page it lands in and the ones after it until the window is filled -- + # exactly, rather than by rounding the offset to something the API likes. + def listed_records(filter) + offset, limit = window(filter&.page) + per_page = Client.bounded_per_page(limit || UNBOUNDED_PAGE_SIZE) + skip = offset % per_page + + collected = collect_pages(first_page: (offset / per_page) + 1, per_page: per_page, + wanted: limit && (skip + limit)) + + limit ? (collected[skip, limit] || []) : collected.drop(skip) + end + + def collect_pages(first_page:, per_page:, wanted:) + records = [] + page = first_page + read = 0 + + loop do + answer = read_offset_page(page: page, per_page: per_page) + records.concat(answer.records) + read += 1 + break if last_page?(answer, page) || (wanted && records.size >= wanted) + break if cap_reached?(read, records.size) + + page += 1 + end + + records + end + + def read_offset_page(page:, per_page:) + client.offset_page(list_path, page: page, per_page: per_page) + end + + def last_page?(answer, page) + answer.records.empty? || (answer.total_pages && page >= answer.total_pages) + end + + def cap_reached?(read, collected) + return false if read < MAX_COLLECTED_PAGES + + ForestAdminDatasourceIntercom.logger.warn( + "[forest_admin_datasource_intercom] Stopped reading #{name} after #{read} page(s) / #{collected} " \ + 'record(s); the rest is left out. This read named no window of its own, and a list view always does.' + ) + true + end + + # A filter with no page asks for every record it matched; nil is how the + # window says so. + def window(page) + return [0, nil] if page.nil? + + limit = page.limit.to_i + [page.offset.to_i.clamp(0, nil), limit.positive? ? limit : nil] + end + + # A record detail is `id equals X`, and a bulk read of related records is + # `id in [...]`. Only a bare leaf on the primary key takes this route: an + # `and` also carrying a scope names a narrower set than the ids do. + def id_lookup(filter) + tree = filter&.condition_tree + return nil unless tree.is_a?(Leaf) && tree.field.to_s == primary_key + + case tree.operator + when Operators::EQUAL then [tree.value].compact.map(&:to_s) + when Operators::IN then Array(tree.value).compact.map(&:to_s) + end + end + + def lookup_condition(filter) + tree = filter&.condition_tree + return nil unless tree.is_a?(Leaf) && tree.operator == Operators::EQUAL + + parameter = lookups[tree.field.to_s] + parameter && { parameter => tree.value.to_s } + end + + def primary_key + @primary_key ||= fields.find do |_name, field| + field.respond_to?(:is_primary_key) && field.is_primary_key + end&.first + end + + # A record the operator can no longer reach -- deleted, or outside the + # token's scope -- reads as "no record" rather than as a failed page. + def records_by_ids(ids) + wanted = ids.first(MAX_ID_READS) + warn_truncated_ids(ids.size) if ids.size > wanted.size + + wanted.filter_map do |id| + client.fetch_record(record_endpoint, id) + rescue APIError => e + raise unless e.status == 404 + + nil + end + end + + # An exact lookup answers few records -- one, for the keys this publishes + # -- so it is read as a single page. More than that page holds is reported + # rather than dropped in silence. + def looked_up_records(params) + answer = client.lookup_page(lookup_path, params: params) + warn_truncated_lookup(params) if answer.next_cursor + + answer.records + end + + def exact_count(page) + return page.total_count if page.total_count + + raise UnsupportedOperatorError, + "#{name} cannot be counted: Intercom answered this listing without a total_count, and counting the " \ + 'pages the agent read would answer a fraction of the collection as if it were the whole of it.' + end + + def refuse_condition!(tree) + offender = nil + tree.some_leaf { |leaf| offender = leaf } + + raise UnsupportedOperatorError, + "#{name} cannot filter #{(offender&.field).inspect}: Intercom exposes no search endpoint for this " \ + "collection and looks a record up by #{lookups.keys.join(", ")} alone -- one exact value at a time, " \ + 'with no combination and no other operator. Filter on one of those, or reach the record from the ' \ + 'collection next door.' + end + + def refuse_unsupported_aggregation!(aggregation) + return if aggregation.is_a?(Aggregation) && aggregation.operation.to_s.casecmp('count').zero? && + Array(aggregation.groups).empty? && aggregation.field.nil? + + raise UnsupportedOperatorError, + "#{name} can only be counted: Intercom exposes no aggregate endpoint, and grouping or summing the " \ + 'pages the agent read would answer a fraction of the collection as if it were the whole of it.' + end + + # Intercom takes no order on this listing at all -- there is no parameter + # for one -- so an order asked for and not applied is reported here or + # nowhere. The ascending primary-key sort the agent injects when a request + # names none is not one of those. + def warn_ignored_sort(sort) + clauses = Array(sort) + return if clauses.empty? || default_pk_sort?(clauses) + + ForestAdminDatasourceIntercom.logger.warn( + "[forest_admin_datasource_intercom] #{name} was asked to sort on " \ + "#{clauses.map { |clause| clause[:field] || clause["field"] }.join(", ")}, and Intercom takes no order " \ + 'on this listing. The rows come back in the order the API imposes.' + ) + end + + def default_pk_sort?(clauses) + return false unless clauses.size == 1 + + clause = clauses.first + return false unless (clause[:field] || clause['field']).to_s == primary_key + + ascending = clause.key?(:ascending) ? clause[:ascending] : clause['ascending'] + ascending != false + end + + def warn_truncated_ids(asked) + ForestAdminDatasourceIntercom.logger.warn( + "[forest_admin_datasource_intercom] #{name} was asked for #{asked} records by id and read the first " \ + "#{MAX_ID_READS}: Intercom reads them one request each. The result is truncated." + ) + end + + def warn_truncated_lookup(params) + ForestAdminDatasourceIntercom.logger.warn( + "[forest_admin_datasource_intercom] #{name} looked up #{params.inspect} and Intercom advertised more " \ + 'records than one page holds; the rest is left out. This lookup is meant for a key that names one record.' + ) + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/relations.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/relations.rb index bcb7dc03b..22e1f731e 100644 --- a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/relations.rb +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/relations.rb @@ -28,6 +28,7 @@ module Collections module Relations # rubocop:disable Metrics/ModuleLength ManyToOneSchema = ForestAdminDatasourceToolkit::Schema::Relations::ManyToOneSchema ManyToManySchema = ForestAdminDatasourceToolkit::Schema::Relations::ManyToManySchema + OneToManySchema = ForestAdminDatasourceToolkit::Schema::Relations::OneToManySchema Filter = ForestAdminDatasourceToolkit::Components::Query::Filter Projection = ForestAdminDatasourceToolkit::Components::Query::Projection Operators = ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Operators @@ -66,6 +67,22 @@ def add_many_to_one(name, foreign_collection:, foreign_key:) is_read_only: true)) end + # The other side of a many-to-one, and the whole point of the 360 degrees: + # a contact's conversations, a company's contacts. The agent serves it by + # listing the target on `origin_key equals `, so the + # target has to be able to answer that condition -- which is what makes + # this declarable here and not everywhere. + # + # Published unfilterable by the agent (`GeneratorField`), so unlike a + # many-to-one it carries no risk of offering a filter this datasource + # would then refuse. + def add_one_to_many(name, foreign_collection:, origin_key:) + add_field(name, OneToManySchema.new(foreign_collection: foreign_collection, + origin_key: origin_key, + origin_key_target: 'id', + is_read_only: true)) + end + def add_many_to_many(name, foreign_collection:, through_collection:, origin_key:, foreign_key:) add_field(name, ManyToManySchema.new(foreign_collection: foreign_collection, through_collection: through_collection, @@ -224,13 +241,25 @@ def check_relation_filterable!(_leaf, _relation); end # over every record Intercom holds rather than over a page of them. def matching_ids(caller, relation, leaf) target = relation.foreign_key_target + filter = Filter.new(condition_tree: leaf, page: match_page) foreign_collection(relation) - .list(caller, Filter.new(condition_tree: leaf), Projection.new([target])) + .list(caller, filter, Projection.new([target])) .filter_map { |row| row[target] } .uniq end + # How much of the target a relation condition may read. Unbounded here: + # the tier that answers one in memory holds every record already, and + # cutting the read short would drop ids its `in` can carry for free. + # + # The tier whose target is a page of something larger overrides it -- see + # `CursorCollection`. Resolving `contact:email contains "@"` against a + # workspace's whole contact list, to then refuse the fan-out it comes to, + # would spend a full cursor walk on a filter that was never going to be + # answered. + def match_page = nil + # The target as the datasource holds it, undecorated -- so a permission # scope or a segment defined on the target does not narrow what a relation # resolves. That is how a native datasource behaves too: it joins the table diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/ticket.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/ticket.rb index f071cd37c..533d5f3ed 100644 --- a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/ticket.rb +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/ticket.rb @@ -18,6 +18,10 @@ class Ticket < CursorCollection include ContactIdentity include Ticket::Serializer include Ticket::DerivedColumns + include CustomAttributes + # The same thread a conversation publishes, and free here: the parts are + # in the response whether or not anything asks for them. + include Timeline # Intercom accepts 150. This is not that: it is what keeps one page of # tickets, timelines included, a response an agent can hold and an operator @@ -43,18 +47,25 @@ def list_key = 'tickets' def searchable = 'tickets' def max_page_size = MAX_TICKETS_PER_PAGE + # The part bodies are HTML written by end customers, and rendering + # third-party HTML inside Forest is neither safe nor useful (R10). Sent on + # the search, where Intercom does not document it: a parameter it ignores + # costs a query string, while the one it honours saves every row of the + # thread from coming back as markup. + def read_params = { 'display_as' => 'plaintext' } + # Intercom exposes no `GET /tickets`, so a list view searches too: with the # filter it was given, or with the predicate that matches everything when # it was given none. - def read_page(per_page:, cursor:, query: nil) - super(per_page: per_page, cursor: cursor, query: query || MATCH_EVERY_TICKET) - end + def searchable_only? = true + def match_all_query = MATCH_EVERY_TICKET def enrich(records, rows, projection) wanted = Array(projection).map(&:to_s) embed_contact_identity(records, rows, wanted) embed_derived_columns(records, rows, wanted) + embed_timeline(records, rows, wanted) end private @@ -85,6 +96,7 @@ def define_schema define_contact_columns define_derived_columns add_column('part_count', 'Number') + add_column('timeline', 'Json') # Before the attribute columns rather than after: a workspace attribute # whose name lands on a relation is then skipped with a warning, the way # one landing on a column already is. Declared after, it would collide @@ -110,14 +122,17 @@ def define_type_columns add_column('ticket_type_name', 'String') end - # The four reference collections a ticket points at. Every target is read - # whole in one request, so a relation resolves for a page at the price of a - # single read. + # The reference collections a ticket points at, and the contact who opened + # it. Every reference target is read whole in one request, so those + # relations resolve for a page at the price of a single read; the contact + # is read from `/contacts/search`, one request for the page as well. # - # Only two of them can be filtered *through*: `/tickets/search` takes a - # filter on `admin_assignee_id`, `team_assignee_id` and `ticket_type_id`, - # and none on a state id -- which the refusal names when a filter reaches - # for it, rather than letting the interface offer what the endpoint drops. + # Which of them can be filtered *through* is the measured table's + # business, not this method's: `/tickets/search` takes a filter on + # `admin_assignee_id`, `team_assignee_id` and `ticket_type_id`, none on a + # state id, and `contact_ids` is a `spec` row the probe has yet to + # confirm. Where the endpoint filters nothing, the traversal is refused by + # name rather than left for the interface to offer and the API to drop. def define_relations add_many_to_one('admin_assignee', foreign_collection: 'IntercomAdmin', foreign_key: 'admin_assignee_id') add_many_to_one('team_assignee', foreign_collection: 'IntercomTeam', foreign_key: 'team_assignee_id') @@ -125,31 +140,27 @@ def define_relations add_many_to_one('previous_state', foreign_collection: 'IntercomTicketState', foreign_key: 'previous_state_id') add_many_to_one('ticket_type', foreign_collection: 'IntercomTicketType', foreign_key: 'ticket_type_id') + add_many_to_one('contact', foreign_collection: 'IntercomContact', foreign_key: 'contact_id') end - # The attribute columns of every ticket type, in union. Read at boot by - # `TicketAttributesIntrospector`, which is also where a workspace's own - # name is turned into one a Forest query string can carry. An attribute - # landing on a native column is skipped rather than overwriting it. - def register_attribute_columns - @attribute_columns = @attributes.reject { |attribute| collides?(attribute) } - @attribute_columns.each { |attribute| add_column(attribute.column_name, attribute.column_type) } - end - - def collides?(attribute) - return false unless fields.key?(attribute.column_name) + # Costs no request, unlike a conversation's: Intercom returns the parts of + # a ticket in the search response and offers no way to ask it not to, so + # the page pays for them whatever the projection says. Building the thread + # out of them is what is guarded here. + # + # An empty list means an empty thread, and says so -- where a conversation + # read from a listing carries no parts at all and its timeline stays nil, + # which reads as unknown. + def embed_timeline(records, rows, projection) + return unless projection.include?('timeline') - ForestAdminDatasourceIntercom.logger.warn( - "[forest_admin_datasource_intercom] #{name} skips the ticket attribute #{attribute.name.inspect}: a " \ - "native column or relation already carries the name #{attribute.column_name.inspect}, and overwriting " \ - 'it would show the attribute where the operator expects the ticket field.' - ) - true + records.each_with_index { |record, index| rows[index]['timeline'] = build_timeline(record) } end - def attribute_columns - @attribute_columns || [] - end + # The attribute columns of every ticket type, in union. Read at boot by + # `TicketAttributesIntrospector`, which is also where a workspace's own + # name is turned into one a Forest query string can carry. + def attribute_kind = 'ticket' end end end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/ticket/serializer.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/ticket/serializer.rb index d547fefb0..a094bbf20 100644 --- a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/ticket/serializer.rb +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/ticket/serializer.rb @@ -7,6 +7,11 @@ class Ticket < CursorCollection module Serializer protected + # Intercom keys the attribute values by **name**, which is what lets a + # single collection display the union of every ticket type's -- and what + # stops it from filtering on them, the filter being written by an id + # that differs from one type to the next. A ticket of another type + # simply does not carry the key, and the column reads as empty. def serialize(ticket) attrs = ticket.is_a?(Hash) ? ticket : {} @@ -14,7 +19,7 @@ def serialize(ticket) .merge(state_of(attrs)) .merge(type_of(attrs['ticket_type'])) .merge(contact_columns_for(attrs)) - .merge(attribute_values_of(attrs['ticket_attributes'])) + .merge(attribute_values(attrs['ticket_attributes'])) .merge(derived_columns_for(attrs)) end @@ -49,32 +54,6 @@ def type_of(ticket_type) { 'ticket_type_id' => stringify_id(attrs['id']), 'ticket_type_name' => attrs['name'] } end - - # Intercom keys the values by attribute **name**, which is what lets a - # single collection display the union of every type's attributes -- and - # what stops it from filtering on them, since the filter is written by id - # and the id differs from one type to the next. - # - # The value is read under the name the workspace gave it and written - # under the column name the schema publishes; the two differ whenever the - # first could not travel through a Forest query string. - # - # A ticket of another type simply does not carry the key: the column - # reads as absent rather than as empty. - def attribute_values_of(values) - held = values.is_a?(Hash) ? values : {} - - attribute_columns.to_h { |attribute| [attribute.column_name, coerce(held[attribute.name], attribute)] } - end - - # A date attribute comes back as epoch seconds like every other Intercom - # date; the rest is handed over as it came. - def coerce(value, attribute) - return nil if value.nil? - return stamp(value) if attribute.column_type == 'Date' && value.is_a?(Numeric) - - value - end end end end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/timeline.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/timeline.rb new file mode 100644 index 000000000..8cb3736d2 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/timeline.rb @@ -0,0 +1,53 @@ +module ForestAdminDatasourceIntercom + module Collections + # The thread of a conversation or of a ticket, as a structured list the + # record view can render: who said what, when, and through which kind of + # event. + # + # `part_type` is kept on every entry: an assignment, an internal note and a + # reply are not the same event, and a thread that flattens them reads as an + # exchange that never happened the way it did. Which also means **the + # internal notes of the team are in there**, alongside what the customer + # was told -- that is what a thread is on Intercom, and hiding half of it + # would be the more surprising answer. + # + # Intercom keeps the 500 most recent parts of either resource, so the entry + # count is what is in hand, never necessarily what exists. The two + # collections differ in where they read those parts and in whether anything + # opens the thread before them -- both are hooks. + module Timeline + private + + def build_timeline(entity) + attrs = entity.is_a?(Hash) ? entity : {} + + [opening_entry(attrs)].compact + (parts_of(attrs) || []).map { |part| part_entry(part) } + end + + # What comes before the parts. A conversation opens on its `source`, which + # is not a part at all; a ticket opens on its first part like any other + # event, so there is nothing to prepend. + def opening_entry(_attrs) = nil + + def part_entry(part) + attrs = part.is_a?(Hash) ? part : {} + + entry(part_type: attrs['part_type'], created_at: attrs['created_at'], author: attrs['author'], + body: attrs['body'], attachments: attrs['attachments']) + .merge('id' => stringify_id(attrs['id']), 'redacted' => attrs['redacted']) + end + + def entry(part_type:, created_at:, author:, body:, attachments:) + writer = author.is_a?(Hash) ? author : {} + + { 'part_type' => part_type, + 'created_at' => stamp(created_at), + 'author_type' => writer['type'], + 'author_name' => writer['name'], + 'author_email' => writer['email'], + 'body' => body, + 'attachment_count' => Array(attachments).size } + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/datasource.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/datasource.rb index c2a4c2bf0..6a689c6e3 100644 --- a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/datasource.rb +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/datasource.rb @@ -36,17 +36,29 @@ def register_collections add_collection(Collections::TeamMembership.new(self)) add_collection(Collections::TicketType.new(self)) add_collection(Collections::TicketState.new(self)) + # Contacts and Companies before the collections that point at them, so a + # relation is declared next to a target the datasource already holds. + # Each carries the custom attributes its workspace defines, read at boot: + # they cannot be discovered from a payload, a contact carrying the values + # of the attributes it happens to have been given. + add_collection(Collections::Contact.new(self, attributes: model_attributes('contact'))) + add_collection(Collections::Company.new(self, attributes: model_attributes('company'))) add_collection(Collections::Conversation.new(self)) - # The one boot-time read of the datasource: the attributes a workspace - # defines on its ticket types, which are columns of the Tickets collection - # and cannot be discovered from a ticket payload -- a ticket carries the - # values of its own type only. It degrades to no attribute column rather - # than to a failed boot. + # The attributes a workspace defines on its ticket types, which are + # columns of the Tickets collection and cannot be discovered from a ticket + # payload either -- a ticket carries the values of its own type only. add_collection(Collections::Ticket.new(self, attributes: ticket_attributes)) end + # The three boot-time reads of the datasource. Each degrades to no attribute + # column rather than to a failed boot: a token missing a permission costs + # the columns it could not read, never the agent. def ticket_attributes Schema::TicketAttributesIntrospector.new(@client).attributes end + + def model_attributes(model) + Schema::DataAttributesIntrospector.new(@client, model: model).attributes + end end end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/query/search_fields.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/query/search_fields.rb index 554b6d175..9f8f9aed8 100644 --- a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/query/search_fields.rb +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/query/search_fields.rb @@ -31,8 +31,13 @@ module SearchFields # `source` says where a row comes from, and `measured?` is what the boot # report and the README section read: a row taken from the documentation is # a candidate the probe has not confirmed. - Field = Struct.new(:column, :field, :type, :operators, :source, keyword_init: true) do + # `sortable` is the one thing here Intercom answers on a single endpoint: + # `/contacts/search` takes a `sort`, the other two take one and ignore it + # without a word. Absent reads as false, so a table saying nothing leaves + # a column unsortable rather than promising an order. + Field = Struct.new(:column, :field, :type, :operators, :source, :sortable, keyword_init: true) do def measured? = source == 'measured' + def sortable? = sortable == true end # A column that stays unfilterable, and why. The reason travels into the @@ -43,7 +48,7 @@ def measured? = source == 'measured' end Endpoint = Struct.new(:name, :path, :measured_at, :fields, :refused, :candidates, :ticket_attributes, - keyword_init: true) do + :custom_attributes, keyword_init: true) do # Whether the probe has run against a real workspace for this endpoint. # False means every `spec` row is still a candidate. def measured? = !measured_at.nil? @@ -51,6 +56,7 @@ def measured? = !measured_at.nil? def field(column) = fields[column] def refusal(column) = refused[column] def filterable_columns = fields.keys + def sortable_columns = fields.values.select(&:sortable?).map(&:column) def unmeasured_fields = fields.values.reject(&:measured?) end @@ -84,14 +90,16 @@ def endpoint(name, definition) fields: fields(name, definition['fields']), refused: refusals(name, definition['refused']), candidates: Array(definition['candidates']).freeze, - ticket_attributes: definition['ticket_attributes'] + ticket_attributes: definition['ticket_attributes'], + custom_attributes: definition['custom_attributes'] ).freeze end def fields(endpoint, declared) (declared || {}).to_h do |column, row| field = Field.new(column: column, field: row.fetch('field'), type: row.fetch('type'), - operators: Array(row['operators']).freeze, source: row.fetch('source')).freeze + operators: Array(row['operators']).freeze, source: row.fetch('source'), + sortable: row.fetch('sortable', false)).freeze validate_field!(endpoint, field) [column, field] @@ -112,6 +120,16 @@ def validate_field!(endpoint, field) validate_source!(endpoint, field.column, field) validate_type!(endpoint, field) validate_operators!(endpoint, field) + validate_sortable!(endpoint, field) + end + + # Anything but a boolean, `"true"` above all: YAML reads it as a string, + # which is truthy in Ruby and would publish a sortable column out of a + # typo the file cannot otherwise show. + def validate_sortable!(endpoint, field) + return if [true, false].include?(field.sortable) + + malformed!(endpoint, field.column, "sortable #{field.sortable.inspect} is neither true nor false") end def validate_type!(endpoint, field) diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/query/search_fields.yml b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/query/search_fields.yml index d2bb0ad2b..c5deb04dc 100644 --- a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/query/search_fields.yml +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/query/search_fields.yml @@ -178,19 +178,26 @@ endpoints: type: boolean operators: ['='] source: spec + # The foreign key of the `contact` relation, and the reason lot 4 could + # turn the denormalized contact columns into one: Intercom matches a + # conversation against one of the contacts it carries. The column is + # singular and the wire field plural on purpose -- the row names its first + # contact, the endpoint asks "is this contact one of yours". + # + # `!=` is deliberately absent: on a field holding a list, "not that + # contact" is a question Intercom's DSL does not clearly answer, and the + # one thing this table may not do is publish a filter whose meaning is a + # guess. + contact_id: + field: contact_ids + type: string + operators: ['='] + source: measured # Columns this collection publishes and will not filter, with the reason an # operator reads when they try. A column absent from both tables is refused # by name too -- these are the ones whose refusal is permanent and has an # explanation worth giving. refused: - contact_ids: - reason: >- - Intercom does match a conversation against one of its contact ids, but - the column holds the list and is therefore typed Json, whose filter - values the agent's own validator requires to be Json too -- an id - would be rejected before this datasource saw it. Filtering by contact - arrives with the Contacts collection and a relation. - source: measured tag_names: reason: >- Intercom filters conversations by tag id, and this column holds the @@ -216,13 +223,10 @@ endpoints: source: measured contact_name: reason: >- - Read from the Contacts endpoint, not from the conversation. Filtering - on it arrives with the Contacts collection. - source: spec - contact_email: - reason: >- - Read from the Contacts endpoint, not from the conversation. Filter on - `source_author_email` instead, which is on the conversation itself. + Read from the Contacts endpoint, not from the conversation. Filter + through the `contact` relation, which resolves against + `/contacts/search` and rewrites onto the contact id -- or on + `source_author_email`, which is on the conversation itself. source: spec contact_count: reason: Counted by the agent from the contacts the payload carries. @@ -305,15 +309,18 @@ endpoints: type: date operators: ['>', '<', '>=', '<=', '=', '!='] source: measured + # The foreign key of the `contact` relation. `spec` rather than + # `measured`, and the difference is visible to an operator: it is what + # decides whether the contacts of a ticket can be filtered at all, and the + # probe has not run on this endpoint. If Intercom refuses it, this row + # moves to the refused table and the relation stays navigable without + # being filterable -- the way `state` already is. + contact_id: + field: contact_ids + type: string + operators: ['='] + source: spec refused: - contact_ids: - reason: >- - Intercom does match a ticket against one of its contact ids, but - the column holds the list and is therefore typed Json, whose filter - values the agent's own validator requires to be Json too -- an id - would be rejected before this datasource saw it. Filtering by contact - arrives with the Contacts collection and a relation. - source: measured company_id: reason: >- Measured during lot 1: `/tickets/search` refuses `company_id` with @@ -354,6 +361,12 @@ endpoints: part_count: reason: Counted by the agent from the parts the payload carries. source: measured + timeline: + reason: >- + Built by the agent from the parts of the ticket, which the search + endpoint does not read. Filter on `last_reply_at` or on the state + instead. + source: measured # R7, and it is a product decision rather than an implementation choice: a # ticket attribute is filtered as `ticket_attribute.{id}`, and a same-named # attribute carries a different id per ticket type (measured: @@ -380,9 +393,229 @@ endpoints: - state_id - is_shared - previous_state_id - - contact_id + - company_ids - source.author.email - source.subject - source.body - title - description + + # The one endpoint of the whole API that sorts. `sortable: true` is what a + # column reads to publish `isSortable`, and it appears nowhere else in this + # file on purpose: `/conversations/search` accepts a `sort` and ignores it + # without a word (measured), so a row that stayed silent there would be a + # column promising an order Intercom never applies. + # + # The set is deliberately narrower than what the documentation implies. It + # says the results may be sorted by any attribute; nothing has been measured, + # and a sort Intercom refuses is a list view that fails rather than one that + # comes back unordered. These are the fields an ops team orders a contact list + # by, and the probe is what may widen the set. + contacts: + path: contacts/search + measured_at: null + fields: + # Measured rather than taken from the documentation: lot 1 reads the + # contacts of a page through `id IN [...]` on this endpoint, which is the + # membership operator every other field refuses. + id: + field: id + type: string + operators: ['=', 'IN'] + source: measured + role: + field: role + type: string + operators: ['=', '!='] + source: spec + name: + field: name + type: text + operators: ['=', '!=', '~', '!~', '^', '$'] + source: spec + sortable: true + email: + field: email + type: text + operators: ['=', '!=', '~', '!~', '^', '$'] + source: spec + sortable: true + # Derived by the agent from the address, and filtered by Intercom all the + # same: the endpoint carries a field of its own for it, which is what + # makes "everyone at this customer" a filter rather than a wildcard. + email_domain: + field: email_domain + type: string + operators: ['=', '!='] + source: spec + phone: + field: phone + type: text + operators: ['=', '!=', '~', '!~', '^', '$'] + source: spec + external_id: + field: external_id + type: string + operators: ['=', '!='] + source: spec + owner_id: + field: owner_id + type: string + operators: ['=', '!='] + source: spec + # Measured on 25 August 2026, API 2.16, and this is the asymmetry the + # per-endpoint table exists for: `/conversations/search` and + # `/tickets/search` accept `>=`, `<=` and `!=` on a date where this + # endpoint answers `data_invalid`. `IN` is refused on a date everywhere. + # + # Nothing is lost that an operator can see: a Date column publishes the + # two bounds alone whatever this table allows -- see the operator table -- + # and the toolkit derives `before`, `after`, `today` and the whole + # `previous_*` family from them. + created_at: + field: created_at + type: date + operators: ['>', '<'] + source: measured + sortable: true + updated_at: + field: updated_at + type: date + operators: ['>', '<'] + source: measured + sortable: true + signed_up_at: + field: signed_up_at + type: date + operators: ['>', '<'] + source: spec + sortable: true + last_seen_at: + field: last_seen_at + type: date + operators: ['>', '<'] + source: spec + sortable: true + last_contacted_at: + field: last_contacted_at + type: date + operators: ['>', '<'] + source: spec + sortable: true + last_replied_at: + field: last_replied_at + type: date + operators: ['>', '<'] + source: spec + sortable: true + last_email_opened_at: + field: last_email_opened_at + type: date + operators: ['>', '<'] + source: spec + last_email_clicked_at: + field: last_email_clicked_at + type: date + operators: ['>', '<'] + source: spec + unsubscribed_from_emails: + field: unsubscribed_from_emails + type: boolean + operators: ['='] + source: spec + has_hard_bounced: + field: has_hard_bounced + type: boolean + operators: ['='] + source: spec + marked_email_as_spam: + field: marked_email_as_spam + type: boolean + operators: ['='] + source: spec + language_override: + field: language_override + type: string + operators: ['=', '!='] + source: spec + browser: + field: browser + type: string + operators: ['=', '!='] + source: spec + browser_language: + field: browser_language + type: string + operators: ['=', '!='] + source: spec + os: + field: os + type: string + operators: ['=', '!='] + source: spec + location_country: + field: location.country + type: string + operators: ['=', '!='] + source: spec + location_region: + field: location.region + type: string + operators: ['=', '!='] + source: spec + location_city: + field: location.city + type: string + operators: ['=', '!='] + source: spec + refused: + avatar: + reason: >- + The url of the contact's picture. Nothing filters an image, and + Intercom's search does not carry the field. + source: spec + company_id: + reason: >- + The first of the companies the contact belongs to, read off the + payload. `/contacts/search` filters no company field, so a company + list is reached from the company side -- open the company and read + its contacts. + source: spec + company_count: + reason: Counted by the agent from the companies the payload carries. + source: spec + session_count: + reason: >- + On the contact payload and not in the search DSL. Filter on + `last_seen_at` instead, which the endpoint does take. + source: spec + # Contact custom attributes are filtered as `custom_attributes.{name}` -- + # by name, unlike a ticket attribute, which carries a different id per + # ticket type. So the obstacle is not R7 here, it is measurement: the + # operator set Intercom answers on a custom attribute of each data type has + # not been probed, and this package does not publish a filter it has not + # seen work. They ship typed and display-only, and the probe is what turns + # that around. + custom_attributes: + filterable: false + reason: >- + Contact custom attributes are published for reading only: which + operators Intercom answers on `custom_attributes.{name}` has not been + measured on a real workspace, and this datasource does not offer a + filter it has not seen work. Filter on a native field instead. + source: spec + candidates: + - tag_ids + - segment_id + - company_id + - companies.id + - companies.name + - android_app_name + - android_last_seen_at + - ios_app_name + - ios_last_seen_at + - utm_source + - utm_campaign + - utm_medium + - referrer + - session_count diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/schema/data_attributes_introspector.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/schema/data_attributes_introspector.rb new file mode 100644 index 000000000..5a2d4422e --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/schema/data_attributes_introspector.rb @@ -0,0 +1,119 @@ +module ForestAdminDatasourceIntercom + module Schema + # The attributes a workspace defines on its contacts and on its companies, + # read once while the datasource is being constructed. + # + # Unlike a ticket attribute, one of these is declared **per model** rather + # than per ticket type, so the union is the whole set and a column maps onto + # exactly one Intercom attribute -- the ambiguity that keeps ticket + # attributes display-only (R7) does not arise here. What keeps these + # display-only is narrower and temporary: which operators Intercom answers + # on `custom_attributes.{name}` has not been measured, and this package + # publishes no filter it has not seen work. + # + # `api_writable` is read and carried although every column of this lot is + # published read-only. It costs nothing now and it is exactly what lot 4b + # needs to tell an attribute it may write from one Intercom fills in + # itself -- re-reading it later would be a second boot-time round trip. + class DataAttributesIntrospector + # Intercom's attribute data types, mapped onto what Forest can render. + COLUMN_TYPES = { + 'string' => 'String', 'integer' => 'Number', 'float' => 'Number', 'decimal' => 'Number', + 'boolean' => 'Boolean', 'date' => 'Date', 'datetime' => 'Date' + }.freeze + + DEFAULT_COLUMN_TYPE = 'String'.freeze + + # What a column name may not contain, and it has nothing to do with + # Intercom: Forest lists the fields of a request in a comma-separated + # query parameter and names a field through a relation with a colon. A + # workspace names its attributes in free text, and a comma in there splits + # the projection into fields no collection has. + UNSAFE_IN_A_COLUMN_NAME = /[,:]/ + + # `name` is the key `custom_attributes` uses, `column_name` the one the + # schema publishes; they differ whenever the workspace's own name cannot + # travel through Forest's query string. + Attribute = Struct.new(:name, :column_name, :column_type, :data_type, :api_writable, keyword_init: true) + + def initialize(client, model:) + @client = client + @model = model + end + + # Degrades to nothing rather than to a failure: a token without the + # permission on this model costs the custom-attribute columns, never the + # boot of the agent. + def attributes + @attributes ||= build + rescue APIError => e + ForestAdminDatasourceIntercom.logger.warn( + "[forest_admin_datasource_intercom] could not read the #{@model} attributes (HTTP #{e.status || "-"}); " \ + 'the collection boots without its custom-attribute columns.' + ) + @attributes = [] + end + + private + + def build + # Read on the boot connection: this happens while Rails is starting, and + # a slow Intercom must not turn that into minutes the operator sits + # through. + @client.fetch_all('data_attributes', params: { 'model' => @model }, boot: true) + .each_with_object({}) { |definition, union| collect(definition, union) } + .values + end + + # The standard attributes are left out: they are columns this datasource + # declares by hand, with the filters the search table measured, and + # publishing them a second time under their `custom_attributes` name would + # show one fact twice -- the unfilterable copy winning nothing. + def collect(definition, union) + return unless definition.is_a?(Hash) && definition['custom'] && !definition['archived'] + + name = definition['name'].to_s + return if name.empty? + + column = column_name_for(name) + return if column.empty? + + add(union, name, column, definition) + end + + def add(union, name, column, definition) + entry = union[column] + return union[column] = attribute_from(name, column, definition) if entry.nil? + return if entry.name == name + + warn_collision(name, entry.name, column) + end + + def attribute_from(name, column, definition) + Attribute.new(name: name, column_name: column, column_type: column_type_for(definition), + data_type: definition['data_type'], api_writable: definition['api_writable'] == true) + end + + # Intercom hands these back HTML-escaped -- `Ce que j'ai vérifié` -- + # which is an artefact of where they were typed, not part of the name. + def column_name_for(name) + CGI.unescapeHTML(name).gsub(UNSAFE_IN_A_COLUMN_NAME, ' ').squeeze(' ').strip + end + + def warn_collision(name, kept, column) + ForestAdminDatasourceIntercom.logger.warn( + "[forest_admin_datasource_intercom] the #{@model} attribute #{name.inspect} is left out: it reads as the " \ + "column #{column.inspect}, which #{kept.inspect} already carries. Rename one of them in Intercom to " \ + 'publish both.' + ) + end + + # An unknown data type reads as a string rather than being dropped: + # showing the value Intercom sent beats hiding a column because its type + # is new. + def column_type_for(definition) + COLUMN_TYPES.fetch(definition['data_type'].to_s, DEFAULT_COLUMN_TYPE) + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/company_spec.rb b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/company_spec.rb new file mode 100644 index 000000000..381399d05 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/company_spec.rb @@ -0,0 +1,374 @@ +module ForestAdminDatasourceIntercom + # The offset tier is exercised through Companies, the one collection Intercom + # paginates that way -- and the only one it does not search at all. + RSpec.describe Collections::Company do + subject(:collection) { datasource.get_collection('IntercomCompany') } + + let(:datasource) { Datasource.new(access_token: 's3cr3t', rate_limiter: nil) } + let(:base) { datasource.configuration.url } + let(:operators) { ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Operators } + + def json(payload, status = 200) + { status: status, body: payload.to_json, headers: { 'Content-Type' => 'application/json' } } + end + + def leaf(field, operator, value = nil) + ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Nodes::ConditionTreeLeaf + .new(field, operator, value) + end + + def branch(aggregator, *conditions) + ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Nodes::ConditionTreeBranch + .new(aggregator, conditions) + end + + def filter(condition_tree: nil, page: nil, sort: nil) + ForestAdminDatasourceToolkit::Components::Query::Filter.new(condition_tree: condition_tree, page: page, + sort: sort) + end + + def page(offset, limit) + ForestAdminDatasourceToolkit::Components::Query::Page.new(offset: offset, limit: limit) + end + + def sort(*clauses) + ForestAdminDatasourceToolkit::Components::Query::Sort.new(clauses) + end + + def count(condition_tree: nil) + collection.aggregate(nil, filter(condition_tree: condition_tree), + ForestAdminDatasourceToolkit::Components::Query::Aggregation.new(operation: 'Count')) + end + + def company(id, overrides = {}) + { 'type' => 'company', 'id' => id, 'company_id' => "erp-#{id}", 'name' => "Company #{id}", + 'plan' => { 'type' => 'plan', 'id' => '9', 'name' => 'Paid' }, 'size' => 85, + 'industry' => 'Manufacturing', 'website' => 'https://acme.test', 'monthly_spend' => 49, + 'session_count' => 26, 'user_count' => 10, 'created_at' => 1_700_000_000, + 'updated_at' => 1_700_000_600, 'last_request_at' => 1_700_000_900, + 'remote_created_at' => 1_394_531_169, 'custom_attributes' => {} }.merge(overrides) + end + + def stub_page(*companies, page: 1, per_page: 15, total_pages: 1, total: nil) + stub_request(:post, "#{base}/companies/list") + .with(query: { 'page' => page.to_s, 'per_page' => per_page.to_s }) + .to_return(json('type' => 'list', 'data' => companies, 'total_count' => total || companies.size, + 'pages' => { 'type' => 'pages', 'page' => page, 'per_page' => per_page, + 'total_pages' => total_pages })) + end + + def rows(projection = %w[id], **options) + collection.list(nil, filter(**options), projection) + end + + describe 'schema' do + it 'is named IntercomCompany' do + expect(collection.name).to eq('IntercomCompany') + end + + it 'publishes the columns of an account' do + expect(collection.fields.keys) + .to include('id', 'company_id', 'name', 'plan_name', 'size', 'industry', 'website', + 'monthly_spend', 'user_count', 'session_count', 'created_at', 'remote_created_at') + end + + it 'publishes every column read-only and unsortable' do + columns = collection.fields.values.grep(ForestAdminDatasourceToolkit::Schema::ColumnSchema) + + expect(columns.map(&:is_read_only).uniq).to eq([true]) + expect(columns.map(&:is_sortable).uniq).to eq([false]) + end + + # Four lookups is what `GET /companies` answers, and two of them name a + # column of this collection. A tag and a segment are collections of their + # own, and filtering by them belongs with the lot that adds them. + it 'offers a filter on the two keys Intercom looks a company up by' do + expect(collection.fields['name'].filter_operators).to eq(%w[equal]) + expect(collection.fields['company_id'].filter_operators).to eq(%w[equal]) + expect(collection.fields['id'].filter_operators).to eq(%w[equal in]) + end + + it 'offers no filter on anything else' do + %w[plan_name size industry website monthly_spend user_count created_at].each do |column| + expect(collection.fields[column].filter_operators).to be_empty, "#{column} advertises a filter" + end + end + + it 'declares the contacts of the account' do + expect(collection.fields['contacts'].origin_key).to eq('company_id') + end + + it 'is countable, total_count being exact on every answer' do + expect(collection.is_countable?).to be(true) + end + end + + describe 'the custom attributes' do + before do + stub_data_attributes('company', + { 'name' => 'arr', 'data_type' => 'float', 'custom' => true, + 'api_writable' => true, 'archived' => false }) + end + + it 'publishes one column per attribute, typed and display-only' do + expect(collection.fields['arr'].column_type).to eq('Number') + expect(collection.fields['arr'].filter_operators).to be_empty + end + + it 'skips an attribute whose name a native column already carries, and says which' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + stub_data_attributes('company', { 'name' => 'name', 'data_type' => 'string', 'custom' => true, + 'api_writable' => true, 'archived' => false }) + stub_page(company('co1', 'custom_attributes' => { 'name' => 'Not the account name' }), + page: 1, per_page: 15) + + expect(rows(nil, page: page(0, 15)).first['name']).to eq('Company co1') + expect(ForestAdminDatasourceIntercom.logger) + .to have_received(:warn).with(/skips the company attribute "name"/) + end + + it 'reads a date attribute as a date' do + stub_data_attributes('company', { 'name' => 'renewal', 'data_type' => 'date', 'custom' => true, + 'api_writable' => true, 'archived' => false }) + stub_page(company('co1', 'custom_attributes' => { 'renewal' => 1_700_000_000 }), page: 1, per_page: 15) + + expect(rows(%w[id renewal], page: page(0, 15)).first['renewal']).to eq('2023-11-14T22:13:20Z') + end + + it 'reads its value off the payload' do + stub_page(company('co1', 'custom_attributes' => { 'arr' => 12_000 })) + + expect(rows(%w[id arr], page: page(0, 15))).to eq([{ 'id' => 'co1', 'arr' => 12_000 }]) + end + end + + describe 'pagination by offset' do + # The one place R1 does not apply: Intercom counts pages, which is what a + # list view asks for. No cursor walked, no page read to be thrown away. + it 'asks for the page the window names, in one request' do + stub_page(company('co1'), page: 3, per_page: 15, total_pages: 3) + + expect(rows(%w[id], page: page(30, 15))).to eq([{ 'id' => 'co1' }]) + expect(WebMock).to have_requested(:post, "#{base}/companies/list") + .with(query: { 'page' => '3', 'per_page' => '15' }).once + end + + # An offset that does not fall on a page boundary is served exactly, by + # reading the page it lands in and the next -- never by rounding the + # window to something the API likes better. + it 'reads across two pages when the window straddles them' do + stub_page(company('a'), company('b'), page: 1, per_page: 2, total_pages: 3) + stub_page(company('c'), company('d'), page: 2, per_page: 2, total_pages: 3) + + expect(rows(%w[id], page: page(1, 2)).map { |row| row['id'] }).to eq(%w[b c]) + end + + it 'stops at the last page rather than asking for one past it' do + stub_page(company('a'), page: 1, per_page: 15, total_pages: 1) + + expect(rows(%w[id], page: page(0, 15)).size).to eq(1) + expect(WebMock).not_to have_requested(:post, "#{base}/companies/list") + .with(query: { 'page' => '2', 'per_page' => '15' }) + end + + it 'stops on a page Intercom answers empty' do + stub_page(page: 1, per_page: 2, total_pages: 9) + + expect(rows(%w[id], page: page(0, 2))).to be_empty + end + + # A read naming no window -- a segment, a customizer -- is the only one + # that can run long, and it is the only one this bounds. + it 'reads full pages when the read names no window' do + stub_page(company('a'), page: 1, per_page: 150, total_pages: 1) + + expect(rows(%w[id]).map { |row| row['id'] }).to eq(%w[a]) + end + + it 'stops such a read after the pages it allows, and says so' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + (1..10).each { |number| stub_page(company("c#{number}"), page: number, per_page: 150, total_pages: 99) } + + expect(rows(%w[id]).size).to eq(10) + expect(ForestAdminDatasourceIntercom.logger).to have_received(:warn).with(/Stopped reading IntercomCompany/) + end + + it 'counts what Intercom counted, in one request' do + stub_page(company('a'), page: 1, per_page: 1, total_pages: 40, total: 597) + + expect(count).to eq([{ 'group' => {}, 'value' => 597 }]) + end + + # Counting the pages read would answer a fraction of the collection as if + # it were the whole of it. + it 'refuses to count a listing Intercom answered without a total' do + stub_request(:post, "#{base}/companies/list").with(query: hash_including({})) + .to_return(json('type' => 'list', 'data' => [company('a')], + 'pages' => {})) + + expect { count }.to raise_error(UnsupportedOperatorError, /cannot be counted/) + end + end + + describe 'the four lookups, and everything past them' do + it 'reads a record detail through its own endpoint' do + stub_request(:get, "#{base}/companies/co1").to_return(json(company('co1'))) + + expect(rows(%w[id], condition_tree: leaf('id', operators::EQUAL, 'co1'))).to eq([{ 'id' => 'co1' }]) + end + + it 'reads a set of ids one request each' do + stub_request(:get, "#{base}/companies/co1").to_return(json(company('co1'))) + stub_request(:get, "#{base}/companies/co2").to_return(json(company('co2'))) + + expect(rows(%w[id], condition_tree: leaf('id', operators::IN, %w[co1 co2])).map { |row| row['id'] }) + .to eq(%w[co1 co2]) + end + + it 'reads a company the token can no longer reach as no record' do + stub_request(:get, "#{base}/companies/co1").to_return(json({ 'type' => 'error.list' }, 404)) + + expect(rows(%w[id], condition_tree: leaf('id', operators::EQUAL, 'co1'))).to be_empty + end + + it 'raises on a failure that is not a missing record' do + stub_request(:get, "#{base}/companies/co1").to_return(json({ 'type' => 'error.list' }, 500)) + + expect { rows(%w[id], condition_tree: leaf('id', operators::EQUAL, 'co1')) }.to raise_error(APIError) + end + + it 'truncates a set of ids larger than it will read, and says so' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + stub_request(:get, %r{#{base}/companies/co\d+}).to_return(json(company('co1'))) + + rows(%w[id], condition_tree: leaf('id', operators::IN, (1..30).map { |n| "co#{n}" })) + + expect(ForestAdminDatasourceIntercom.logger).to have_received(:warn).with(/asked for 30 records by id/) + end + + it 'counts the records the ids named' do + stub_request(:get, "#{base}/companies/co1").to_return(json(company('co1'))) + + expect(count(condition_tree: leaf('id', operators::EQUAL, 'co1'))).to eq([{ 'group' => {}, 'value' => 1 }]) + end + + # `GET /companies?name=` answers the company itself where a listing would + # answer an envelope: a record is read as a page of one rather than as a + # shape every caller has to test for. + it 'looks a company up by name, and reads the record Intercom answers' do + stub_request(:get, "#{base}/companies").with(query: { 'name' => 'Acme' }) + .to_return(json(company('co1'))) + + expect(rows(%w[id], condition_tree: leaf('name', operators::EQUAL, 'Acme'))).to eq([{ 'id' => 'co1' }]) + end + + it 'looks one up by the identifier the workspace gave it' do + stub_request(:get, "#{base}/companies").with(query: { 'company_id' => 'erp-co1' }) + .to_return(json('type' => 'list', 'data' => [company('co1')], + 'total_count' => 1, 'pages' => {})) + + expect(rows(%w[id], condition_tree: leaf('company_id', operators::EQUAL, 'erp-co1'))) + .to eq([{ 'id' => 'co1' }]) + end + + it 'counts what a lookup found' do + stub_request(:get, "#{base}/companies").with(query: { 'name' => 'Acme' }) + .to_return(json(company('co1'))) + + expect(count(condition_tree: leaf('name', operators::EQUAL, 'Acme'))) + .to eq([{ 'group' => {}, 'value' => 1 }]) + end + + it 'reports a lookup Intercom answered with more than one page' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + stub_request(:get, "#{base}/companies").with(query: { 'name' => 'Acme' }) + .to_return(json('type' => 'list', 'data' => [company('co1')], + 'pages' => { 'next' => { 'starting_after' => 'zzz' } })) + + rows(%w[id], condition_tree: leaf('name', operators::EQUAL, 'Acme')) + + expect(ForestAdminDatasourceIntercom.logger).to have_received(:warn).with(/advertised more records/) + end + + # A filter this collection cannot look up is refused by name. A page + # served in answer to a filter it ignored is the one failure this + # datasource is built to avoid. + it 'refuses a filter on a column Intercom does not look up' do + expect { rows(%w[id], condition_tree: leaf('industry', operators::EQUAL, 'Manufacturing')) } + .to raise_error(UnsupportedOperatorError, /cannot filter "industry".*name, company_id alone/m) + end + + it 'refuses an operator the lookup has no equivalent for' do + expect { rows(%w[id], condition_tree: leaf('name', operators::CONTAINS, 'Acm')) } + .to raise_error(UnsupportedOperatorError, /cannot filter "name"/) + end + + it 'refuses a combination, the lookup answering one value at a time' do + expect do + rows(%w[id], condition_tree: branch('And', leaf('name', operators::EQUAL, 'Acme'), + leaf('company_id', operators::EQUAL, 'erp-co1'))) + end.to raise_error(UnsupportedOperatorError, /one exact value at a time/) + end + + it 'refuses to count what it refuses to list' do + expect { count(condition_tree: leaf('industry', operators::EQUAL, 'Manufacturing')) } + .to raise_error(UnsupportedOperatorError, /cannot filter "industry"/) + end + end + + describe 'what it will not do' do + it 'refuses to group, Intercom exposing no aggregate endpoint' do + expect do + collection.aggregate(nil, filter, ForestAdminDatasourceToolkit::Components::Query::Aggregation + .new(operation: 'Count', groups: [{ field: 'industry' }])) + end.to raise_error(UnsupportedOperatorError, /can only be counted/) + end + + # There is no order parameter on this listing at all, so an order asked + # for and not applied is reported here or nowhere. + it 'reports an order it cannot apply' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + stub_page(company('a'), page: 1, per_page: 15) + + rows(%w[id], page: page(0, 15), sort: sort({ field: 'name', ascending: true })) + + expect(ForestAdminDatasourceIntercom.logger).to have_received(:warn).with(/takes no order on this listing/) + end + + it 'says nothing of the ascending primary-key order the agent injects' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + stub_page(company('a'), page: 1, per_page: 15) + + rows(%w[id], page: page(0, 15), sort: sort({ field: 'id', ascending: true })) + + expect(ForestAdminDatasourceIntercom.logger).not_to have_received(:warn) + end + + it 'reports an explicit descending order on the key, which it cannot apply either' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + stub_page(company('a'), page: 1, per_page: 15) + + rows(%w[id], page: page(0, 15), sort: sort({ field: 'id', ascending: false })) + + expect(ForestAdminDatasourceIntercom.logger).to have_received(:warn).with(/sort on id/) + end + end + + describe 'the row' do + it 'flattens the payload, the plan read off the object that carries it' do + stub_page(company('co1'), page: 1, per_page: 15) + + expect(rows(nil, page: page(0, 15)).first) + .to include('id' => 'co1', 'company_id' => 'erp-co1', 'name' => 'Company co1', 'plan_name' => 'Paid', + 'size' => 85, 'monthly_spend' => 49, 'user_count' => 10, + 'created_at' => '2023-11-14T22:13:20Z', 'remote_created_at' => '2014-03-11T09:46:09Z') + end + + it 'reads a company carrying no plan without failing' do + stub_page(company('co1', 'plan' => nil), page: 1, per_page: 15) + + expect(rows(nil, page: page(0, 15)).first).to include('plan_name' => nil) + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/contact_spec.rb b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/contact_spec.rb new file mode 100644 index 000000000..d461612fb --- /dev/null +++ b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/contact_spec.rb @@ -0,0 +1,481 @@ +module ForestAdminDatasourceIntercom + RSpec.describe Collections::Contact do + subject(:collection) { datasource.get_collection('IntercomContact') } + + let(:datasource) { Datasource.new(access_token: 's3cr3t', rate_limiter: nil) } + let(:base) { datasource.configuration.url } + let(:operators) { ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Operators } + + def json(payload, status = 200) + { status: status, body: payload.to_json, headers: { 'Content-Type' => 'application/json' } } + end + + def leaf(field, operator, value = nil) + ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Nodes::ConditionTreeLeaf + .new(field, operator, value) + end + + def branch(aggregator, *conditions) + ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Nodes::ConditionTreeBranch + .new(aggregator, conditions) + end + + def filter(condition_tree: nil, page: nil, sort: nil, search: nil) + ForestAdminDatasourceToolkit::Components::Query::Filter.new(condition_tree: condition_tree, page: page, + sort: sort, search: search) + end + + def page(offset, limit) + ForestAdminDatasourceToolkit::Components::Query::Page.new(offset: offset, limit: limit) + end + + def sort(*clauses) + ForestAdminDatasourceToolkit::Components::Query::Sort.new(clauses) + end + + def contact(id, overrides = {}) + { 'type' => 'contact', 'id' => id, 'role' => 'user', 'name' => "Contact #{id}", + 'email' => "#{id}@acme.test", 'phone' => nil, 'external_id' => "ext-#{id}", 'owner_id' => 493_881, + 'created_at' => 1_700_000_000, 'updated_at' => 1_700_000_600, 'session_count' => 3, + 'unsubscribed_from_emails' => false, 'has_hard_bounced' => false, 'marked_email_as_spam' => false, + 'browser' => 'chrome', 'browser_language' => 'fr', 'os' => 'OS X', 'language_override' => nil, + 'location' => { 'type' => 'location', 'country' => 'France', 'region' => 'IdF', 'city' => 'Paris' }, + 'companies' => { 'type' => 'list', 'data' => [{ 'type' => 'company', 'id' => 'co1' }], + 'total_count' => 2 }, + 'custom_attributes' => {} }.merge(overrides) + end + + def stub_list(*contacts, cursor: nil) + pages = cursor ? { 'next' => { 'starting_after' => cursor } } : {} + stub_request(:get, "#{base}/contacts").with(query: hash_including({})) + .to_return(json('type' => 'list', 'data' => contacts, + 'total_count' => contacts.size, 'pages' => pages)) + end + + def stub_search(*contacts, total: nil) + stub_request(:post, "#{base}/contacts/search") + .to_return(json('type' => 'list', 'data' => contacts, 'total_count' => total || contacts.size, + 'pages' => {})) + end + + def rows(projection = %w[id], **options) + collection.list(nil, filter(**options), projection) + end + + describe 'schema' do + it 'is named IntercomContact' do + expect(collection.name).to eq('IntercomContact') + end + + it 'publishes the columns of a contact, the account it belongs to included' do + expect(collection.fields.keys) + .to include('id', 'role', 'name', 'email', 'email_domain', 'phone', 'external_id', 'avatar', + 'owner_id', 'company_id', 'company_count', 'session_count', 'created_at', + 'last_seen_at', 'unsubscribed_from_emails', 'location_country') + end + + it 'publishes every column read-only, this lot writing nothing' do + columns = collection.fields.values.grep(ForestAdminDatasourceToolkit::Schema::ColumnSchema) + + expect(columns.map(&:is_read_only).uniq).to eq([true]) + end + + # The measured asymmetry: `/contacts/search` refuses `>=`, `<=`, `!=` and + # `IN` on a date where the other two endpoints take them. A Date column + # publishes the two bounds alone anyway -- declaring `equal` would make + # the toolkit republish `in`, which its own validator then refuses + # (PRD-989) -- so what the restriction really has to guarantee is that + # nothing wider reaches the wire. + it 'offers the two bounds on a date, and nothing the endpoint refuses' do + expect(collection.fields['created_at'].filter_operators).to eq(%w[greater_than less_than]) + expect(collection.fields['last_seen_at'].filter_operators).to eq(%w[greater_than less_than]) + end + + it 'offers on a text column exactly what the table measured' do + expect(collection.fields['email'].filter_operators) + .to eq(%w[equal not_equal contains i_contains not_contains starts_with ends_with]) + expect(collection.fields['role'].filter_operators).to eq(%w[equal not_equal]) + end + + # A column the table does not carry advertises nothing, which is how a + # refusal is spelled in a schema. + it 'advertises no filter on a column the endpoint does not filter' do + %w[avatar company_id company_count session_count].each do |column| + expect(collection.fields[column].filter_operators).to be_empty, "#{column} advertises a filter" + end + end + + # The one collection of the whole API Intercom sorts, and the reason this + # tier reads a `sortable` flag off the measured table at all. + it 'is sortable on the columns the table measured, and on no others' do + sortable = collection.fields.select { |_, f| f.respond_to?(:is_sortable) && f.is_sortable }.keys + + expect(sortable).to contain_exactly('name', 'email', 'created_at', 'updated_at', 'signed_up_at', + 'last_seen_at', 'last_contacted_at', 'last_replied_at') + end + + it 'declares the relations the 360 degrees is walked through' do + expect(collection.fields['owner']).to be_a(ForestAdminDatasourceToolkit::Schema::Relations::ManyToOneSchema) + expect(collection.fields['company']).to be_a(ForestAdminDatasourceToolkit::Schema::Relations::ManyToOneSchema) + expect(collection.fields['conversations']) + .to be_a(ForestAdminDatasourceToolkit::Schema::Relations::OneToManySchema) + expect(collection.fields['tickets'].origin_key).to eq('contact_id') + end + + it 'is countable, total_count being exact on every answer' do + expect(collection.is_countable?).to be(true) + end + + it 'is searchable on the address an ops team types' do + expect(collection.is_searchable?).to be(true) + end + end + + describe 'the custom attributes' do + subject(:collection) { datasource.get_collection('IntercomContact') } + + before do + stub_data_attributes('contact', + { 'name' => 'paid_subscriber', 'data_type' => 'boolean', 'custom' => true, + 'api_writable' => true, 'archived' => false }, + { 'name' => 'email', 'data_type' => 'string', 'custom' => false, + 'api_writable' => false, 'archived' => false }) + end + + it 'publishes one column per custom attribute, typed from the introspection' do + expect(collection.fields['paid_subscriber'].column_type).to eq('Boolean') + end + + # Display-only: which operators Intercom answers on + # `custom_attributes.{name}` has not been measured, and this package + # publishes no filter it has not seen work. + it 'publishes it unfilterable and unsortable' do + expect(collection.fields['paid_subscriber'].filter_operators).to be_empty + expect(collection.fields['paid_subscriber'].is_sortable).to be(false) + end + + # Measured on a real workspace, and it took the agent's boot with it: the + # toolkit refuses a field declared twice, so an attribute landing on a + # native column has to be skipped -- and skipped rather than renamed, + # since the operator expects the record's own key under `id`. + it 'skips an attribute whose name a native column already carries, and says which' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + stub_data_attributes('contact', { 'name' => 'id', 'data_type' => 'string', 'custom' => true, + 'api_writable' => true, 'archived' => false }) + stub_list(contact('1', 'custom_attributes' => { 'id' => 'erp-42' })) + + expect { collection }.not_to raise_error + expect(rows(nil).first['id']).to eq('1') + expect(ForestAdminDatasourceIntercom.logger) + .to have_received(:warn).with(/skips the contact attribute "id"/) + end + + # Relations are declared before these columns for exactly this reason. + it 'skips one landing on a relation name too' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + stub_data_attributes('contact', { 'name' => 'company', 'data_type' => 'string', 'custom' => true, + 'api_writable' => true, 'archived' => false }) + + expect(collection.fields['company']) + .to be_a(ForestAdminDatasourceToolkit::Schema::Relations::ManyToOneSchema) + end + + # A custom date arrives as epoch seconds like every other Intercom date, + # and a Date column handed an integer renders as one. + it 'reads a date attribute as a date' do + stub_data_attributes('contact', { 'name' => 'renewal', 'data_type' => 'date', 'custom' => true, + 'api_writable' => true, 'archived' => false }) + stub_list(contact('1', 'custom_attributes' => { 'renewal' => 1_700_000_000 })) + + expect(rows(%w[id renewal]).first['renewal']).to eq('2023-11-14T22:13:20Z') + end + + it 'reads its value off the payload, nil where the contact carries none' do + stub_list(contact('1', 'custom_attributes' => { 'paid_subscriber' => true }), contact('2')) + + expect(rows(%w[id paid_subscriber])) + .to eq([{ 'id' => '1', 'paid_subscriber' => true }, { 'id' => '2', 'paid_subscriber' => nil }]) + end + end + + describe '#list' do + it 'walks the listing endpoint when nothing is filtered, sorted or searched' do + stub_list(contact('1'), contact('2')) + + expect(rows).to eq([{ 'id' => '1' }, { 'id' => '2' }]) + expect(WebMock).not_to have_requested(:post, "#{base}/contacts/search") + end + + it 'flattens the payload onto the row' do + stub_list(contact('1')) + + expect(rows(nil).first) + .to include('id' => '1', 'role' => 'user', 'email' => '1@acme.test', 'email_domain' => 'acme.test', + 'owner_id' => '493881', 'session_count' => 3, 'created_at' => '2023-11-14T22:13:20Z', + 'location_country' => 'France', 'location_city' => 'Paris') + end + + # A contact belongs to several accounts: the row names the first and says + # how many there are, which Intercom counts itself -- the nested list is + # capped and a contact of twelve accounts must not read as one of ten. + it 'names the first account and takes the count from Intercom' do + stub_list(contact('1')) + + expect(rows(nil).first).to include('company_id' => 'co1', 'company_count' => 2) + end + + it 'counts the accounts it can see when Intercom sends no count' do + stub_list(contact('1', 'companies' => { 'type' => 'list', 'data' => [{ 'id' => 'co1' }] })) + + expect(rows(nil).first).to include('company_count' => 1) + end + + it 'leaves the account columns empty on a contact belonging to none' do + stub_list(contact('1', 'companies' => nil)) + + expect(rows(nil).first).to include('company_id' => nil, 'company_count' => 0) + end + + it 'reads no domain out of an address that has none' do + stub_list(contact('1', 'email' => nil)) + + expect(rows(nil).first).to include('email_domain' => nil) + end + + it 'translates a filter into the search DSL' do + stub_search(contact('1')) + + expect(rows(%w[id], condition_tree: leaf('role', operators::EQUAL, 'lead'))).to eq([{ 'id' => '1' }]) + expect(WebMock).to have_requested(:post, "#{base}/contacts/search") + .with(body: hash_including('query' => { 'field' => 'role', 'operator' => '=', 'value' => 'lead' })) + end + + it 'answers a free-text search on the address, per word' do + stub_search(contact('1')) + + rows(%w[id], search: 'acme.test') + + expect(WebMock).to have_requested(:post, "#{base}/contacts/search") + .with(body: hash_including('query' => { 'field' => 'email', 'operator' => '~', 'value' => 'acme.test' })) + end + end + + describe 'the one collection Intercom sorts' do + # A list view asking for an order has no condition to send, and the + # listing endpoint does not sort: the order is what routes the read + # through the search, with the predicate that matches everything. + it 'sends the order to the search endpoint, with the match-all predicate' do + stub_search(contact('1')) + + rows(%w[id], sort: sort({ field: 'last_seen_at', ascending: false })) + + expect(WebMock).to have_requested(:post, "#{base}/contacts/search") + .with(body: hash_including('query' => described_class::MATCH_EVERY_CONTACT, + 'sort' => { 'field' => 'last_seen_at', 'order' => 'descending' })) + end + + it 'sends it alongside the filter when there is one' do + stub_search(contact('1')) + + rows(%w[id], condition_tree: leaf('role', operators::EQUAL, 'user'), + sort: sort({ field: 'name', ascending: true })) + + expect(WebMock).to have_requested(:post, "#{base}/contacts/search") + .with(body: hash_including('query' => { 'field' => 'role', 'operator' => '=', 'value' => 'user' }, + 'sort' => { 'field' => 'name', 'order' => 'ascending' })) + end + + # The ascending primary-key sort the agent injects when a request names + # none is not an order anybody asked for, and this endpoint does not sort + # on an id anyway. + it 'keeps the listing route for the default primary-key order, and says nothing' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + stub_list(contact('1')) + + rows(%w[id], sort: sort({ field: 'id', ascending: true })) + + expect(WebMock).to have_requested(:get, "#{base}/contacts").with(query: hash_including({})) + expect(ForestAdminDatasourceIntercom.logger).not_to have_received(:warn) + end + + it 'reports an order on a column Intercom does not sort, rather than dropping it' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + stub_list(contact('1')) + + rows(%w[id], sort: sort({ field: 'browser', ascending: true })) + + expect(ForestAdminDatasourceIntercom.logger).to have_received(:warn).with(/sort on browser/) + expect(WebMock).not_to have_requested(:post, "#{base}/contacts/search") + end + + # Intercom takes a single `{ field, order }`: honouring the first clause + # alone would order the page by something the operator did not ask for. + it 'reports a composite order rather than honouring half of it' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + stub_list(contact('1')) + + rows(%w[id], sort: sort({ field: 'name', ascending: true }, { field: 'email', ascending: false })) + + expect(ForestAdminDatasourceIntercom.logger).to have_received(:warn).with(/sort on name, email/) + end + end + + describe 'reading records by id' do + # One request per hundred ids rather than one per id: this endpoint + # answers `id IN [...]`, which is what makes a related list of contacts + # affordable at all. + it 'reads a set of ids in one request through the search' do + stub_search(contact('1'), contact('2')) + + expect(rows(%w[id], condition_tree: leaf('id', operators::IN, %w[1 2]))) + .to eq([{ 'id' => '1' }, { 'id' => '2' }]) + expect(WebMock).to have_requested(:post, "#{base}/contacts/search") + .with(body: hash_including('query' => { 'field' => 'id', 'operator' => 'IN', 'value' => %w[1 2] })).once + end + + it 'reads a record detail the same way' do + stub_search(contact('1')) + + expect(rows(%w[id], condition_tree: leaf('id', operators::EQUAL, '1'))).to eq([{ 'id' => '1' }]) + end + + # A contact merged into another disappears from the search: the row reads + # as gone rather than as an error, which is what a merge means. + it 'answers no row for a contact that was merged away' do + stub_search + + expect(rows(%w[id], condition_tree: leaf('id', operators::EQUAL, 'merged'))).to be_empty + end + + it 'counts what the ids named' do + stub_search(contact('1')) + + expect(collection.aggregate(nil, filter(condition_tree: leaf('id', operators::EQUAL, '1')), + ForestAdminDatasourceToolkit::Components::Query::Aggregation + .new(operation: 'Count'))) + .to eq([{ 'group' => {}, 'value' => 1 }]) + end + + it 'truncates a set larger than it will read, and says so' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + stub_search(contact('1')) + + rows(%w[id], condition_tree: leaf('id', operators::IN, (1..400).map(&:to_s))) + + expect(ForestAdminDatasourceIntercom.logger).to have_received(:warn).with(/asked for 400 records by id/) + expect(WebMock).to have_requested(:post, "#{base}/contacts/search").times(3) + end + end + + describe 'the contacts of an account' do + # `/contacts/search` filters no company field, and `GET + # /companies/{id}/contacts` is what answers the one relation an ops team + # walks the most. Without this route it would be a refusal. + it 'reads them from the company endpoint rather than from the search' do + stub_request(:get, "#{base}/companies/co1/contacts").with(query: hash_including({})) + .to_return(json('type' => 'list', 'data' => [contact('1')], + 'total_count' => 1, 'pages' => {})) + + expect(rows(%w[id], condition_tree: leaf('company_id', operators::EQUAL, 'co1'))) + .to eq([{ 'id' => '1' }]) + expect(WebMock).not_to have_requested(:post, "#{base}/contacts/search") + end + + it 'counts them from the same endpoint' do + stub_request(:get, "#{base}/companies/co1/contacts").with(query: hash_including({})) + .to_return(json('type' => 'list', 'data' => [contact('1')], + 'total_count' => 42, 'pages' => {})) + + expect(collection.aggregate(nil, filter(condition_tree: leaf('company_id', operators::EQUAL, 'co1')), + ForestAdminDatasourceToolkit::Components::Query::Aggregation + .new(operation: 'Count'))) + .to eq([{ 'group' => {}, 'value' => 42 }]) + end + + it 'reports an order this route cannot apply' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + stub_request(:get, "#{base}/companies/co1/contacts").with(query: hash_including({})) + .to_return(json('type' => 'list', 'data' => [contact('1')], + 'total_count' => 1, 'pages' => {})) + + rows(%w[id], condition_tree: leaf('company_id', operators::EQUAL, 'co1'), + sort: sort({ field: 'name', ascending: true })) + + expect(ForestAdminDatasourceIntercom.logger).to have_received(:warn).with(/sort on name/) + end + + # An `and` also carrying a scope names a narrower set than the account + # does, and answering it with the account alone would serve contacts the + # scope excludes. + it 'refuses to take the route for anything but a bare equality' do + expect do + rows(%w[id], condition_tree: branch('And', leaf('company_id', operators::EQUAL, 'co1'), + leaf('role', operators::EQUAL, 'user'))) + end.to raise_error(UnsupportedOperatorError, /cannot filter "company_id"/) + end + end + + describe 'a condition through a relation' do + def stub_admins(*admins) + stub_request(:get, "#{base}/admins").to_return(json('type' => 'admin.list', 'admins' => admins)) + end + + # The owner is a teammate, read whole in one request, and + # `/contacts/search` filters on the key: readable, navigable and + # filterable alike. + it 'resolves the owner against the teammates and filters on the key' do + stub_admins({ 'id' => '493881', 'name' => 'Marie' }) + stub_search(contact('1')) + + rows(%w[id], condition_tree: leaf('owner:name', operators::EQUAL, 'Marie')) + + expect(WebMock).to have_requested(:post, "#{base}/contacts/search") + .with(body: hash_including('query' => { 'field' => 'owner_id', 'operator' => '=', 'value' => '493881' })) + end + + it 'nests the owner under the relation when the projection names it' do + stub_admins({ 'id' => '493881', 'name' => 'Marie' }) + stub_list(contact('1')) + + expect(rows(%w[id owner:name]).first).to eq('id' => '1', + 'owner' => { 'id' => '493881', 'name' => 'Marie' }) + end + + # The endpoint filters no company field, so the relation is there to be + # read and navigated. Refused by name, before the target is read: a + # refusal that spends a request costs exactly what it refuses to do. + it 'refuses a condition through the company, and says what to filter instead' do + expect { rows(%w[id], condition_tree: leaf('company:name', operators::EQUAL, 'Acme')) } + .to raise_error(UnsupportedOperatorError, + %r{resolves to "company_id", on which contacts/search takes no filter}) + expect(WebMock).not_to have_requested(:post, /companies/) + end + end + + describe 'what it will not do' do + it 'refuses to group, Intercom exposing no aggregate endpoint' do + expect do + collection.aggregate(nil, filter, ForestAdminDatasourceToolkit::Components::Query::Aggregation + .new(operation: 'Count', groups: [{ field: 'role' }])) + end.to raise_error(UnsupportedOperatorError, /can only be counted/) + end + + it 'counts what the filter names in one request' do + stub_search(contact('1'), total: 812) + + expect(collection.aggregate(nil, filter(condition_tree: leaf('role', operators::EQUAL, 'user')), + ForestAdminDatasourceToolkit::Components::Query::Aggregation + .new(operation: 'Count'))) + .to eq([{ 'group' => {}, 'value' => 812 }]) + end + end + + describe 'paging' do + it 'asks Intercom for the window the list view named' do + stub_list(contact('1'), contact('2'), contact('3')) + + expect(rows(%w[id], page: page(1, 2)).map { |row| row['id'] }).to eq(%w[2 3]) + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/conversation_spec.rb b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/conversation_spec.rb index a6cfe8db5..754df26d9 100644 --- a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/conversation_spec.rb +++ b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/conversation_spec.rb @@ -120,7 +120,7 @@ def ids(rows) # and the contact identity are all read from somewhere the endpoint does # not filter. it 'advertises no filter on a column the endpoint does not filter' do - %w[tag_names company_name contact_email timeline contact_ids].each do |column| + %w[tag_names company_name contact_name timeline contact_count].each do |column| expect(collection.fields[column].filter_operators).to be_empty, "#{column} advertises a filter" end end @@ -199,6 +199,33 @@ def ids(rows) end end + # The other half of the 360 degrees: `IntercomContact#conversations` is a + # one-to-many, and the agent serves it by listing this collection on the key + # -- so what makes that list possible is this endpoint filtering on a + # contact id at all. The column is singular and the wire field plural: the + # row names its first contact, the endpoint asks whether a contact is one of + # the conversation's. + describe 'the conversations of a contact' do + it 'filters on the contact id the relation resolves to' do + stub_search(conversation('1')) + + collection.list(nil, filter(condition_tree: leaf('contact_id', operators::EQUAL, 'c1')), %w[id]) + + expect(WebMock).to have_requested(:post, "#{base}/conversations/search") + .with(query: hash_including({}), + body: hash_including('query' => { 'field' => 'contact_ids', 'operator' => '=', 'value' => 'c1' })) + end + + it 'counts them in one request' do + stub_search(conversation('1'), total: 37) + + expect(collection.aggregate(nil, filter(condition_tree: leaf('contact_id', operators::EQUAL, 'c1')), + ForestAdminDatasourceToolkit::Components::Query::Aggregation + .new(operation: 'Count'))) + .to eq([{ 'group' => {}, 'value' => 37 }]) + end + end + describe '#list' do it 'reads the listing endpoint as plain text and pages by cursor' do stub_list(conversation('1')) @@ -245,13 +272,14 @@ def ids(rows) .to include('closed_at' => nil, 'reopen_count' => nil) end - # A group conversation has several contacts: the row names how many rather - # than presenting one of them as the one. - it 'carries the contact ids and their count' do + # A group conversation has several contacts: the row names the first, says + # how many there are, and the relation resolves that same first one -- the + # column and the relation cannot disagree. + it 'names the first contact and counts them' do stub_list(conversation('1')) expect(collection.list(nil, filter, nil).first) - .to include('contact_ids' => %w[c1 c2], 'contact_count' => 2) + .to include('contact_id' => 'c1', 'contact_count' => 2) end it 'narrows the row to the projection' do @@ -573,16 +601,20 @@ def aggregation(operation, field: nil, groups: []) 'data' => [{ 'id' => 'c1', 'name' => 'Camille', 'email' => 'camille@acme.test' }])) end - # Denormalized rather than declared as a relation: the Contacts collection - # arrives in lot 4, and a relation whose target is missing is a schema the - # agent refuses to boot on. + # One label on the row plus the relation to navigate, which is the rule + # lot 2.5 set for the ticket labels: `contact_email` is gone, it is a hop + # away on `contact:email`. it 'reads the identity of the page in one request and puts it on the row' do - row = collection.list(nil, filter, %w[id contact_name contact_email]).first + row = collection.list(nil, filter, %w[id contact_name]).first - expect(row).to include('contact_name' => 'Camille', 'contact_email' => 'camille@acme.test') + expect(row).to include('contact_name' => 'Camille') expect(WebMock).to have_requested(:post, "#{base}/contacts/search").once end + it 'no longer publishes the e-mail the relation carries' do + expect(collection.fields.keys).not_to include('contact_email', 'contact_ids') + end + it 'asks for the contacts of the page by id' do collection.list(nil, filter, %w[id contact_name]) diff --git a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/ticket_spec.rb b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/ticket_spec.rb index 3e5530c25..ca03247b5 100644 --- a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/ticket_spec.rb +++ b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/ticket_spec.rb @@ -81,7 +81,7 @@ def stub_search(*records, total: nil, body: nil) answer = { 'type' => 'ticket.list', 'tickets' => records, 'total_count' => total || records.size, 'pages' => { 'type' => 'pages', 'page' => 1 } } - request = stub_request(:post, "#{base}/tickets/search") + request = stub_request(:post, "#{base}/tickets/search").with(query: hash_including({})) request = request.with(body: hash_including(body)) if body request.to_return(json(answer)) end @@ -173,7 +173,7 @@ def columns rows(%w[id]) - expect(WebMock).to have_requested(:post, "#{base}/tickets/search") + expect(WebMock).to have_requested(:post, %r{#{base}/tickets/search}) .with(body: hash_including('query' => { 'field' => 'created_at', 'operator' => '>', 'value' => '0' })) end @@ -186,7 +186,7 @@ def columns pagination = hash_including('per_page' => described_class::MAX_TICKETS_PER_PAGE) - expect(WebMock).to have_requested(:post, "#{base}/tickets/search") + expect(WebMock).to have_requested(:post, %r{#{base}/tickets/search}) .with(body: hash_including('pagination' => pagination)) end @@ -251,7 +251,7 @@ def columns # A record detail goes to its own endpoint, which is not the search one. it 'reads one ticket through the record endpoint' do - stub_request(:get, "#{base}/tickets/1").to_return(json(ticket('1'))) + stub_request(:get, "#{base}/tickets/1").with(query: hash_including({})).to_return(json(ticket('1'))) expect(rows(%w[id], condition_tree: leaf('id', operators::EQUAL, '1')).map { |row| row['id'] }).to eq(%w[1]) end @@ -380,6 +380,77 @@ def columns # exists. What Intercom is really filtered on is the foreign key: the target # says which of its records match -- over every record it holds, not over a # page -- and the ids it names are what the search carries. + # `IntercomContact#tickets` is a one-to-many the agent serves by listing this + # collection on the key. The row it rests on is `spec`, not `measured`: + # whether `/tickets/search` filters on a contact id at all is one of the + # probe's questions, and the answer moves the row into the table or into the + # refusals. + describe 'the tickets of a contact' do + it 'filters on the contact id the relation resolves to' do + stub_search(ticket('1')) + + collection.list(nil, filter(condition_tree: leaf('contact_id', operators::EQUAL, 'c1')), %w[id]) + + expect(WebMock).to have_requested(:post, %r{#{base}/tickets/search}) + .with(body: hash_including('query' => { 'field' => 'contact_ids', 'operator' => '=', + 'value' => 'c1' })) + end + end + + # The exchange itself, which lot 1 paid for and did not publish: the parts + # ride along in every ticket response, so the thread costs no request. + describe 'the thread' do + it 'names who said what, when, and through which kind of event' do + stub_search(ticket('1', parts(comment(at: 1_700_001_000, by: 'Camille', type: 'contact'), + comment(at: 1_700_002_000, by: 'Alice')))) + + expect(rows(%w[id timeline]).first['timeline']) + .to eq([{ 'id' => 'c1700001000', 'part_type' => 'comment', 'created_at' => '2023-11-14T22:30:00Z', + 'author_type' => 'contact', 'author_name' => 'Camille', 'author_email' => nil, + 'body' => 'Je regarde.', 'attachment_count' => 0, 'redacted' => nil }, + { 'id' => 'c1700002000', 'part_type' => 'comment', 'created_at' => '2023-11-14T22:46:40Z', + 'author_type' => 'admin', 'author_name' => 'Alice', 'author_email' => nil, + 'body' => 'Je regarde.', 'attachment_count' => 0, 'redacted' => nil }]) + end + + # An assignment, an internal note and a reply are not the same event, and + # a thread that flattens them reads as an exchange that never happened the + # way it did. Which also means the team's internal notes are in there. + it 'keeps the internal notes and the state changes, each under its own type' do + stub_search(ticket('1', parts(comment(at: 1_700_001_000, part_type: 'note'), + state_change('resolved', at: 1_700_002_000)))) + + expect(rows(%w[id timeline]).first['timeline'].map { |entry| entry['part_type'] }) + .to eq(%w[note ticket_state_updated_by_admin]) + end + + # Empty means empty here, and says so: the parts are in every response, + # unlike a conversation read from a listing, whose timeline stays nil + # because nothing is known. + it 'answers an empty thread on a ticket nothing happened to' do + stub_search(ticket('1')) + + expect(rows(%w[id timeline]).first['timeline']).to eq([]) + end + + it 'builds nothing when the projection does not name it' do + stub_search(ticket('1', parts(comment(at: 1_700_001_000)))) + + expect(rows(%w[id]).first).to eq('id' => '1') + end + + # The bodies are HTML written by end customers, and rendering third-party + # HTML inside Forest is neither safe nor useful (R10). + it 'asks Intercom for plain text rather than markup' do + stub_search(ticket('1')) + + rows(%w[id timeline]) + + expect(WebMock).to have_requested(:post, %r{#{base}/tickets/search}) + .with(query: { 'display_as' => 'plaintext' }) + end + end + describe 'a condition through a relation' do it 'filters on the foreign key the target resolved to' do stub_admins('id' => '493881', 'name' => 'Alice') @@ -387,7 +458,7 @@ def columns rows(%w[id], condition_tree: leaf('admin_assignee:name', operators::EQUAL, 'Alice')) - expect(WebMock).to have_requested(:post, "#{base}/tickets/search") + expect(WebMock).to have_requested(:post, %r{#{base}/tickets/search}) .with(body: hash_including('query' => { 'field' => 'admin_assignee_id', 'operator' => '=', 'value' => '493881' })) end @@ -400,7 +471,7 @@ def columns rows(%w[id], condition_tree: leaf('admin_assignee:name', operators::EQUAL, 'Alice')) - expect(WebMock).to have_requested(:post, "#{base}/tickets/search") + expect(WebMock).to have_requested(:post, %r{#{base}/tickets/search}) .with(body: hash_including('query' => { 'operator' => 'OR', 'value' => [{ 'field' => 'admin_assignee_id', 'operator' => '=', @@ -417,7 +488,7 @@ def columns stub_admins('id' => '493881', 'name' => 'Alice') expect(rows(%w[id], condition_tree: leaf('admin_assignee:name', operators::EQUAL, 'Zoe'))).to be_empty - expect(WebMock).not_to have_requested(:post, "#{base}/tickets/search") + expect(WebMock).not_to have_requested(:post, %r{#{base}/tickets/search}) end it 'counts none of them either, without a request' do @@ -428,7 +499,7 @@ def columns operators::EQUAL, 'Zoe')), aggregation) expect(counted).to eq([{ 'group' => {}, 'value' => 0 }]) - expect(WebMock).not_to have_requested(:post, "#{base}/tickets/search") + expect(WebMock).not_to have_requested(:post, %r{#{base}/tickets/search}) end # A relation group nested inside the tree the agent assembled: a scope, a @@ -442,7 +513,7 @@ def columns collection.list(nil, filter(condition_tree: tree), %w[id]) - expect(WebMock).to have_requested(:post, "#{base}/tickets/search") + expect(WebMock).to have_requested(:post, %r{#{base}/tickets/search}) .with(body: hash_including('query' => { 'operator' => 'AND', 'value' => [{ 'field' => 'category', 'operator' => '=', @@ -469,7 +540,7 @@ def columns collection.list(nil, filter(condition_tree: tree), %w[id]) - expect(WebMock).to have_requested(:post, "#{base}/tickets/search") + expect(WebMock).to have_requested(:post, %r{#{base}/tickets/search}) .with(body: hash_including('query' => { 'operator' => 'AND', 'value' => [{ 'field' => 'category', 'operator' => '=', @@ -495,7 +566,7 @@ def columns collection.list(nil, filter(condition_tree: tree), %w[id]) - expect(WebMock).to(have_requested(:post, "#{base}/tickets/search").with do |request| + expect(WebMock).to(have_requested(:post, %r{#{base}/tickets/search}).with do |request| query = JSON.parse(request.body)['query'] query['value'].size == 15 && query['value'].last['operator'] == 'OR' && query['value'].last['value'].size == 3 @@ -508,17 +579,21 @@ def columns leaf('admin_assignee:name', operators::EQUAL, 'Zoe')) expect(collection.list(nil, filter(condition_tree: tree), %w[id])).to be_empty - expect(WebMock).not_to have_requested(:post, "#{base}/tickets/search") + expect(WebMock).not_to have_requested(:post, %r{#{base}/tickets/search}) end # Fifteen conditions per group is Intercom's limit, and a relation reaches # it without trying. Refused by name rather than sent and answered with a # 400 naming neither the limit nor the filter that hit it. + # + # "more than fifteen" rather than a count: the target is read one record + # past what a group holds and no further, so what is known is that it does + # not fit -- see `match_page`. it 'refuses a relation condition matching more records than a group holds' do - stub_admins(*(1..16).map { |index| { 'id' => index.to_s, 'name' => 'Alice' } }) + stub_admins(*(1..20).map { |index| { 'id' => index.to_s, 'name' => 'Alice' } }) expect { rows(%w[id], condition_tree: leaf('admin_assignee:name', operators::EQUAL, 'Alice')) } - .to raise_error(UnsupportedOperatorError, /names 16 records.*15 conditions per group/m) + .to raise_error(UnsupportedOperatorError, /names more than 15 records.*15 conditions per group/m) end # `/tickets/search` filters no state id -- the table carries none -- so the diff --git a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/datasource_spec.rb b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/datasource_spec.rb index 91ad2f84f..0dc29ba1d 100644 --- a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/datasource_spec.rb +++ b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/datasource_spec.rb @@ -7,24 +7,28 @@ module ForestAdminDatasourceIntercom end # The reference collections come first: they are what turns an assignee id - # into a teammate and a state id into a label. Conversations follow, Tickets - # next. The membership sits with the two collections it joins: a many-to-many + # into a teammate and a state id into a label. Contacts and Companies + # follow, before the two collections whose relations point at them. + # The membership sits with the two collections it joins: a many-to-many # needs a collection to travel through, and Intercom exposes none. it 'publishes the collections of the lot' do expect(datasource.collections.keys) .to eq(%w[IntercomAdmin IntercomTeam IntercomTeamMembership IntercomTicketType IntercomTicketState - IntercomConversation IntercomTicket]) + IntercomContact IntercomCompany IntercomConversation IntercomTicket]) end - # The one read a boot performs: the attributes a workspace declares on its - # ticket types are columns of the Tickets collection, and a ticket payload - # carries the values of its own type only, so they cannot be discovered from - # the records. - it 'introspects the ticket-type attributes while registering, and reads nothing else' do + # The three reads a boot performs, and no fourth: the attributes a workspace + # declares on its ticket types, on its contacts and on its companies are + # columns of those collections, and a payload carries the values of the + # attributes that record happens to have been given, never their + # definitions. + it 'introspects the workspace attributes while registering, and reads nothing else' do datasource expect(WebMock).to have_requested(:get, /ticket_types/).once - expect(WebMock).not_to have_requested(:get, /conversations|admins|teams/) + expect(WebMock).to have_requested(:get, /data_attributes/).with(query: { 'model' => 'contact' }).once + expect(WebMock).to have_requested(:get, /data_attributes/).with(query: { 'model' => 'company' }).once + expect(WebMock).not_to have_requested(:get, %r{conversations|admins|teams|companies/list}) end # A token without that permission costs the attribute columns, never the @@ -37,8 +41,22 @@ module ForestAdminDatasourceIntercom expect(datasource.get_collection('IntercomTicket').fields.keys).not_to include('_default_title_') end + # The same guarantee on the other two models, and it is what the acceptance + # criterion of lot 4 asks for: a token missing a permission costs the + # columns it could not read, and the collection still boots. + it 'boots the contact and company collections when their introspection is refused' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + stub_request(:get, /data_attributes/).to_return(status: 403, body: '{}', + headers: { 'Content-Type' => 'application/json' }) + + expect(datasource.get_collection('IntercomContact').fields.keys).to include('email') + expect(datasource.get_collection('IntercomCompany').fields.keys).to include('name') + end + it 'configures a client from the options it is handed' do stub_ticket_types(base: 'https://api.eu.intercom.io') + stub_data_attributes('contact', base: 'https://api.eu.intercom.io') + stub_data_attributes('company', base: 'https://api.eu.intercom.io') configured = described_class.new(access_token: 's3cr3t', region: :eu, rate_limiter: nil) expect(configured.configuration.url).to eq('https://api.eu.intercom.io') diff --git a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/query/search_fields_spec.rb b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/query/search_fields_spec.rb index b203c5545..9b36bf607 100644 --- a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/query/search_fields_spec.rb +++ b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/query/search_fields_spec.rb @@ -15,8 +15,27 @@ def field_row(overrides = {}) # The schema derives its filters from this file, so a malformed row is a # boot failure rather than a column nobody can explain. Reading it here is # what turns that guarantee into a test. - it 'reads the two search endpoints' do - expect(described_class.endpoints).to eq(%w[conversations tickets]) + it 'reads the three search endpoints' do + expect(described_class.endpoints).to eq(%w[conversations tickets contacts]) + end + + # The asymmetry the per-endpoint table exists for, asserted on the file + # that ships rather than on a table a spec built: a date is bounded on + # every endpoint, and only two of them take the closed bounds and the + # inequality. + it 'carries the measured date restrictions of /contacts/search' do + expect(described_class.fetch('contacts').field('created_at').operators).to eq(['>', '<']) + expect(described_class.fetch('conversations').field('created_at').operators) + .to include('>=', '<=', '!=') + end + + # Intercom sorts one endpoint and ignores the sort it is sent on the other + # two, so a sortable column outside contacts would promise an order that + # never happens. + it 'declares a sortable column on the one endpoint that sorts' do + expect(described_class.fetch('contacts').sortable_columns).not_to be_empty + expect(described_class.fetch('conversations').sortable_columns).to be_empty + expect(described_class.fetch('tickets').sortable_columns).to be_empty end it 'names the path each endpoint is searched through' do @@ -85,7 +104,8 @@ def field_row(overrides = {}) end it 'lists exactly the columns each endpoint filters' do - { 'IntercomConversation' => 'conversations', 'IntercomTicket' => 'tickets' }.each do |collection, endpoint| + { 'IntercomConversation' => 'conversations', 'IntercomTicket' => 'tickets', + 'IntercomContact' => 'contacts' }.each do |collection, endpoint| row = filterable.lines.find { |line| line.start_with?("| `#{collection}` |") } listed = row.to_s.scan(/`([a-z_]+)`/).flatten @@ -111,6 +131,14 @@ def field_row(overrides = {}) .to raise_error(ConfigurationError, /belongs in the refused table/) end + # YAML reads an unquoted `true` as a boolean and a quoted one as a string, + # which is truthy in Ruby: a typo would publish a sortable column the + # endpoint never sorts. + it 'refuses a sortable flag that is not a boolean' do + expect { table(fields: { 'created_at' => field_row('sortable' => 'true') }) } + .to raise_error(ConfigurationError, /sortable "true" is neither true nor false/) + end + it 'refuses a provenance that is neither measured nor read off the documentation' do expect { table(fields: { 'created_at' => field_row('source' => 'guessed') }) } .to raise_error(ConfigurationError, /source "guessed" is neither measured nor spec/) @@ -140,8 +168,8 @@ def field_row(overrides = {}) end it 'refuses an endpoint nothing declares, rather than filtering nothing' do - expect { described_class.fetch('contacts') } - .to raise_error(ConfigurationError, /Unknown Intercom search endpoint "contacts"/) + expect { described_class.fetch('companies') } + .to raise_error(ConfigurationError, /Unknown Intercom search endpoint "companies"/) end it 'is measured once the probe has stamped a date on it' do diff --git a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/schema/data_attributes_introspector_spec.rb b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/schema/data_attributes_introspector_spec.rb new file mode 100644 index 000000000..6896ccf33 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/schema/data_attributes_introspector_spec.rb @@ -0,0 +1,136 @@ +module ForestAdminDatasourceIntercom + module Schema + RSpec.describe DataAttributesIntrospector do + subject(:introspector) { described_class.new(Client.new(configuration), model: 'contact') } + + let(:configuration) { Configuration.new(access_token: 's3cr3t', rate_limiter: nil) } + let(:base) { configuration.url } + + def json(payload, status = 200) + { status: status, body: payload.to_json, headers: { 'Content-Type' => 'application/json' } } + end + + def attribute(name, overrides = {}) + { 'type' => 'data_attribute', 'model' => 'contact', 'name' => name, 'label' => name, + 'data_type' => 'string', 'custom' => true, 'archived' => false, + 'api_writable' => true }.merge(overrides) + end + + def stub_attributes(*attributes, model: 'contact') + stub_request(:get, "#{base}/data_attributes").with(query: { 'model' => model }) + .to_return(json('type' => 'list', 'data' => attributes)) + end + + it 'reads the attributes of the model it was built for' do + stub_attributes(attribute('paid_subscriber'), model: 'company') + + expect(described_class.new(Client.new(configuration), model: 'company').attributes.map(&:name)) + .to eq(['paid_subscriber']) + end + + # The standard attributes are columns the collection declares by hand, + # with the filters the search table measured. Publishing them again under + # their `custom_attributes` name would show one fact twice, the + # unfilterable copy winning nothing. + it 'leaves out the attributes Intercom defines itself' do + stub_attributes(attribute('paid_subscriber'), attribute('email', 'custom' => false)) + + expect(introspector.attributes.map(&:name)).to eq(['paid_subscriber']) + end + + it 'leaves out an archived attribute, which the workspace stopped offering' do + stub_attributes(attribute('paid_subscriber'), attribute('old_plan', 'archived' => true)) + + expect(introspector.attributes.map(&:name)).to eq(['paid_subscriber']) + end + + it 'leaves out an attribute with no name to be a column of' do + stub_attributes(attribute(''), 'not a hash') + + expect(introspector.attributes).to be_empty + end + + # Read and carried although every column of this lot is published + # read-only: it is what lot 4b needs to tell an attribute it may write + # from one Intercom fills in itself, and reading it again then would be a + # second boot-time round trip. + it 'carries api_writable for the lot that writes' do + stub_attributes(attribute('paid_subscriber'), attribute('lifetime_value', 'api_writable' => false)) + + expect(introspector.attributes.map { |a| [a.name, a.api_writable] }) + .to eq([['paid_subscriber', true], ['lifetime_value', false]]) + end + + it 'maps each Intercom data type onto what Forest renders' do + stub_attributes(attribute('a', 'data_type' => 'integer'), attribute('b', 'data_type' => 'float'), + attribute('c', 'data_type' => 'boolean'), attribute('d', 'data_type' => 'date')) + + expect(introspector.attributes.map(&:column_type)).to eq(%w[Number Number Boolean Date]) + end + + # Showing the value Intercom sent beats hiding a column because its type + # is new. + it 'reads an unknown data type as a string rather than dropping the column' do + stub_attributes(attribute('paid_subscriber', 'data_type' => 'quantum')) + + expect(introspector.attributes.first.column_type).to eq('String') + end + + # Forest lists the fields of a request in a comma-separated query + # parameter and names a field through a relation with a colon: either one + # in a column name breaks the projection before the page is read. + it 'takes the commas and colons out of a column name, keeping the name the payload uses' do + stub_attributes(attribute('Plan, tier: current')) + + expect(introspector.attributes.map { |a| [a.name, a.column_name] }) + .to eq([['Plan, tier: current', 'Plan tier current']]) + end + + it 'unescapes a name Intercom handed back escaped' do + stub_attributes(attribute('Ce que j'ai vérifié')) + + expect(introspector.attributes.first.column_name).to eq("Ce que j'ai vérifié") + end + + # Two attributes landing on one column would share an entry, and the + # second's values would be read under the first's name -- wrong values + # rather than missing ones. + it 'leaves out an attribute colliding with one already kept, and says which' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + stub_attributes(attribute('Plan, tier'), attribute('Plan: tier')) + + expect(introspector.attributes.map(&:name)).to eq(['Plan, tier']) + expect(ForestAdminDatasourceIntercom.logger) + .to have_received(:warn).with(/"Plan: tier" is left out.*"Plan tier"/) + end + + it 'keeps an attribute whose name is nothing but separators out of the schema' do + stub_attributes(attribute(',,')) + + expect(introspector.attributes).to be_empty + end + + # A token without the permission costs the columns, never the boot. + it 'answers no attribute when Intercom refuses the read, and says so' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + stub_request(:get, "#{base}/data_attributes").with(query: { 'model' => 'contact' }) + .to_return(json( + { 'type' => 'error.list', + 'errors' => [{ 'code' => 'forbidden' }] }, 403 + )) + + expect(introspector.attributes).to eq([]) + expect(ForestAdminDatasourceIntercom.logger) + .to have_received(:warn).with(/could not read the contact attributes \(HTTP 403\)/) + end + + it 'reads Intercom once, however many times it is asked' do + stub_attributes(attribute('paid_subscriber')) + + 2.times { introspector.attributes } + + expect(WebMock).to have_requested(:get, /data_attributes/).once + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/spec/probe_search_fields_spec.rb b/packages/forest_admin_datasource_intercom/spec/probe_search_fields_spec.rb index 405d2f85f..105c4eac7 100644 --- a/packages/forest_admin_datasource_intercom/spec/probe_search_fields_spec.rb +++ b/packages/forest_admin_datasource_intercom/spec/probe_search_fields_spec.rb @@ -151,7 +151,7 @@ def run_probe(field, type) end it 'refuses an endpoint the table does not declare' do - expect { described_class.call(['--endpoint', 'contacts', '--token', 's3cr3t']) } + expect { described_class.call(['--endpoint', 'companies', '--token', 's3cr3t']) } .to raise_error(ConfigurationError, /Unknown Intercom search endpoint/) end @@ -160,6 +160,8 @@ def run_probe(field, type) stub_search(code: 'invalid_field') stub_request(:post, "#{base}/conversations/search") .to_return(json({ 'type' => 'error.list', 'errors' => [{ 'code' => 'invalid_field' }] }, 400)) + stub_request(:post, "#{base}/contacts/search") + .to_return(json({ 'type' => 'error.list', 'errors' => [{ 'code' => 'invalid_field' }] }, 400)) expect { described_class.call(['--token', 's3cr3t', '--out', out]) }.to output(/NOT FILTERABLE/).to_stdout expect(YAML.safe_load_file(out)['endpoint']).to eq('conversations') diff --git a/packages/forest_admin_datasource_intercom/spec/spec_helper.rb b/packages/forest_admin_datasource_intercom/spec/spec_helper.rb index 056ee440e..4cbaea580 100644 --- a/packages/forest_admin_datasource_intercom/spec/spec_helper.rb +++ b/packages/forest_admin_datasource_intercom/spec/spec_helper.rb @@ -27,16 +27,25 @@ # and a fixture is read by everyone who clones the repo. WebMock.disable_net_connect!(allow_localhost: true) -# A datasource introspects the ticket-type attributes while it registers its -# collections, so every spec building one issues that read. The base url is not -# taken from the datasource on purpose: reading it would build the datasource, -# and boot the very read this stubs. +# A datasource introspects the ticket-type attributes and the contact and +# company attributes while it registers its collections, so every spec building +# one issues those three reads. The base url is not taken from the datasource on +# purpose: reading it would build the datasource, and boot the very reads this +# stubs. module IntercomBootStubs def stub_ticket_types(*types, base: ForestAdminDatasourceIntercom::Configuration::REGION_HOSTS[:us]) stub_request(:get, "#{base}/ticket_types") .to_return(status: 200, body: { 'type' => 'list', 'data' => types }.to_json, headers: { 'Content-Type' => 'application/json' }) end + + def stub_data_attributes(model, *attributes, + base: ForestAdminDatasourceIntercom::Configuration::REGION_HOSTS[:us]) + stub_request(:get, "#{base}/data_attributes").with(query: { 'model' => model }) + .to_return(status: 200, + body: { 'type' => 'list', 'data' => attributes }.to_json, + headers: { 'Content-Type' => 'application/json' }) + end end RSpec.configure do |config| @@ -55,5 +64,7 @@ def stub_ticket_types(*types, base: ForestAdminDatasourceIntercom::Configuration config.before do WebMock.reset! stub_ticket_types + stub_data_attributes('contact') + stub_data_attributes('company') end end From 955300b597b9b171bd7f22cc1c297813639d67c1 Mon Sep 17 00:00:00 2001 From: Brun Christophe Date: Wed, 9 Sep 2026 14:25:22 +0200 Subject: [PATCH 5/6] fix(datasource): refuse rather than half-answer on Intercom Review pass on PR #387. Correctness * a relation read slices the fan-out by what the target resolves in one read, and refuses past what it resolves at all, instead of resolving the first 25 rows of a page and leaving the rest nil; * the contacts of an account with a scope or a segment filtered alongside are refused with a message naming that condition, rather than with the table reason telling the operator to read the account's contacts, which is what they asked for; * the `id in [...]` route reports the order it cannot apply, refuses a count larger than a bulk read fetches, and deduplicates its ids; * the cursor walk logs the two defensive stops it took silently; * a date filter naming no day, a time of day or a month, is refused rather than completed from the clock of whichever host ran it; * the offset tier bounds a windowed read by a record budget instead of by the page cap meant for a read with no window of its own; * a projection reaching through two relations is refused by name, like the filter that reaches that deep. Consistency * the four enrichment hooks read the projection through `column_asked?`, so an all-columns read fills the columns it publishes; * `search_fields.yml` fails at load on a column filed as both filterable and refused, and on operators that publish nothing for its type, which is what its own docstring promised; * `active_support` and `active_support/time` are required rather than relied on through an incidental require of the toolkit; * a base_url carrying credentials is masked in every `inspect`, and a cleartext one is reported as carrying the bearer in clear; * `RecordsById` and `AttributeNaming` hold what the paginated tiers and the two introspectors were duplicating, messages included. Docs: nine collections rather than eleven, 19 of 85 table rows measured, free-text search on contacts too, what each relation really costs, and every limit above. 560 examples, 0 failures, 100% line coverage; rubocop clean on 912 files; toolkit, customizer and agent suites green. Co-Authored-By: Claude Opus 5 (1M context) --- .../README.md | 132 +++++++++++++++--- .../lib/forest_admin_datasource_intercom.rb | 12 ++ .../client.rb | 2 +- .../collections/base_collection.rb | 51 +++++++ .../collections/contact.rb | 63 ++++++++- .../collections/contact_identity.rb | 2 +- .../collections/conversation.rb | 10 +- .../collections/cursor_collection.rb | 126 +++++------------ .../collections/fetch_all_collection.rb | 9 +- .../collections/offset_collection.rb | 129 ++++++----------- .../collections/records_by_id.rb | 126 +++++++++++++++++ .../collections/relations.rb | 69 ++++++++- .../collections/ticket.rb | 10 +- .../collections/ticket/derived_columns.rb | 6 +- .../configuration.rb | 38 ++++- .../pagination/cursor_walker.rb | 63 ++++++--- .../query/condition_tree_translator.rb | 17 ++- .../query/filter_value.rb | 28 +++- .../query/search_fields.rb | 37 ++++- .../rate_limiter.rb | 13 ++ .../schema/attribute_naming.rb | 52 +++++++ .../schema/data_attributes_introspector.rb | 33 +---- .../schema/ticket_attributes_introspector.rb | 34 +---- .../collections/company_spec.rb | 83 ++++++++++- .../collections/contact_spec.rb | 60 +++++++- .../collections/conversation_spec.rb | 23 +++ .../collections/ticket_spec.rb | 21 +++ .../configuration_spec.rb | 38 +++++ .../pagination/cursor_walker_spec.rb | 24 +++- .../query/condition_tree_translator_spec.rb | 11 +- .../query/filter_value_spec.rb | 23 +++ .../query/search_fields_spec.rb | 19 +++ 32 files changed, 1040 insertions(+), 324 deletions(-) create mode 100644 packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/records_by_id.rb create mode 100644 packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/schema/attribute_naming.rb diff --git a/packages/forest_admin_datasource_intercom/README.md b/packages/forest_admin_datasource_intercom/README.md index 7ca2838d0..1a005706e 100644 --- a/packages/forest_admin_datasource_intercom/README.md +++ b/packages/forest_admin_datasource_intercom/README.md @@ -109,16 +109,33 @@ exact lookups and nothing else. See [Companies](#companies). ## Relations Intercom joins nothing: a ticket carries an assignee id, and the teammate behind it is a second read -of a second endpoint. What makes eight relations affordable is that every collection on the far end -is read whole in one request — so a relation resolves for a **whole page at the price of one read**, -never one read per row. The price is per target *collection*, not per relation: a ticket's `state` -and `previous_state` are one read of `/ticket_states`, over the ids both of them name. +of a second endpoint. What makes these relations affordable is what the collection on the far end +costs to read by id, and the three tiers do not cost the same: -*Exactly*, with one bound worth naming: "read whole" is what the endpoint answers, and `fetch_all` -stops after [`MAX_COLLECTED_PAGES`](lib/forest_admin_datasource_intercom/client.rb) pages if Intercom -paginates one of these on its own — it logs when it does. A workspace whose `/admins` or `/teams` -runs past that cap resolves the relations pointing at the records it dropped as empty. The figure is -sized for reference collections, which is what every target here is. +| Target | Read by id | A page of rows costs | Fan-out | +| --- | --- | --- | --- | +| `IntercomAdmin`, `IntercomTeam`, `IntercomTicketState`, `IntercomTicketType` | read whole | **one request**, whatever the page holds | unbounded | +| `IntercomContact` | `id IN [...]`, 100 at a time | one request per 100 distinct contacts | unbounded | +| `IntercomCompany` | `GET /companies/{id}` | **one request per distinct account** | bounded, see below | + +The price is per target *collection*, not per relation: a ticket's `state` and `previous_state` are +one read of `/ticket_states`, over the ids both of them name. + +**The one relation with a ceiling is `IntercomContact#company`.** There is no bulk read for a +company — `/companies/scroll` is [deliberately rejected](#companies) — so each distinct account on +the page costs a request, and past +[`MAX_RELATION_READS`](lib/forest_admin_datasource_intercom/collections/offset_collection.rb) +distinct accounts the read is **refused by name** rather than resolved for the first slice and left +nil for the rest. Every page size a list view offers sits under that figure; an export or a segment +resolved whole does not, and the message says to leave the column out or read fewer rows at a time. +The alternative — a nil where an account exists — is the one answer this datasource must not give, +and a log line nobody reads is not a substitute for it. + +*Exactly*, with one more bound worth naming: "read whole" is what the endpoint answers, and +`fetch_all` stops after [`MAX_COLLECTED_PAGES`](lib/forest_admin_datasource_intercom/client.rb) pages +if Intercom paginates one of these on its own — it logs when it does. A workspace whose `/admins` or +`/teams` runs past that cap resolves the relations pointing at the records it dropped as empty. The +figure is sized for reference collections, which is what the first tier above is. | Collection | Relation | Target | Filterable through | | --- | --- | --- | --- | @@ -201,9 +218,12 @@ arrive as a 400 carrying the text. - **No offset pagination, except on companies.** Intercom hands out the page after a cursor and documents that jumping to page N is unsupported, so reaching page 20 costs 20 sequential requests. - The walk is capped at 50 pages / 7 500 records and every truncation is logged, naming the window - it stopped in. `POST /companies/list` is the exception and takes a page number, which is why - companies escape the walker and its caps entirely. + The walk is capped at 50 pages / 7 500 records and **every route out of it that is short of what + was asked for is logged**, naming the window it stopped in: the two caps, and the two defensive + stops — a page that advertises a next cursor and holds nothing, and a cursor already followed. + Intercom does neither of the last two today, which is exactly why they are reported rather than + taken for the end of the data. `POST /companies/list` is the exception and takes a page number, + which is why companies escape the walker and its caps entirely. - **Duplicates on a moving dataset.** Intercom documents that records modified between two paginated requests can be served twice; the walk deduplicates by id. The missed counterpart is inherent to cursor pagination and cannot be repaired — it is documented rather than papered over. @@ -213,7 +233,9 @@ arrive as a 400 carrying the text. Since that is undetectable at runtime, no column of `IntercomConversation`, `IntercomTicket` or `IntercomCompany` is declared sortable and a requested order is reported in the log. The collections read whole sort in memory, exactly, and Contacts sort server-side on the columns the - measured table declares — see [Contacts](#contacts). + measured table declares — see [Contacts](#contacts). One route of Contacts cannot carry it either: + a read by id cuts the window in the order the ids were named, so ordering what comes back would + order a slice picked by something else. That order is reported in the log too. - **No aggregate endpoint.** Counting is free and exact — `total_count` counts what the query names, not what a page held — so the record counter is one request. Anything beyond a count is refused on the cursor collections: grouping over the pages a walk collected would look exact while answering @@ -236,7 +258,7 @@ else is **refused with a message naming what to change** — a condition dropped back as an unfiltered page that looks filtered, which is the one answer this datasource must not give. A refusal costs no request: it is raised before anything leaves the process. -### The table is measured, not documented +### The table is data, and it says where each row comes from The fields a search endpoint filters are not the fields its specification lists. Measured: `/tickets/search` refuses `company_id` with `invalid_field` although every ticket carries one. So @@ -250,11 +272,38 @@ the source of truth is a committed table — `lib/forest_admin_datasource_interc Every `filter_operators` a column publishes is **derived** from that table, so a column cannot advertise a filter the translator would then refuse, and a column the table does not carry -advertises nothing at all. - -To measure a workspace of your own: +advertises nothing at all. That is the mechanism, and it holds whatever the rows say. + +**What the rows say today is mostly `spec`: 19 of 85 are measured, and no endpoint has been probed +end to end** — all three carry `measured_at: null`, which is what `Endpoint#measured?` reports. The +measured rows are the ones a spike went out of its way to check: the date operators on each +endpoint, which disagree between them, and `id IN` on `/contacts/search`. Everything else is +Intercom's documentation, and the disagreement above is why that is a candidate rather than a +promise. + +So the first thing to do against a customer's workspace is to run the probe. The rows worth watching +first, in the order they will hurt: + +1. **`admin_assignee_id` and `team_assignee_id`**, on both search endpoints. Typed `string` here; + Intercom documents them as `Integer` and answers `data_invalid` on a value whose type it does not + accept. These carry the `admin_assignee` and `team_assignee` relations — the filter an ops team + reaches for first — so a wrong type here is the most expensive `spec` row in the file; +2. **`state_id` on `/tickets/search`** — the table carries no filter on it at all, which is what + keeps the `state` relation read-only. If the endpoint does filter one, a support queue becomes + filterable by state; +3. **`contact_ids` on `/tickets/search`** — the contact relation of a ticket rests on it; +4. **the operators Intercom answers on `custom_attributes.{name}`**, per data type, which is the only + thing keeping those columns display-only; +5. **whether `POST /conversations/search` honours `display_as=plaintext`** — it is sent either way, + and an ignored parameter costs a query string where the honoured one saves every filtered row + from coming back as markup. + +To measure a workspace of your own. The probe is a repo tool, not part of the published gem — `bin/` +is excluded from `spec.files` — so it runs from a clone of `agent-ruby`, in this package's +directory: ```bash +cd packages/forest_admin_datasource_intercom INTERCOM_ACCESS_TOKEN=... bin/probe_search_fields --endpoint tickets --out measured.yml ``` @@ -301,10 +350,17 @@ answered by a search: `id equals X` and `id in [...]` read the record endpoint d per record. The search answers it only when something else is filtered alongside it — a permission scope, a segment, or a second filter. -**Free-text search** is answered on `IntercomConversation` only, through `~` on `source.body` — the -message that opened the conversation. Intercom matches it **per word, not as a substring**: searching -`fact` does not find `facture`. `IntercomTicket` exposes no text column this endpoint matches and -refuses a search by name. +**Free-text search** is answered on two collections, each on the one column its endpoint matches text +on: + +| Collection | Searched on | +| --- | --- | +| `IntercomConversation` | `~` on `source.body` — the message that opened the conversation | +| `IntercomContact` | `~` on `email` — what an ops team types when they are looking for someone | + +Intercom matches `~` **per word, not as a substring**: searching `fact` does not find `facture`, and +searching `acme` does not find `camille@acme.test`. `IntercomTicket` exposes no text column its +endpoint matches and refuses a search by name; `IntercomCompany` has no search endpoint at all. ### What is not filterable, and why @@ -315,7 +371,18 @@ refuses a search by name. that adds those collections; - **a contact's `company_id`, `company_count`, `avatar` and `session_count`** — the endpoint filters none of them. Reach the contacts of an account from the account instead, through its `contacts` - relation, which is one request; + relation, which `GET /companies/{id}/contacts` answers in one request — **and answers alone**. That + endpoint returns the contacts of the account whole and narrows nothing, so a `company_id equals X` + carrying anything else cannot be answered at all: there is no request that takes both halves. + Which means the related list of an account **is refused as soon as a permission scope or a segment + is defined on `IntercomContact`**, since that is what the agent intersects into the condition. The + refusal names the condition it could not carry alongside the account. Resolving it properly would + mean reading the account's contact ids first and handing `id IN [...]` plus the rest of the tree to + `/contacts/search` — which that endpoint does answer, and which is not in this lot; +- **a set of ids, counted.** `id in [...]` reads one record per id (100 at a time on contacts), so + counting a set means reading it, and past what a bulk read fetches the count is refused rather + than answered with the number the truncation left. A collection that advertises an exact count does + not answer 25 to a question about forty records; - **the custom attributes of a contact or a company.** They are filtered as `custom_attributes.{name}`, by name — the ambiguity that keeps ticket attributes display-only does not arise here — but which operators Intercom answers on each data type has not been measured, and @@ -494,6 +561,20 @@ so without that route the contacts of an account would be a refusal rather than equality only: an `and` also carrying a permission scope names a narrower set than the account does, and answering it with the account alone would serve contacts the scope excludes. +That is a limit worth knowing before scoping permissions, because it is not a slower route but no +route at all: the account endpoint returns its contacts whole and narrows nothing, and the search +filters no company field, so **a scope or a segment on this collection turns the related list of an +account into a refusal**. The message names the condition it could not carry alongside the account, +rather than telling the operator to open the account and read its contacts — which is what they were +doing. What would answer it is a read of the account's contact ids followed by `id IN [...]` plus the +rest of the tree on `/contacts/search`, which that endpoint takes; it is not in this lot. + +One more thing the two routes do not agree on, and it is visible: the `company_id` column names **the +first** of the accounts a contact belongs to, and the `company` relation resolves that same first +one — while `company_id equals X` returns **every** contact of X. So an account's related list can +show a contact whose `company` points somewhere else. The column is the payload's reading; the filter +is the account endpoint's, and it is the more useful of the two. + **A merged contact reads as gone, not as an error.** Intercom drops it from the listing and from the search, and the record lives on under the id it was merged into. A row pointing at the old id comes back empty rather than failing the page. @@ -518,6 +599,13 @@ A contact carries its accounts as a list of ids and nothing else, so **projectin a contact list costs one request per distinct account on the page**. Reading the account from the contact's record page, or listing contacts from the account, both cost one. +Which is why that projection is the one relation of the datasource with a ceiling: past +[`MAX_RELATION_READS`](lib/forest_admin_datasource_intercom/collections/offset_collection.rb) +distinct accounts on a single read, it is **refused rather than resolved for part of the rows**. Every +page size a list view offers stays under it. A read that does not — an export, which batches a +thousand rows at a time, or a segment resolved whole — has to leave the column out. See +[Relations](#relations). + Custom attributes are introspected at boot the same way, from `GET /data_attributes?model=company`, and published display-only for the same reason. diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom.rb index b90198801..e87d432a6 100644 --- a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom.rb +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom.rb @@ -7,6 +7,18 @@ require 'uri' require 'yaml' require 'zeitwerk' +# `Time.zone` and `Time.use_zone`, which is how a date filter is read in the +# timezone the caller wrote it in -- see `Query::CallerZone`. Required here +# rather than relied on: the toolkit happens to pull ActiveSupport in far +# enough for these to exist, but that is its business and not a contract, and a +# `require` of this gem on its own left them undefined. +# +# Both lines: `active_support/time` brings `Time.zone`, and the bare +# `active_support` brings the autoload table `Time.use_zone` reaches into for +# `ActiveSupport::IsolatedExecutionState`. Without the second, the first raises +# a NameError on the first date filter rather than at load. +require 'active_support' +require 'active_support/time' require 'faraday' require 'faraday/retry' require 'forest_admin_datasource_toolkit' diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/client.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/client.rb index 46b9008d3..47551b209 100644 --- a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/client.rb +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/client.rb @@ -148,7 +148,7 @@ def self.bounded_per_page(size) # The client holds the connections whose headers carry the access token in # clear, and Faraday prints those headers on `inspect`. def inspect - "#<#{self.class.name} url=#{@configuration.url.inspect}>" + "#<#{self.class.name} url=#{@configuration.redacted_url.inspect}>" end private diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/base_collection.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/base_collection.rb index 30d11d7e7..6f08e0067 100644 --- a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/base_collection.rb +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/base_collection.rb @@ -27,6 +27,17 @@ def client datasource.client end + # What one relation read may ask of this collection as its *target*: how + # many ids it resolves in a single `list`, and how many it resolves at + # all. Both nil here -- the tier read whole answers any number of ids for + # the one request it already pays. `Relations` asks the target rather than + # assuming, the cost of an id being the target's business. + # + # Public because that is who calls them: the collection holding the + # relation, on the collection at the other end of it. + def ids_per_read = nil + def max_resolvable_ids = nil + protected def define_schema = raise(NotImplementedError, "#{self.class} did not implement define_schema") @@ -61,6 +72,14 @@ def column_asked?(projection, column) asked.empty? || asked.include?(column) end + # Whether a projection asks for any of a group of columns, read the same + # way: no projection at all asks for every one of them. + def any_column_asked?(projection, columns) + asked = Array(projection).map(&:to_s) + + asked.empty? || columns.any? { |column| asked.include?(column) } + end + # The window a list view asked for, cut out of records already in hand. # # A filter with no page -- or a page naming no limit -- asks for every @@ -116,6 +135,38 @@ def stamp(seconds) Time.at(seconds).utc.iso8601 end + + # --- Reading a sort clause ------------------------------------------- + # + # Forest hands a clause keyed with symbols or with strings depending on + # where it was written, and all three tiers have to read one. + + def primary_key + @primary_key ||= fields.find do |_name, field| + field.respond_to?(:is_primary_key) && field.is_primary_key + end&.first + end + + def sort_field(clause) = clause[:field] || clause['field'] + + # `key?` rather than `||`: a descending clause carries `false`, which an + # `||` fallback reads as "absent" -- so an explicit `?sort=-id` would be + # taken for the ascending default the agent injects. + def ascending?(clause) + clause.key?(:ascending) ? clause[:ascending] : clause['ascending'] + end + + # The ascending primary-key sort the agent injects when a request names + # none. Not an order anybody asked for, so a tier that cannot honour one + # stays quiet about this one. + def default_pk_sort?(clauses) + return false unless clauses.size == 1 + + clause = clauses.first + return false unless sort_field(clause).to_s == primary_key + + ascending?(clause) != false + end end end end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/contact.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/contact.rb index 871591733..f10fb71cf 100644 --- a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/contact.rb +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/contact.rb @@ -49,12 +49,22 @@ def initialize(datasource, attributes: []) enable_search end + # This endpoint answers `id IN [...]`, so a hundred ids cost one request + # rather than a hundred: a relation pointing here resolves a whole page of + # rows for a handful of requests, and there is no fan-out to refuse. That + # is why `max_resolvable_ids` is nil where the other two tiers bound it -- + # the read is proportional to the rows already collected, and the walk + # that collected them is bounded. + def ids_per_read = IDS_PER_READ + def max_resolvable_ids = nil + protected def list_endpoint = 'contacts' def searchable = 'contacts' def search_column = 'email' def match_all_query = MATCH_EVERY_CONTACT + def max_id_reads = MAX_IDS_READ private @@ -142,7 +152,10 @@ def define_relations # `GET /companies/{id}/contacts` can. Anything else goes the usual way. def fetch_records(caller, filter, sort = nil) company = company_lookup(filter) - return super unless company + unless company + refuse_narrowed_company!(filter) if narrowed_company?(filter) + return super + end warn_ignored_sort(Array(filter&.sort)) if sort offset, limit = translate_page(filter&.page) @@ -154,7 +167,10 @@ def fetch_records(caller, filter, sort = nil) def count_records(caller, filter) company = company_lookup(filter) - return super unless company + unless company + refuse_narrowed_company!(filter) if narrowed_company?(filter) + return super + end exact_count(read_company_page(company, per_page: 1, cursor: nil)) end @@ -170,6 +186,45 @@ def company_lookup(filter) tree.value&.to_s end + # The same equality, and something filtered alongside it. There is no + # request that answers both halves: `GET /companies/{id}/contacts` narrows + # nothing it returns, and `/contacts/search` filters no company field at + # all -- which is what the related list of an account runs into the moment + # a permission scope or a segment is defined on this collection. + # + # Refused here rather than left to the translator, whose reason for + # `company_id` is "open the company and read its contacts" -- which is + # exactly what this caller was doing. + def narrowed_company?(filter) + tree = filter&.condition_tree + return false if tree.nil? || tree.is_a?(Leaf) + + tree.some_leaf { |leaf| leaf.field.to_s == 'company_id' && leaf.operator == Operators::EQUAL } + end + + def refuse_narrowed_company!(filter) + raise UnsupportedOperatorError, + "#{name} cannot answer this filter: the contacts of an account are read through " \ + 'GET /companies/{id}/contacts, which returns them whole and narrows nothing, and ' \ + "#{search_endpoint.path} filters no company field -- so the account and the " \ + "#{narrowing_cause(filter)} cannot be asked for in one request. Read the account's contacts " \ + 'without it, or filter on a column this collection is searched on: ' \ + "#{search_endpoint.filterable_columns.join(", ")}." + end + + # A scope and a segment are what put a condition next to the account's in + # practice, and the operator can act on neither the same way -- so the + # message names what it can see rather than guessing. + def narrowing_cause(filter) + others = [] + filter.condition_tree.some_leaf do |leaf| + others << leaf.field.to_s unless leaf.field.to_s == 'company_id' + false + end + + others.empty? ? 'condition filtered alongside it' : "condition on #{others.uniq.join(", ")}" + end + def read_company_page(company, per_page:, cursor:) client.list_page("companies/#{Faraday::Utils.escape(company)}/contacts", per_page: [per_page, max_page_size].min, starting_after: cursor) @@ -180,7 +235,7 @@ def read_company_page(company, per_page:, cursor:) # through. A contact that was merged away is simply absent from the # answer -- the row reads as gone, not as a failure. def records_by_ids(ids) - wanted = ids.first(MAX_IDS_READ) + wanted = ids.first(max_id_reads) warn_truncated_ids(ids.size) if ids.size > wanted.size wanted.each_slice(IDS_PER_READ).flat_map do |chunk| @@ -193,7 +248,7 @@ def records_by_ids(ids) def warn_truncated_ids(asked) ForestAdminDatasourceIntercom.logger.warn( "[forest_admin_datasource_intercom] #{name} was asked for #{asked} records by id and read the first " \ - "#{MAX_IDS_READ}: Intercom reads them #{IDS_PER_READ} at a time. The result is truncated." + "#{max_id_reads}: Intercom reads them #{IDS_PER_READ} at a time. The result is truncated." ) end end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/contact_identity.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/contact_identity.rb index c190f459d..280635b19 100644 --- a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/contact_identity.rb +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/contact_identity.rb @@ -50,7 +50,7 @@ def first_contact_id(record) end def embed_contact_identity(records, rows, projection) - return unless (COLUMNS & projection).any? + return unless any_column_asked?(projection, COLUMNS) identities = contact_identities(records) records.each_with_index do |record, index| diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/conversation.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/conversation.rb index 09c8b390a..bf9060f2a 100644 --- a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/conversation.rb +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/conversation.rb @@ -10,7 +10,7 @@ module Collections # customers, and rendering third-party HTML inside Forest is neither safe nor # useful (R10). # Long by line count only: most of it declares the columns, one call each. - class Conversation < CursorCollection # rubocop:disable Metrics/ClassLength + class Conversation < CursorCollection include ContactIdentity include Conversation::Serializer # The shared thread, then the hooks this collection puts over it. @@ -49,10 +49,8 @@ def read_params = { 'display_as' => 'plaintext' } # names it: neither is on the conversation payload, and a page that never # asked for them must not pay for them. def enrich(records, rows, projection) - wanted = Array(projection).map(&:to_s) - - embed_contact_identity(records, rows, wanted) - embed_timeline(records, rows, wanted) + embed_contact_identity(records, rows, projection) + embed_timeline(records, rows, projection) end private @@ -145,7 +143,7 @@ def define_statistics_columns # its timeline is free; one read from the listing does not, and pays a # request. Rows past the cap keep the nil the projection put there. def embed_timeline(records, rows, projection) - return unless projection.include?('timeline') + return unless column_asked?(projection, 'timeline') budget = MAX_TIMELINE_READS missing = 0 diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/cursor_collection.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/cursor_collection.rb index cb7ec67d9..cbb637cad 100644 --- a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/cursor_collection.rb +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/cursor_collection.rb @@ -25,12 +25,7 @@ module Collections # Long by line count only: half of it is the refusals, and a refusal that # does not say what to do instead is a refusal an operator cannot act on. class CursorCollection < BaseCollection # rubocop:disable Metrics/ClassLength - Aggregation = ForestAdminDatasourceToolkit::Components::Query::Aggregation - - # How many records an `id in [...]` read may fetch. One request per id -- - # Intercom has no "read these records" endpoint -- so the fan-out is - # bounded rather than turned into a rate limit halfway through a page. - MAX_ID_READS = 25 + include RecordsById # Countable, and exactly: unlike the pages a walk collected, `total_count` # is the whole dataset the filter names. @@ -39,6 +34,13 @@ def initialize(datasource, name) enable_count end + # One request per id here, so a relation pointing at this tier resolves a + # slice at a time and is refused past what a slice may hold -- see + # `Relations#target_rows`. The collection that reads its ids in bulk + # widens both; see `Contact`. + def ids_per_read = max_id_reads + def max_resolvable_ids = max_id_reads + def list(caller, filter, projection) records = fetch_records(caller, filter, server_sort(filter)) # Serialized whole and projected afterwards rather than the other way @@ -166,6 +168,11 @@ def walker @walker ||= Pagination::CursorWalker.new end + # How many records a bulk read by id may fetch. A hook rather than the + # constant, since the collection that reads its ids in bulk answers far + # more of them for the same request count. + def max_id_reads = MAX_ID_READS + private def column_operators(name, is_primary_key) @@ -183,7 +190,16 @@ def fetch_records(caller, filter, sort = nil) # would pay for a whole page to hand back a slice of it -- and page 2 of # a set larger than the cap would come back empty, the records it names # having been dropped by the truncation before the window was applied. - return records_by_ids(page_window(ids, filter)) if ids + if ids + # Which is also why an order cannot be honoured on this route: the + # window is cut in the order the ids were named, so sorting what comes + # back would order a slice picked by something else. Reported rather + # than dropped in silence, like every other order this tier cannot + # apply -- `server_sort` stayed quiet, having found a column Intercom + # does sort. + warn_unordered_ids(sort) if sort + return records_by_ids(page_window(ids, filter)) + end query = translate(caller, filter) # A condition through a relation the target matched no record with names @@ -281,46 +297,10 @@ def search_condition(filter) Leaf.new(search_column, Operators::CONTAINS, filter.search.to_s.strip) end - def blank_search?(filter) - search = filter.respond_to?(:search) ? filter.search : nil - search.nil? || search.to_s.strip.empty? - end - - # A record detail is `id equals X`, and a bulk read of related records is - # `id in [...]`. Only a bare leaf on the primary key takes this route: an - # `and` also carrying a scope names a narrower set than the ids do, and - # answering it with the ids alone would serve records the scope excludes. - def id_lookup(filter) - tree = filter&.condition_tree - return nil unless tree.is_a?(Leaf) && tree.field.to_s == primary_key - return nil unless blank_search?(filter) - - case tree.operator - when Operators::EQUAL then [tree.value].compact.map(&:to_s) - when Operators::IN then Array(tree.value).compact.map(&:to_s) - end - end - - def primary_key - @primary_key ||= fields.find do |_name, field| - field.respond_to?(:is_primary_key) && field.is_primary_key - end&.first - end - - # A record the operator can no longer reach -- deleted, or outside the - # token's scope -- reads as "no record" rather than as a failed page. - def records_by_ids(ids) - wanted = ids.first(MAX_ID_READS) - warn_truncated_ids(ids.size) if ids.size > wanted.size - - wanted.filter_map do |id| - client.fetch_record(record_endpoint, id, params: read_params) - rescue APIError => e - raise unless e.status == 404 - - nil - end - end + # The records of this tier carry bodies written by end customers, so every + # read of one asks Intercom for plain text (R10). `BaseCollection` reads + # this on the way to the record endpoint. + def record_read_params = read_params def listed_records(filter, query, sort = nil) offset, limit = translate_page(filter&.page) @@ -344,7 +324,7 @@ def translate_page(page) # which is cheaper still. def count_records(caller, filter) ids = id_lookup(filter) - return records_by_ids(ids).size if ids + return count_by_ids(ids) if ids query = translate(caller, filter) return 0 if query == NOTHING @@ -352,27 +332,6 @@ def count_records(caller, filter) exact_count(read_page(per_page: 1, cursor: nil, query: query)) end - # The count Intercom answered, or nothing at all. Counting the pages a - # walk collected would answer a fraction of the collection as if it were - # the whole of it, which is the one thing this tier does not do. - def exact_count(page) - return page.total_count if page.total_count - - raise UnsupportedOperatorError, - "#{name} cannot be counted: Intercom answered this listing without a total_count, and counting the " \ - 'pages the agent walked would answer a fraction of the collection as if it were the whole of it.' - end - - def refuse_unsupported_aggregation!(aggregation) - return if aggregation.is_a?(Aggregation) && aggregation.operation.to_s.casecmp('count').zero? && - Array(aggregation.groups).empty? && aggregation.field.nil? - - raise UnsupportedOperatorError, - "#{name} can only be counted: Intercom exposes no aggregate endpoint, and grouping or summing the " \ - 'pages the agent walked would answer a fraction of the collection as if it were the whole of it. ' \ - 'Chart it on a collection read whole, or wait for the bounded group-by of the reporting lot.' - end - def refuse_search! raise UnsupportedOperatorError, "#{name} cannot answer a free-text search: #{search_endpoint.path} matches values field by field, " \ @@ -421,29 +380,14 @@ def warn_ignored_sort(clauses) ) end - def sort_field(clause) = clause[:field] || clause['field'] - - # `key?` rather than `||`: a descending clause carries `false`, which an - # `||` fallback reads as "absent" -- so an explicit `?sort=-id` would be - # taken for the ascending default the agent injects, and the one order - # Intercom silently drops would go unreported. - def ascending?(clause) - clause.key?(:ascending) ? clause[:ascending] : clause['ascending'] - end - - def default_pk_sort?(clauses) - return false unless clauses.size == 1 - - clause = clauses.first - return false unless sort_field(clause).to_s == primary_key - - ascending?(clause) != false - end - - def warn_truncated_ids(asked) + # `sort` here is the clause `server_sort` found the endpoint does honour, + # which is why nothing has reported it yet: it is this route, not the + # column, that cannot carry it. + def warn_unordered_ids(sort) ForestAdminDatasourceIntercom.logger.warn( - "[forest_admin_datasource_intercom] #{name} was asked for #{asked} records by id and read the first " \ - "#{MAX_ID_READS}: Intercom reads them one request each. The result is truncated." + "[forest_admin_datasource_intercom] #{name} was asked to sort on #{sort[:field].inspect} while reading " \ + 'records by id, which Intercom reads by id and in no order. The rows come back in the order the ids ' \ + 'were named.' ) end end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/fetch_all_collection.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/fetch_all_collection.rb index 2cb9d8b30..c3d39180f 100644 --- a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/fetch_all_collection.rb +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/fetch_all_collection.rb @@ -184,16 +184,9 @@ def sort_clauses(sort) known, unknown = Array(sort).partition { |clause| fields.key?(sort_field(clause)) } warn_unsortable(unknown) unless unknown.empty? - known.map do |clause| - # `key?` rather than `||`: a descending clause carries `false`, which an - # `||` fallback would read as "absent" and turn back into ascending. - ascending = clause.key?(:ascending) ? clause[:ascending] : clause['ascending'] - [sort_field(clause), ascending != false] - end + known.map { |clause| [sort_field(clause), ascending?(clause) != false] } end - def sort_field(clause) = clause[:field] || clause['field'] - def warn_unsortable(clauses) ForestAdminDatasourceIntercom.logger.warn( "[forest_admin_datasource_intercom] #{name} was asked to sort on " \ diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/offset_collection.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/offset_collection.rb index 3c11ddd7e..e1247b94d 100644 --- a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/offset_collection.rb +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/offset_collection.rb @@ -20,13 +20,7 @@ module Collections # Long by line count only: half of it is the refusals, and a refusal that # does not say what to do instead is one an operator cannot act on. class OffsetCollection < BaseCollection # rubocop:disable Metrics/ClassLength - Aggregation = ForestAdminDatasourceToolkit::Components::Query::Aggregation - - # How many records an `id in [...]` read may fetch. One request per id -- - # Intercom has no "read these records" endpoint here either -- so the - # fan-out is bounded rather than turned into a rate limit halfway through - # a page. - MAX_ID_READS = 25 + include RecordsById # What one page holds when the read names no window: a relation resolving # its target, a segment, a customizer. A list view always names one. @@ -36,11 +30,31 @@ class OffsetCollection < BaseCollection # rubocop:disable Metrics/ClassLength # figure only ever applies to a read with no window of its own. MAX_COLLECTED_PAGES = 10 + # What a read that *does* name a window may collect before it is cut + # short. A window is its own bound, so the page count above must not apply + # to it: a window needing eleven pages would come back one page short + # while the warning blamed it for naming no window -- which it did name. + # This is the backstop for a window nothing sane asked for, and it is the + # cursor walker's own record budget, that tier being bounded the same way. + MAX_COLLECTED_RECORDS = Pagination::CursorWalker::MAX_RECORDS + + # How many records of this collection one relation read may resolve. + # Higher than `MAX_ID_READS`, which bounds a single request batch, and + # deliberately above every page size a list view offers: a page naming one + # account per row resolves, and a read naming more -- an export, a + # segment resolved whole -- is refused by name rather than answered with + # a nil where an account exists. There is no bulk read for a company, so + # each one costs a request and the figure is what an operator waits for. + MAX_RELATION_READS = 100 + def initialize(datasource, name) super enable_count end + def ids_per_read = MAX_ID_READS + def max_resolvable_ids = MAX_RELATION_READS + def list(caller, filter, projection) warn_ignored_sort(filter&.sort) @@ -110,7 +124,7 @@ def fetch_records(filter) def count_records(filter) ids = id_lookup(filter) - return records_by_ids(ids).size if ids + return count_by_ids(ids) if ids lookup = lookup_condition(filter) return looked_up_records(lookup).size if lookup @@ -145,7 +159,7 @@ def collect_pages(first_page:, per_page:, wanted:) records.concat(answer.records) read += 1 break if last_page?(answer, page) || (wanted && records.size >= wanted) - break if cap_reached?(read, records.size) + break if cap_reached?(read, records.size, wanted) page += 1 end @@ -161,14 +175,31 @@ def last_page?(answer, page) answer.records.empty? || (answer.total_pages && page >= answer.total_pages) end - def cap_reached?(read, collected) - return false if read < MAX_COLLECTED_PAGES + # Which of the two caps applies, and the reason that goes with it: a read + # with no window of its own is stopped by the page count, a read that + # named one by the record budget. Telling a list view it named no window + # is exactly the confusion these two keep apart. + def cap_reached?(read, collected, wanted) + if wanted + return false if collected < MAX_COLLECTED_RECORDS + + warn_capped("#{collected} record(s)", + 'this read named a window larger than one answer may hold') + else + return false if read < MAX_COLLECTED_PAGES + + warn_capped("#{read} page(s) / #{collected} record(s)", + 'this read named no window of its own, and a list view always does') + end + true + end + + def warn_capped(reached, reason) ForestAdminDatasourceIntercom.logger.warn( - "[forest_admin_datasource_intercom] Stopped reading #{name} after #{read} page(s) / #{collected} " \ - 'record(s); the rest is left out. This read named no window of its own, and a list view always does.' + "[forest_admin_datasource_intercom] Stopped reading #{name} after #{reached}; the rest is left out: " \ + "#{reason}." ) - true end # A filter with no page asks for every record it matched; nil is how the @@ -180,19 +211,6 @@ def window(page) [page.offset.to_i.clamp(0, nil), limit.positive? ? limit : nil] end - # A record detail is `id equals X`, and a bulk read of related records is - # `id in [...]`. Only a bare leaf on the primary key takes this route: an - # `and` also carrying a scope names a narrower set than the ids do. - def id_lookup(filter) - tree = filter&.condition_tree - return nil unless tree.is_a?(Leaf) && tree.field.to_s == primary_key - - case tree.operator - when Operators::EQUAL then [tree.value].compact.map(&:to_s) - when Operators::IN then Array(tree.value).compact.map(&:to_s) - end - end - def lookup_condition(filter) tree = filter&.condition_tree return nil unless tree.is_a?(Leaf) && tree.operator == Operators::EQUAL @@ -201,27 +219,6 @@ def lookup_condition(filter) parameter && { parameter => tree.value.to_s } end - def primary_key - @primary_key ||= fields.find do |_name, field| - field.respond_to?(:is_primary_key) && field.is_primary_key - end&.first - end - - # A record the operator can no longer reach -- deleted, or outside the - # token's scope -- reads as "no record" rather than as a failed page. - def records_by_ids(ids) - wanted = ids.first(MAX_ID_READS) - warn_truncated_ids(ids.size) if ids.size > wanted.size - - wanted.filter_map do |id| - client.fetch_record(record_endpoint, id) - rescue APIError => e - raise unless e.status == 404 - - nil - end - end - # An exact lookup answers few records -- one, for the keys this publishes # -- so it is read as a single page. More than that page holds is reported # rather than dropped in silence. @@ -232,14 +229,6 @@ def looked_up_records(params) answer.records end - def exact_count(page) - return page.total_count if page.total_count - - raise UnsupportedOperatorError, - "#{name} cannot be counted: Intercom answered this listing without a total_count, and counting the " \ - 'pages the agent read would answer a fraction of the collection as if it were the whole of it.' - end - def refuse_condition!(tree) offender = nil tree.some_leaf { |leaf| offender = leaf } @@ -251,15 +240,6 @@ def refuse_condition!(tree) 'collection next door.' end - def refuse_unsupported_aggregation!(aggregation) - return if aggregation.is_a?(Aggregation) && aggregation.operation.to_s.casecmp('count').zero? && - Array(aggregation.groups).empty? && aggregation.field.nil? - - raise UnsupportedOperatorError, - "#{name} can only be counted: Intercom exposes no aggregate endpoint, and grouping or summing the " \ - 'pages the agent read would answer a fraction of the collection as if it were the whole of it.' - end - # Intercom takes no order on this listing at all -- there is no parameter # for one -- so an order asked for and not applied is reported here or # nowhere. The ascending primary-key sort the agent injects when a request @@ -270,28 +250,11 @@ def warn_ignored_sort(sort) ForestAdminDatasourceIntercom.logger.warn( "[forest_admin_datasource_intercom] #{name} was asked to sort on " \ - "#{clauses.map { |clause| clause[:field] || clause["field"] }.join(", ")}, and Intercom takes no order " \ + "#{clauses.map { |clause| sort_field(clause) }.join(", ")}, and Intercom takes no order " \ 'on this listing. The rows come back in the order the API imposes.' ) end - def default_pk_sort?(clauses) - return false unless clauses.size == 1 - - clause = clauses.first - return false unless (clause[:field] || clause['field']).to_s == primary_key - - ascending = clause.key?(:ascending) ? clause[:ascending] : clause['ascending'] - ascending != false - end - - def warn_truncated_ids(asked) - ForestAdminDatasourceIntercom.logger.warn( - "[forest_admin_datasource_intercom] #{name} was asked for #{asked} records by id and read the first " \ - "#{MAX_ID_READS}: Intercom reads them one request each. The result is truncated." - ) - end - def warn_truncated_lookup(params) ForestAdminDatasourceIntercom.logger.warn( "[forest_admin_datasource_intercom] #{name} looked up #{params.inspect} and Intercom advertised more " \ diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/records_by_id.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/records_by_id.rb new file mode 100644 index 000000000..23e56ed20 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/records_by_id.rb @@ -0,0 +1,126 @@ +module ForestAdminDatasourceIntercom + module Collections + # `id equals X` is a record detail and `id in [...]` a bulk read of related + # records, and both are answered by the record endpoint rather than by + # whatever the collection is filtered through. What the two paginated tiers + # share, and what the tier read whole has no use for -- it holds every + # record already, so an id is a lookup in memory rather than a request. + # + # Which is also why the counting lives here: a count over the pages a read + # collected would answer a fraction of the collection as if it were the + # whole of it, and that risk only exists where what is in hand is a page. + module RecordsById + Operators = ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Operators + Leaf = ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Nodes::ConditionTreeLeaf + Aggregation = ForestAdminDatasourceToolkit::Components::Query::Aggregation + + # How many records a bulk read by id may fetch. Intercom exposes no "read + # these records" endpoint, so a set of ids is one request each -- the + # fan-out is bounded rather than turned into a rate limit halfway through + # a page. The collection that reads its ids in bulk raises the figure. + MAX_ID_READS = 25 + + protected + + # The endpoint one record is read from. + def record_endpoint = raise(NotImplementedError, "#{self.class} did not implement record_endpoint") + + # What rides along on that read. `display_as=plaintext` for the tiers + # whose records carry bodies written by end customers; nothing elsewhere. + def record_read_params = {} + + # How many records a bulk read may fetch, as a hook rather than the + # constant: the collection that reads its ids in bulk answers far more of + # them for the same request count. + def max_id_reads = MAX_ID_READS + + # Only a bare leaf on the primary key takes this route: an `and` also + # carrying a scope names a narrower set than the ids do, and answering it + # with the ids alone would serve records the scope excludes. A free-text + # search alongside is the same problem -- the ids would answer a question + # nobody asked. + # + # Deduplicated: Intercom reads a record by id, so a value named twice + # would be fetched twice and handed back as two rows carrying one id -- + # where the `in` this comes from is a membership, matching a record once. + def id_lookup(filter) + tree = filter&.condition_tree + return nil unless tree.is_a?(Leaf) && tree.field.to_s == primary_key + return nil unless blank_search?(filter) + + case tree.operator + when Operators::EQUAL then [tree.value].compact.map(&:to_s).uniq + when Operators::IN then Array(tree.value).compact.map(&:to_s).uniq + end + end + + # A record the operator can no longer reach -- deleted, or outside the + # token's scope -- reads as "no record" rather than as a failed page. + def records_by_ids(ids) + wanted = ids.first(max_id_reads) + warn_truncated_ids(ids.size) if ids.size > wanted.size + + wanted.filter_map do |id| + client.fetch_record(record_endpoint, id, params: record_read_params) + rescue APIError => e + raise unless e.status == 404 + + nil + end + end + + # How many of a set of ids name a record, which means reading them: + # Intercom has no "how many of these exist" endpoint. Past what a bulk + # read may fetch the count is refused rather than answered with the number + # the truncation left -- both paginated tiers advertise an exact count, + # and "25" where the question named forty records is not one. + def count_by_ids(ids) + refuse_id_count!(ids.size) if ids.size > max_id_reads + + records_by_ids(ids).size + end + + # The count Intercom answered, or nothing at all. Counting the pages a + # read collected would answer a fraction of the collection as if it were + # the whole of it, which is the one thing the paginated tiers do not do. + def exact_count(page) + return page.total_count if page.total_count + + raise UnsupportedOperatorError, + "#{name} cannot be counted: Intercom answered this listing without a total_count, and counting the " \ + 'pages the agent read would answer a fraction of the collection as if it were the whole of it.' + end + + def refuse_unsupported_aggregation!(aggregation) + return if aggregation.is_a?(Aggregation) && aggregation.operation.to_s.casecmp('count').zero? && + Array(aggregation.groups).empty? && aggregation.field.nil? + + raise UnsupportedOperatorError, + "#{name} can only be counted: Intercom exposes no aggregate endpoint, and grouping or summing the " \ + 'pages the agent read would answer a fraction of the collection as if it were the whole of it. ' \ + 'Chart it on a collection read whole, or wait for the bounded group-by of the reporting lot.' + end + + def refuse_id_count!(asked) + raise UnsupportedOperatorError, + "#{name} cannot count #{asked} records by id: Intercom exposes no endpoint that counts a set of ids, " \ + "so counting them means reading them, and this reads #{max_id_reads} at most. Counting the ones it " \ + 'read would answer a number smaller than the question. Narrow the condition, or count the ' \ + 'collection with a filter it answers instead of a list of ids.' + end + + def warn_truncated_ids(asked) + ForestAdminDatasourceIntercom.logger.warn( + "[forest_admin_datasource_intercom] #{name} was asked for #{asked} records by id and read the first " \ + "#{max_id_reads}: Intercom reads them one request each. The result is truncated." + ) + end + + def blank_search?(filter) + search = filter.respond_to?(:search) ? filter.search : nil + + search.nil? || search.to_s.strip.empty? + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/relations.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/relations.rb index 22e1f731e..48ba492cc 100644 --- a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/relations.rb +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/relations.rb @@ -151,14 +151,33 @@ def relations_asked(projection) # not it was asked for: it is what the rows are indexed by here, and what # makes the nested row a link rather than a label in the interface. def many_to_one_asked(projection) - relations_asked(projection).filter_map do |name, sub_projection| - relation = fields[name] + relations_asked(projection).filter_map do |relation_name, sub_projection| + relation = fields[relation_name] next unless relation.is_a?(ManyToOneSchema) - [name, relation, Array(sub_projection).map(&:to_s).union([relation.foreign_key_target])] + columns = Array(sub_projection).map(&:to_s) + refuse_two_hop_projection!(relation_name, relation, columns) + + [relation_name, relation, columns.union([relation.foreign_key_target])] end end + # `admin_assignee:teams:name` is a projection Forest's own parser builds + # and this cannot answer: it nests one target row under the relation name, + # not a tree of them, so the second hop would be dropped by the target's + # projection and the column would come back missing from a row that looks + # complete. Refused by name, like the filter that reaches that deep -- + # `refuse_two_hops!` is its counterpart. + def refuse_two_hop_projection!(relation_name, relation, columns) + deep = columns.select { |column| column.include?(':') } + return if deep.empty? + + raise UnsupportedOperatorError, + "#{name} cannot read #{deep.map { |column| "#{relation_name}:#{column}".inspect }.join(", ")}: it " \ + 'reaches through two relations, and this datasource resolves one. Read the column from a record ' \ + "page of #{relation.foreign_collection}, one hop away." + end + def indexed_targets(caller, records, asked) asked.group_by { |_, relation, _| relation.foreign_collection } .transform_values { |group| target_rows(caller, records, group) } @@ -166,16 +185,56 @@ def indexed_targets(caller, records, asked) # One read per target collection, over the ids every relation pointing at # it names and the union of the columns they asked for. + # + # The ids are sliced by what the target resolves in one read, and the + # fan-out is refused past what it resolves at all. Neither is a detail: + # a bulk read by id is bounded on every tier -- Intercom offers no "read + # these records" endpoint, so one of them pays a request per id -- and + # handing the whole page's keys to that bound would resolve the first + # slice and leave the rest of the rows carrying a nil where a record + # exists. A relation answered for part of a page is the failure this + # datasource is built to avoid, so it is named instead. def target_rows(caller, records, group) relation = group.first[1] ids = group.flat_map { |_, rel, _| records.filter_map { |record| record[rel.foreign_key] } }.uniq return {} if ids.empty? + collection = foreign_collection(relation) + refuse_relation_fan_out!(group, collection, ids.size) if beyond_reach?(collection, ids.size) + target = relation.foreign_key_target wanted = Projection.new(group.flat_map { |_, _, columns| columns }.uniq) - filter = Filter.new(condition_tree: Leaf.new(target, Operators::IN, ids)) - foreign_collection(relation).list(caller, filter, wanted).to_h { |row| [row[target], row] } + read_targets(caller, collection, target, ids, wanted).to_h { |row| [row[target], row] } + end + + def beyond_reach?(collection, count) + limit = collection.max_resolvable_ids + + !limit.nil? && count > limit + end + + def read_targets(caller, collection, target, ids, wanted) + chunk = collection.ids_per_read + slices = chunk ? ids.each_slice(chunk).to_a : [ids] + + slices.flat_map do |slice| + collection.list(caller, Filter.new(condition_tree: Leaf.new(target, Operators::IN, slice)), wanted) + end + end + + # Names the relations rather than the key they resolve to: what the + # operator can act on is the column they put in the view or the export, + # and how many rows they asked for at once. + def refuse_relation_fan_out!(group, collection, count) + asked = group.map { |relation_name, _, _| relation_name }.join(', ') + + raise UnsupportedOperatorError, + "#{name} cannot resolve #{asked} over this read: it names #{count} distinct #{collection.name} " \ + 'records and Intercom has no bulk read for that collection, so this resolves at most ' \ + "#{collection.max_resolvable_ids} of them per read -- one request each. Read fewer rows at a time, " \ + 'or leave the relation out of the projection: a relation resolved for part of a page would show no ' \ + 'record where one exists.' end def rewrite_branch(caller, branch, &builder) diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/ticket.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/ticket.rb index 533d5f3ed..3afc81497 100644 --- a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/ticket.rb +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/ticket.rb @@ -61,11 +61,9 @@ def searchable_only? = true def match_all_query = MATCH_EVERY_TICKET def enrich(records, rows, projection) - wanted = Array(projection).map(&:to_s) - - embed_contact_identity(records, rows, wanted) - embed_derived_columns(records, rows, wanted) - embed_timeline(records, rows, wanted) + embed_contact_identity(records, rows, projection) + embed_derived_columns(records, rows, projection) + embed_timeline(records, rows, projection) end private @@ -152,7 +150,7 @@ def define_relations # read from a listing carries no parts at all and its timeline stays nil, # which reads as unknown. def embed_timeline(records, rows, projection) - return unless projection.include?('timeline') + return unless column_asked?(projection, 'timeline') records.each_with_index { |record, index| rows[index]['timeline'] = build_timeline(record) } end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/ticket/derived_columns.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/ticket/derived_columns.rb index 41911f280..db18cb873 100644 --- a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/ticket/derived_columns.rb +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/ticket/derived_columns.rb @@ -31,6 +31,10 @@ module DerivedColumns # the person waiting. REPLY_PART = 'comment'.freeze + # The columns a truncated timeline leaves unknown rather than empty, and + # therefore the ones whose presence in a projection is worth a warning. + CLOSURE_COLUMNS = %w[closed_at closed_by_name].freeze + private def define_derived_columns @@ -60,7 +64,7 @@ def derived_columns_for(attrs) # operator when a value is missing because the timeline was truncated # rather than because the event never happened. def embed_derived_columns(records, _rows, projection) - return unless (%w[closed_at closed_by_name] & projection).any? + return unless any_column_asked?(projection, CLOSURE_COLUMNS) unknown = records.count { |record| closure_unknown?(record) } warn_unknown_closures(unknown) if unknown.positive? diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/configuration.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/configuration.rb index 65a3013c8..fa7c771b4 100644 --- a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/configuration.rb +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/configuration.rb @@ -12,6 +12,10 @@ class Configuration DEFAULT_REGION = :us + # Where a cleartext base_url reaches nobody else's network, so the bearer + # header crossing it in clear is not worth a word. + LOOPBACK_HOSTS = %w[localhost 127.0.0.1 ::1 [::1] 0.0.0.0].freeze + # Without an explicit version a request follows the workspace's own default, # which an operator can change on Intercom's side -- and the payloads change # shape under us. Pinned to what the spike ran against; 2.14 and 2.16 both @@ -60,6 +64,15 @@ def base_path @base_path ||= URI.parse(url).path end + # The url with any credentials in it masked. `URI` accepts them -- + # `https://user:pass@proxy/...` -- and a credentialed egress proxy is one of + # the two reasons to set a `base_url` at all, so printing the url verbatim + # would put a second secret exactly where this class keeps the first one + # from going. Read by `Client#inspect` too, which prints the same url. + def redacted_url + @redacted_url ||= url.sub(%r{\A([a-zA-Z][\w+.-]*://)[^/@]*@}, '\1[FILTERED]@') + end + # `access_token` is a bearer credential, and nothing prints a Configuration # on purpose: what reaches an `inspect` is a Rails error page, or a # `logger.debug` of something holding one. The default would put the token @@ -67,7 +80,8 @@ def base_path # reason -- together they cut every path from an object this package hands # out to the credential. def inspect - "#<#{self.class.name} url=#{url.inspect} api_version=#{@api_version.inspect} access_token=[FILTERED]>" + "#<#{self.class.name} url=#{redacted_url.inspect} api_version=#{@api_version.inspect} " \ + 'access_token=[FILTERED]>' end private @@ -96,7 +110,7 @@ def validate_base_url! return if @base_url.nil? uri = URI.parse(@base_url) - return if uri.is_a?(URI::HTTP) && !blank?(uri.host) + return warn_cleartext!(uri) if uri.is_a?(URI::HTTP) && !blank?(uri.host) raise ConfigurationError, "ForestAdminDatasourceIntercom base_url must be an absolute http(s) url, got #{@base_url.inspect}" @@ -105,6 +119,26 @@ def validate_base_url! "ForestAdminDatasourceIntercom base_url is not a valid url: #{@base_url.inspect}" end + # `URI::HTTPS` is a `URI::HTTP`, so plain http passes above -- and it is + # worth passing: a mock server in a test suite is one, and that is half of + # what `base_url` is for. What it costs is `Authorization: Bearer ` + # travelling in clear to whatever sits at the other end, and that token + # reads the whole workspace. Named rather than allowed in silence; a + # loopback host is nobody else's network, so it says nothing there. + def warn_cleartext!(uri) + return if uri.scheme == 'https' || loopback?(uri.host) + + ForestAdminDatasourceIntercom.logger.warn( + "[forest_admin_datasource_intercom] base_url #{redacted_url.inspect} is not https, and every request " \ + 'carries the Intercom access token as a bearer header. Anything on the path can read it and use it ' \ + 'against the workspace. Use https, or terminate TLS before the network this crosses.' + ) + end + + def loopback?(host) + LOOPBACK_HOSTS.include?(host.to_s.downcase) || host.to_s.downcase.end_with?('.localhost') + end + def blank?(value) value.nil? || value.to_s.strip.empty? end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/pagination/cursor_walker.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/pagination/cursor_walker.rb index 5132f0d8a..e701c3bc4 100644 --- a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/pagination/cursor_walker.rb +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/pagination/cursor_walker.rb @@ -62,13 +62,12 @@ def collect(offset, limit) records.concat(fresh(page.records, seen_ids)) pages += 1 - break if stop?(page, seen_cursors) + # The only silent exits: Intercom said there is no page left, or the + # window asked for is covered. Everything below hands back less than + # was asked for, and says so. + break if page.next_cursor.nil? break if needed && records.size >= needed - - if capped?(pages, records.size) - log_truncation(offset: offset, limit: limit, pages: pages, collected: records.size) - break - end + break if cut_short?(page, seen_cursors, pages, records: records.size, window: window_of(offset, limit)) cursor = page.next_cursor end @@ -76,6 +75,23 @@ def collect(offset, limit) records end + # Whether the walk is handing back less than it was asked for, and which + # of the two reasons it is. Both are logged here rather than by the + # caller: a page that looks like the whole answer and is not is the + # failure this datasource exists to avoid, so there is no route out of + # this walk that is short and quiet. + def cut_short?(page, seen_cursors, pages, records:, window:) + if stalled?(page, seen_cursors) + log_stalled(window: window, pages: pages, collected: records) + elsif capped?(pages, records) + log_truncation(window: window, pages: pages, collected: records) + else + return false + end + + true + end + # Intercom documents that "if items are modified between paginated # requests it is possible to see duplicate or missed records" -- and # conversations move constantly, so a deep walk over them will see the @@ -90,14 +106,18 @@ def fresh(records, seen_ids) records.select { |record| record['id'].nil? || seen_ids.add?(record['id']) } end - # An empty page, a cursor that does not move and a cursor already followed - # all stop the walk. Intercom does none of the three today -- `pages.next` - # is simply absent on the last page -- but a walk driven by a remote value - # stops on its own terms rather than on the caps only: a cycle wider than - # one page would otherwise collect the same pages until a cap cut it - # short. - def stop?(page, seen_cursors) - page.next_cursor.nil? || page.records.empty? || !seen_cursors.add?(page.next_cursor) + # A page that advertises a next cursor and holds nothing, and a cursor + # already followed. Intercom does neither today -- `pages.next` is simply + # absent on the last page -- but a walk driven by a remote value stops on + # its own terms rather than on the caps only: a cycle wider than one page + # would otherwise collect the same pages until a cap cut it short. + # + # Which is exactly why it is reported. Stopping here means Intercom said + # there was more and this could not follow it, so the answer is short of + # what was asked -- a truncation like the caps below, and the same rule + # applies: never silent. + def stalled?(page, seen_cursors) + page.records.empty? || !seen_cursors.add?(page.next_cursor) end def capped?(pages, collected) @@ -113,14 +133,25 @@ def batch_size(needed, collected) Client.bounded_per_page(budget) end - def log_truncation(offset:, limit:, pages:, collected:) - window = limit ? "offset=#{offset} limit=#{limit}" : "every record past offset=#{offset}" + def log_truncation(window:, pages:, collected:) ForestAdminDatasourceIntercom.logger.warn( "[forest_admin_datasource_intercom] Stopped paginating after #{pages} page(s) / " \ "#{collected} record(s) while fetching #{window}; results are truncated. " \ 'Narrow the filter to reach records past this point.' ) end + + def log_stalled(window:, pages:, collected:) + ForestAdminDatasourceIntercom.logger.warn( + "[forest_admin_datasource_intercom] Stopped paginating after #{pages} page(s) / " \ + "#{collected} record(s) while fetching #{window}: Intercom advertised a next page this could not " \ + 'follow -- an empty page, or a cursor already read. Results are truncated.' + ) + end + + def window_of(offset, limit) + limit ? "offset=#{offset} limit=#{limit}" : "every record past offset=#{offset}" + end end end end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/query/condition_tree_translator.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/query/condition_tree_translator.rb index aa33b085d..60ecdbd56 100644 --- a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/query/condition_tree_translator.rb +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/query/condition_tree_translator.rb @@ -139,13 +139,16 @@ def unfilterable_reason(column) "#{@endpoint.path} takes no filter on it. Filter on one of: #{@endpoint.filterable_columns.join(", ")}." end - # A relation reaches the translator as `relation:field`. None of the - # collections this endpoint serves declares one yet, so the condition can - # only come from a scope or a segment written against a schema this - # datasource does not have. - def relation_reason(column) - "#{@collection} declares no relation, so #{column.inspect} names a field it cannot reach. Filter on one " \ - "of its own columns: #{@endpoint.filterable_columns.join(", ")}." + # A relation reaches the translator as `relation:field`, and it should not: + # `Relations#rewrite_relation_conditions` trades every one of them for a + # condition on the foreign key before a tree gets this far, or refuses it + # by name -- with the relations the collection does declare. So this is + # the message for a caller that skipped that pass, and it names what this + # endpoint filters rather than a relation nobody resolved. + def relation_reason(_column) + "#{@endpoint.path} filters columns, not paths through a relation -- those are resolved against the " \ + 'collection they point at before a condition reaches here. Filter on one of: ' \ + "#{@endpoint.filterable_columns.join(", ")}." end def refuse_operator!(leaf, field) diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/query/filter_value.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/query/filter_value.rb index c14be8fc3..9940489b6 100644 --- a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/query/filter_value.rb +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/query/filter_value.rb @@ -14,6 +14,9 @@ class FilterValue # timezone of whoever wrote the filter rather than in the server's. DATE_ONLY = /\A\d{4}-\d{2}-\d{2}\z/ + # What `Date._parse` has to have found for a string to name a day at all. + DATE_PARTS = %i[year mon mday].freeze + INTEGER = /\A-?\d+\z/ def initialize(collection:, timezone: nil) @@ -81,19 +84,34 @@ def seconds(value, leaf) refuse_value!(leaf, value, 'a date') end - # `Time.parse` is called for what it refuses, not for what it returns: - # it raises on a string naming no date, where `Time.zone.parse` answers - # today -- a filter on `last tuesday` coming back as a filter on today is - # the silent wrong answer this datasource exists not to give. + # What the string names is read before it is trusted, rather than being + # handed to a parser that completes what is missing. `Time.zone.parse` + # answers today for a string naming no date at all -- a filter on `last + # tuesday` coming back as a filter on today is the silent wrong answer + # this datasource exists not to give -- and `Time.parse`, which raises on + # that one, *accepts* a time of day on its own and fills the date in from + # the server's clock: `"12:00"` would have travelled as a bound on + # whichever day the request happened to run. + # + # A date is a year, a month and a day. Anything short of the three is + # refused rather than completed, `"Jan 2026"` included -- what it means is + # the caller's to say, not this class's to guess. def parse(value, leaf) return @zone.start_of_day(Date.parse(value)) if DATE_ONLY.match?(value) - Time.parse(value) + refuse_value!(leaf, value, 'a date') unless dated?(value) + @zone.timestamp(value) rescue ArgumentError, TypeError refuse_value!(leaf, value, 'a date') end + def dated?(value) + parts = Date._parse(value) + + DATE_PARTS.all? { |part| parts.key?(part) } + end + # The agent casts every Number column with `to_f`, so an integer field # would be filtered with `42.0` -- a form none of its values carry. A float # with nothing after the point travels as the integer it is. diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/query/search_fields.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/query/search_fields.rb index 9f8f9aed8..64b3ff042 100644 --- a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/query/search_fields.rb +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/query/search_fields.rb @@ -83,18 +83,36 @@ def build(raw) private def endpoint(name, definition) + filterable = fields(name, definition['fields']) + refused = refusals(name, definition['refused']) + validate_no_overlap!(name, filterable, refused) + Endpoint.new( name: name, path: definition.fetch('path'), measured_at: definition['measured_at'], - fields: fields(name, definition['fields']), - refused: refusals(name, definition['refused']), + fields: filterable, + refused: refused, candidates: Array(definition['candidates']).freeze, ticket_attributes: definition['ticket_attributes'], custom_attributes: definition['custom_attributes'] ).freeze end + # A column filed as both filterable and refused. `Endpoint#field` is + # consulted first, so the filterable row would win and the refusal -- + # with the reason an operator reads -- would be ignored in silence. The + # file is rewritten by a script, so this is the shape a bad rewrite + # takes, and it fails at load rather than producing a schema nobody can + # explain. + def validate_no_overlap!(endpoint, filterable, refused) + both = filterable.keys & refused.keys + return if both.empty? + + malformed!(endpoint, both.join(', '), + 'it is declared filterable and refused at once; the refusal would be ignored') + end + def fields(endpoint, declared) (declared || {}).to_h do |column, row| field = Field.new(column: column, field: row.fetch('field'), type: row.fetch('type'), @@ -120,9 +138,24 @@ def validate_field!(endpoint, field) validate_source!(endpoint, field.column, field) validate_type!(endpoint, field) validate_operators!(endpoint, field) + validate_publishable!(endpoint, field) validate_sortable!(endpoint, field) end + # Operators the DSL spells, none of which this column's *type* can carry: + # a `date` row declaring `~` passes the alphabet above and publishes + # nothing, `OperatorTable` mapping no Forest operator onto it. The result + # is a column the table calls filterable that no filter can reach, and + # the translator refusing every request on it -- the same + # advertise-then-refuse this package exists to prevent, one layer lower. + def validate_publishable!(endpoint, field) + return unless OperatorTable.forest_operators(field).empty? + + malformed!(endpoint, field.column, + "none of #{field.operators.join(", ")} is an operator Intercom answers on a " \ + "#{field.type}, so the column would publish no filter at all") + end + # Anything but a boolean, `"true"` above all: YAML reads it as a string, # which is truthy in Ruby and would publish a sortable column out of a # typo the file cannot otherwise show. diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/rate_limiter.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/rate_limiter.rb index 6b37a5794..82dcbd12f 100644 --- a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/rate_limiter.rb +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/rate_limiter.rb @@ -18,6 +18,19 @@ module ForestAdminDatasourceIntercom # # One limiter per Configuration, hence per token, since that is what Intercom # meters. + # + # **What it does not do**, and deliberately: it paces, it does not queue. A + # caller that finds the window spent computes the delay to the reset and + # sleeps it, and it reserves nothing while it waits -- so a hundred callers + # arriving in a spent window all compute the same delay and all wake at the + # reset, and past that instant `remaining` is still zero while the wait + # computes as elapsed, which lets them through unmetered until the first + # response reports the new window. Two things make that acceptable rather + # than a bug to work around: the burst is bounded by the callers that were + # already blocked, and the 429 retry behind this is what absorbs it -- this + # sits in front of that retry, it does not replace it. A real reservation + # would mean guessing the boundaries of a window Intercom has not reported + # yet, and a wrong guess suppresses the figures it later sends. class RateLimiter # Intercom's allocation window. Only used as the ceiling below: the reset # instant itself always comes from the response. diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/schema/attribute_naming.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/schema/attribute_naming.rb new file mode 100644 index 000000000..baa3e8f28 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/schema/attribute_naming.rb @@ -0,0 +1,52 @@ +module ForestAdminDatasourceIntercom + module Schema + # What the two introspectors do identically: turn the name a workspace gave + # an attribute into a name a Forest schema can carry, and say which + # attribute was left out when two of them land on one column. + # + # They differ in where the attributes come from -- `/ticket_types` per + # ticket type, `/data_attributes` per model -- and in the data types they + # map, which is why `COLUMN_TYPES` stays with each of them. + module AttributeNaming + # What a column name may not contain, and it has nothing to do with + # Intercom: Forest lists the fields of a request in a **comma-separated** + # query parameter, and uses a colon to name a field through a relation. + # A workspace names its attributes in free text -- measured, one is called + # `ID de l'objet en question (immo, facture, user)` -- and a comma in + # there splits the projection into fields no collection has, which the + # agent rejects as a 400 before the page is ever read. + UNSAFE_IN_A_COLUMN_NAME = /[,:]/ + + private + + # Intercom hands these back HTML-escaped -- `Ce que j'ai vérifié` -- + # which is an artefact of where they were typed, not part of the name. + def column_name_for(name) + CGI.unescapeHTML(name).gsub(UNSAFE_IN_A_COLUMN_NAME, ' ').squeeze(' ').strip + end + + # An unknown data type reads as a string rather than being dropped: + # showing the value Intercom sent beats hiding a column because its type + # is new. + def column_type_for(definition) + self.class::COLUMN_TYPES.fetch(definition['data_type'].to_s, self.class::DEFAULT_COLUMN_TYPE) + end + + # Two attributes reading as one column: the second is left out rather than + # sharing the first's entry, which would show its values under the first's + # name -- wrong values rather than missing ones, and worse. + # + # `attribute_kind` is the workspace's own vocabulary for these, which is + # what the operator has to go and rename. + def warn_collision(name, kept, column) + ForestAdminDatasourceIntercom.logger.warn( + "[forest_admin_datasource_intercom] the #{attribute_kind} attribute #{name.inspect} is left out: it " \ + "reads as the column #{column.inspect}, which #{kept.inspect} already carries. Rename one of them in " \ + 'Intercom to publish both.' + ) + end + + def attribute_kind = raise(NotImplementedError, "#{self.class} did not implement attribute_kind") + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/schema/data_attributes_introspector.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/schema/data_attributes_introspector.rb index 5a2d4422e..9a1c9cc6b 100644 --- a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/schema/data_attributes_introspector.rb +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/schema/data_attributes_introspector.rb @@ -16,6 +16,8 @@ module Schema # needs to tell an attribute it may write from one Intercom fills in # itself -- re-reading it later would be a second boot-time round trip. class DataAttributesIntrospector + include AttributeNaming + # Intercom's attribute data types, mapped onto what Forest can render. COLUMN_TYPES = { 'string' => 'String', 'integer' => 'Number', 'float' => 'Number', 'decimal' => 'Number', @@ -24,13 +26,6 @@ class DataAttributesIntrospector DEFAULT_COLUMN_TYPE = 'String'.freeze - # What a column name may not contain, and it has nothing to do with - # Intercom: Forest lists the fields of a request in a comma-separated - # query parameter and names a field through a relation with a colon. A - # workspace names its attributes in free text, and a comma in there splits - # the projection into fields no collection has. - UNSAFE_IN_A_COLUMN_NAME = /[,:]/ - # `name` is the key `custom_attributes` uses, `column_name` the one the # schema publishes; they differ whenever the workspace's own name cannot # travel through Forest's query string. @@ -94,26 +89,10 @@ def attribute_from(name, column, definition) data_type: definition['data_type'], api_writable: definition['api_writable'] == true) end - # Intercom hands these back HTML-escaped -- `Ce que j'ai vérifié` -- - # which is an artefact of where they were typed, not part of the name. - def column_name_for(name) - CGI.unescapeHTML(name).gsub(UNSAFE_IN_A_COLUMN_NAME, ' ').squeeze(' ').strip - end - - def warn_collision(name, kept, column) - ForestAdminDatasourceIntercom.logger.warn( - "[forest_admin_datasource_intercom] the #{@model} attribute #{name.inspect} is left out: it reads as the " \ - "column #{column.inspect}, which #{kept.inspect} already carries. Rename one of them in Intercom to " \ - 'publish both.' - ) - end - - # An unknown data type reads as a string rather than being dropped: - # showing the value Intercom sent beats hiding a column because its type - # is new. - def column_type_for(definition) - COLUMN_TYPES.fetch(definition['data_type'].to_s, DEFAULT_COLUMN_TYPE) - end + # These are declared per model, so that is what the log calls them: it is + # the workspace's own vocabulary, and where the operator goes to rename + # the one that was left out. + def attribute_kind = @model end end end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/schema/ticket_attributes_introspector.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/schema/ticket_attributes_introspector.rb index 85f597e5f..933004c1b 100644 --- a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/schema/ticket_attributes_introspector.rb +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/schema/ticket_attributes_introspector.rb @@ -17,6 +17,8 @@ module Schema # translation of the next lot will need, and reading them again would cost a # second boot-time round trip. class TicketAttributesIntrospector + include AttributeNaming + # Intercom's attribute data types, mapped onto what Forest can render. A # `list` is a single choice among values the workspace defined, so it reads # as a string rather than as a Json blob; `files` is a list of attachments @@ -28,15 +30,6 @@ class TicketAttributesIntrospector DEFAULT_COLUMN_TYPE = 'String'.freeze - # What a column name may not contain, and it has nothing to do with - # Intercom: Forest lists the fields of a request in a **comma-separated** - # query parameter, and uses a colon to name a field through a relation. - # A workspace names its ticket attributes in free text -- measured, one is - # called `ID de l'objet en question (immo, facture, user)` -- and a comma - # in there splits the projection into fields no collection has, which the - # agent rejects as a 400 before the page is ever read. - UNSAFE_IN_A_COLUMN_NAME = /[,:]/ - # `name` is the key the payload uses, `column_name` the one the schema # publishes; they differ whenever the workspace's own name cannot travel # through Forest's query string. @@ -108,20 +101,6 @@ def attribute_from(name, column, definition) data_type: definition['data_type'], ids_by_ticket_type: {}) end - # Intercom hands these back HTML-escaped -- `Ce que j'ai vérifié` -- - # which is an artefact of where they were typed, not part of the name. - def column_name_for(name) - CGI.unescapeHTML(name).gsub(UNSAFE_IN_A_COLUMN_NAME, ' ').squeeze(' ').strip - end - - def warn_collision(name, kept, column) - ForestAdminDatasourceIntercom.logger.warn( - "[forest_admin_datasource_intercom] the ticket attribute #{name.inspect} is left out: it reads as the " \ - "column #{column.inspect}, which #{kept.inspect} already carries. Rename one of them in Intercom to " \ - 'publish both.' - ) - end - def definitions(ticket_type) return [] unless ticket_type.is_a?(Hash) @@ -132,11 +111,10 @@ def definitions(ticket_type) list.is_a?(Array) ? list : [] end - # An unknown data type reads as a string rather than being dropped: showing - # the value Intercom sent beats hiding a column because its type is new. - def column_type_for(definition) - COLUMN_TYPES.fetch(definition['data_type'].to_s, DEFAULT_COLUMN_TYPE) - end + # These are declared per ticket type, and that is what the log calls + # them: the workspace's own vocabulary, and where the operator goes to + # rename the one that was left out. + def attribute_kind = 'ticket' end end end diff --git a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/company_spec.rb b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/company_spec.rb index 381399d05..942976dd4 100644 --- a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/company_spec.rb +++ b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/company_spec.rb @@ -190,7 +190,29 @@ def rows(projection = %w[id], **options) (1..10).each { |number| stub_page(company("c#{number}"), page: number, per_page: 150, total_pages: 99) } expect(rows(%w[id]).size).to eq(10) - expect(ForestAdminDatasourceIntercom.logger).to have_received(:warn).with(/Stopped reading IntercomCompany/) + expect(ForestAdminDatasourceIntercom.logger) + .to have_received(:warn).with(/Stopped reading IntercomCompany.*named no window of its own/) + end + + # A window is its own bound, so the page cap must not apply to it: a + # window needing eleven pages would come back one page short while the + # warning blamed it for naming no window -- which it did name. The record + # budget is what bounds a window nothing sane asked for, and it says so + # in its own words. + it 'reads past the page cap for a window that needs it' do + (1..12).each { |number| stub_page(company("c#{number}"), page: number, per_page: 150, total_pages: 12) } + + expect(rows(%w[id], page: page(0, 1800)).size).to eq(12) + end + + it 'stops a window on the record budget, and says which cap it hit' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + stub_const('ForestAdminDatasourceIntercom::Collections::OffsetCollection::MAX_COLLECTED_RECORDS', 2) + (1..4).each { |number| stub_page(company("c#{number}"), page: number, per_page: 150, total_pages: 99) } + + expect(rows(%w[id], page: page(0, 1800)).size).to eq(2) + expect(ForestAdminDatasourceIntercom.logger) + .to have_received(:warn).with(/named a window larger than one answer may hold/) end it 'counts what Intercom counted, in one request' do @@ -252,6 +274,65 @@ def rows(projection = %w[id], **options) expect(count(condition_tree: leaf('id', operators::EQUAL, 'co1'))).to eq([{ 'group' => {}, 'value' => 1 }]) end + # Counting a set of ids means reading it, so past what a bulk read fetches + # the count is refused rather than answered with the number the truncation + # left: this collection advertises an exact count, and 25 where the + # question named forty records is not one. + it 'refuses to count more ids than it will read' do + expect { count(condition_tree: leaf('id', operators::IN, (1..40).map { |n| "co#{n}" })) } + .to raise_error(UnsupportedOperatorError, /cannot count 40 records by id/) + end + + # The `in` this comes from is a membership: it matches a record once, + # where a read by id would fetch a repeated value twice and hand it back + # as two rows carrying one id. + it 'reads a value named twice once' do + stub_request(:get, "#{base}/companies/co1").to_return(json(company('co1'))) + + expect(rows(%w[id], condition_tree: leaf('id', operators::IN, %w[co1 co1]))).to eq([{ 'id' => 'co1' }]) + expect(WebMock).to have_requested(:get, "#{base}/companies/co1").once + end + end + + # A relation pointing at this collection resolves it by id, and Intercom has + # no bulk read for a company: the fan-out is sliced by what one batch holds + # and refused past what the tier resolves at all, rather than answered with a + # nil where an account exists. + describe 'as the target of a relation' do + subject(:contacts) { datasource.get_collection('IntercomContact') } + + # An unfiltered, unsorted contact list reads the listing endpoint; the + # search is for the routes that filter. + def stub_contact_list(*records) + stub_request(:get, "#{base}/contacts") + .with(query: hash_including({})) + .to_return(json('type' => 'list', 'data' => records, 'total_count' => records.size, 'pages' => {})) + end + + def contact_of(company_id) + { 'type' => 'contact', 'id' => "c#{company_id}", + 'companies' => { 'type' => 'list', 'data' => [{ 'type' => 'company', 'id' => company_id }], + 'total_count' => 1 } } + end + + it 'slices the fan-out into batches rather than truncating it' do + stub_contact_list(*(1..30).map { |n| contact_of("co#{n}") }) + stub_request(:get, %r{#{base}/companies/co\d+}).to_return do |request| + json(company(request.uri.path.split('/').last)) + end + + rows = contacts.list(nil, filter, %w[id company:name]) + + expect(rows.filter_map { |row| row['company'] }.size).to eq(30) + end + + it 'refuses a fan-out past what it resolves, naming the relation' do + stub_contact_list(*(1..120).map { |n| contact_of("co#{n}") }) + + expect { contacts.list(nil, filter, %w[id company:name]) } + .to raise_error(UnsupportedOperatorError, /cannot resolve company over this read: it names 120/) + end + # `GET /companies?name=` answers the company itself where a listing would # answer an envelope: a record is read as a page of one rather than as a # shape every caller has to test for. diff --git a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/contact_spec.rb b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/contact_spec.rb index d461612fb..8fb68d3a4 100644 --- a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/contact_spec.rb +++ b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/contact_spec.rb @@ -365,6 +365,41 @@ def rows(projection = %w[id], **options) expect(ForestAdminDatasourceIntercom.logger).to have_received(:warn).with(/asked for 400 records by id/) expect(WebMock).to have_requested(:post, "#{base}/contacts/search").times(3) end + + # The window is cut in the order the ids were named, so ordering what + # comes back would order a slice picked by something else. This is the one + # route of the collection Intercom does sort where an order cannot be + # honoured, so `server_sort` stayed quiet and this has to speak. + it 'reports an order this route cannot apply, on a column Intercom does sort' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + stub_search(contact('1')) + + rows(%w[id], condition_tree: leaf('id', operators::IN, %w[1 2]), + sort: sort({ field: 'name', ascending: true })) + + expect(ForestAdminDatasourceIntercom.logger).to have_received(:warn).with(/sort on "name" while reading/) + end + + # The `in` this comes from is a membership: it matches a record once. + it 'reads a value named twice once' do + stub_search(contact('1')) + + rows(%w[id], condition_tree: leaf('id', operators::IN, %w[1 1])) + + expect(WebMock).to have_requested(:post, "#{base}/contacts/search") + .with(body: hash_including('query' => { 'field' => 'id', 'operator' => 'IN', 'value' => %w[1] })) + end + + # Counting a set of ids means reading it, so past what a bulk read fetches + # the count is refused rather than answered with the number the truncation + # left -- this collection advertises an exact count. + it 'refuses to count more ids than it will read' do + expect do + collection.aggregate(nil, filter(condition_tree: leaf('id', operators::IN, (1..400).map(&:to_s))), + ForestAdminDatasourceToolkit::Components::Query::Aggregation + .new(operation: 'Count')) + end.to raise_error(UnsupportedOperatorError, /cannot count 400 records by id/) + end end describe 'the contacts of an account' do @@ -406,12 +441,33 @@ def rows(projection = %w[id], **options) # An `and` also carrying a scope names a narrower set than the account # does, and answering it with the account alone would serve contacts the - # scope excludes. + # scope excludes. There is no request that answers both halves either: + # the account endpoint narrows nothing and the search filters no company + # field, which is what a related list runs into the moment a scope or a + # segment exists on this collection. it 'refuses to take the route for anything but a bare equality' do expect do rows(%w[id], condition_tree: branch('And', leaf('company_id', operators::EQUAL, 'co1'), leaf('role', operators::EQUAL, 'user'))) - end.to raise_error(UnsupportedOperatorError, /cannot filter "company_id"/) + end.to raise_error(UnsupportedOperatorError, %r{GET /companies/\{id\}/contacts}) + end + + # The refusal names the half that does not fit rather than telling the + # operator to read the account's contacts -- which is what they asked for. + it 'names the condition filtered alongside the account' do + expect do + rows(%w[id], condition_tree: branch('And', leaf('company_id', operators::EQUAL, 'co1'), + leaf('role', operators::EQUAL, 'user'))) + end.to raise_error(UnsupportedOperatorError, /condition on role/) + end + + it 'refuses the count of a narrowed account the same way' do + expect do + collection.aggregate(nil, + filter(condition_tree: branch('And', leaf('company_id', operators::EQUAL, 'co1'), + leaf('role', operators::EQUAL, 'user'))), + ForestAdminDatasourceToolkit::Components::Query::Aggregation.new(operation: 'Count')) + end.to raise_error(UnsupportedOperatorError, %r{GET /companies/\{id\}/contacts}) end end diff --git a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/conversation_spec.rb b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/conversation_spec.rb index 754df26d9..d671f7824 100644 --- a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/conversation_spec.rb +++ b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/conversation_spec.rb @@ -227,6 +227,17 @@ def ids(rows) end describe '#list' do + # No projection at all asks for every declared column, `contact_name` and + # `timeline` included, so an all-columns read pays for both enrichments -- + # the identity read of the page, and the detail read the parts are only + # returned by. Guarding them on a column being *named* would leave nil the + # very columns the row publishes. + before do + stub_request(:post, "#{base}/contacts/search") + .to_return(json('type' => 'list', 'data' => [{ 'id' => 'c1', 'name' => 'Camille' }])) + stub_record('1', conversation('1')) + end + it 'reads the listing endpoint as plain text and pages by cursor' do stub_list(conversation('1')) @@ -288,6 +299,18 @@ def ids(rows) expect(collection.list(nil, filter, %w[id state])).to eq([{ 'id' => '1', 'state' => 'closed' }]) end + # The counterpart of the guard above: a row that publishes a column has + # that column filled, rather than carrying the nil the projection put + # there. + it 'fills the enriched columns of an all-columns read' do + stub_list(conversation('1')) + + row = collection.list(nil, filter, nil).first + + expect(row['contact_name']).to eq('Camille') + expect(row['timeline']).to be_an(Array) + end + it 'walks the cursor until the window is covered' do first = { 'conversations' => [conversation('1'), conversation('2')], 'pages' => { 'next' => { 'starting_after' => 'c2' } } } diff --git a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/ticket_spec.rb b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/ticket_spec.rb index ca03247b5..6bd18731d 100644 --- a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/ticket_spec.rb +++ b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/ticket_spec.rb @@ -95,6 +95,15 @@ def columns collection.fields.select { |_, field| field.type == 'Column' } end + # No projection at all asks for every declared column, `contact_name` + # included, so the reads below pay the identity read of the page -- guarding + # it on the column being *named* would leave nil the very column the row + # publishes. The blocks that assert on the identity itself stub over this. + before do + stub_request(:post, "#{base}/contacts/search") + .to_return(json('type' => 'list', 'data' => [{ 'id' => 'c1', 'name' => 'Camille' }])) + end + describe 'schema' do it 'is named IntercomTicket' do expect(collection.name).to eq('IntercomTicket') @@ -344,6 +353,18 @@ def columns expect(WebMock).not_to have_requested(:get, "#{base}/admins") end + # Forest's own parser builds a two-hop path, and this nests one target row + # under the relation name rather than a tree of them: the second hop would + # be dropped by the target's projection and the column would come back + # missing from a row that looks complete. Refused by name, like the filter + # that reaches that deep. + it 'refuses a projection reaching through two relations' do + stub_search(ticket('1')) + + expect { collection.list(nil, filter, ['id', 'admin_assignee:teams:name']) } + .to raise_error(UnsupportedOperatorError, /reaches through two relations/) + end + # The price of a relation is one read per target *collection*, not one per # relation: `state` and `previous_state` name the same endpoint, and it is # read once, over the ids both of them point at. diff --git a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/configuration_spec.rb b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/configuration_spec.rb index 64b9079a1..957f6ffff 100644 --- a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/configuration_spec.rb +++ b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/configuration_spec.rb @@ -84,6 +84,34 @@ module ForestAdminDatasourceIntercom end end + # `URI::HTTPS` is a `URI::HTTP`, so a cleartext base_url is accepted -- a + # mock server is one, and that is half of what the parameter is for. What it + # costs is the bearer header crossing that network in clear, and the token + # reads the whole workspace. + describe 'a cleartext base_url' do + before { allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) } + + it 'is accepted, and says the access token travels in clear over it' do + described_class.new(access_token: 's3cr3t', base_url: 'http://proxy.internal/intercom') + + expect(ForestAdminDatasourceIntercom.logger) + .to have_received(:warn).with(/is not https, and every request carries the Intercom access token/) + end + + it 'says nothing about a loopback host, which is nobody else s network' do + described_class.new(access_token: 's3cr3t', base_url: 'http://localhost:4010') + described_class.new(access_token: 's3cr3t', base_url: 'http://intercom.localhost') + + expect(ForestAdminDatasourceIntercom.logger).not_to have_received(:warn) + end + + it 'says nothing about https' do + described_class.new(access_token: 's3cr3t', base_url: 'https://intercom.test') + + expect(ForestAdminDatasourceIntercom.logger).not_to have_received(:warn) + end + end + describe '#inspect' do it 'never prints the bearer token' do expect(configuration.inspect).to include('[FILTERED]') @@ -93,6 +121,16 @@ module ForestAdminDatasourceIntercom it 'still names the host and version, which is what one inspects it for' do expect(configuration.inspect).to include('https://api.intercom.io', '2.16') end + + # A credentialed egress proxy is the other reason to set a base_url, and + # `URI` accepts its credentials in the url: printing it verbatim would put + # a second secret exactly where this keeps the first one from going. + it 'masks the credentials of a base_url that carries them' do + configured = described_class.new(access_token: 's3cr3t', base_url: 'https://bob:hunter2@proxy.test/api') + + expect(configured.inspect).not_to include('hunter2', 'bob') + expect(configured.inspect).to include('https://[FILTERED]@proxy.test/api') + end end end end diff --git a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/pagination/cursor_walker_spec.rb b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/pagination/cursor_walker_spec.rb index 70529cc32..06df6b725 100644 --- a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/pagination/cursor_walker_spec.rb +++ b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/pagination/cursor_walker_spec.rb @@ -65,15 +65,23 @@ def ids(records) expect(asked.size).to eq(1) end - it 'stops on an empty page' do + # Stopping here is a truncation like a cap: Intercom said there was a next + # page and this could not follow it, so the window is short of what was + # asked -- and a short answer is never silent. + it 'stops on an empty page, and says the result is truncated' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + records = walker.walk(offset: 0, limit: 10, &source(page([], next_cursor: 'c1'))) expect(records).to be_empty + expect(ForestAdminDatasourceIntercom.logger) + .to have_received(:warn).with(/advertised a next page this could not follow/) end # None of this happens against Intercom today, but a walk driven by a # remote value stops on its own terms rather than on the caps only. - it 'stops on a cursor it has already followed' do + it 'stops on a cursor it has already followed, and says the result is truncated' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) pages = source(page([record('a')], next_cursor: 'loop'), page([record('b')], next_cursor: 'loop'), page([record('c')], next_cursor: 'loop')) @@ -81,6 +89,18 @@ def ids(records) walker.walk(offset: 0, limit: 10, &pages) expect(asked.size).to eq(2) + expect(ForestAdminDatasourceIntercom.logger) + .to have_received(:warn).with(/advertised a next page this could not follow/) + end + + # The window was covered: there is nothing short about the answer, so the + # cursor that did not move is not worth a word. + it 'stays quiet when a stalled cursor comes after the window was covered' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + + walker.walk(offset: 0, limit: 1, &source(page([record('a')], next_cursor: 'loop'))) + + expect(ForestAdminDatasourceIntercom.logger).not_to have_received(:warn) end # Intercom documents duplicates on a dataset that moves between two diff --git a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/query/condition_tree_translator_spec.rb b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/query/condition_tree_translator_spec.rb index 797efa12a..cd891d7de 100644 --- a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/query/condition_tree_translator_spec.rb +++ b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/query/condition_tree_translator_spec.rb @@ -185,11 +185,14 @@ def leaves(count) .to raise_error(UnsupportedOperatorError, /takes no filter on it. Filter on one of: id, state, open/) end - # None of these collections declares a relation yet, so a `relation:field` - # can only come from a scope or a segment written against another schema. - it 'refuses a condition on a relation by name' do + # A `relation:field` is traded for a condition on the foreign key before a + # tree reaches this translation, or refused by the collection with the + # relations it does declare. Reaching here means that pass was skipped, so + # the message names what the endpoint filters rather than a relation + # nobody resolved. + it 'refuses a path through a relation, naming what it does filter' do expect { translate(leaf('contact:email', operators::EQUAL, 'camille@acme.test')) } - .to raise_error(UnsupportedOperatorError, /declares no relation/) + .to raise_error(UnsupportedOperatorError, /filters columns, not paths through a relation/) end it 'refuses an operator the endpoint does not answer on that field' do diff --git a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/query/filter_value_spec.rb b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/query/filter_value_spec.rb index 13ee73781..30cd4f71f 100644 --- a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/query/filter_value_spec.rb +++ b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/query/filter_value_spec.rb @@ -144,6 +144,29 @@ def bound(value, spelling) .to raise_error(UnsupportedOperatorError, /Intercom expects a date on this field/) end + # `Time.parse` accepts a time of day on its own and fills the date in from + # the server's clock, so this would have travelled as a bound on whichever + # day the request happened to run -- a filter whose answer changes at + # midnight and never says why. + it 'refuses a time of day rather than completing it from today' do + expect { call('date', '12:00', spelling: '>') } + .to raise_error(UnsupportedOperatorError, /Intercom expects a date on this field/) + end + + # A month names a range, and which end of it the operator meant is theirs + # to say rather than this class's to guess. + it 'refuses a date missing its day' do + expect { call('date', 'Jan 2026', spelling: '>') } + .to raise_error(UnsupportedOperatorError, /Intercom expects a date on this field/) + end + + # It carries the three parts a date needs and still names no day: the + # parser is what says so, and its complaint is not a bound to send on. + it 'refuses a date whose parts are out of range' do + expect { call('date', '2026-13-45', spelling: '>') } + .to raise_error(UnsupportedOperatorError, /Intercom expects a date on this field/) + end + # Reading either as epoch seconds raises a FloatDomainError, which would # leave the read with an error naming a float where the operator asked # for a date. The number branch already refuses them; a date is no diff --git a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/query/search_fields_spec.rb b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/query/search_fields_spec.rb index 9b36bf607..c641cb11f 100644 --- a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/query/search_fields_spec.rb +++ b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/query/search_fields_spec.rb @@ -149,6 +149,25 @@ def field_row(overrides = {}) .to raise_error(ConfigurationError, /tickets.company_id/) end + # Operators the DSL does spell, none of which the column's *type* can + # carry: `OperatorTable` maps no Forest operator onto `~` for a date, so + # the column would be called filterable and publish nothing, and the + # translator would refuse every request on it. + it 'refuses operators the column type cannot publish' do + expect { table(fields: { 'created_at' => field_row('operators' => ['~']) }) } + .to raise_error(ConfigurationError, /none of ~ is an operator Intercom answers on a date/) + end + + # `Endpoint#field` is consulted before `#refusal`, so the filterable row + # would win and the reason an operator reads would be dropped in silence. + # The file is script-rewritten, which is the shape a bad rewrite takes. + it 'refuses a column declared filterable and refused at once' do + expect do + table(fields: { 'created_at' => field_row }, + refused: { 'created_at' => { 'reason' => 'no', 'source' => 'spec' } }) + end.to raise_error(ConfigurationError, /filterable and refused at once/) + end + it 'names the endpoint and the column it choked on' do expect { table(fields: { 'created_at' => field_row('type' => 'timestamp') }) } .to raise_error(ConfigurationError, /search_fields\.yml is malformed at tickets\.created_at/) From 69ee4f8fa11e74f5dd2a4dc2c7d35b5f4021f92d Mon Sep 17 00:00:00 2001 From: Brun Christophe Date: Thu, 10 Sep 2026 14:49:30 +0200 Subject: [PATCH 6/6] fix(datasource): run the Intercom checks it documents Second review pass on PR #387. Four defects, each one a place where the package promised something it did not do. * a lookup naming no record is an empty page rather than a 500. Intercom answers `GET /companies?name=` with a 404 where a search endpoint would answer an empty list, and nothing read it: an operator typing an account name that matches nothing broke the page instead of seeing no row. Read the way a record read by its id already was, and only a 404 -- anything else still raises. The related list of an account Intercom no longer answers for reads the same way; * the pinned API version is checked at boot. `Client#me` reads the header Intercom echoes the served version in, and nothing called it: the check the configuration claimed ran was code that never ran, while a workspace serving another version answers payloads of another shape. Called from the datasource on the boot connection, degrading to a warning like the three attribute reads; * the table names every column of the collections it answers for. Seven columns had no row at all -- `state_id` among them, which the README ranks second on the list to probe -- so they fell back on the generic "takes no filter on it", the one refusal an operator cannot act on; and `state_external_label` outlived by two lots the column it refused. A spec now reads the schema against the table in both directions, which is what the reasons are worth; * the probe ships with the gem. `bin/` is excluded from `spec.files`, so the tool the README makes the first step against a customer's workspace reached nobody who had not cloned the repository. It moves to `exe/forest_admin_intercom_probe` -- and with it its guard, RubyGems loading an executable from a stub of its own, where `$PROGRAM_NAME == __FILE__` would have shipped a command that exits without doing anything. Consistency * `ContactIdentity` reads the Contacts path off the table rather than writing it a second time; * the account-contacts route reports an order it cannot apply with the route as the reason and not the column: this is the one collection Intercom does sort; * the attribute families of the table stop being data nothing reads. `Endpoint#attribute_refusal` is validated at load and read by the translator, so a scope filtering on a workspace attribute is refused with the R7 arbitration rather than with the message for a column nobody declared -- those columns being named by the workspace, no row can name them; * the README figures for the measured rows are derived from the file by a spec: 18 of 89, and they cannot drift from it again. 579 examples, 0 failures, 100% line coverage; rubocop clean on 912 files; toolkit, agent and customizer suites green. Co-Authored-By: Claude Opus 5 (1M context) --- .../README.md | 45 +++++---- .../forest_admin_intercom_probe} | 23 ++++- .../collections/base_collection.rb | 7 ++ .../collections/contact.rb | 26 +++++- .../collections/contact_identity.rb | 14 ++- .../collections/cursor_collection.rb | 3 +- .../collections/custom_attributes.rb | 6 ++ .../collections/offset_collection.rb | 9 ++ .../datasource.rb | 28 +++++- .../query/condition_tree_translator.rb | 20 +++- .../query/search_fields.rb | 49 ++++++++-- .../query/search_fields.yml | 46 +++++++++- .../collections/company_spec.rb | 35 +++++++ .../collections/contact_spec.rb | 30 +++++- .../collections/ticket_spec.rb | 10 ++ .../datasource_spec.rb | 37 +++++++- .../query/search_fields_spec.rb | 92 +++++++++++++++++-- .../spec/probe_search_fields_spec.rb | 21 ++++- .../spec/spec_helper.rb | 18 +++- 19 files changed, 459 insertions(+), 60 deletions(-) rename packages/forest_admin_datasource_intercom/{bin/probe_search_fields => exe/forest_admin_intercom_probe} (88%) diff --git a/packages/forest_admin_datasource_intercom/README.md b/packages/forest_admin_datasource_intercom/README.md index 1a005706e..952014b0b 100644 --- a/packages/forest_admin_datasource_intercom/README.md +++ b/packages/forest_admin_datasource_intercom/README.md @@ -267,19 +267,21 @@ the source of truth is a committed table — `lib/forest_admin_datasource_interc | `source` | What it means | | --- | --- | -| `measured` | observed against a real workspace, by `bin/probe_search_fields` or during the spike | +| `measured` | observed against a real workspace, by `forest_admin_intercom_probe` or during the spike | | `spec` | read off Intercom's documentation, and therefore still a candidate | Every `filter_operators` a column publishes is **derived** from that table, so a column cannot advertise a filter the translator would then refuse, and a column the table does not carry advertises nothing at all. That is the mechanism, and it holds whatever the rows say. -**What the rows say today is mostly `spec`: 19 of 85 are measured, and no endpoint has been probed +**What the rows say today is mostly `spec`: 18 of 89 are measured, and no endpoint has been probed end to end** — all three carry `measured_at: null`, which is what `Endpoint#measured?` reports. The measured rows are the ones a spike went out of its way to check: the date operators on each -endpoint, which disagree between them, and `id IN` on `/contacts/search`. Everything else is -Intercom's documentation, and the disagreement above is why that is a candidate rather than a -promise. +endpoint, which disagree between them, `id IN` on `/contacts/search`, `contact_ids` on +`/conversations/search`, and the refusals a read confirmed — `company_id` on `/tickets/search` +above all, alongside the columns the agent derives rather than reads. Everything else is Intercom's +documentation, and the disagreement above is why that is a candidate rather than a promise. Those +two figures are asserted against the file, so they cannot drift from it. So the first thing to do against a customer's workspace is to run the probe. The rows worth watching first, in the order they will hurt: @@ -298,13 +300,13 @@ first, in the order they will hurt: and an ignored parameter costs a query string where the honoured one saves every filtered row from coming back as markup. -To measure a workspace of your own. The probe is a repo tool, not part of the published gem — `bin/` -is excluded from `spec.files` — so it runs from a clone of `agent-ruby`, in this package's -directory: +To measure a workspace of your own. The probe ships with the gem — it is what measures the +customer's workspace, and whoever runs it there has the gem installed rather than a clone of +`agent-ruby` — so `bundle install` puts it on the path of the application the datasource is +mounted in: ```bash -cd packages/forest_admin_datasource_intercom -INTERCOM_ACCESS_TOKEN=... bin/probe_search_fields --endpoint tickets --out measured.yml +INTERCOM_ACCESS_TOKEN=... bundle exec forest_admin_intercom_probe --endpoint tickets --out measured.yml ``` It sends one search per (field, operator) cell, reads Intercom's refusal codes — `invalid_field` for @@ -642,15 +644,22 @@ The body of a conversation is raw personal data, and this datasource is built on ## Boot-time introspection -Constructing the datasource performs exactly **three** reads, and they are all of the same kind: -`GET /ticket_types` for the attribute columns of `IntercomTicket`, and +Constructing the datasource performs exactly **four** reads. + +Three are of one kind: `GET /ticket_types` for the attribute columns of `IntercomTicket`, and `GET /data_attributes?model=contact` and `?model=company` for those of `IntercomContact` and `IntercomCompany`. A payload carries the values of the attributes that record happens to have been given, never their definitions, which is why they cannot be discovered from the records. -All three run on the boot connection — short timeouts, one quick retry — so a slow Intercom cannot -turn a Rails boot into minutes the operator sits through, and each degrades to no attribute column -rather than to a failed boot. +The fourth is `GET /me`, and it reads no column: Intercom echoes in a response header the API +version it served, and it serves the workspace's own default when the pin is not honoured — whose +payloads are shaped differently from the ones this expects. That echo is the only place the +substitution shows, so it is checked while the agent starts and reported as a warning. + +All four run on the boot connection — short timeouts, one quick retry — so a slow Intercom cannot +turn a Rails boot into minutes the operator sits through, and each degrades to a warning rather +than to a failed boot: a token missing a permission costs the columns it could not read, or the +version check, never the agent. `api_writable` is read alongside each attribute and kept, although every column of this lot is published read-only: it is what tells an attribute the API may write from one Intercom fills in @@ -668,7 +677,7 @@ Everything else is read when a collection is listed, so an agent boots whatever | 6 | Bounded group-by and the reporting export | Two questions this lot leaves in the table rather than in an assumption, both for -`bin/probe_search_fields` to answer against the customer's workspace: whether `/tickets/search` +`forest_admin_intercom_probe` to answer against the customer's workspace: whether `/tickets/search` filters on `contact_ids`, and which operators `/contacts/search` answers on a custom attribute. ## Development @@ -680,6 +689,10 @@ BUNDLE_GEMFILE=Gemfile-test bundle exec rspec bundle exec rubocop # from the repository root ``` +`exe/forest_admin_intercom_probe` ships with the gem rather than living in the repository alone: +what it measures is the customer's workspace, and whoever runs it there has the gem installed and +not a clone of this repository. From a checkout it runs in place, `exe/forest_admin_intercom_probe`. + Specs stub the HTTP layer with WebMock. Every payload they feed in is **hand-written from the OpenAPI 2.16 specification**, never captured from a workspace: a conversation body is personal data, and a fixture is read by everyone who clones the repository. diff --git a/packages/forest_admin_datasource_intercom/bin/probe_search_fields b/packages/forest_admin_datasource_intercom/exe/forest_admin_intercom_probe similarity index 88% rename from packages/forest_admin_datasource_intercom/bin/probe_search_fields rename to packages/forest_admin_datasource_intercom/exe/forest_admin_intercom_probe index 2ba05fabf..4d004c9d2 100755 --- a/packages/forest_admin_datasource_intercom/bin/probe_search_fields +++ b/packages/forest_admin_datasource_intercom/exe/forest_admin_intercom_probe @@ -7,8 +7,12 @@ # off the documentation: measured during lot 1, `/tickets/search` refuses # `company_id` with `invalid_field` although a ticket carries one. # -# INTERCOM_ACCESS_TOKEN=... bin/probe_search_fields --endpoint tickets -# INTERCOM_ACCESS_TOKEN=... bin/probe_search_fields --region eu --out measured.yml +# INTERCOM_ACCESS_TOKEN=... forest_admin_intercom_probe --endpoint tickets +# INTERCOM_ACCESS_TOKEN=... forest_admin_intercom_probe --region eu --out measured.yml +# +# Shipped with the gem rather than left in the repository: what it measures is +# the customer's workspace, and the operator running it has the gem installed +# and not a clone of the agent. # # It sends one search per (field, operator) cell, asking for a single record, # and reads Intercom's refusal codes: `invalid_field` for a field the endpoint @@ -30,6 +34,17 @@ require 'optparse' module ProbeSearchFields SearchFields = ForestAdminDatasourceIntercom::Query::SearchFields + # Whether this file is being run as the command rather than loaded by a + # spec. The basenames and not the paths: RubyGems installs an executable + # behind a stub of its own and `load`s this file from it, so `$PROGRAM_NAME` + # is the stub while `__FILE__` is this file -- comparing the two would make + # the shipped command exit without doing anything, which is the one thing + # shipping it was for. A spec loading it runs under `rspec`, whose basename + # is not this one. + def self.invoked_as_command?(program_name) + File.basename(program_name.to_s) == File.basename(__FILE__) + end + # A cell is probed with the value shapes its field plausibly takes, most # likely first: a wrong shape is refused with `data_invalid` just like an # unsupported operator, so a single attempt would report an operator as @@ -189,7 +204,7 @@ module ProbeSearchFields options = { endpoints: SearchFields.endpoints, region: 'us', token: ENV.fetch('INTERCOM_ACCESS_TOKEN', nil) } OptionParser.new do |parser| - parser.banner = 'Usage: INTERCOM_ACCESS_TOKEN=... bin/probe_search_fields [options]' + parser.banner = 'Usage: INTERCOM_ACCESS_TOKEN=... forest_admin_intercom_probe [options]' parser.on('--endpoint NAME', "one of #{SearchFields.endpoints.join(", ")}") { |v| options[:endpoints] = [v] } parser.on('--region NAME', 'us (default), eu or au') { |v| options[:region] = v } parser.on('--token TOKEN', 'defaults to $INTERCOM_ACCESS_TOKEN') { |v| options[:token] = v } @@ -249,4 +264,4 @@ module ProbeSearchFields end end -exit(ProbeSearchFields::CLI.call(ARGV)) if $PROGRAM_NAME == __FILE__ +exit(ProbeSearchFields::CLI.call(ARGV)) if ProbeSearchFields.invoked_as_command?($PROGRAM_NAME) diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/base_collection.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/base_collection.rb index 6f08e0067..0a9bc16c8 100644 --- a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/base_collection.rb +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/base_collection.rb @@ -101,6 +101,13 @@ def column_names @column_names ||= fields.select { |_, field| field.is_a?(ColumnSchema) }.keys end + # The columns a workspace's own attributes became. None here; the + # collections that carry any override this through `CustomAttributes`. + # The translator reads them to answer a filter on one with the refusal + # covering the family rather than with the message for a column nobody + # declared. + def attribute_column_names = [] + # The timezone in-memory date comparisons are evaluated in. The caller's, # since that is whose "today" the filter was written against. def timezone_for(caller) diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/contact.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/contact.rb index f10fb71cf..cba095747 100644 --- a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/contact.rb +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/contact.rb @@ -157,7 +157,7 @@ def fetch_records(caller, filter, sort = nil) return super end - warn_ignored_sort(Array(filter&.sort)) if sort + warn_unordered_company_contacts(Array(filter&.sort)) if sort offset, limit = translate_page(filter&.page) walker.walk(offset: offset, limit: limit) do |per_page, cursor| @@ -225,9 +225,18 @@ def narrowing_cause(filter) others.empty? ? 'condition filtered alongside it' : "condition on #{others.uniq.join(", ")}" end + # An account Intercom no longer answers for -- deleted, or moved outside + # the token's reach between the moment the row was rendered and the + # moment its related list was opened -- reads as an account with no + # contact rather than as a failed page, the way a record read by its id + # already does. def read_company_page(company, per_page:, cursor:) client.list_page("companies/#{Faraday::Utils.escape(company)}/contacts", per_page: [per_page, max_page_size].min, starting_after: cursor) + rescue APIError => e + raise unless e.status == 404 + + Client::Page.new(records: [], next_cursor: nil, total_count: 0) end # One request per hundred ids instead of one per id: this endpoint answers @@ -245,6 +254,21 @@ def records_by_ids(ids) end end + # `sort` here is the clause `server_sort` found `/contacts/search` does + # honour, which is why nothing has reported it yet: it is this route that + # cannot carry the order, not the column. Saying Intercom does not sort + # this collection would be wrong -- it is the one collection it sorts -- + # so this is the counterpart of `warn_unordered_ids`, for the other route + # that leaves the search behind. + def warn_unordered_company_contacts(clauses) + ForestAdminDatasourceIntercom.logger.warn( + "[forest_admin_datasource_intercom] #{name} was asked to sort on " \ + "#{clauses.map { |clause| sort_field(clause) }.join(", ")} while reading the contacts of an account, " \ + 'which Intercom answers through GET /companies/{id}/contacts -- a route that takes no order, unlike ' \ + 'the search this collection is otherwise read through. The rows come back in the order the API imposes.' + ) + end + def warn_truncated_ids(asked) ForestAdminDatasourceIntercom.logger.warn( "[forest_admin_datasource_intercom] #{name} was asked for #{asked} records by id and read the first " \ diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/contact_identity.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/contact_identity.rb index 280635b19..a00cac835 100644 --- a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/contact_identity.rb +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/contact_identity.rb @@ -49,6 +49,14 @@ def first_contact_id(record) contact.is_a?(Hash) ? stringify_id(contact['id']) : nil end + # The Contacts endpoint as the table spells it, rather than a path written + # a second time here: this is the same `/contacts/search` the Contacts + # collection reads itself through, and a table that renamed it would + # otherwise leave this one behind. + def contact_search_path + @contact_search_path ||= Query::SearchFields.fetch('contacts').path + end + def embed_contact_identity(records, rows, projection) return unless any_column_asked?(projection, COLUMNS) @@ -66,9 +74,9 @@ def contact_identities(records) return {} if ids.empty? ids.each_slice(CONTACT_CHUNK).with_object({}) do |chunk, indexed| - page = client.search_page('contacts/search', per_page: chunk.size, - query: { 'field' => 'id', 'operator' => 'IN', - 'value' => chunk }) + page = client.search_page(contact_search_path, per_page: chunk.size, + query: { 'field' => 'id', 'operator' => 'IN', + 'value' => chunk }) page.records.each { |contact| indexed[contact['id'].to_s] = contact } end rescue APIError => e diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/cursor_collection.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/cursor_collection.rb index cbb637cad..1f942c581 100644 --- a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/cursor_collection.rb +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/cursor_collection.rb @@ -222,7 +222,8 @@ def translate(caller, filter) return NOTHING if tree == NOTHING Query::ConditionTreeTranslator.call(tree, endpoint: search_endpoint, collection: name, - timezone: timezone_for(caller)) + timezone: timezone_for(caller), + attribute_columns: attribute_column_names) end # The ids the target matched, written as the filter Intercom does take on diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/custom_attributes.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/custom_attributes.rb index 8437079d6..771931a79 100644 --- a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/custom_attributes.rb +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/custom_attributes.rb @@ -25,6 +25,12 @@ def register_attribute_columns def attribute_columns = @attribute_columns || [] + # What the translator needs of them: the names the schema published, so a + # filter reaching one is refused with the reason the table carries for + # the whole family rather than with the message for a column that is not + # in it. + def attribute_column_names = attribute_columns.map(&:column_name) + # What the log calls these, which is the workspace's own vocabulary: a # ticket attribute is declared per ticket type, a contact attribute per # model. diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/offset_collection.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/offset_collection.rb index e1247b94d..4c7373ac5 100644 --- a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/offset_collection.rb +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/offset_collection.rb @@ -222,11 +222,20 @@ def lookup_condition(filter) # An exact lookup answers few records -- one, for the keys this publishes # -- so it is read as a single page. More than that page holds is reported # rather than dropped in silence. + # + # A lookup naming no record is an empty page, not a failure: Intercom + # answers this route with a 404 where a search endpoint would answer an + # empty list, and a filter matching nothing is the most ordinary thing a + # list view does. Read the way a record read by its id already is. def looked_up_records(params) answer = client.lookup_page(lookup_path, params: params) warn_truncated_lookup(params) if answer.next_cursor answer.records + rescue APIError => e + raise unless e.status == 404 + + [] end def refuse_condition!(tree) diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/datasource.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/datasource.rb index 6a689c6e3..704ea1383 100644 --- a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/datasource.rb +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/datasource.rb @@ -7,6 +7,7 @@ def initialize(access_token:, **options) @configuration = Configuration.new(access_token: access_token, **options) @client = Client.new(@configuration) + verify_api_version register_collections end @@ -50,9 +51,30 @@ def register_collections add_collection(Collections::Ticket.new(self, attributes: ticket_attributes)) end - # The three boot-time reads of the datasource. Each degrades to no attribute - # column rather than to a failed boot: a token missing a permission costs - # the columns it could not read, never the agent. + # The one boot read that is not about a column, and the only place the + # pinned API version can be checked: Intercom serves the workspace's own + # default when the pin is not honoured, echoes what it served in a response + # header, and the payload shapes differ between versions -- the silent + # drift this package refuses everywhere else. `Client#me` reads that echo + # and reports a mismatch; nothing else in the datasource calls it, so + # leaving it uncalled left the check as code that never ran. + # + # Degrades like the three attribute reads below: a token that cannot reach + # `/me` costs the check, never the boot. + def verify_api_version + @client.me(boot: true) + rescue APIError => e + ForestAdminDatasourceIntercom.logger.warn( + "[forest_admin_datasource_intercom] could not read /me at boot (HTTP #{e.status || "-"}); the API " \ + 'version Intercom serves was not checked against the pinned one, and a workspace serving another one ' \ + 'answers payloads of another shape.' + ) + end + + # The three attribute reads of the boot, alongside the version check above. + # Each degrades to no attribute column rather than to a failed boot: a + # token missing a permission costs the columns it could not read, never + # the agent. def ticket_attributes Schema::TicketAttributesIntrospector.new(@client).attributes end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/query/condition_tree_translator.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/query/condition_tree_translator.rb index 60ecdbd56..4556015fd 100644 --- a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/query/condition_tree_translator.rb +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/query/condition_tree_translator.rb @@ -29,15 +29,21 @@ class ConditionTreeTranslator MAX_DEPTH = 2 MAX_GROUP_SIZE = 15 - def self.call(condition_tree, endpoint:, collection:, timezone: nil) + def self.call(condition_tree, endpoint:, collection:, timezone: nil, attribute_columns: []) return nil if condition_tree.nil? - new(endpoint: endpoint, collection: collection, timezone: timezone).translate(condition_tree) + new(endpoint: endpoint, collection: collection, timezone: timezone, + attribute_columns: attribute_columns).translate(condition_tree) end - def initialize(endpoint:, collection:, timezone: nil) + # `attribute_columns` are the columns a workspace's own attributes became, + # which no row of the table can name: they are discovered at boot. They + # share one refusal, carried by the endpoint, and passing them here is + # what lets it be read instead of the generic message. + def initialize(endpoint:, collection:, timezone: nil, attribute_columns: []) @endpoint = endpoint @collection = collection + @attribute_columns = Array(attribute_columns).map(&:to_s) @value = FilterValue.new(collection: collection, timezone: timezone) end @@ -133,7 +139,7 @@ def refuse_unfilterable!(column) def unfilterable_reason(column) return relation_reason(column) if column.include?(':') - refusal = @endpoint.refusal(column) + refusal = @endpoint.refusal(column) || attribute_refusal(column) return refusal.reason if refusal "#{@endpoint.path} takes no filter on it. Filter on one of: #{@endpoint.filterable_columns.join(", ")}." @@ -151,6 +157,12 @@ def relation_reason(_column) "#{@endpoint.filterable_columns.join(", ")}." end + # A column the table cannot carry a row for, the workspace having named it: + # the endpoint's one refusal for the whole family answers for it. + def attribute_refusal(column) + @attribute_columns.include?(column) ? @endpoint.attribute_refusal : nil + end + def refuse_operator!(leaf, field) supported = OperatorTable.forest_operators(field) diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/query/search_fields.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/query/search_fields.rb index 64b3ff042..2f3f44a4a 100644 --- a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/query/search_fields.rb +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/query/search_fields.rb @@ -4,7 +4,7 @@ module Query # filter, and hands it to the schema and to the translator as objects rather # than as nested hashes. # - # The table is data rather than code for one reason: `bin/probe_search_fields` + # The table is data rather than code for one reason: `forest_admin_intercom_probe` # rewrites it from a real workspace. Anything derived from it -- which # columns are filterable, with which Forest operators, and what an operator # is told about the ones that are not -- therefore follows a measurement @@ -47,8 +47,8 @@ def sortable? = sortable == true def measured? = source == 'measured' end - Endpoint = Struct.new(:name, :path, :measured_at, :fields, :refused, :candidates, :ticket_attributes, - :custom_attributes, keyword_init: true) do + Endpoint = Struct.new(:name, :path, :measured_at, :fields, :refused, :candidates, :attribute_refusal, + keyword_init: true) do # Whether the probe has run against a real workspace for this endpoint. # False means every `spec` row is still a candidate. def measured? = !measured_at.nil? @@ -60,7 +60,10 @@ def sortable_columns = fields.values.select(&:sortable?).map(&:column) def unmeasured_fields = fields.values.reject(&:measured?) end - class << self + # Long by line count only: it is one parse method and one validation per + # thing the file can get wrong, and a validation that does not say what + # is wrong is one nobody can act on. + class << self # rubocop:disable Metrics/ClassLength def fetch(name) table[name.to_s] || raise(ConfigurationError, "Unknown Intercom search endpoint #{name.inspect}; " \ @@ -94,11 +97,45 @@ def endpoint(name, definition) fields: filterable, refused: refused, candidates: Array(definition['candidates']).freeze, - ticket_attributes: definition['ticket_attributes'], - custom_attributes: definition['custom_attributes'] + attribute_refusal: attribute_refusal(name, definition) ).freeze end + # The refusal that covers a whole family of columns rather than one: + # the attributes a workspace defines, whose names are unknown until the + # datasource boots and which therefore cannot have a row each. Spelled + # `ticket_attributes` on the endpoint that carries them per ticket type + # and `custom_attributes` on the ones that carry them per model, since + # that is the workspace's own vocabulary and what an operator goes and + # renames. + # + # Read by the translator, which is the point: without it a filter on + # such a column falls back on the generic "takes no filter on it", + # where this says why -- and the reason is the arbitration, not an + # oversight. + def attribute_refusal(endpoint, definition) + column = %w[ticket_attributes custom_attributes].find { |key| definition.key?(key) } + return nil if column.nil? + + row = definition.fetch(column) + refusal = Refusal.new(column: column, reason: squish(row.fetch('reason')), source: row.fetch('source')) + validate_source!(endpoint, column, refusal) + validate_attributes_unfilterable!(endpoint, column, row) + + refusal.freeze + end + + # `filterable: true` is a state nothing here implements: the whole + # block is a refusal, and a rewrite flipping the flag would publish + # nothing new while making the file say the opposite of what it does. + def validate_attributes_unfilterable!(endpoint, column, row) + return if row.fetch('filterable') == false + + malformed!(endpoint, column, + "filterable #{row["filterable"].inspect} is not something this reads; the attribute columns " \ + 'are published for display only, and the block exists to say why') + end + # A column filed as both filterable and refused. `Endpoint#field` is # consulted first, so the filterable row would win and the refusal -- # with the reason an operator reads -- would be ignored in silence. The diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/query/search_fields.yml b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/query/search_fields.yml index c5deb04dc..a97b5b610 100644 --- a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/query/search_fields.yml +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/query/search_fields.yml @@ -6,7 +6,7 @@ # # `source` is the provenance of a row, and it is not decoration: # -# measured -- observed against a real workspace, by `bin/probe_search_fields` +# measured -- observed against a real workspace, by `forest_admin_intercom_probe` # or during a spike, and recorded here with the date; # spec -- read off Intercom's documentation and nothing else. # @@ -231,7 +231,13 @@ endpoints: contact_count: reason: Counted by the agent from the contacts the payload carries. source: measured - # What `bin/probe_search_fields` enumerates on top of the fields above: + source_author_name: + reason: >- + The endpoint matches the author of the opening message by e-mail + alone -- `source.author.email` is a field, `source.author.name` is + not. Filter on `source_author_email`. + source: spec + # What `forest_admin_intercom_probe` enumerates on top of the fields above: # names the documentation mentions, or that an ops team would plausibly # search on. A candidate that turns out to be filterable becomes a field # above -- with a column to expose it on, or with none, in which case it is @@ -350,14 +356,44 @@ endpoints: filters a ticket state at all, and under which name, is one of the probe's questions. source: spec - state_external_label: - reason: Read off the state object the ticket embeds. - source: spec ticket_type_name: reason: >- Read off the type object the ticket embeds. Filter on `ticket_type_id`, which the endpoint does take. source: spec + ticket_id: + reason: >- + The number the support team says out loud, which is not the id the + API answers by. The search endpoint carries no field for it. Filter + on `id`, which takes the value the url and the record page use. + source: spec + is_shared: + reason: >- + Whether the ticket is visible to the customer. On the payload and not + in the search DSL; the probe lists it as a candidate. + source: spec + state_id: + reason: >- + Read off the state object the ticket embeds. The endpoint takes no + filter on a ticket state under any name this table knows, which is + what keeps the `state` relation navigable without being filterable. + Whether it filters one at all is the probe's second question. Filter + on `open` or on `category` meanwhile. + source: spec + previous_state_id: + reason: >- + The state the ticket left, read off its payload. Filterable no more + than the state it is in, and for the same reason. + source: spec + contact_name: + reason: >- + Read from the Contacts endpoint, not from the ticket. Filter through + the `contact` relation, which resolves against `/contacts/search` and + rewrites onto `contact_ids`. + source: spec + contact_count: + reason: Counted by the agent from the contacts the payload carries. + source: spec part_count: reason: Counted by the agent from the parts the payload carries. source: measured diff --git a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/company_spec.rb b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/company_spec.rb index 942976dd4..dec029a7f 100644 --- a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/company_spec.rb +++ b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/company_spec.rb @@ -360,6 +360,41 @@ def contact_of(company_id) .to eq([{ 'group' => {}, 'value' => 1 }]) end + # Intercom answers this route with a 404 where a search endpoint would + # answer an empty list, and a filter matching nothing is the most + # ordinary thing a list view does: it reads as no record rather than as a + # failure the operator sees as "Unexpected error". + it 'answers a lookup naming no record with an empty page' do + stub_request(:get, "#{base}/companies").with(query: { 'name' => 'Nope' }) + .to_return(json({ 'type' => 'error.list', + 'errors' => [{ 'code' => 'company_not_found', + 'message' => 'Company Not Found' }] }, + 404)) + + expect(rows(%w[id], condition_tree: leaf('name', operators::EQUAL, 'Nope'))).to eq([]) + end + + it 'counts a lookup naming no record as none' do + stub_request(:get, "#{base}/companies").with(query: { 'name' => 'Nope' }) + .to_return(json({ 'type' => 'error.list', + 'errors' => [{ 'code' => 'company_not_found', + 'message' => 'Company Not Found' }] }, + 404)) + + expect(count(condition_tree: leaf('name', operators::EQUAL, 'Nope'))) + .to eq([{ 'group' => {}, 'value' => 0 }]) + end + + # A 404 is the only status read as an absence: anything else is a failure + # this must not answer an empty page to. + it 'raises on a lookup Intercom refused for another reason' do + stub_request(:get, "#{base}/companies").with(query: { 'name' => 'Acme' }) + .to_return(json({ 'type' => 'error.list' }, 500)) + + expect { rows(%w[id], condition_tree: leaf('name', operators::EQUAL, 'Acme')) } + .to raise_error(APIError, /HTTP 500/) + end + it 'reports a lookup Intercom answered with more than one page' do allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) stub_request(:get, "#{base}/companies").with(query: { 'name' => 'Acme' }) diff --git a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/contact_spec.rb b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/contact_spec.rb index 8fb68d3a4..fc9926424 100644 --- a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/contact_spec.rb +++ b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/contact_spec.rb @@ -436,7 +436,35 @@ def rows(projection = %w[id], **options) rows(%w[id], condition_tree: leaf('company_id', operators::EQUAL, 'co1'), sort: sort({ field: 'name', ascending: true })) - expect(ForestAdminDatasourceIntercom.logger).to have_received(:warn).with(/sort on name/) + # Named for what it is: this collection *is* the one Intercom sorts, so + # the reason is the route, not the column. + expect(ForestAdminDatasourceIntercom.logger) + .to have_received(:warn).with(/sort on name while reading the contacts of an account/) + expect(ForestAdminDatasourceIntercom.logger) + .to have_received(:warn).with(/a route that takes no order, unlike the search/) + end + + # An account Intercom no longer answers for -- deleted, or moved out of + # the token's reach between the moment the row was rendered and the + # moment its related list was opened -- reads as an account with no + # contact rather than as a failed page. + it 'reads an account Intercom no longer answers for as one with no contact' do + stub_request(:get, "#{base}/companies/gone/contacts").with(query: hash_including({})) + .to_return(json({ 'type' => 'error.list' }, 404)) + + expect(rows(%w[id], condition_tree: leaf('company_id', operators::EQUAL, 'gone'))).to eq([]) + expect(collection.aggregate(nil, filter(condition_tree: leaf('company_id', operators::EQUAL, 'gone')), + ForestAdminDatasourceToolkit::Components::Query::Aggregation + .new(operation: 'Count'))) + .to eq([{ 'group' => {}, 'value' => 0 }]) + end + + it 'raises where Intercom refused the account for another reason' do + stub_request(:get, "#{base}/companies/co1/contacts").with(query: hash_including({})) + .to_return(json({ 'type' => 'error.list' }, 500)) + + expect { rows(%w[id], condition_tree: leaf('company_id', operators::EQUAL, 'co1')) } + .to raise_error(APIError, /HTTP 500/) end # An `and` also carrying a scope names a narrower set than the account diff --git a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/ticket_spec.rb b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/ticket_spec.rb index 6bd18731d..89139ab1d 100644 --- a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/ticket_spec.rb +++ b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/ticket_spec.rb @@ -156,6 +156,16 @@ def columns expect(collection.fields['_default_title_'].filter_operators).to be_empty end + # The schema advertises no filter, but a scope, a segment or a customizer + # can still send one -- and the reason it is refused with is the + # arbitration itself rather than "this column is not in the table", which + # is what an attribute column would otherwise get: its name is the + # workspace's, so no row can carry it. + it 'refuses a filter on a ticket attribute with the arbitration as its reason' do + expect { rows(%w[id], condition_tree: leaf('Due', operators::EQUAL, 'x')) } + .to raise_error(UnsupportedOperatorError, /differs from one ticket type to the next/) + end + # Intercom matches text field by field, and this endpoint exposes none # this collection carries. it 'is not searchable' do diff --git a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/datasource_spec.rb b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/datasource_spec.rb index 0dc29ba1d..290bc4123 100644 --- a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/datasource_spec.rb +++ b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/datasource_spec.rb @@ -17,20 +17,48 @@ module ForestAdminDatasourceIntercom IntercomContact IntercomCompany IntercomConversation IntercomTicket]) end - # The three reads a boot performs, and no fourth: the attributes a workspace - # declares on its ticket types, on its contacts and on its companies are - # columns of those collections, and a payload carries the values of the + # The four reads a boot performs, and no fifth: `/me`, which is where + # Intercom echoes the API version it served, and the attributes a workspace + # declares on its ticket types, on its contacts and on its companies -- + # columns of those collections, since a payload carries the values of the # attributes that record happens to have been given, never their # definitions. - it 'introspects the workspace attributes while registering, and reads nothing else' do + it 'checks the version and introspects the workspace attributes, and reads nothing else' do datasource + expect(WebMock).to have_requested(:get, %r{/me}).once expect(WebMock).to have_requested(:get, /ticket_types/).once expect(WebMock).to have_requested(:get, /data_attributes/).with(query: { 'model' => 'contact' }).once expect(WebMock).to have_requested(:get, /data_attributes/).with(query: { 'model' => 'company' }).once expect(WebMock).not_to have_requested(:get, %r{conversations|admins|teams|companies/list}) end + # Intercom serves the workspace's own default when the pin is not honoured + # and the payload shapes differ between versions. The echo is the only + # place that shows, and reading it at boot is what turns the check from + # code that exists into code that runs. + it 'reports a workspace serving another API version than the pinned one' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + stub_me(version: '2.11') + + datasource + + expect(ForestAdminDatasourceIntercom.logger) + .to have_received(:warn).with(/asked Intercom for API version 2\.16 and it served 2\.11/) + end + + # Degrades like the attribute reads: a token that cannot reach `/me` costs + # the check, never the agent. + it 'boots without the check when the token cannot read /me' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + stub_request(:get, %r{/me}).to_return(status: 403, body: '{}', + headers: { 'Content-Type' => 'application/json' }) + + expect(datasource.collections.keys).to include('IntercomTicket') + expect(ForestAdminDatasourceIntercom.logger) + .to have_received(:warn).with(%r{could not read /me at boot \(HTTP 403\)}) + end + # A token without that permission costs the attribute columns, never the # agent. it 'boots without the attribute columns when the introspection is refused' do @@ -54,6 +82,7 @@ module ForestAdminDatasourceIntercom end it 'configures a client from the options it is handed' do + stub_me(base: 'https://api.eu.intercom.io') stub_ticket_types(base: 'https://api.eu.intercom.io') stub_data_attributes('contact', base: 'https://api.eu.intercom.io') stub_data_attributes('company', base: 'https://api.eu.intercom.io') diff --git a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/query/search_fields_spec.rb b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/query/search_fields_spec.rb index c641cb11f..e6edc9088 100644 --- a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/query/search_fields_spec.rb +++ b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/query/search_fields_spec.rb @@ -1,10 +1,15 @@ module ForestAdminDatasourceIntercom RSpec.describe Query::SearchFields do - def table(fields: {}, refused: {}, path: 'tickets/search', measured_at: nil, candidates: []) - described_class.build( - 'endpoints' => { 'tickets' => { 'path' => path, 'measured_at' => measured_at, 'fields' => fields, - 'refused' => refused, 'candidates' => candidates } } - )['tickets'] + def table(fields: {}, refused: {}, measured_at: nil, attributes: nil) + definition = { 'path' => 'tickets/search', 'measured_at' => measured_at, 'fields' => fields, + 'refused' => refused, 'candidates' => [] } + definition['ticket_attributes'] = attributes if attributes + + described_class.build('endpoints' => { 'tickets' => definition })['tickets'] + end + + def attributes_row(overrides = {}) + { 'filterable' => false, 'reason' => 'display only', 'source' => 'spec' }.merge(overrides) end def field_row(overrides = {}) @@ -61,8 +66,26 @@ def field_row(overrides = {}) 'last_responder_type') end + # The columns a workspace's own attributes become cannot have a row each + # -- their names are discovered at boot -- so the endpoint carries one + # refusal for the whole family, and the translator reads it in place of + # the message for a column nobody declared. it 'keeps the ticket attributes unfilterable while the arbitration stands' do - expect(described_class.fetch('tickets').ticket_attributes['filterable']).to be(false) + refusal = described_class.fetch('tickets').attribute_refusal + + expect(refusal.reason).to include('differs from one ticket type to the next') + expect(refusal).to be_measured + end + + it 'carries the same refusal for the custom attributes of a contact' do + expect(described_class.fetch('contacts').attribute_refusal.reason) + .to include('custom_attributes.{name}') + end + + # `/conversations/search` has no attribute family of its own, and a nil + # here is what makes the translator fall back on the column message. + it 'carries none where the endpoint declares no attribute family' do + expect(described_class.fetch('conversations').attribute_refusal).to be_nil end # Until the probe runs against the customer's workspace, the date rows are @@ -93,6 +116,36 @@ def field_row(overrides = {}) end end + # The table is checked in one direction by the schema itself: a column + # publishes exactly the operators its row allows, so it cannot advertise a + # filter the translator would refuse. This is the other direction, and it + # is what the refusal reasons are worth: a column the table says nothing + # about falls back on the generic "takes no filter on it", which is the one + # refusal an operator cannot act on -- and a row whose column was dropped + # survives its own collection, which is how `state_external_label` outlived + # the schema by two lots. + # + # The datasource boots here with no workspace attribute, those columns + # being named by the workspace rather than by the table; they are covered + # by `attribute_refusal` instead. + describe 'the columns of the collections it answers for' do + let(:datasource) { Datasource.new(access_token: 's3cr3t', rate_limiter: nil) } + + { 'IntercomConversation' => 'conversations', 'IntercomTicket' => 'tickets', + 'IntercomContact' => 'contacts' }.each do |collection_name, endpoint_name| + it "names every column of #{collection_name}, as filterable or as refused" do + endpoint = described_class.fetch(endpoint_name) + columns = datasource.get_collection(collection_name).fields + .select { |_, field| field.is_a?(ForestAdminDatasourceToolkit::Schema::ColumnSchema) } + .keys + named = endpoint.filterable_columns + endpoint.refused.keys + + expect(columns - named).to be_empty + expect(named - columns).to be_empty + end + end + end + # The README is where an operator reads what they may filter on before the # interface shows it to them, so it is checked against the table rather than # left to drift from it. @@ -112,6 +165,20 @@ def field_row(overrides = {}) expect(listed).to match_array(described_class.fetch(endpoint).filterable_columns) end end + + # How much of the table a measurement backs is the first thing the README + # says about it, and a figure typed by hand is a figure that drifts the + # next time a row moves. + it 'counts the measured rows the way the file does' do + rows = described_class.endpoints.flat_map do |name| + endpoint = described_class.fetch(name) + + endpoint.fields.values + endpoint.refused.values + end + readme = File.read(File.expand_path('../../../README.md', __dir__), encoding: 'UTF-8') + + expect(readme).to include("#{rows.count(&:measured?)} of #{rows.size} are measured") + end end describe 'a table that cannot be trusted' do @@ -168,6 +235,19 @@ def field_row(overrides = {}) end.to raise_error(ConfigurationError, /filterable and refused at once/) end + # The block is a refusal from end to end, so a rewrite flipping the flag + # would leave the file saying the opposite of what the package does with + # it -- publish nothing. + it 'refuses an attribute family declared filterable' do + expect { table(attributes: attributes_row('filterable' => true)) } + .to raise_error(ConfigurationError, /tickets.ticket_attributes: filterable true is not something/) + end + + it 'refuses an attribute family with no provenance of its own' do + expect { table(attributes: attributes_row('source' => 'hearsay')) } + .to raise_error(ConfigurationError, /tickets.ticket_attributes: source "hearsay"/) + end + it 'names the endpoint and the column it choked on' do expect { table(fields: { 'created_at' => field_row('type' => 'timestamp') }) } .to raise_error(ConfigurationError, /search_fields\.yml is malformed at tickets\.created_at/) diff --git a/packages/forest_admin_datasource_intercom/spec/probe_search_fields_spec.rb b/packages/forest_admin_datasource_intercom/spec/probe_search_fields_spec.rb index 105c4eac7..b7d79b175 100644 --- a/packages/forest_admin_datasource_intercom/spec/probe_search_fields_spec.rb +++ b/packages/forest_admin_datasource_intercom/spec/probe_search_fields_spec.rb @@ -1,6 +1,6 @@ require 'tmpdir' -load File.expand_path('../bin/probe_search_fields', __dir__) +load File.expand_path('../exe/forest_admin_intercom_probe', __dir__) module ForestAdminDatasourceIntercom RSpec.describe ProbeSearchFields do @@ -8,6 +8,25 @@ module ForestAdminDatasourceIntercom let(:endpoint) { Query::SearchFields.fetch('tickets') } let(:client) { Client.new(Configuration.new(access_token: 's3cr3t', rate_limiter: nil)) } + # The gem installs this as `forest_admin_intercom_probe`, and RubyGems runs + # it through a stub that `load`s it -- so `$PROGRAM_NAME` is the stub and + # not this file. A guard comparing the two paths would ship a command that + # exits without doing anything. + describe '.invoked_as_command?' do + it 'runs behind the stub RubyGems installs' do + expect(described_class.invoked_as_command?('/usr/local/bin/forest_admin_intercom_probe')).to be(true) + end + + it 'runs when the file is the command itself' do + expect(described_class.invoked_as_command?(File.expand_path('../exe/forest_admin_intercom_probe', + __dir__))).to be(true) + end + + it 'stays out of the way of whatever else loads it' do + expect(described_class.invoked_as_command?('/usr/local/bin/rspec')).to be(false) + end + end + def json(payload, status = 200) { status: status, body: payload.to_json, headers: { 'Content-Type' => 'application/json' } } end diff --git a/packages/forest_admin_datasource_intercom/spec/spec_helper.rb b/packages/forest_admin_datasource_intercom/spec/spec_helper.rb index 4cbaea580..bdf9ae346 100644 --- a/packages/forest_admin_datasource_intercom/spec/spec_helper.rb +++ b/packages/forest_admin_datasource_intercom/spec/spec_helper.rb @@ -27,12 +27,19 @@ # and a fixture is read by everyone who clones the repo. WebMock.disable_net_connect!(allow_localhost: true) -# A datasource introspects the ticket-type attributes and the contact and -# company attributes while it registers its collections, so every spec building -# one issues those three reads. The base url is not taken from the datasource on -# purpose: reading it would build the datasource, and boot the very reads this -# stubs. +# A datasource checks the API version it was served and introspects the +# ticket-type attributes and the contact and company attributes while it +# registers its collections, so every spec building one issues those four +# reads. The base url is not taken from the datasource on purpose: reading it +# would build the datasource, and boot the very reads this stubs. module IntercomBootStubs + def stub_me(base: ForestAdminDatasourceIntercom::Configuration::REGION_HOSTS[:us], + version: ForestAdminDatasourceIntercom::Configuration::DEFAULT_API_VERSION) + stub_request(:get, "#{base}/me") + .to_return(status: 200, body: { 'type' => 'admin', 'id' => '1', 'email' => 'ops@example.test' }.to_json, + headers: { 'Content-Type' => 'application/json', 'Intercom-Version' => version }) + end + def stub_ticket_types(*types, base: ForestAdminDatasourceIntercom::Configuration::REGION_HOSTS[:us]) stub_request(:get, "#{base}/ticket_types") .to_return(status: 200, body: { 'type' => 'list', 'data' => types }.to_json, @@ -63,6 +70,7 @@ def stub_data_attributes(model, *attributes, config.before do WebMock.reset! + stub_me stub_ticket_types stub_data_attributes('contact') stub_data_attributes('company')