diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f4fe543c2..a88277e4b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -70,6 +70,7 @@ jobs: - forest_admin_datasource_snowflake - forest_admin_datasource_mambu_payments - forest_admin_datasource_graphql_hasura + - forest_admin_datasource_intercom services: mongodb: image: mongo:latest @@ -153,7 +154,7 @@ jobs: with: verbose: true oidc: true - files: ${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_agent/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_datasource_active_record/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_datasource_customizer/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_datasource_toolkit/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_rails/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_datasource_mongoid/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_rpc_agent/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_datasource_rpc/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_datasource_zendesk/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_datasource_snowflake/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_datasource_mambu_payments/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_datasource_graphql_hasura/coverage.json + files: ${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_agent/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_datasource_active_record/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_datasource_customizer/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_datasource_toolkit/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_rails/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_datasource_mongoid/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_rpc_agent/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_datasource_rpc/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_datasource_zendesk/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_datasource_snowflake/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_datasource_mambu_payments/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_datasource_graphql_hasura/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_datasource_intercom/coverage.json deploy: name: Release package diff --git a/.releaserc.js b/.releaserc.js index 76e57c8f0..479152e69 100644 --- a/.releaserc.js +++ b/.releaserc.js @@ -31,7 +31,8 @@ module.exports = { 'sed -i \'s/VERSION = ".*"/VERSION = "${nextRelease.version}"/g\' packages/forest_admin_datasource_zendesk/lib/forest_admin_datasource_zendesk/version.rb; '+ 'sed -i \'s/VERSION = ".*"/VERSION = "${nextRelease.version}"/g\' packages/forest_admin_datasource_snowflake/lib/forest_admin_datasource_snowflake/version.rb; '+ 'sed -i \'s/VERSION = ".*"/VERSION = "${nextRelease.version}"/g\' packages/forest_admin_datasource_mambu_payments/lib/forest_admin_datasource_mambu_payments/version.rb; '+ - 'sed -i \'s/VERSION = ".*"/VERSION = "${nextRelease.version}"/g\' packages/forest_admin_datasource_graphql_hasura/lib/forest_admin_datasource_graphql_hasura/version.rb; ', + 'sed -i \'s/VERSION = ".*"/VERSION = "${nextRelease.version}"/g\' packages/forest_admin_datasource_graphql_hasura/lib/forest_admin_datasource_graphql_hasura/version.rb; '+ + 'sed -i \'s/VERSION = ".*"/VERSION = "${nextRelease.version}"/g\' packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/version.rb; ', successCmd: '( cd packages/forest_admin_agent && gem build && gem push forest_admin_agent-*.gem );' + '( cd packages/forest_admin_datasource_active_record && gem build && gem push forest_admin_datasource_active_record-*.gem );' + @@ -45,7 +46,8 @@ module.exports = { '( cd packages/forest_admin_datasource_zendesk && gem build && gem push forest_admin_datasource_zendesk-*.gem );' + '( cd packages/forest_admin_datasource_snowflake && gem build && gem push forest_admin_datasource_snowflake-*.gem );' + '( cd packages/forest_admin_datasource_mambu_payments && gem build && gem push forest_admin_datasource_mambu_payments-*.gem );' + - '( cd packages/forest_admin_datasource_graphql_hasura && gem build && gem push forest_admin_datasource_graphql_hasura-*.gem );' , + '( cd packages/forest_admin_datasource_graphql_hasura && gem build && gem push forest_admin_datasource_graphql_hasura-*.gem );' + + '( cd packages/forest_admin_datasource_intercom && gem build && gem push forest_admin_datasource_intercom-*.gem );' , }, ], [ @@ -68,6 +70,7 @@ module.exports = { 'packages/forest_admin_datasource_snowflake/lib/forest_admin_datasource_snowflake/version.rb', 'packages/forest_admin_datasource_mambu_payments/lib/forest_admin_datasource_mambu_payments/version.rb', 'packages/forest_admin_datasource_graphql_hasura/lib/forest_admin_datasource_graphql_hasura/version.rb', + 'packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/version.rb', 'package.json' ], }, diff --git a/.rubocop.yml b/.rubocop.yml index c6985f27d..b2cc222be 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -42,6 +42,7 @@ Gemspec/RequireMFA: - 'packages/forest_admin_datasource_snowflake/forest_admin_datasource_snowflake.gemspec' - 'packages/forest_admin_datasource_mambu_payments/forest_admin_datasource_mambu_payments.gemspec' - 'packages/forest_admin_datasource_graphql_hasura/forest_admin_datasource_graphql_hasura.gemspec' + - 'packages/forest_admin_datasource_intercom/forest_admin_datasource_intercom.gemspec' # Offense count: 1 # This cop supports unsafe autocorrection (--autocorrect-all). @@ -133,6 +134,7 @@ Style/MutableConstant: - 'packages/forest_admin_datasource_zendesk/lib/forest_admin_datasource_zendesk/version.rb' - 'packages/forest_admin_datasource_snowflake/lib/forest_admin_datasource_snowflake/version.rb' - 'packages/forest_admin_datasource_mambu_payments/lib/forest_admin_datasource_mambu_payments/version.rb' + - 'packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/version.rb' # Offense count: 38 # This cop supports safe autocorrection (--autocorrect). @@ -217,6 +219,7 @@ Style/StringLiterals: - 'packages/forest_admin_datasource_zendesk/lib/forest_admin_datasource_zendesk/version.rb' - 'packages/forest_admin_datasource_snowflake/lib/forest_admin_datasource_snowflake/version.rb' - 'packages/forest_admin_datasource_mambu_payments/lib/forest_admin_datasource_mambu_payments/version.rb' + - 'packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/version.rb' # Offense count: 1 # This cop supports safe autocorrection (--autocorrect). @@ -261,6 +264,8 @@ Metrics/ParameterLists: - 'packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/capture.rb' - 'packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/store.rb' - 'packages/forest_admin_datasource_graphql_hasura/lib/forest_admin_datasource_graphql_hasura/configuration.rb' + - 'packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/configuration.rb' + - 'packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/client.rb' - 'packages/forest_admin_datasource_zendesk/lib/forest_admin_datasource_zendesk/collections/base_collection.rb' - 'packages/forest_admin_datasource_snowflake/lib/forest_admin_datasource_snowflake/datasource.rb' - 'packages/forest_admin_agent/lib/forest_admin_agent/routes/query_handler.rb' @@ -296,6 +301,7 @@ Metrics/ModuleLength: - 'packages/forest_admin_datasource_mambu_payments/spec/**/*' - 'packages/forest_admin_rails/spec/**/*' - 'packages/forest_admin_rpc_agent/spec/**/*' + - 'packages/forest_admin_datasource_intercom/spec/**/*' - 'packages/forest_admin_datasource_mongoid/lib/forest_admin_datasource_mongoid/utils/helpers.rb' Metrics/MethodLength: diff --git a/packages/forest_admin_datasource_intercom/.gitignore b/packages/forest_admin_datasource_intercom/.gitignore new file mode 100644 index 000000000..06cfcfb83 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/.gitignore @@ -0,0 +1,8 @@ +*.gem +.bundle/ +Gemfile.lock +Gemfile-test.lock +coverage/ +pkg/ +tmp/ +.rspec_status diff --git a/packages/forest_admin_datasource_intercom/.rspec b/packages/forest_admin_datasource_intercom/.rspec new file mode 100644 index 000000000..34c5164d9 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/.rspec @@ -0,0 +1,3 @@ +--format documentation +--color +--require spec_helper diff --git a/packages/forest_admin_datasource_intercom/Gemfile b/packages/forest_admin_datasource_intercom/Gemfile new file mode 100644 index 000000000..c229ff1d5 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/Gemfile @@ -0,0 +1,16 @@ +source 'https://rubygems.org' + +gemspec + +gem 'forest_admin_datasource_customizer' +gem 'forest_admin_datasource_toolkit' +gem 'rake', '~> 13.0' +gem 'rubocop', '1.86.1' +gem 'rubocop-performance', '1.26.1' +gem 'rubocop-rspec', '3.9.0' + +group :development, :test do + gem 'rspec', '~> 3.0' + gem 'simplecov', '~> 0.22', require: false + gem 'webmock', '~> 3.0' +end diff --git a/packages/forest_admin_datasource_intercom/Gemfile-test b/packages/forest_admin_datasource_intercom/Gemfile-test new file mode 100644 index 000000000..8b433e2b5 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/Gemfile-test @@ -0,0 +1,19 @@ +source 'https://rubygems.org' + +# Specify your gem's dependencies in forest_admin_datasource_intercom.gemspec +gemspec + +gem 'rake', '~> 13.0' +gem 'rubocop', '1.86.1' +gem 'rubocop-performance', '1.26.1' +gem 'rubocop-rspec', '3.9.0' + +group :development, :test do + gem 'forest_admin_datasource_customizer', path: '../forest_admin_datasource_customizer' + gem 'forest_admin_datasource_toolkit', path: '../forest_admin_datasource_toolkit' + gem 'rspec', '~> 3.0' + gem 'simplecov', '~> 0.22', require: false + gem 'simplecov-html', '~> 0.12.3' + gem 'simplecov_json_formatter', '~> 0.1.4' + gem 'webmock', '~> 3.0' +end diff --git a/packages/forest_admin_datasource_intercom/README.md b/packages/forest_admin_datasource_intercom/README.md new file mode 100644 index 000000000..952014b0b --- /dev/null +++ b/packages/forest_admin_datasource_intercom/README.md @@ -0,0 +1,698 @@ +# Forest — Intercom datasource + +Surface [Intercom](https://www.intercom.com) conversations, tickets, teammates, teams, ticket types +and ticket states as Forest collections. + +This is **lot 1: read only**. Rows, record details, exact counts and the conversation thread work; +server-side filtering, writes, business actions, contacts and companies arrive in the lots after it +— see "What is not here yet". + +## Installation + +```ruby +# Gemfile +gem 'forest_admin_datasource_intercom' +``` + +## Usage + +```ruby +# app/lib/forest_admin_rails/create_agent.rb +ForestAdminAgent::Builder::AgentFactory.instance.add_datasource( + ForestAdminDatasourceIntercom::Datasource.new( + access_token: ENV['INTERCOM_ACCESS_TOKEN'], + region: :eu # :us (default), :eu or :au + ) +) +``` + +The token is the access token of a private app, created in Intercom's Developer Hub under +*Configure › Authentication*. OAuth is out of scope: it belongs to a control plane distributing a +connector, not to an agent reading one workspace. + +`Client#me` is the health check — it returns the admin the token belongs to, and is the one call +that verifies the pinned API version was honoured. + +### Configuration + +| Option | Default | What it is for | +| --- | --- | --- | +| `access_token` | — | Required. The private app's bearer token. | +| `region` | `:us` | `:us`, `:eu`, `:au`. A workspace answers in its own region only. | +| `base_url` | from `region` | Wins over `region`. For an egress proxy or a mock server. | +| `api_version` | `'2.16'` | Sent as `Intercom-Version` on every request. | +| `open_timeout` / `timeout` | `5` / `30` | A request that already has a page on screen. | +| `boot_open_timeout` / `boot_timeout` | `3` / `10` | The one read performed while the agent starts. | +| `retry_policy` | `RetryPolicy.new` | Statuses, verbs and backoff. | +| `boot_retry_policy` | `RetryPolicy.boot` | One quick retry; gives up rather than waiting a 429 out. | +| `rate_limiter` | `RateLimiter.new` | `nil` takes the pacing out of the stack. | + +**Pin the region explicitly.** `api.intercom.io` does route to the right one, but a workspace under +GDPR wants its requests reaching the European host and nothing else. + +**The version is pinned on purpose.** Without the header a request follows the workspace's own +default version, which an operator can change on Intercom's side — and the payloads change shape +underneath. Intercom echoes the version it served, so `me` compares the two and logs a warning when +the pin was not honoured, rather than raising: running against a version we did not ask for still +beats not running. + +### Token permissions + +A read-only token is enough, and is what to recommend for this lot. A permission the token lacks +costs **columns or a collection, never the boot of the agent**: the three boot-time introspections +each degrade to no attribute column, a collection whose endpoint answers 403 fails its own page, and +a token that cannot read `/admins` or `/teams` leaves the `admin_names` / `team_names` column empty +rather than failing the page it is on. A token denied contacts or companies costs those two +collections and the `contact_name` column, and leaves everything else standing. + +A **relation is the exception**, and it is worth knowing before scoping a token: resolving one reads +the target endpoint, and that read is not guarded the way the names above are. A token denied +`/admins` fails any page projecting `admin_assignee:name`, and fails the related list behind +`IntercomTeam#admins` — the failure lands on the collection being read, not on the one that was +denied. Scope the token to the endpoints in the table below, or to none of them. + +## Collections + +| Collection | Endpoint | Paginated | Countable | +| --- | --- | --- | --- | +| `IntercomConversation` | `GET /conversations`, `POST /conversations/search`, `GET /conversations/{id}` | cursor | yes, exactly | +| `IntercomTicket` | `POST /tickets/search`, `GET /tickets/{id}` | cursor | yes, exactly | +| `IntercomAdmin` | `GET /admins` | read whole | yes, exactly | +| `IntercomTeam` | `GET /teams` | read whole | yes, exactly | +| `IntercomTeamMembership` | `GET /teams` | read whole | yes, exactly | +| `IntercomTicketType` | `GET /ticket_types` | read whole | yes, exactly | +| `IntercomTicketState` | `GET /ticket_states` | read whole | yes, exactly | +| `IntercomContact` | `GET /contacts`, `POST /contacts/search`, `GET /companies/{id}/contacts` | cursor | yes, exactly | +| `IntercomCompany` | `POST /companies/list`, `GET /companies?...`, `GET /companies/{id}` | **offset** | yes, exactly | + +Three tiers, and they behave differently on purpose. + +**Read whole** — admins, teams, team memberships, ticket types, ticket states. Their endpoints answer in one response, +so filtering, sorting, paging and counting them in memory is *exact*: the records in hand are every +record Intercom holds. These are the only collections that can be filtered, sorted and grouped in +this lot, and the only ones a chart may group by. The cost is bandwidth, not correctness. + +**Cursor** — conversations, tickets and contacts. What is in hand is a page of something far larger, +so nothing is filtered or sorted in memory. Three routes and no fourth: no condition walks the +listing, `id equals X` reads the record through its own endpoint, and anything else is translated +into Intercom's search DSL and walked through the search endpoint. What the translation cannot +express is **refused by name** — see [Filtering](#filtering). Contacts add two routes of their own, +both described under [Contacts](#contacts). + +**Offset** — companies, and nothing else. `POST /companies/list` takes a **page number**, which is +what a list view asks for: page 7 is one request rather than six pages walked to reach it, with no +cap and no truncation warning. It is the one place the [first limitation +below](#what-the-api-cannot-do-and-what-this-does-about-it) does not apply. What it pays for that is +filtering — there is no company search endpoint at all, so what a filter may say is a handful of +exact lookups and nothing else. See [Companies](#companies). + +## Relations + +Intercom joins nothing: a ticket carries an assignee id, and the teammate behind it is a second read +of a second endpoint. What makes these relations affordable is what the collection on the far end +costs to read by id, and the three tiers do not cost the same: + +| Target | Read by id | A page of rows costs | Fan-out | +| --- | --- | --- | --- | +| `IntercomAdmin`, `IntercomTeam`, `IntercomTicketState`, `IntercomTicketType` | read whole | **one request**, whatever the page holds | unbounded | +| `IntercomContact` | `id IN [...]`, 100 at a time | one request per 100 distinct contacts | unbounded | +| `IntercomCompany` | `GET /companies/{id}` | **one request per distinct account** | bounded, see below | + +The price is per target *collection*, not per relation: a ticket's `state` and `previous_state` are +one read of `/ticket_states`, over the ids both of them name. + +**The one relation with a ceiling is `IntercomContact#company`.** There is no bulk read for a +company — `/companies/scroll` is [deliberately rejected](#companies) — so each distinct account on +the page costs a request, and past +[`MAX_RELATION_READS`](lib/forest_admin_datasource_intercom/collections/offset_collection.rb) +distinct accounts the read is **refused by name** rather than resolved for the first slice and left +nil for the rest. Every page size a list view offers sits under that figure; an export or a segment +resolved whole does not, and the message says to leave the column out or read fewer rows at a time. +The alternative — a nil where an account exists — is the one answer this datasource must not give, +and a log line nobody reads is not a substitute for it. + +*Exactly*, with one more bound worth naming: "read whole" is what the endpoint answers, and +`fetch_all` stops after [`MAX_COLLECTED_PAGES`](lib/forest_admin_datasource_intercom/client.rb) pages +if Intercom paginates one of these on its own — it logs when it does. A workspace whose `/admins` or +`/teams` runs past that cap resolves the relations pointing at the records it dropped as empty. The +figure is sized for reference collections, which is what the first tier above is. + +| Collection | Relation | Target | Filterable through | +| --- | --- | --- | --- | +| `IntercomConversation` | `admin_assignee`, `closed_by` | `IntercomAdmin` | yes | +| `IntercomConversation` | `team_assignee` | `IntercomTeam` | yes | +| `IntercomTicket` | `admin_assignee` | `IntercomAdmin` | yes | +| `IntercomTicket` | `team_assignee` | `IntercomTeam` | yes | +| `IntercomTicket` | `ticket_type` | `IntercomTicketType` | yes | +| `IntercomTicket` | `state`, `previous_state` | `IntercomTicketState` | **no** — read and navigate only | +| `IntercomTeam` | `admins` | `IntercomAdmin` | no (many-to-many) | +| `IntercomAdmin` | `teams` | `IntercomTeam` | no (many-to-many) | +| `IntercomTeamMembership` | `team`, `admin` | `IntercomTeam`, `IntercomAdmin` | yes | +| `IntercomConversation` | `contact` | `IntercomContact` | yes | +| `IntercomTicket` | `contact` | `IntercomContact` | **spec, unprobed** — see below | +| `IntercomContact` | `owner` | `IntercomAdmin` | yes | +| `IntercomContact` | `company` | `IntercomCompany` | **no** — read and navigate only | +| `IntercomContact` | `conversations`, `tickets` | `IntercomConversation`, `IntercomTicket` | no (one-to-many) | +| `IntercomCompany` | `contacts` | `IntercomContact` | no (one-to-many) | + +Every one of them is **read-only**: this lot writes nothing, and Intercom exposes no endpoint that +writes a team membership at all. + +**`IntercomTeamMembership` exists because Intercom's does not.** The workspace carries the +membership on the team (`admin_ids`) and on the teammate (`team_ids`) both and exposes no resource +for the pair, while a many-to-many needs a collection to travel through. It is synthesized from +`GET /teams`, one record per pair, keyed `teamId:adminId`. Without it, both sides read as an array of +ids nobody can click. + +Two consequences of travelling through it are worth knowing. A **related list of teammates is +ordered by the membership, not by the teammate**: the agent hands the through collection the columns +of the collection the relation reaches, so an order on `name` or `email` cannot be resolved there and +is logged rather than silently dropped. And a `admin_ids` entry naming a teammate `/admins` does not +answer -- one who left, one outside the token's reach -- **drops out of the related list** instead of +appearing as an empty row. + +Alongside it, a team names its teammates (`admin_names`) and a teammate its teams (`team_names`) on +the row itself, so a list view reads without a join. **Those replace the arrays of ids** the first +lots published: one readable form plus a relation to navigate, rather than two ways to read one fact. +They are read only when a projection asks for them, and a token that cannot read the other side +costs the column and nothing else — never the page, and never the relation. + +**The 360 degrees is those last four rows.** From a ticket or a conversation, `contact` reaches the +person who wrote in; from them, `conversations` and `tickets` list everything they ever opened, and +`company` reaches their account, whose `contacts` lists their colleagues. Each of those lists is one +request: `/conversations/search` matches a conversation against one of its contact ids, and +`GET /companies/{id}/contacts` answers the contacts of an account — which is the one relation +`/contacts/search` could not have resolved, filtering no company field. + +**A conversation has several contacts, and the relation names the first of them** — the same one +`contact_name` and `contact_count` describe, so the column and the relation cannot disagree. The +others are a hop away: open that contact and read their conversations. The alternative, a +many-to-many through a join collection, would have been the honest cardinality at the price of three +collections of plumbing in the interface; naming the first contact and counting them is what lot 1 +already published, and lot 4 promotes it rather than replacing it. + +Two of these carry a caveat worth reading before scoping a token or writing a segment. The **ticket +side is a `spec` row the probe has not confirmed**: whether `/tickets/search` filters on +`contact_ids` at all is unmeasured, and if it does not, the relation stays navigable and the filter +moves to the refusal table — exactly what happened to the ticket `state`. And **the company +traversal is refused by name**: `/contacts/search` filters no company field, so `company:name` is +answered with a message saying to filter from the company side instead. + +The same rule settled the ticket labels: `state_label` and `ticket_type_name` stay on the row, +`state_category` and `state_external_label` are gone — they are a hop away, on the `state` relation, +and neither was ever filterable, so no segment, scope or saved filter could rest on them. + +A relation reads its target **undecorated**, so a permission scope or a segment defined on the target +does not narrow what a relation resolves — the same way a native datasource joins a table without +applying the scopes of the collection mapped to it. + +One semantic worth stating plainly: **a row whose foreign key is null matches no relation filter**, +the way a join drops it, negated filters included. A ticket with no assignee is not "assigned to +someone other than Marie". + +## What the API cannot do, and what this does about it + +Where Forest asks for something Intercom has no equivalent for, this datasource **refuses with a +message naming the reason** rather than answering something that looks right and is not. Those +arrive as a 400 carrying the text. + +- **No offset pagination, except on companies.** Intercom hands out the page after a cursor and + documents that jumping to page N is unsupported, so reaching page 20 costs 20 sequential requests. + The walk is capped at 50 pages / 7 500 records and **every route out of it that is short of what + was asked for is logged**, naming the window it stopped in: the two caps, and the two defensive + stops — a page that advertises a next cursor and holds nothing, and a cursor already followed. + Intercom does neither of the last two today, which is exactly why they are reported rather than + taken for the end of the data. `POST /companies/list` is the exception and takes a page number, + which is why companies escape the walker and its caps entirely. +- **Duplicates on a moving dataset.** Intercom documents that records modified between two paginated + requests can be served twice; the walk deduplicates by id. The missed counterpart is inherent to + cursor pagination and cannot be repaired — it is documented rather than papered over. +- **One endpoint sorts, and it is `/contacts/search`.** Everything else comes back in the order the + API imposes: `POST /companies/list` has no order parameter at all, and the other two search + endpoints **accept a `sort` and ignore it** — measured, it raises nothing and changes nothing. + Since that is undetectable at runtime, no column of `IntercomConversation`, `IntercomTicket` or + `IntercomCompany` is declared sortable and a requested order is reported in the log. The + collections read whole sort in memory, exactly, and Contacts sort server-side on the columns the + measured table declares — see [Contacts](#contacts). One route of Contacts cannot carry it either: + a read by id cuts the window in the order the ids were named, so ordering what comes back would + order a slice picked by something else. That order is reported in the log too. +- **No aggregate endpoint.** Counting is free and exact — `total_count` counts what the query names, + not what a page held — so the record counter is one request. Anything beyond a count is refused on + the cursor collections: grouping over the pages a walk collected would look exact while answering + a fraction. +- **`per_page` is refused past 150**, with `invalid_per_page` and no silent downgrade, so the page + size is bounded before the request leaves. Tickets are bounded far lower still: **25**, because + the search response carries the whole timeline of every ticket and Intercom offers no field + selection. Provisional, pending measurement against real response sizes. +- **No `GET /tickets` at all.** Even an unfiltered ticket list goes through `POST /tickets/search` + with a predicate matching everything. +- **The envelope key is not always `data`.** Measured: `/tickets/search` answers under `tickets`, + `/admins` under `admins`, `/teams` under `teams`. A response carrying neither the expected key nor + `data` is refused rather than read as an empty page. + +## Filtering + +`POST /conversations/search` and `POST /tickets/search` answer the condition trees Forest sends, on +the fields Intercom really filters and with the operators each endpoint really validates. Anything +else is **refused with a message naming what to change** — a condition dropped on the way out comes +back as an unfiltered page that looks filtered, which is the one answer this datasource must not +give. A refusal costs no request: it is raised before anything leaves the process. + +### The table is data, and it says where each row comes from + +The fields a search endpoint filters are not the fields its specification lists. Measured: +`/tickets/search` refuses `company_id` with `invalid_field` although every ticket carries one. So +the source of truth is a committed table — `lib/forest_admin_datasource_intercom/query/search_fields.yml` +— one row per column, each carrying its provenance: + +| `source` | What it means | +| --- | --- | +| `measured` | observed against a real workspace, by `forest_admin_intercom_probe` or during the spike | +| `spec` | read off Intercom's documentation, and therefore still a candidate | + +Every `filter_operators` a column publishes is **derived** from that table, so a column cannot +advertise a filter the translator would then refuse, and a column the table does not carry +advertises nothing at all. That is the mechanism, and it holds whatever the rows say. + +**What the rows say today is mostly `spec`: 18 of 89 are measured, and no endpoint has been probed +end to end** — all three carry `measured_at: null`, which is what `Endpoint#measured?` reports. The +measured rows are the ones a spike went out of its way to check: the date operators on each +endpoint, which disagree between them, `id IN` on `/contacts/search`, `contact_ids` on +`/conversations/search`, and the refusals a read confirmed — `company_id` on `/tickets/search` +above all, alongside the columns the agent derives rather than reads. Everything else is Intercom's +documentation, and the disagreement above is why that is a candidate rather than a promise. Those +two figures are asserted against the file, so they cannot drift from it. + +So the first thing to do against a customer's workspace is to run the probe. The rows worth watching +first, in the order they will hurt: + +1. **`admin_assignee_id` and `team_assignee_id`**, on both search endpoints. Typed `string` here; + Intercom documents them as `Integer` and answers `data_invalid` on a value whose type it does not + accept. These carry the `admin_assignee` and `team_assignee` relations — the filter an ops team + reaches for first — so a wrong type here is the most expensive `spec` row in the file; +2. **`state_id` on `/tickets/search`** — the table carries no filter on it at all, which is what + keeps the `state` relation read-only. If the endpoint does filter one, a support queue becomes + filterable by state; +3. **`contact_ids` on `/tickets/search`** — the contact relation of a ticket rests on it; +4. **the operators Intercom answers on `custom_attributes.{name}`**, per data type, which is the only + thing keeping those columns display-only; +5. **whether `POST /conversations/search` honours `display_as=plaintext`** — it is sent either way, + and an ignored parameter costs a query string where the honoured one saves every filtered row + from coming back as markup. + +To measure a workspace of your own. The probe ships with the gem — it is what measures the +customer's workspace, and whoever runs it there has the gem installed rather than a clone of +`agent-ruby` — so `bundle install` puts it on the path of the application the datasource is +mounted in: + +```bash +INTERCOM_ACCESS_TOKEN=... bundle exec forest_admin_intercom_probe --endpoint tickets --out measured.yml +``` + +It sends one search per (field, operator) cell, reads Intercom's refusal codes — `invalid_field` for +a field the endpoint does not filter, `data_invalid` for an operator it refuses on that field — and +prints what the committed table promises that Intercom refuses, plus what Intercom accepts that the +table does not know about. It writes evidence rather than rewriting the table, which carries the +prose a generated file would drop. + +### A date filter is day-granular, and the day is the UTC one + +Intercom truncates a date search to the day, at the **UTC** boundary — measured, and against its own +documentation, which promises the workspace's timezone. `> V` answers from the start of the day +*after* V; `< V` answers before the start of V's own day. + +Sent as they come, the two bounds an interval is rewritten into cancel each other out: `today` +reaches the datasource as `> 00:00` and `< 23:59` of one day, which Intercom reads as "from +tomorrow" *and* "before today" — no rows at all, to the most ordinary filter there is. So each bound +is moved to the boundary that makes Intercom answer the day the filter named. + +What follows from that: + +- a bound naming a time of day matches **from the start of that day, or through the end of it**. It + is the granularity the Intercom interface itself filters on; +- a caller in UTC gets exactly the day they asked for; +- a caller in another timezone gets the UTC days their window overlaps — up to a day wider at each + end — and the agent logs that once per filter; +- a date column publishes `>` and `<` only, and no equality. Everything an operator actually uses — + `before`, `after`, `today`, `yesterday`, `past`, `future`, the whole `previous_*` family — is + rewritten by the agent into a pair of those bounds. An equality on an instant is what stays out, + and a day-granular filter could not have honoured it anyway. + +### What is filterable + +| Collection | Filterable on | +| --- | --- | +| `IntercomConversation` | `id`, `state`, `priority`, `open`, `read`, `title`, `admin_assignee_id`, `team_assignee_id`, `source_type`, `source_subject`, `source_body`, `source_delivered_as`, `source_author_email`, `closed_by_id`, `reopen_count`, `part_count`, `ai_agent_participated`, `contact_id`, and the dates `created_at`, `updated_at`, `waiting_since`, `snoozed_until`, `closed_at`, `first_closed_at`, `first_contact_reply_at`, `last_contact_reply_at`, `last_admin_reply_at` | +| `IntercomTicket` | `id`, `open`, `category`, `ticket_type_id`, `admin_assignee_id`, `team_assignee_id`, `contact_id`, `created_at`, `updated_at` | +| `IntercomContact` | `id`, `role`, `name`, `email`, `email_domain`, `phone`, `external_id`, `owner_id`, `unsubscribed_from_emails`, `has_hard_bounced`, `marked_email_as_spam`, `language_override`, `browser`, `browser_language`, `os`, `location_country`, `location_region`, `location_city`, and the dates `created_at`, `updated_at`, `signed_up_at`, `last_seen_at`, `last_contacted_at`, `last_replied_at`, `last_email_opened_at`, `last_email_clicked_at` | +| `IntercomCompany` | `id`, `company_id`, `name` — four lookups and no search endpoint, see [Companies](#companies) | + +**The primary key** is filterable like any other column, but a filter naming it *alone* is not +answered by a search: `id equals X` and `id in [...]` read the record endpoint directly, one request +per record. The search answers it only when something else is filtered alongside it — a permission +scope, a segment, or a second filter. + +**Free-text search** is answered on two collections, each on the one column its endpoint matches text +on: + +| Collection | Searched on | +| --- | --- | +| `IntercomConversation` | `~` on `source.body` — the message that opened the conversation | +| `IntercomContact` | `~` on `email` — what an ops team types when they are looking for someone | + +Intercom matches `~` **per word, not as a substring**: searching `fact` does not find `facture`, and +searching `acme` does not find `camille@acme.test`. `IntercomTicket` exposes no text column its +endpoint matches and refuses a search by name; `IntercomCompany` has no search endpoint at all. + +### What is not filterable, and why + +- **every column of a company but two.** There is no `/companies/search`: Intercom looks a company + up by `name`, by `company_id`, by `tag_id` or by `segment_id`, one exact value at a time, and the + first two are the ones that name a column of the collection. Everything else — the industry, the + plan, the monthly spend — is refused by name. Filtering by tag or by segment belongs with the lot + that adds those collections; +- **a contact's `company_id`, `company_count`, `avatar` and `session_count`** — the endpoint filters + none of them. Reach the contacts of an account from the account instead, through its `contacts` + relation, which `GET /companies/{id}/contacts` answers in one request — **and answers alone**. That + endpoint returns the contacts of the account whole and narrows nothing, so a `company_id equals X` + carrying anything else cannot be answered at all: there is no request that takes both halves. + Which means the related list of an account **is refused as soon as a permission scope or a segment + is defined on `IntercomContact`**, since that is what the agent intersects into the condition. The + refusal names the condition it could not carry alongside the account. Resolving it properly would + mean reading the account's contact ids first and handing `id IN [...]` plus the rest of the tree to + `/contacts/search` — which that endpoint does answer, and which is not in this lot; +- **a set of ids, counted.** `id in [...]` reads one record per id (100 at a time on contacts), so + counting a set means reading it, and past what a bulk read fetches the count is refused rather + than answered with the number the truncation left. A collection that advertises an exact count does + not answer 25 to a question about forty records; +- **the custom attributes of a contact or a company.** They are filtered as + `custom_attributes.{name}`, by name — the ambiguity that keeps ticket attributes display-only does + not arise here — but which operators Intercom answers on each data type has not been measured, and + this package publishes no filter it has not seen work. They ship typed and display-only, and the + probe is what turns that around; +- **the columns a ticket derives from its parts** — `closed_at`, `closed_by_name`, `last_reply_at`, + `last_responder_name`, `last_responder_type`. They exist nowhere in Intercom; `/tickets/search` + filters none of them and ignores a sort on them without a word; +- **the account of a ticket** — `company_id`, refused by the endpoint itself with `invalid_field`; +- **the state of a ticket** — the measured table carries no filter on a state id, so `state_id`, + `previous_state_id` and the `state` relation are read and navigated rather than filtered. Whether + the endpoint filters one at all is one of the probe's open questions; +- **the ticket attributes** — filtered as `ticket_attribute.{id}`, and the same attribute carries a + different id per ticket type, so a union column has no single id to translate to. See + [Tickets](#tickets); +- **the tag names, the company name and the contact identity of a conversation** — read from + somewhere the search endpoint does not filter, or filtered by an id the column does not hold; +- **absence** — `present`, `blank` and `missing` are derived by the agent from an equality and + rewritten into a comparison with an empty value. Intercom's search matches values and has no + operator for the lack of one, so the rewritten condition is refused rather than sent as a + comparison against the empty string; +- **group-by**, on either cursor collection: there is no aggregate endpoint, and grouping over the + pages a walk collected would look exact while answering a fraction. + +### Through a relation + +A relation is published filterable as soon as *any* column of its target is — the agent decides that, +not this datasource — so the interface offers `admin_assignee:name` the moment the relation exists. +What Intercom is really filtered on is the foreign key: the **target says which of its records +match**, over every record it holds rather than over a page, and the ids it names become the +condition the search carries. + +That is exact, and it has four visible edges: + +- **The target is read one record past what a group may hold, and no further.** Against a collection + read whole that costs nothing — every record is in hand — but Contacts are a page of something + far larger, and resolving `contact:email contains "@"` over a whole workspace to then refuse the + fan-out it comes to would spend a full cursor walk on a filter that was never going to be + answered. So the read is bounded, and the refusal says "more than fifteen" rather than a count it + deliberately did not go and measure. +- Intercom takes no membership operator on these fields, so several matches become **one equality per + match**, inside an `OR` — which counts against the fifteen conditions a group allows. A relation + condition matching more records than that is refused by name rather than sent and answered with a + 400 naming neither the limit nor the filter that hit it. That `OR` is **inlined into a parent that + aggregates the same way**, so it costs no level of nesting where it does not have to: the two + levels Intercom allows are spent on the filter that was written, not on the expansion of a + relation. Where inlining it would take the parent past fifteen conditions it stays nested, width + being the scarcer of the two. +- A condition the target matched **no record** with names no row, and the DSL cannot say so: the + search is skipped entirely rather than sent as a filter that would come back with everything. +- A relation whose foreign key the endpoint does not filter — the ticket `state`, a contact's + `company` — is refused with a message saying which of the two it is: the relation is there to be + read and navigated. Whether `/tickets/search` filters a state id or a contact id at all is one of + the probe's open questions; the answer lands in the table, not in an assumption. + +On the collections read whole the same condition costs nothing: they filter in memory, so the ids go +in as a plain membership and none of the DSL's limits apply. + +A **many-to-many is published unfilterable** — `admins` and `teams` — and a condition written on one +anyway, in a scope or a segment, is refused before it reaches this datasource: the agent's own +validator answers a 400 naming the field and its type. Filter on a column of the collection next +door instead. + +### The limits of a search, checked before the request leaves + +Intercom nests a search **two levels** deep and takes **fifteen conditions per group**. Past either +it answers a 400 whose body names neither the limit nor the part of the filter that reached it, so +both are checked here and refused with a message naming what to simplify. + +Fifteen is reached without trying: a scope, a segment and an operator's own filter add up, and a +condition naming several values arrives expanded into **one condition per value** — Intercom accepts +no membership operator on these fields. Branches carrying a single condition are unwrapped and spend +no level. + +## Conversations + +The row carries what a queue is read for: state, priority, assignee and team ids, the company, the +tags, and the lifecycle Intercom keeps in `statistics` — `closed_at`, `closed_by_id`, +`first_contact_reply_at`, `last_contact_reply_at`, `last_admin_reply_at`, `reopen_count`. + +**The timeline opens on `source`, not on the parts.** The message that started the conversation +lives there; a thread built from the parts alone opens on the first reply and loses what the +customer actually asked. Every entry keeps its `part_type` — an assignment, a note and a reply are +different events. + +Intercom returns the parts **only when retrieving a single conversation**, so: + +- a record detail gets its timeline for free; +- a list view asking for the `timeline` column pays one request per row, bounded to 10. The rows past + that keep a `nil`, which reads as *unknown* — never as an empty thread. + +A conversation is capped at its **500 most recent parts**; a very long thread is therefore partial, +and says so nowhere but here. + +**The internal notes of the team are in the thread**, next to what the customer was told. That is +what a thread is on Intercom, and publishing half of it would be the more surprising answer — but it +is worth knowing before opening the collection to a role that should not read them. + +The contact's name is denormalized onto the row by **one bulk read per page**, not one per row, and +only when the projection names it. A failure there costs that column, not the page. The e-mail is a +hop away, on the `contact` relation. + +## Tickets + +A ticket carries **no `statistics` block** — measured against a workspace of 81 142 tickets — so +neither a closure date nor a last responder exists as a field. Both are derived from the parts, +which ride along in the search response whether or not anything asks for them, and therefore cost +nothing: + +| Column | Derived from | +| --- | --- | +| `closed_at`, `closed_by_name` | the last transition into a state of category `resolved` | +| `last_reply_at`, `last_responder_name`, `last_responder_type` | the last `comment` part | + +Four things to know about them: + +- a ticket is not "closed" on Intercom, it enters a **resolved** state; +- the state-change event is matched on its **prefix**, not on `ticket_state_updated_by_admin`: a + workspace running workflows closes tickets through other variants, and an invisible closure is + worse than an absent column; +- a transition whose target equals the previous state is ignored — measured, they exist; +- **a resolved ticket showing no closure date may have been closed all the same**: past the 500-part + ceiling the transition falls out of the window. That case is detected and logged, since a Date + column cannot say "unknown". + +Both are **display only**, and not temporarily: `/tickets/search` filters on neither and ignores a +sort, so neither advertises an operator. + +**The thread is published too, and it is free here.** The same `timeline` column a conversation +carries — who said what, when, and through which kind of event, internal notes and state changes +included — built from the parts the response already holds. No request per row and no cap: where a +conversation read from a listing carries no parts at all and leaves the column `nil` for *unknown*, +a ticket always carries them, so an empty list means an empty thread. Both reads ask Intercom for +`display_as=plaintext`: the bodies are HTML written by end customers, and rendering third-party +markup inside Forest is neither safe nor useful. The 500-part ceiling applies here as well, which is +the same truncation that can hide a closure date. + +The attributes a workspace declares on its ticket types are introspected once at boot and published +as the **union** of every type's, keyed by name the way the payload is. Filtering one is a different +matter: Intercom filters an attribute by id (`ticket_attribute.{id}`), and the same name carries a +different id from one ticket type to the next — measured, `_default_title_` is `14162161` on one +type and `14162165` on another. A union column has no single id to translate to, so filtering on a +ticket attribute means one collection per ticket type — more collections in the interface, and a +schema that changes shape whenever the customer adds a type. Until that trade is worth paying for, +the attributes stay display-only and advertise no operator. The ids are kept per type so the day the +answer changes costs no second boot round trip. + +## Contacts + +The people who write in, users and leads alike. Cursor-paginated like conversations and tickets, +with two routes of its own and one thing no other collection has. + +**Intercom sorts this one.** `POST /contacts/search` is the only endpoint of the whole API that +takes a `sort` and applies it, so these are the only sortable columns of the datasource: `name`, +`email`, `created_at`, `updated_at`, `signed_up_at`, `last_seen_at`, `last_contacted_at`, +`last_replied_at`. The set is deliberately narrower than the documentation implies — nothing has +been measured, and a sort Intercom refuses is a list view that fails rather than one that comes back +unordered. A sort on any other column, or on two columns at once, is reported in the log and the +rows come back in the API's order: Intercom takes a single `{ field, order }`, and honouring the +first clause of two would order the page by something nobody asked for. + +An order is also what routes a plain list view through the search endpoint, the listing sorting +nothing: the read then carries the predicate matching everything that Tickets already send. + +**Its date operators are narrower than the other two endpoints'** — measured, 25 August 2026: +`/contacts/search` refuses `>=`, `<=` and `!=` on a date where `/conversations/search` and +`/tickets/search` take them. Nothing is lost that an operator can see, a Date column publishing the +two bounds alone everywhere in this datasource, but it is why the operator table is per endpoint. + +**A set of ids is read in one request** — `id IN [...]`, which this endpoint answers and no other +does — a hundred at a time, rather than one request per record. It is what makes a related list of +contacts affordable. + +**`company_id equals X` reads `GET /companies/{id}/contacts`.** The search filters no company field, +so without that route the contacts of an account would be a refusal rather than a list. It is a bare +equality only: an `and` also carrying a permission scope names a narrower set than the account does, +and answering it with the account alone would serve contacts the scope excludes. + +That is a limit worth knowing before scoping permissions, because it is not a slower route but no +route at all: the account endpoint returns its contacts whole and narrows nothing, and the search +filters no company field, so **a scope or a segment on this collection turns the related list of an +account into a refusal**. The message names the condition it could not carry alongside the account, +rather than telling the operator to open the account and read its contacts — which is what they were +doing. What would answer it is a read of the account's contact ids followed by `id IN [...]` plus the +rest of the tree on `/contacts/search`, which that endpoint takes; it is not in this lot. + +One more thing the two routes do not agree on, and it is visible: the `company_id` column names **the +first** of the accounts a contact belongs to, and the `company` relation resolves that same first +one — while `company_id equals X` returns **every** contact of X. So an account's related list can +show a contact whose `company` points somewhere else. The column is the payload's reading; the filter +is the account endpoint's, and it is the more useful of the two. + +**A merged contact reads as gone, not as an error.** Intercom drops it from the listing and from the +search, and the record lives on under the id it was merged into. A row pointing at the old id comes +back empty rather than failing the page. + +The custom attributes a workspace declares on its contacts are introspected once at boot from +`GET /data_attributes?model=contact`, typed from `data_type`, and published display-only. + +## Companies + +The accounts contacts belong to, and the collection that behaves least like the others. + +**Paginated by offset**, which is the tier above. **Looked up, not searched**: `name` and +`company_id` — the identifier the customer's own system gave the account, not Intercom's — are the +two filters it publishes, each an exact equality, and anything else is refused by name. A record is +read through `GET /companies/{id}`, and a set of ids one request each, capped at 25 with the +truncation logged. + +`GET /companies/scroll` exists and is **deliberately rejected**: one open scroll per application, +expiring after a minute, cannot serve two operators looking at a list at the same time. + +A contact carries its accounts as a list of ids and nothing else, so **projecting `company:name` on +a contact list costs one request per distinct account on the page**. Reading the account from the +contact's record page, or listing contacts from the account, both cost one. + +Which is why that projection is the one relation of the datasource with a ceiling: past +[`MAX_RELATION_READS`](lib/forest_admin_datasource_intercom/collections/offset_collection.rb) +distinct accounts on a single read, it is **refused rather than resolved for part of the rows**. Every +page size a list view offers stays under it. A read that does not — an export, which batches a +thousand rows at a time, or a segment resolved whole — has to leave the column out. See +[Relations](#relations). + +Custom attributes are introspected at boot the same way, from `GET /data_attributes?model=company`, +and published display-only for the same reason. + +## Rate limits + +Intercom meters the app and, above it, the whole workspace — 25 000 requests a minute shared with +every other private app the customer runs — and allocates that budget in **10-second windows**: the +measured `x-ratelimit-limit` is 1667, not 10 000. A burst therefore takes a 429 while the minute's +budget is barely touched, which is why what matters is the instantaneous rate. + +The limiter is driven by the headers Intercom returns on every response rather than by a table: it +waits out the reset when the window is spent, and counts its own in-flight requests down so several +of them do not go out on the same stale figure. A reset further out than a window is a clock +disagreement rather than a window emptying — the request goes through and the log says so, once per +window. + +It sits **in front of** the 429 retry, not instead of it: the retry remains the defence against the +part of the workspace budget spent by traffic this process cannot see. Pass `rate_limiter: nil` to +meter on your own side instead. + +## Privacy + +The body of a conversation is raw personal data, and this datasource is built on that assumption. + +- **Nothing logs a body.** Logs carry the operation, the counts and Intercom's request id — never + content. A response that fails to parse is reported by name, never quoted: a JSON parser opens its + message with the characters it choked on, and on a 200 those are the payload. +- **`display_as=plaintext` on every conversation read.** The bodies are HTML written by end + customers; rendering third-party HTML inside Forest is neither safe nor useful. +- **The regional host is configurable** so a workspace's data stays in its region. +- Ticket list pages carry customer message bodies whether or not anything asks for them — Intercom + offers no field selection. Restrict the body columns with Forest's field-level permissions where + that matters. + +## Boot-time introspection + +Constructing the datasource performs exactly **four** reads. + +Three are of one kind: `GET /ticket_types` for the attribute columns of `IntercomTicket`, and +`GET /data_attributes?model=contact` and `?model=company` for those of `IntercomContact` and +`IntercomCompany`. A payload carries the values of the attributes that record happens to have been +given, never their definitions, which is why they cannot be discovered from the records. + +The fourth is `GET /me`, and it reads no column: Intercom echoes in a response header the API +version it served, and it serves the workspace's own default when the pin is not honoured — whose +payloads are shaped differently from the ones this expects. That echo is the only place the +substitution shows, so it is checked while the agent starts and reported as a warning. + +All four run on the boot connection — short timeouts, one quick retry — so a slow Intercom cannot +turn a Rails boot into minutes the operator sits through, and each degrades to a warning rather +than to a failed boot: a token missing a permission costs the columns it could not read, or the +version check, never the agent. + +`api_writable` is read alongside each attribute and kept, although every column of this lot is +published read-only: it is what tells an attribute the API may write from one Intercom fills in +itself, and reading it again later would be a second boot-time round trip. + +Everything else is read when a collection is listed, so an agent boots whatever Intercom is doing. + +## What is not here yet + +| Lot | What it brings | +| --- | --- | +| 3 | Writes and business actions on tickets and conversations: reply, close, snooze, reopen, assign, tag, convert | +| 4b | Writes on contacts and companies: create, update, archive, block, merge, attach and detach | +| 5 | Notes, tags, segments | +| 6 | Bounded group-by and the reporting export | + +Two questions this lot leaves in the table rather than in an assumption, both for +`forest_admin_intercom_probe` to answer against the customer's workspace: whether `/tickets/search` +filters on `contact_ids`, and which operators `/contacts/search` answers on a custom attribute. + +## Development + +```bash +cd packages/forest_admin_datasource_intercom +BUNDLE_GEMFILE=Gemfile-test bundle install +BUNDLE_GEMFILE=Gemfile-test bundle exec rspec +bundle exec rubocop # from the repository root +``` + +`exe/forest_admin_intercom_probe` ships with the gem rather than living in the repository alone: +what it measures is the customer's workspace, and whoever runs it there has the gem installed and +not a clone of this repository. From a checkout it runs in place, `exe/forest_admin_intercom_probe`. + +Specs stub the HTTP layer with WebMock. Every payload they feed in is **hand-written from the +OpenAPI 2.16 specification**, never captured from a workspace: a conversation body is personal data, +and a fixture is read by everyone who clones the repository. diff --git a/packages/forest_admin_datasource_intercom/Rakefile b/packages/forest_admin_datasource_intercom/Rakefile new file mode 100644 index 000000000..4c774a2bf --- /dev/null +++ b/packages/forest_admin_datasource_intercom/Rakefile @@ -0,0 +1,6 @@ +require 'bundler/gem_tasks' +require 'rspec/core/rake_task' + +RSpec::Core::RakeTask.new(:spec) + +task default: :spec diff --git a/packages/forest_admin_datasource_intercom/exe/forest_admin_intercom_probe b/packages/forest_admin_datasource_intercom/exe/forest_admin_intercom_probe new file mode 100755 index 000000000..4d004c9d2 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/exe/forest_admin_intercom_probe @@ -0,0 +1,267 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +# Enumerates what Intercom's search endpoints really filter, and with which +# operators, against a real workspace. The answer is what `search_fields.yml` +# holds and what the schema derives its filters from -- and it cannot be read +# off the documentation: measured during lot 1, `/tickets/search` refuses +# `company_id` with `invalid_field` although a ticket carries one. +# +# INTERCOM_ACCESS_TOKEN=... forest_admin_intercom_probe --endpoint tickets +# INTERCOM_ACCESS_TOKEN=... forest_admin_intercom_probe --region eu --out measured.yml +# +# Shipped with the gem rather than left in the repository: what it measures is +# the customer's workspace, and the operator running it has the gem installed +# and not a clone of the agent. +# +# It sends one search per (field, operator) cell, asking for a single record, +# and reads Intercom's refusal codes: `invalid_field` for a field the endpoint +# does not filter at all -- the rest of its row is then skipped -- `data_invalid` +# for an operator it refuses on that field, `invalid_value` for a value shape it +# refuses. Read-only: a search changes nothing, and no payload is printed or +# written, only field names, operators and error codes. +# +# What it writes with `--out` is *evidence*, deliberately not the table itself: +# `search_fields.yml` carries the prose that says why a column stays refused, +# and a generated file would drop it. Merge the measurements into the table by +# hand, which is also where deciding to expose a newly discovered field belongs. + +$LOAD_PATH.unshift(File.expand_path('../lib', __dir__)) + +require 'forest_admin_datasource_intercom' +require 'optparse' + +module ProbeSearchFields + SearchFields = ForestAdminDatasourceIntercom::Query::SearchFields + + # Whether this file is being run as the command rather than loaded by a + # spec. The basenames and not the paths: RubyGems installs an executable + # behind a stub of its own and `load`s this file from it, so `$PROGRAM_NAME` + # is the stub while `__FILE__` is this file -- comparing the two would make + # the shipped command exit without doing anything, which is the one thing + # shipping it was for. A spec loading it runs under `rspec`, whose basename + # is not this one. + def self.invoked_as_command?(program_name) + File.basename(program_name.to_s) == File.basename(__FILE__) + end + + # A cell is probed with the value shapes its field plausibly takes, most + # likely first: a wrong shape is refused with `data_invalid` just like an + # unsupported operator, so a single attempt would report an operator as + # refused when only the value was wrong. + VALUES = { + 'date' => ['1', 1, '2026-01-01'], + 'number' => [0, '0'], + 'boolean' => [true, 'true'], + 'string' => ['forest-probe', 0], + 'text' => ['forest-probe'] + }.freeze + + LIST_OPERATORS = %w[IN NIN].freeze + + # A candidate carries no declared type -- discovering it is the point -- so + # one is guessed from its name, and the fallbacks above cover a wrong guess. + def self.guess_type(field) + case field + when /(_at|_since|_until)\z/ then 'date' + when /\Acount_|_count\z|\Atime_to_|_time\z/ then 'number' + when /\A(open|read|is_|has_|.*_participated)/ then 'boolean' + else 'string' + end + end + + class Probe + def initialize(client, endpoint) + @client = client + @endpoint = endpoint + end + + # Every operator of the alphabet on every field of the row: what the table + # already declares, plus the candidates. Nothing is assumed from the + # declared operators -- an operator missing from the table is exactly what + # this is here to find. + def run(field, type) + results = {} + + SearchFields::KNOWN_OPERATORS.each do |operator| + outcome = probe(field, operator, type) + return { unfilterable: outcome } if outcome[:code] == 'invalid_field' + + results[operator] = outcome + end + + { operators: results } + end + + private + + def probe(field, operator, type) + last = nil + + VALUES.fetch(type, VALUES['string']).each do |value| + outcome = attempt(field, operator, operator_value(operator, value)) + # `invalid_field` is about the field, not about what was sent with it: + # another value shape cannot make a field the endpoint does not filter + # appear, so the row ends here rather than paying a request per shape. + return outcome if outcome[:ok] || outcome[:code] == 'invalid_field' + + last = outcome + end + + last + end + + def attempt(field, operator, value) + @client.search_page(@endpoint.path, query: { 'field' => field, 'operator' => operator, 'value' => value }, + per_page: 1, list_key: list_key) + { ok: true } + rescue ForestAdminDatasourceIntercom::APIError => e + { ok: false, code: error_code(e), status: e.status } + end + + def operator_value(operator, value) + LIST_OPERATORS.include?(operator) ? [value] : value + end + + # `/tickets/search` answers under `tickets` and `/conversations/search` + # under `conversations`; a body read under the wrong key raises before the + # outcome is classified. + def list_key = @endpoint.name + + def error_code(error) + errors = error.body.is_a?(Hash) ? error.body['errors'] : nil + first = Array(errors).first + + (first.is_a?(Hash) ? first['code'] : nil) || "http_#{error.status}" + end + end + + class Report + def initialize(endpoint) + @endpoint = endpoint + @rows = {} + end + + def record(field, column, result) + @rows[field] = { 'column' => column, 'result' => result } + end + + # What the run says about the committed table, which is the only reason to + # run it: an operator the table promises and Intercom refuses is a filter + # the interface offers and the read cannot honour. + def print_diff + @rows.each do |field, row| + column = row['column'] + result = row['result'] + + if result[:unfilterable] + declared = column ? " <- table declares it on column '#{column}'" : '' + puts " #{field.ljust(42)} NOT FILTERABLE (#{result[:unfilterable][:code]})#{declared}" + next + end + + print_operators(field, column, result[:operators]) + end + end + + def to_yaml_document + { 'endpoint' => @endpoint.name, 'path' => @endpoint.path, 'measured_at' => Time.now.utc.strftime('%Y-%m-%d'), + 'fields' => @rows.to_h { |field, row| [field, measured_row(row)] } }.to_yaml + end + + private + + def print_operators(field, column, outcomes) + supported = outcomes.select { |_, outcome| outcome[:ok] }.keys + declared = column ? Array(@endpoint.field(column)&.operators) : [] + puts " #{field.ljust(42)} #{supported.empty? ? "(no operator answered)" : supported.join(" ")}" + + report_drift(declared - supported, supported - declared) + end + + def report_drift(promised, discovered) + puts " ! the table promises #{promised.join(", ")} here, Intercom refuses it" unless promised.empty? + puts " + Intercom also accepts #{discovered.join(", ")}" unless discovered.empty? + end + + def measured_row(row) + result = row['result'] + return { 'filterable' => false, 'code' => result[:unfilterable][:code] } if result[:unfilterable] + + { 'filterable' => true, + 'operators' => result[:operators].select { |_, outcome| outcome[:ok] }.keys, + 'refused' => result[:operators].reject { |_, outcome| outcome[:ok] } + .transform_values { |outcome| outcome[:code] } } + end + end + + class CLI + def self.call(argv) + new(options(argv)).run + end + + def self.options(argv) + options = { endpoints: SearchFields.endpoints, region: 'us', token: ENV.fetch('INTERCOM_ACCESS_TOKEN', nil) } + + OptionParser.new do |parser| + parser.banner = 'Usage: INTERCOM_ACCESS_TOKEN=... forest_admin_intercom_probe [options]' + parser.on('--endpoint NAME', "one of #{SearchFields.endpoints.join(", ")}") { |v| options[:endpoints] = [v] } + parser.on('--region NAME', 'us (default), eu or au') { |v| options[:region] = v } + parser.on('--token TOKEN', 'defaults to $INTERCOM_ACCESS_TOKEN') { |v| options[:token] = v } + parser.on('--out PATH', 'write the measurements as YAML evidence') { |v| options[:out] = v } + end.parse!(argv) + + options + end + + def initialize(options) + @options = options + end + + def run + abort('No token: set INTERCOM_ACCESS_TOKEN or pass --token.') if @options[:token].to_s.empty? + + documents = @options[:endpoints].map { |name| probe_endpoint(SearchFields.fetch(name)) } + write(documents) if @options[:out] + 0 + end + + private + + def probe_endpoint(endpoint) + puts "\n#{endpoint.path} -- #{endpoint.fields.size} declared field(s), #{endpoint.candidates.size} candidate(s)" + probe = Probe.new(client, endpoint) + report = Report.new(endpoint) + + targets(endpoint).each { |field, column, type| report.record(field, column, probe.run(field, type)) } + report.print_diff + report.to_yaml_document + end + + # The declared fields first, with the type the table gives them; then the + # candidates, whose type is guessed. A candidate already covered by a + # declared field is not probed twice. + def targets(endpoint) + declared = endpoint.fields.values.map { |field| [field.field, field.column, field.type] } + candidates = (endpoint.candidates - declared.map(&:first)).map { |field| [field, nil, guess_type(field)] } + + declared + candidates + end + + def guess_type(field) = ProbeSearchFields.guess_type(field) + + def client + @client ||= ForestAdminDatasourceIntercom::Client.new( + ForestAdminDatasourceIntercom::Configuration.new(access_token: @options[:token], region: @options[:region]) + ) + end + + def write(documents) + File.write(@options[:out], documents.join("\n")) + puts "\nWritten to #{@options[:out]}. Merge the measurements into search_fields.yml by hand: " \ + 'the table carries the prose a generated file would drop.' + end + end +end + +exit(ProbeSearchFields::CLI.call(ARGV)) if ProbeSearchFields.invoked_as_command?($PROGRAM_NAME) diff --git a/packages/forest_admin_datasource_intercom/forest_admin_datasource_intercom.gemspec b/packages/forest_admin_datasource_intercom/forest_admin_datasource_intercom.gemspec new file mode 100644 index 000000000..14ce48bb4 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/forest_admin_datasource_intercom.gemspec @@ -0,0 +1,36 @@ +lib = File.expand_path('lib', __dir__) +$LOAD_PATH.unshift lib unless $LOAD_PATH.include?(lib) + +require_relative 'lib/forest_admin_datasource_intercom/version' + +Gem::Specification.new do |spec| + spec.name = 'forest_admin_datasource_intercom' + spec.version = ForestAdminDatasourceIntercom::VERSION + spec.authors = ['Forest Admin'] + spec.email = ['contact@forestadmin.com'] + spec.homepage = 'https://www.forestadmin.com' + spec.summary = 'Intercom datasource for Forest Admin Ruby agent.' + spec.description = 'Surface Intercom conversations, tickets, contacts and companies as Forest Admin collections.' + spec.license = 'GPL-3.0' + spec.required_ruby_version = '>= 3.0.0' + + spec.metadata['homepage_uri'] = spec.homepage + spec.metadata['source_code_uri'] = 'https://github.com/ForestAdmin/agent-ruby' + spec.metadata['changelog_uri'] = 'https://github.com/ForestAdmin/agent-ruby/blob/main/CHANGELOG.md' + spec.metadata['rubygems_mfa_required'] = 'false' + + spec.files = Dir.chdir(__dir__) do + `git ls-files -z`.split("\x0").reject do |f| + (File.expand_path(f) == __FILE__) || + f.start_with?(*%w[bin/ test/ spec/ features/ .git .circleci appveyor Gemfile]) + end + end + spec.bindir = 'exe' + spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) } + spec.require_paths = ['lib'] + + spec.add_dependency 'activesupport', '>= 6.1' + spec.add_dependency 'faraday', '~> 2.0' + spec.add_dependency 'faraday-retry', '~> 2.0' + spec.add_dependency 'zeitwerk', '~> 2.3' +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom.rb new file mode 100644 index 000000000..e87d432a6 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom.rb @@ -0,0 +1,72 @@ +require_relative 'forest_admin_datasource_intercom/version' +require 'cgi/escape' +require 'json' +require 'logger' +require 'set' +require 'time' +require 'uri' +require 'yaml' +require 'zeitwerk' +# `Time.zone` and `Time.use_zone`, which is how a date filter is read in the +# timezone the caller wrote it in -- see `Query::CallerZone`. Required here +# rather than relied on: the toolkit happens to pull ActiveSupport in far +# enough for these to exist, but that is its business and not a contract, and a +# `require` of this gem on its own left them undefined. +# +# Both lines: `active_support/time` brings `Time.zone`, and the bare +# `active_support` brings the autoload table `Time.use_zone` reaches into for +# `ActiveSupport::IsolatedExecutionState`. Without the second, the first raises +# a NameError on the first date filter rather than at load. +require 'active_support' +require 'active_support/time' +require 'faraday' +require 'faraday/retry' +require 'forest_admin_datasource_toolkit' + +loader = Zeitwerk::Loader.for_gem +loader.setup + +module ForestAdminDatasourceIntercom + class Error < StandardError; end + class ConfigurationError < Error; end + + # A filter Intercom cannot express exactly: an operator its search DSL refuses + # on that field, a tree deeper than the two levels it allows, or a group past + # its fifteen filters. It descends from the toolkit's ValidationError rather + # than from this package's Error so the agent answers 400 carrying the message + # instead of a 500 "Unexpected error" -- each one names something the operator + # set and can change, and the message is the only place they learn which. + # + # This datasource refuses rather than approximates: a result that looks + # filtered and is not is worse than an explicit refusal. + class UnsupportedOperatorError < ForestAdminDatasourceToolkit::Exceptions::ValidationError; end + + # Raised when an Intercom API call fails. Carries the HTTP status and the + # parsed response body so callers -- smart actions in particular -- can + # surface Intercom's own error message instead of a generic string. + class APIError < Error + attr_reader :status, :body + + def initialize(message, status: nil, body: nil) + super(message) + @status = status + @body = body + end + end + + class << self + attr_writer :logger + + def logger + @logger ||= default_logger + end + + private + + def default_logger + return Rails.logger if defined?(Rails) && Rails.respond_to?(:logger) && Rails.logger + + Logger.new($stderr).tap { |l| l.progname = 'forest_admin_datasource_intercom' } + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/client.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/client.rb new file mode 100644 index 000000000..47551b209 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/client.rb @@ -0,0 +1,455 @@ +module ForestAdminDatasourceIntercom + # Every call to Intercom goes through here. Hand-written on Faraday rather + # than through the official `intercom` gem: that one maps the JSON onto + # objects, and what this datasource needs is the parts it hides -- the raw + # payload, because a custom attribute is a key nobody declared in advance; + # the quota headers, because the pacing is driven by them; and Intercom's own + # error body, because that text is what an operator reads when an action + # fails. + # Long by line count only: the public surface is one method per endpoint, and + # the rest is the envelope and error handling every one of them shares. + class Client # rubocop:disable Metrics/ClassLength + # `per_page=200` is refused with `invalid_per_page` -- "must be an integer + # between 0 and 150". There is no silent downgrade, so a page size is bounded + # before it is sent or the list view breaks rather than shrinks. + MAX_PER_PAGE = 150 + + # Bounds `fetch_all`, which asks for a whole reference collection rather than + # a window: those endpoints answer in one response, so reaching this many + # pages means Intercom started paginating on its own and the read is spending + # more than the answer is worth. + MAX_COLLECTED_PAGES = 10 + + # `next_cursor` is nil as soon as Intercom stops advertising a next page, so + # callers never have to know how the absence is spelled on the wire. + # `total_count` is exact, filter included, which is what makes Forest's + # record counter and its "number of" charts one request each. + # `total_pages` is filled by the one endpoint that paginates by offset and + # nil everywhere else: a cursor page has no notion of how many there are. + Page = Struct.new(:records, :next_cursor, :total_count, :total_pages, keyword_init: true) + + def initialize(configuration) + @configuration = configuration + end + + # Health check: the admin the token belongs to, plus its workspace. Enough + # to prove the credentials are usable, and the one call that verifies the + # pinned API version was honoured -- Intercom echoes the version it served + # in a response header. + # + # `boot: true` runs it on the short-timeout connection, for a caller + # checking the token while the agent is still starting. + def me(boot: false) + must_succeed('me') do + response = get('me', boot: boot) + verify_pinned_version(response) + response.body + end + end + + # One page of a cursor-paginated listing. `starting_after` is what the + # previous page advertised; nil asks for the first one. + # + # `CursorWalker` is what turns the offset/limit window a list view asks for + # into a sequence of these. + # + # `list_key` is the key the endpoint puts its records under, and it is not + # `data` everywhere: `/tickets/search` answers under `tickets` (measured), so + # a caller names its own and `data` stays the fallback. + def list_page(path, per_page:, starting_after: nil, params: {}, list_key: 'data', boot: false) + query = params.merge('per_page' => self.class.bounded_per_page(per_page)) + query['starting_after'] = starting_after unless blank?(starting_after) + + must_succeed(path) { to_page(get(path, query, boot: boot).body, path, list_key) } + end + + # One page of a search endpoint. The query is the one the condition-tree + # translator wrote, and it travels in the body; `params` is what still + # belongs in the query string -- `display_as` above all, which is not part + # of the search payload. + # `sort` is honoured by `/contacts/search` alone. The other two search + # endpoints accept one and ignore it without a word -- measured -- so the + # collections that read them never send one, and the parameter is here for + # the one that does. `{ field:, ascending: }`, translated to Intercom's own + # spelling on the way out. + def search_page(path, query:, per_page:, starting_after: nil, list_key: 'data', params: {}, sort: nil) + pagination = { 'per_page' => self.class.bounded_per_page(per_page) } + pagination['starting_after'] = starting_after unless blank?(starting_after) + body = { 'query' => query, 'pagination' => pagination } + body['sort'] = sort_clause(sort) if sort + + must_succeed(path) { to_page(post(path, body, params: params).body, path, list_key) } + end + + # One page of an endpoint that paginates by **offset** rather than by + # cursor. `POST /companies/list` is the only one, and it is what lets the + # Companies collection answer page 7 of a list view with one request + # instead of walking six pages to reach it. Intercom counts pages from 1. + def offset_page(path, page:, per_page:, params: {}, list_key: 'data') + query = params.merge('page' => [page.to_i, 1].max, + 'per_page' => self.class.bounded_per_page(per_page)) + + must_succeed(path) { to_page(post(path, {}, params: query).body, path, list_key) } + end + + # An exact lookup, and the shape surprise that comes with it: `GET + # /companies?name=` answers the company itself where `?tag_id=` answers a + # list. A record is read here as a page of one, so a caller writes one route + # rather than testing the envelope. + def lookup_page(path, params:, list_key: 'data') + must_succeed(path) do + body = get(path, params).body + next Page.new(records: [body], next_cursor: nil, total_count: 1) if single_record?(body, list_key) + + to_page(body, path, list_key) + end + end + + # One record from its own endpoint. Raises on a 404 like on any other + # failure: what a missing record means -- a stale link, a record outside the + # token's scope, a deletion -- is the caller's to decide, not the client's. + def fetch_record(path, id, params: {}, boot: false) + operation = "#{path}/#{id}" + + must_succeed(operation) do + body = get("#{path}/#{Faraday::Utils.escape(id)}", params, boot: boot).body + body.is_a?(Hash) ? body : refuse_body_shape(operation, 'the response is not a record') + end + end + + # Every record of an endpoint that answers in one response: the reference + # collections -- admins, teams, ticket types, ticket states -- whose paths + # declare no pagination parameter at all. + # + # `list_key` is the key the endpoint puts its records under. Intercom is not + # consistent about it: `/admins` answers `{"type": "admin.list", "admins": + # [...]}` where `/ticket_types` answers the `data` envelope every paginated + # listing uses, so the collection names its own and `data` is the fallback. + # + # A cursor is followed if one is advertised, defensively: no pagination + # parameter in the specification is not a promise that a large workspace + # answers in one response, and a truncated reference collection would show + # an operator a state list missing its last states. + # `params` is what narrows the endpoint rather than what pages it: + # `/data_attributes` answers the attributes of contacts and those of + # companies under `?model=`, and both are read whole. + def fetch_all(path, list_key: 'data', params: {}, boot: false) + must_succeed(path) { collect_pages(path, list_key: list_key, params: params, boot: boot) } + end + + # The page size Intercom accepts, whatever was asked for. + def self.bounded_per_page(size) + value = size.to_i + return 1 if value < 1 + + [value, MAX_PER_PAGE].min + end + + # The client holds the connections whose headers carry the access token in + # clear, and Faraday prints those headers on `inspect`. + def inspect + "#<#{self.class.name} url=#{@configuration.redacted_url.inspect}>" + end + + private + + # The raw response rather than its body: the quota headers are read by the + # throttle, and the version echo by `verify_pinned_version`. + def get(path, params = nil, boot: false) + (boot ? boot_connection : connection).get(path, params) + end + + def post(path, body, params: {}, boot: false) + http = boot ? boot_connection : connection + + http.post(path) do |request| + request.params.update(params) unless params.nil? || params.empty? + request.body = body + end + end + + # Intercom serves the version its workspace defaults to when the pin is not + # honoured, and the payloads differ between versions. The echo is the only + # way to notice, and noticing at boot is worth more than a schema that + # drifts silently -- so this reports rather than raises: the agent still runs + # against a version it did not ask for, which is better than not running. + def verify_pinned_version(response) + served = response.headers['intercom-version'] if response.respond_to?(:headers) + return if served.nil? || served.to_s == @configuration.api_version + + ForestAdminDatasourceIntercom.logger.warn( + "[forest_admin_datasource_intercom] asked Intercom for API version #{@configuration.api_version} " \ + "and it served #{served}. Payload shapes may differ from the ones this datasource expects; check the " \ + "workspace's default version in the Developer Hub." + ) + end + + # Intercom spells an order `{ "field": "...", "order": "descending" }`, + # and answers `data_invalid` on anything else -- so the clause is written + # here rather than by the caller, whose vocabulary is Forest's. + def sort_clause(sort) + { 'field' => sort[:field].to_s, 'order' => sort[:ascending] == false ? 'descending' : 'ascending' } + end + + # A record rather than a listing: no list under either key, and an id where + # a record carries one. An empty listing is not one of these -- it answers + # `data` as an empty array, which is a page of nothing rather than a record. + def single_record?(body, list_key) + body.is_a?(Hash) && !body[list_key].is_a?(Array) && !body['data'].is_a?(Array) && body.key?('id') + end + + def collect_pages(path, list_key:, params:, boot:) + records = [] + cursor = nil + pages = 0 + + loop do + query = cursor.nil? ? params : params.merge('starting_after' => cursor) + body = get(path, query.empty? ? nil : query, boot: boot).body + records.concat(extract_entities(body, path, list_key)) + pages += 1 + cursor = next_cursor(body, path) + break if cursor.nil? + + if pages >= MAX_COLLECTED_PAGES + log_collection_cap(path, pages, records.size) + break + end + end + + records + end + + # The records under the key the endpoint uses, or under `data`. Anything + # else is refused rather than read as an empty page: `Array()` would turn the + # envelope into `[key, value]` pairs a collection would serialize into rows + # holding nothing -- a page that looks answered and is empty -- and a + # reference collection silently read as empty is a state column with no + # values and an assignee shown as a raw id. + def extract_entities(body, operation, list_key) + return [] if body.nil? || body == '' + + entities = body.is_a?(Hash) ? (body[list_key] || body['data']) : nil + return entities if entities.is_a?(Array) + + detail = list_key == 'data' ? "'data' is not a list" : "neither '#{list_key}' nor 'data' is a list" + refuse_body_shape(operation, detail) + end + + def log_collection_cap(path, pages, collected) + ForestAdminDatasourceIntercom.logger.warn( + "[forest_admin_datasource_intercom] Stopped reading #{path} after #{pages} page(s) / " \ + "#{collected} record(s); the rest is left out. This endpoint is read whole on purpose, so a workspace " \ + 'this large needs the collection bounded rather than listed.' + ) + end + + # Intercom wraps a listing in `{ "type": "list", "data": [...], + # "total_count": N, "pages": { "next": { "starting_after": "..." } } }`. + def to_page(body, operation, list_key) + Page.new(records: extract_entities(body, operation, list_key), + next_cursor: next_cursor(body, operation), + total_count: extract_count(body), + total_pages: extract_total_pages(body)) + end + + # Absent on the last page, which is how the walk knows it is done. An older + # API version spells it as a url instead of an object -- and one can be + # served despite the pin, which is what the version echo warns about -- so + # the cursor is read out of its query string rather than the page being + # taken for the last one. + # + # Anything else is refused, an advertised page whose cursor cannot be read + # included: taking it for the last page would truncate the answer silently, + # which is worse than a failure naming what it could not read. + def next_cursor(body, operation) + advertised = body.is_a?(Hash) && body['pages'].is_a?(Hash) ? body['pages']['next'] : nil + return nil if advertised.nil? + + cursor = case advertised + when Hash then advertised['starting_after'] + when String then cursor_from_url(advertised) + end + + presence(cursor) || refuse_body_shape(operation, "'pages.next' carries no cursor this can follow") + end + + def cursor_from_url(url) + Faraday::Utils.parse_query(URI.parse(url).query.to_s)['starting_after'] + rescue URI::InvalidURIError + nil + end + + # nil rather than 0 when Intercom sends no count: zero is an answer, and + # this is the absence of one. + def extract_count(body) + count = body['total_count'] if body.is_a?(Hash) + count.is_a?(Numeric) ? count.to_i : nil + end + + # How many pages the offset tier has to read through, when the endpoint + # counts them. nil on a cursor page, which counts nothing. + def extract_total_pages(body) + pages = body['pages'] if body.is_a?(Hash) + total = pages['total_pages'] if pages.is_a?(Hash) + total.is_a?(Numeric) ? total.to_i : nil + end + + def refuse_body_shape(operation, detail) + raise APIError.new("Intercom API call failed: #{operation}: unexpected response shape, #{detail}", status: nil) + end + + def presence(value) + blank?(value) ? nil : value + end + + def blank?(value) + value.nil? || value.to_s.empty? + end + + # `JSON::ParserError` alongside Faraday's own errors: on its own it would + # reach the catch-all below, whose message is the exception's -- and a JSON + # parser opens its message with what it choked on. + def must_succeed(operation) + yield + rescue Faraday::Error, JSON::ParserError => e + raise api_error(operation, e) + rescue APIError + # Already mapped, with its status intact; re-wrapping would erase it -- + # a 404 read as "no such record" rather than as a failure, above all. + raise + rescue StandardError => e + raise APIError, "Intercom API call failed: #{operation}: #{e.class}: #{e.message}" + end + + # Builds an APIError preserving the HTTP status and Intercom's own error body + # so a smart action can show the operator the real reason instead of + # "failed". + def api_error(operation, error) + response = response_of(error) + status = response[:status] + body = parse_body(response[:body]) + + APIError.new("Intercom API call failed: #{operation}: #{failure_detail(error, status, body)}", + status: status, body: body) + end + + # Faraday hands the status and the body back in a plain hash on most errors + # and in its own `Env` on a parsing error; both answer `[]`. + def response_of(error) + response = error.respond_to?(:response) ? error.response : nil + return { status: nil, body: nil } unless response.respond_to?(:[]) + + { status: response[:status], body: response[:body] } + end + + # A body that could not be parsed is named, never quoted: the parser's own + # message opens with the characters it choked on, and on a 200 those are the + # payload -- a conversation body, most of the time (R10). + def failure_detail(error, status, body) + return unreadable_detail(status) if parse_failure?(error) + return "#{error.class}: #{error.message}" unless status + + "HTTP #{status} #{error_message(body)}".strip + end + + def unreadable_detail(status) + detail = 'the response could not be read as JSON' + status ? "#{detail} (HTTP #{status})" : detail + end + + def parse_failure?(error) + error.is_a?(Faraday::ParsingError) || error.is_a?(JSON::ParserError) + end + + # Intercom answers a failure with `{ "type": "error.list", "request_id": + # "...", "errors": [{ "code": ..., "message": ... }] }`. The request id is + # what its support asks for first, so it is appended after the truncation + # rather than being what a long body pushes out. + # + # A body of any other shape is *not* echoed here. This message travels into + # the interface and into whatever collects the agent's errors, and the body + # of a response that failed to parse is a payload rather than an error -- + # conversation bodies included (R10). Its size is reported instead, and the + # body itself stays on the exception for whoever inspects one. + def error_message(parsed) + return "(unreadable body, #{parsed.to_s.bytesize} bytes)" unless parsed.is_a?(Hash) + + message = join_errors(parsed['errors']) + message = parsed.to_json if message.empty? + + append_request_id(message[0, 500], parsed['request_id']) + end + + def join_errors(errors) + Array(errors).filter_map do |error| + next error unless error.is_a?(Hash) + + [error['code'], error['message']].compact.join(': ') + end.join('; ') + end + + def append_request_id(message, request_id) + return message unless request_id + + "#{message} (request_id: #{request_id})" + end + + def parse_body(body) + return body unless body.is_a?(String) && !body.empty? + + JSON.parse(body) + rescue JSON::ParserError + body + end + + def connection + @connection ||= build_connection( + retry_policy: @configuration.retry_policy, + timeout: @configuration.timeout, + open_timeout: @configuration.open_timeout + ) + end + + # For what is read while the datasource is being constructed -- the + # custom-attribute introspection above all: short timeouts and one quick + # retry, so a slow Intercom cannot turn a Rails boot into minutes of + # waiting. Memoized separately from `connection`, which keeps the patience + # every later request is entitled to. + def boot_connection + @boot_connection ||= build_connection( + retry_policy: @configuration.boot_retry_policy, + timeout: @configuration.boot_timeout, + open_timeout: @configuration.boot_open_timeout + ) + end + + # Middleware order is deliberate: `raise_error` sits outside the JSON parser + # so it raises with an already-parsed body, and `retry` sits innermost so it + # inspects raw statuses -- behind `raise_error` it would never see a 429. + # + # The throttle goes inside `retry`, which is what makes a replay wait for + # the window like a first attempt, and what lets the 429's own headers reach + # the limiter: outside it, the middleware would run once for a request that + # reached Intercom three times. + # + # How long a request may take is the caller's to state; everything else is + # the same on every connection this builds, the limiter included -- a second + # limiter would meter in a window of its own and spend the budget twice. + def build_connection(retry_policy:, timeout:, open_timeout:) + Faraday.new(url: @configuration.url) do |f| + f.request :json + f.response :raise_error + f.response :json + f.request :retry, **retry_policy.to_faraday_options + f.use Throttle, limiter: @configuration.rate_limiter if @configuration.rate_limiter + f.headers['Authorization'] = "Bearer #{@configuration.access_token}" + f.headers['Accept'] = 'application/json' + f.headers['Intercom-Version'] = @configuration.api_version + f.headers['User-Agent'] = "forest_admin_datasource_intercom/#{VERSION}" + f.options.open_timeout = open_timeout + f.options.timeout = timeout + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/admin.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/admin.rb new file mode 100644 index 000000000..9cba4a2d3 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/admin.rb @@ -0,0 +1,81 @@ +module ForestAdminDatasourceIntercom + module Collections + # The teammates of the workspace: who a conversation or a ticket is assigned + # to. Without this collection an assignee is a raw id on every row. + class Admin < FetchAllCollection + def initialize(datasource) + super(datasource, 'IntercomAdmin') + end + + protected + + # `/admins` puts its records under `admins`, not under the `data` envelope + # the paginated listings use. + def fetch_all + client.fetch_all('admins', list_key: 'admins') + end + + def serialize(admin) + attrs = admin.is_a?(Hash) ? admin : {} + + { 'id' => stringify_id(attrs['id']), + 'name' => attrs['name'], + 'email' => attrs['email'], + 'job_title' => attrs['job_title'], + 'away_mode_enabled' => attrs['away_mode_enabled'], + 'away_mode_reassign' => attrs['away_mode_reassign'], + 'has_inbox_seat' => attrs['has_inbox_seat'], + # Not a column: it is what the names below are read from, and what the + # membership collection turns into the relation. + 'team_ids' => Array(attrs['team_ids']).map { |id| stringify_id(id) }, + 'team_names' => nil } + end + + # The teams the teammate belongs to, by name, and only when a projection + # asked for them: one read of `/teams` for the whole page. A token that + # cannot read them costs the column and nothing else. + def enrich(records, rows, projection) + return unless column_asked?(projection, 'team_names') + + names = team_names + records.each_with_index do |record, index| + rows[index]['team_names'] = Array(record['team_ids']).filter_map { |id| names[id] } + end + end + + private + + def team_names + client.fetch_all('teams', list_key: 'teams') + .to_h { |team| [stringify_id(team['id']), team['name']] } + rescue APIError => e + ForestAdminDatasourceIntercom.logger.warn( + "[forest_admin_datasource_intercom] #{name} could not read the teams of the workspace (HTTP " \ + "#{e.status || "-"}); the names are left empty. The relation to IntercomTeam is unaffected." + ) + {} + end + + def define_schema + add_column('id', 'String', is_primary_key: true) + add_column('name', 'String') + add_column('email', 'String') + add_column('job_title', 'String') + # Whether the teammate is away, and whether their conversations get + # reassigned while they are: the two an ops lead looks at before + # assigning anything. + add_column('away_mode_enabled', 'Boolean') + add_column('away_mode_reassign', 'Boolean') + add_column('has_inbox_seat', 'Boolean') + # A list, so neither filterable nor sortable -- as the array of ids it + # replaces was. It reads the teammate without a join; the relation below + # is what navigates it, through the membership collection that gives the + # many-to-many the join Intercom does not expose. + add_column('team_names', 'Json') + add_many_to_many('teams', foreign_collection: 'IntercomTeam', + through_collection: 'IntercomTeamMembership', + origin_key: 'admin_id', foreign_key: 'team_id') + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/base_collection.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/base_collection.rb new file mode 100644 index 000000000..0a9bc16c8 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/base_collection.rb @@ -0,0 +1,179 @@ +module ForestAdminDatasourceIntercom + module Collections + # What every Intercom collection shares: how a schema is declared, how a + # record is narrowed to the projection asked for, and how a window is cut + # out of records already in hand. + # + # Read-only for now: the writes and the business actions arrive with lot 3. + # The relations towards Contacts and Companies wait for lot 4, those two + # collections not existing yet -- a relation whose target collection is + # missing is a schema the agent refuses to boot on. The relations between the + # collections this datasource already serves are declared and answered here, + # through `Relations`. + class BaseCollection < ForestAdminDatasourceToolkit::Collection + include Relations + + ColumnSchema = ForestAdminDatasourceToolkit::Schema::ColumnSchema + Operators = ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Operators + Equivalent = ForestAdminDatasourceToolkit::Components::Query::ConditionTree::ConditionTreeEquivalent + Leaf = ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Nodes::ConditionTreeLeaf + + def initialize(datasource, name) + super + define_schema + end + + def client + datasource.client + end + + # What one relation read may ask of this collection as its *target*: how + # many ids it resolves in a single `list`, and how many it resolves at + # all. Both nil here -- the tier read whole answers any number of ids for + # the one request it already pays. `Relations` asks the target rather than + # assuming, the cost of an id being the target's business. + # + # Public because that is who calls them: the collection holding the + # relation, on the collection at the other end of it. + def ids_per_read = nil + def max_resolvable_ids = nil + + protected + + def define_schema = raise(NotImplementedError, "#{self.class} did not implement define_schema") + + # A record narrowed to the columns that were asked for. A projection naming + # a field the record does not carry yields nil rather than nothing at all: + # the agent asked for a column, and an absent key would read as a record + # missing it. + # + # A path through a relation is not a column and is skipped here: it is + # answered by `embed_relations`, which nests a whole row under the relation + # name once the page is in hand. + # + # No projection at all asks for every column -- which is not the same as + # every key a serialized record happens to carry: a couple of them hold + # what a column is read *from*, the ids behind a membership for one, and + # publishing those would show the operator the plumbing. + def project(record, projection) + asked = Array(projection).map(&:to_s) + return record.slice(*column_names) if asked.empty? + + asked.reject { |field| field.include?(':') }.to_h { |field| [field, record[field]] } + end + + # Whether a projection asks for a column. No projection at all asks for + # every declared column, which is how `project` reads it -- an enrichment + # guarded on the column being named would leave nil the very column the + # projection publishes. + def column_asked?(projection, column) + asked = Array(projection).map(&:to_s) + + asked.empty? || asked.include?(column) + end + + # Whether a projection asks for any of a group of columns, read the same + # way: no projection at all asks for every one of them. + def any_column_asked?(projection, columns) + asked = Array(projection).map(&:to_s) + + asked.empty? || columns.any? { |column| asked.include?(column) } + end + + # The window a list view asked for, cut out of records already in hand. + # + # A filter with no page -- or a page naming no limit -- asks for every + # record it matched, and there is nothing to cut. How far the read that + # collected them went is a different question, answered by the walker and + # its caps. + def page_window(records, filter) + page = filter&.page + return records if page.nil? + + offset = page.offset.to_i.clamp(0, nil) + limit = page.limit.to_i + return records.drop(offset) unless limit.positive? + + records[offset, limit] || [] + end + + def column_names + @column_names ||= fields.select { |_, field| field.is_a?(ColumnSchema) }.keys + end + + # The columns a workspace's own attributes became. None here; the + # collections that carry any override this through `CustomAttributes`. + # The translator reads them to answer a filter on one with the refusal + # covering the family rather than with the message for a column nobody + # declared. + def attribute_column_names = [] + + # The timezone in-memory date comparisons are evaluated in. The caller's, + # since that is whose "today" the filter was written against. + def timezone_for(caller) + caller.respond_to?(:timezone) ? caller.timezone : nil + end + + # Ids reach this datasource as strings -- a filter value from Forest, a + # segment, a url -- while Intercom types them inconsistently: a team id is + # a string, the same team's id inside `admin_ids` is a number. Left as it + # comes, an integer id would never match the string the filter carries. + def stringify_id(value) + value&.to_s + end + + # Intercom nests its lists twice -- `{"type": "contact.list", "contacts": + # [...]}` -- and answers a null instead of an empty list when there is + # nothing. + def nested_list(container, key) + return [] unless container.is_a?(Hash) + + list = container[key] + list.is_a?(Array) ? list : [] + end + + # Intercom dates travel as epoch seconds; Forest reads a Date column as an + # ISO8601 string, and a filter carries one too, so comparing the two is the + # ordering itself. UTC deliberately: that is where Intercom stores and + # truncates, and rendering a local time here would hide the very shift that + # makes a day-granular date filter wrong. + def stamp(seconds) + return nil unless seconds.is_a?(Numeric) && seconds.positive? + + Time.at(seconds).utc.iso8601 + end + + # --- Reading a sort clause ------------------------------------------- + # + # Forest hands a clause keyed with symbols or with strings depending on + # where it was written, and all three tiers have to read one. + + def primary_key + @primary_key ||= fields.find do |_name, field| + field.respond_to?(:is_primary_key) && field.is_primary_key + end&.first + end + + def sort_field(clause) = clause[:field] || clause['field'] + + # `key?` rather than `||`: a descending clause carries `false`, which an + # `||` fallback reads as "absent" -- so an explicit `?sort=-id` would be + # taken for the ascending default the agent injects. + def ascending?(clause) + clause.key?(:ascending) ? clause[:ascending] : clause['ascending'] + end + + # The ascending primary-key sort the agent injects when a request names + # none. Not an order anybody asked for, so a tier that cannot honour one + # stays quiet about this one. + def default_pk_sort?(clauses) + return false unless clauses.size == 1 + + clause = clauses.first + return false unless sort_field(clause).to_s == primary_key + + ascending?(clause) != false + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/company.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/company.rb new file mode 100644 index 000000000..5377cd4dc --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/company.rb @@ -0,0 +1,85 @@ +module ForestAdminDatasourceIntercom + module Collections + # The accounts the contacts belong to. + # + # The one collection Intercom paginates by offset, and the one place R1 -- + # a window the API cannot express -- does not apply: `POST /companies/list` + # takes a page number, which is what a list view asks for. See + # `OffsetCollection`. + # + # In exchange it is the least filterable collection of the datasource. + # There is no `/companies/search`, and what `GET /companies` answers is four + # exact lookups: by `name`, by `company_id` -- the workspace's own + # identifier, not Intercom's -- by `tag_id` and by `segment_id`. Two of them + # are published as filters here, the two that name a column of this + # collection; a tag and a segment are collections of their own and arrive + # with lot 5, which is where filtering by them belongs. + # + # `GET /companies/scroll` is deliberately rejected rather than used: one + # open scroll per application, expiring after a minute, cannot serve two + # operators looking at a list at the same time. + class Company < OffsetCollection + include Company::Serializer + include CustomAttributes + + # The column each lookup is written on, and the query parameter Intercom + # answers it under. They happen to share a name; keeping the mapping + # explicit is what lets a column be renamed without silently dropping the + # lookup. + LOOKUPS = { 'name' => 'name', 'company_id' => 'company_id' }.freeze + + def initialize(datasource, attributes: []) + @attributes = attributes + super(datasource, 'IntercomCompany') + end + + protected + + def list_path = 'companies/list' + def record_endpoint = 'companies' + def lookups = LOOKUPS + + private + + def define_schema + add_column('id', 'String', is_primary_key: true) + # Intercom's id and the workspace's own are two different things, and an + # ops team knows the second one: it is what their billing system calls + # the account. + add_column('company_id', 'String') + add_column('name', 'String') + define_profile_columns + define_activity_columns + # Before the attribute columns, so an attribute whose name lands on the + # relation is skipped with a warning rather than taking the boot with + # it. + add_one_to_many('contacts', foreign_collection: 'IntercomContact', origin_key: 'company_id') + register_attribute_columns + end + + def define_profile_columns + add_column('plan_name', 'String') + add_column('size', 'Number') + add_column('industry', 'String') + add_column('website', 'String') + add_column('monthly_spend', 'Number') + end + + def define_activity_columns + add_column('user_count', 'Number') + add_column('session_count', 'Number') + add_column('created_at', 'Date') + add_column('updated_at', 'Date') + add_column('last_request_at', 'Date') + # When the account was created in the customer's own system, which is + # not when Intercom heard about it. + add_column('remote_created_at', 'Date') + end + + # Typed from `GET /data_attributes?model=company`, and unfilterable for + # the same reason as everything else here: this collection is looked up, + # not searched. + def attribute_kind = 'company' + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/company/serializer.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/company/serializer.rb new file mode 100644 index 000000000..c449d1070 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/company/serializer.rb @@ -0,0 +1,44 @@ +module ForestAdminDatasourceIntercom + module Collections + class Company < OffsetCollection + # One Intercom company flattened into the row the schema declares. + module Serializer + protected + + def serialize(company) + attrs = company.is_a?(Hash) ? company : {} + plan = attrs['plan'].is_a?(Hash) ? attrs['plan'] : {} + + identity(attrs).merge( + 'plan_name' => plan['name'], + 'user_count' => attrs['user_count'], + 'session_count' => attrs['session_count'] + ).merge(dates_of(attrs)).merge(attribute_values(attrs['custom_attributes'])) + end + + private + + def identity(attrs) + { + 'id' => stringify_id(attrs['id']), + 'company_id' => stringify_id(attrs['company_id']), + 'name' => attrs['name'], + 'size' => attrs['size'], + 'industry' => attrs['industry'], + 'website' => attrs['website'], + 'monthly_spend' => attrs['monthly_spend'] + } + end + + def dates_of(attrs) + { + 'created_at' => stamp(attrs['created_at']), + 'updated_at' => stamp(attrs['updated_at']), + 'last_request_at' => stamp(attrs['last_request_at']), + 'remote_created_at' => stamp(attrs['remote_created_at']) + } + end + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/contact.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/contact.rb new file mode 100644 index 000000000..cba095747 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/contact.rb @@ -0,0 +1,280 @@ +module ForestAdminDatasourceIntercom + module Collections + # The people who write to the workspace: users and leads alike. + # + # Cursor-paginated like conversations and tickets, with one thing no other + # collection of this datasource has -- **Intercom sorts it**. + # `POST /contacts/search` is the only endpoint of the whole API that takes a + # `sort` and applies it, so this is the only collection whose columns are + # published sortable, and the measured table is what says which ones. + # + # Two routes of its own, on top of the three the tier already has: + # + # * a set of ids is read in one request -- `id IN [...]`, which this + # endpoint answers and no other does -- rather than one request per id; + # * `company_id equals X` reads `GET /companies/{id}/contacts`, which is + # what serves the contacts of an account. `/contacts/search` filters no + # company field, so without this route the one relation an ops team walks + # the most would be a refusal. + # + # A contact merged into another **disappears** from the search and from the + # listing: a row whose contact was merged reads as gone rather than as an + # error, which is what a merge means -- the record still exists, under the + # id it was merged into. + # Long by line count only: most of it declares the columns, one call each. + class Contact < CursorCollection # rubocop:disable Metrics/ClassLength + include Contact::Serializer + include CustomAttributes + + # `/contacts/search` demands a query, so a read with no condition of its + # own -- a list view asking for an order -- sends the least noisy + # predicate that matches everything. Every contact has a creation date, + # and a bound at the epoch keeps whatever the day-granular truncation does + # to it harmless. The same predicate `Ticket` sends, for the same reason. + MATCH_EVERY_CONTACT = { 'field' => 'created_at', 'operator' => '>', 'value' => '0' }.freeze + + # How many ids one bulk read carries, and how many a single `id in [...]` + # may name. Both are far above what a page asks for; they keep a scope or + # a customizer naming thousands of ids from turning one list view into a + # rate limit. + IDS_PER_READ = 100 + MAX_IDS_READ = 300 + + def initialize(datasource, attributes: []) + @attributes = attributes + super(datasource, 'IntercomContact') + # Answered on the e-mail address, which is what an ops team types when + # they are looking for someone. Per word, not as a substring -- see the + # README. + enable_search + end + + # This endpoint answers `id IN [...]`, so a hundred ids cost one request + # rather than a hundred: a relation pointing here resolves a whole page of + # rows for a handful of requests, and there is no fan-out to refuse. That + # is why `max_resolvable_ids` is nil where the other two tiers bound it -- + # the read is proportional to the rows already collected, and the walk + # that collected them is bounded. + def ids_per_read = IDS_PER_READ + def max_resolvable_ids = nil + + protected + + def list_endpoint = 'contacts' + def searchable = 'contacts' + def search_column = 'email' + def match_all_query = MATCH_EVERY_CONTACT + def max_id_reads = MAX_IDS_READ + + private + + def define_schema + add_column('id', 'String', is_primary_key: true) + define_identity_columns + define_date_columns + define_reachability_columns + define_device_columns + # Before the attribute columns rather than after: a workspace attribute + # whose name lands on a relation is then skipped with a warning, the way + # one landing on a column already is. Declared after, it would collide + # and take the boot with it. + define_relations + register_attribute_columns + end + + def define_identity_columns + add_column('role', 'String') + add_column('name', 'String') + add_column('email', 'String') + add_column('email_domain', 'String') + add_column('phone', 'String') + add_column('external_id', 'String') + add_column('avatar', 'String') + add_column('owner_id', 'String') + add_column('session_count', 'Number') + # The first of the accounts the contact belongs to, and how many there + # are: the same reading a conversation gives its contacts, and the + # foreign key the `company` relation is built on. + add_column('company_id', 'String') + add_column('company_count', 'Number') + end + + def define_date_columns + add_column('created_at', 'Date') + add_column('updated_at', 'Date') + add_column('signed_up_at', 'Date') + add_column('last_seen_at', 'Date') + add_column('last_contacted_at', 'Date') + add_column('last_replied_at', 'Date') + add_column('last_email_opened_at', 'Date') + add_column('last_email_clicked_at', 'Date') + end + + def define_reachability_columns + add_column('unsubscribed_from_emails', 'Boolean') + add_column('has_hard_bounced', 'Boolean') + add_column('marked_email_as_spam', 'Boolean') + end + + def define_device_columns + add_column('language_override', 'String') + add_column('browser', 'String') + add_column('browser_language', 'String') + add_column('os', 'String') + add_column('location_country', 'String') + add_column('location_region', 'String') + add_column('location_city', 'String') + end + + # Typed from `GET /data_attributes?model=contact` rather than guessed from + # a payload, and published unfilterable: which operators Intercom answers + # on `custom_attributes.{name}` has not been measured, and this package + # offers no filter it has not seen work. `api_writable` travels on the + # introspected attribute for lot 4b, not on the column -- everything here + # is read-only. + def attribute_kind = 'contact' + + # The owner is a teammate, read whole in one request, and + # `/contacts/search` filters on the key -- so that relation is readable, + # navigable and filterable alike. + # + # The company is none of those last two: the endpoint filters no company + # field, so the traversal is refused by name (see `check_relation_filterable!` + # on the tier). The two lists are the 360 degrees this lot exists for. + def define_relations + add_many_to_one('owner', foreign_collection: 'IntercomAdmin', foreign_key: 'owner_id') + add_many_to_one('company', foreign_collection: 'IntercomCompany', foreign_key: 'company_id') + add_one_to_many('conversations', foreign_collection: 'IntercomConversation', origin_key: 'contact_id') + add_one_to_many('tickets', foreign_collection: 'IntercomTicket', origin_key: 'contact_id') + end + + # The contacts of an account, which `/contacts/search` cannot answer and + # `GET /companies/{id}/contacts` can. Anything else goes the usual way. + def fetch_records(caller, filter, sort = nil) + company = company_lookup(filter) + unless company + refuse_narrowed_company!(filter) if narrowed_company?(filter) + return super + end + + warn_unordered_company_contacts(Array(filter&.sort)) if sort + offset, limit = translate_page(filter&.page) + + walker.walk(offset: offset, limit: limit) do |per_page, cursor| + read_company_page(company, per_page: per_page, cursor: cursor) + end + end + + def count_records(caller, filter) + company = company_lookup(filter) + unless company + refuse_narrowed_company!(filter) if narrowed_company?(filter) + return super + end + + exact_count(read_company_page(company, per_page: 1, cursor: nil)) + end + + # A bare equality and nothing else: an `and` also carrying a scope names a + # narrower set than the account does, and answering it with the account + # alone would serve contacts the scope excludes. + def company_lookup(filter) + tree = filter&.condition_tree + return nil unless tree.is_a?(Leaf) && tree.field.to_s == 'company_id' + return nil unless tree.operator == Operators::EQUAL && blank_search?(filter) + + tree.value&.to_s + end + + # The same equality, and something filtered alongside it. There is no + # request that answers both halves: `GET /companies/{id}/contacts` narrows + # nothing it returns, and `/contacts/search` filters no company field at + # all -- which is what the related list of an account runs into the moment + # a permission scope or a segment is defined on this collection. + # + # Refused here rather than left to the translator, whose reason for + # `company_id` is "open the company and read its contacts" -- which is + # exactly what this caller was doing. + def narrowed_company?(filter) + tree = filter&.condition_tree + return false if tree.nil? || tree.is_a?(Leaf) + + tree.some_leaf { |leaf| leaf.field.to_s == 'company_id' && leaf.operator == Operators::EQUAL } + end + + def refuse_narrowed_company!(filter) + raise UnsupportedOperatorError, + "#{name} cannot answer this filter: the contacts of an account are read through " \ + 'GET /companies/{id}/contacts, which returns them whole and narrows nothing, and ' \ + "#{search_endpoint.path} filters no company field -- so the account and the " \ + "#{narrowing_cause(filter)} cannot be asked for in one request. Read the account's contacts " \ + 'without it, or filter on a column this collection is searched on: ' \ + "#{search_endpoint.filterable_columns.join(", ")}." + end + + # A scope and a segment are what put a condition next to the account's in + # practice, and the operator can act on neither the same way -- so the + # message names what it can see rather than guessing. + def narrowing_cause(filter) + others = [] + filter.condition_tree.some_leaf do |leaf| + others << leaf.field.to_s unless leaf.field.to_s == 'company_id' + false + end + + others.empty? ? 'condition filtered alongside it' : "condition on #{others.uniq.join(", ")}" + end + + # An account Intercom no longer answers for -- deleted, or moved outside + # the token's reach between the moment the row was rendered and the + # moment its related list was opened -- reads as an account with no + # contact rather than as a failed page, the way a record read by its id + # already does. + def read_company_page(company, per_page:, cursor:) + client.list_page("companies/#{Faraday::Utils.escape(company)}/contacts", + per_page: [per_page, max_page_size].min, starting_after: cursor) + rescue APIError => e + raise unless e.status == 404 + + Client::Page.new(records: [], next_cursor: nil, total_count: 0) + end + + # One request per hundred ids instead of one per id: this endpoint answers + # `id IN [...]`, which is what lot 1 already reads the contacts of a page + # through. A contact that was merged away is simply absent from the + # answer -- the row reads as gone, not as a failure. + def records_by_ids(ids) + wanted = ids.first(max_id_reads) + warn_truncated_ids(ids.size) if ids.size > wanted.size + + wanted.each_slice(IDS_PER_READ).flat_map do |chunk| + client.search_page(search_endpoint.path, per_page: chunk.size, + query: { 'field' => 'id', 'operator' => 'IN', + 'value' => chunk }).records + end + end + + # `sort` here is the clause `server_sort` found `/contacts/search` does + # honour, which is why nothing has reported it yet: it is this route that + # cannot carry the order, not the column. Saying Intercom does not sort + # this collection would be wrong -- it is the one collection it sorts -- + # so this is the counterpart of `warn_unordered_ids`, for the other route + # that leaves the search behind. + def warn_unordered_company_contacts(clauses) + ForestAdminDatasourceIntercom.logger.warn( + "[forest_admin_datasource_intercom] #{name} was asked to sort on " \ + "#{clauses.map { |clause| sort_field(clause) }.join(", ")} while reading the contacts of an account, " \ + 'which Intercom answers through GET /companies/{id}/contacts -- a route that takes no order, unlike ' \ + 'the search this collection is otherwise read through. The rows come back in the order the API imposes.' + ) + end + + def warn_truncated_ids(asked) + ForestAdminDatasourceIntercom.logger.warn( + "[forest_admin_datasource_intercom] #{name} was asked for #{asked} records by id and read the first " \ + "#{max_id_reads}: Intercom reads them #{IDS_PER_READ} at a time. The result is truncated." + ) + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/contact/serializer.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/contact/serializer.rb new file mode 100644 index 000000000..509351034 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/contact/serializer.rb @@ -0,0 +1,106 @@ +module ForestAdminDatasourceIntercom + module Collections + class Contact < CursorCollection + # One Intercom contact flattened into the row the schema declares. + # Nothing here reads a sub-resource: every value comes from the payload + # the search already returned. + module Serializer + protected + + def serialize(contact) + attrs = contact.is_a?(Hash) ? contact : {} + + native(attrs) + .merge(account_of(attrs['companies'])) + .merge(location_of(attrs['location'])) + .merge(attribute_values(attrs['custom_attributes'])) + end + + private + + def native(attrs) + identity(attrs).merge(dates_of(attrs)).merge(flags(attrs)).merge(device_of(attrs)) + end + + def identity(attrs) + { + 'id' => stringify_id(attrs['id']), + 'role' => attrs['role'], + 'name' => attrs['name'], + 'email' => attrs['email'], + # Derived rather than read, and filterable all the same: the search + # endpoint carries a field of its own for it, which is what turns + # "everyone at this customer" into a filter instead of a wildcard. + 'email_domain' => domain_of(attrs['email']), + 'phone' => attrs['phone'], + 'external_id' => attrs['external_id'], + 'avatar' => attrs['avatar'], + 'owner_id' => stringify_id(attrs['owner_id']), + 'session_count' => attrs['session_count'] + } + end + + def dates_of(attrs) + { + 'created_at' => stamp(attrs['created_at']), + 'updated_at' => stamp(attrs['updated_at']), + 'signed_up_at' => stamp(attrs['signed_up_at']), + 'last_seen_at' => stamp(attrs['last_seen_at']), + 'last_contacted_at' => stamp(attrs['last_contacted_at']), + 'last_replied_at' => stamp(attrs['last_replied_at']), + 'last_email_opened_at' => stamp(attrs['last_email_opened_at']), + 'last_email_clicked_at' => stamp(attrs['last_email_clicked_at']) + } + end + + def flags(attrs) + { + 'unsubscribed_from_emails' => attrs['unsubscribed_from_emails'], + 'has_hard_bounced' => attrs['has_hard_bounced'], + 'marked_email_as_spam' => attrs['marked_email_as_spam'] + } + end + + def device_of(attrs) + { + 'language_override' => attrs['language_override'], + 'browser' => attrs['browser'], + 'browser_language' => attrs['browser_language'], + 'os' => attrs['os'] + } + end + + # A contact belongs to several accounts, and the row names the first of + # them and counts them -- the same reading a conversation gives its + # contacts. The whole list is a hop away, on the `company` relation. + def account_of(companies) + list = nested_list(companies, 'data') + first = list.first.is_a?(Hash) ? list.first : {} + + { 'company_id' => stringify_id(first['id']), 'company_count' => account_count(companies, list) } + end + + # Intercom caps the accounts it nests on a contact and says how many + # there really are, so the count is read rather than measured on the + # list -- a contact belonging to twelve accounts must not read as + # belonging to the ten the payload had room for. + def account_count(companies, list) + declared = companies['total_count'] if companies.is_a?(Hash) + + declared.is_a?(Numeric) ? declared : list.size + end + + def location_of(location) + attrs = location.is_a?(Hash) ? location : {} + + { 'location_country' => attrs['country'], 'location_region' => attrs['region'], + 'location_city' => attrs['city'] } + end + + def domain_of(email) + email.to_s[/@(.+)\z/, 1] + end + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/contact_identity.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/contact_identity.rb new file mode 100644 index 000000000..a00cac835 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/contact_identity.rb @@ -0,0 +1,91 @@ +module ForestAdminDatasourceIntercom + module Collections + # The contact of a conversation or of a ticket, on the row itself. + # + # Intercom nests only the ids -- `{"type": "contact.list", "contacts": + # [{"id": "..."}]}` -- so a name costs a read. That read is done once per + # page, for every row at once, and never per row: a page of 25 rows is one + # request, not 25. + # + # **Three columns, where lot 1 published four.** Now that the Contacts + # collection exists, the identity is a relation, and the rule lot 2.5 set + # for the ticket labels applies here too: one readable label on the row plus + # the relation to navigate, rather than two ways to read one fact. + # `contact_email` is gone -- it is one hop away, on `contact:email` -- and + # `contact_ids` gave way to `contact_id`, which is a foreign key rather than + # a Json blob no filter could reach. The list of every contact of a group + # conversation is the `contacts` relation. + module ContactIdentity + COLUMNS = %w[contact_name].freeze + + # How many ids one `id in [...]` read carries. A page holds fewer than this + # in practice; the chunk keeps the request bounded if it ever does not. + CONTACT_CHUNK = 100 + + private + + def define_contact_columns + add_column('contact_id', 'String') + add_column('contact_count', 'Number') + add_column('contact_name', 'String') + end + + # A group conversation, or a ticket opened for several people, has more + # than one contact: the row names the first and counts them, rather than + # presenting one of several as the one. The `contact` relation resolves + # that same first contact, so the column and the relation cannot disagree; + # the others are reached through the contact's own conversations. + def contact_columns_for(attrs) + ids = nested_list(attrs['contacts'], 'contacts').filter_map { |contact| stringify_id(contact['id']) } + + { 'contact_id' => ids.first, 'contact_count' => ids.size, + # Filled by the bulk read below, and left nil when the projection did + # not ask for it. + 'contact_name' => nil } + end + + def first_contact_id(record) + contact = nested_list((record || {})['contacts'], 'contacts').first + contact.is_a?(Hash) ? stringify_id(contact['id']) : nil + end + + # The Contacts endpoint as the table spells it, rather than a path written + # a second time here: this is the same `/contacts/search` the Contacts + # collection reads itself through, and a table that renamed it would + # otherwise leave this one behind. + def contact_search_path + @contact_search_path ||= Query::SearchFields.fetch('contacts').path + end + + def embed_contact_identity(records, rows, projection) + return unless any_column_asked?(projection, COLUMNS) + + identities = contact_identities(records) + records.each_with_index do |record, index| + identity = identities[first_contact_id(record)] || {} + rows[index]['contact_name'] = identity['name'] if rows[index].key?('contact_name') + end + end + + # A failure costs the column and nothing else: an identity that could not + # be read is not a page that could not be served. + def contact_identities(records) + ids = records.filter_map { |record| first_contact_id(record) }.uniq + return {} if ids.empty? + + ids.each_slice(CONTACT_CHUNK).with_object({}) do |chunk, indexed| + page = client.search_page(contact_search_path, per_page: chunk.size, + query: { 'field' => 'id', 'operator' => 'IN', + 'value' => chunk }) + page.records.each { |contact| indexed[contact['id'].to_s] = contact } + end + rescue APIError => e + ForestAdminDatasourceIntercom.logger.warn( + "[forest_admin_datasource_intercom] #{name} could not read the contacts of this page (HTTP " \ + "#{e.status || "-"}); the name column is left empty for it." + ) + {} + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/conversation.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/conversation.rb new file mode 100644 index 000000000..bf9060f2a --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/conversation.rb @@ -0,0 +1,183 @@ +module ForestAdminDatasourceIntercom + module Collections + # The conversations of the workspace: what a support team actually works on. + # + # Read through `GET /conversations`, whose records Intercom puts under + # `conversations` rather than under the `data` envelope -- `/tickets/search` + # does the same with `tickets`, so the key is named rather than assumed. + # + # `display_as=plaintext` on every read: the bodies are HTML written by end + # customers, and rendering third-party HTML inside Forest is neither safe nor + # useful (R10). + # Long by line count only: most of it declares the columns, one call each. + class Conversation < CursorCollection + include ContactIdentity + include Conversation::Serializer + # The shared thread, then the hooks this collection puts over it. + include Collections::Timeline + include Conversation::Timeline + + # How many conversations of one page may have their timeline read. The + # parts are absent from the listing response -- Intercom returns them only + # when retrieving a single conversation -- so a timeline asked for in a + # list view costs one request per row. Bounded rather than turned into a + # page the operator waits half a minute for; rows past the cap are left at + # nil, which reads as "unknown", never as "this conversation is empty". + MAX_TIMELINE_READS = 10 + + def initialize(datasource) + super(datasource, 'IntercomConversation') + # The one collection of this datasource Intercom matches text on: `~` on + # `source.body`, which is the message that opened the conversation. + enable_search + end + + protected + + def list_endpoint = 'conversations' + def list_key = 'conversations' + def searchable = 'conversations' + def search_column = 'source_body' + + # Sent on the search too, where Intercom does not document it: the bodies + # are HTML written by end customers (R10), and a parameter it ignores costs + # a query string while the one it honours saves every filtered row from + # coming back as markup. + def read_params = { 'display_as' => 'plaintext' } + + # The contact identity and the timeline, each read only when the projection + # names it: neither is on the conversation payload, and a page that never + # asked for them must not pay for them. + def enrich(records, rows, projection) + embed_contact_identity(records, rows, projection) + embed_timeline(records, rows, projection) + end + + private + + def define_schema + add_column('id', 'String', is_primary_key: true) + add_column('title', 'String') + # Left plain strings rather than enums: the values a workspace really + # serves are worth measuring before the interface offers them as a + # closed list, and nothing filters on them in this lot anyway. + add_column('state', 'String') + add_column('priority', 'String') + add_column('open', 'Boolean') + add_column('read', 'Boolean') + add_column('created_at', 'Date') + add_column('updated_at', 'Date') + add_column('waiting_since', 'Date') + add_column('snoozed_until', 'Date') + define_assignment_columns + define_contact_columns + define_source_columns + define_statistics_columns + add_column('tag_names', 'Json') + add_column('ai_agent_participated', 'Boolean') + add_column('timeline', 'Json') + define_relations + end + + # Who the conversation sits with, and who closed it. All three targets are + # read whole in one request, and `/conversations/search` takes a filter on + # each of the three keys -- so these relations can be read, navigated and + # filtered through alike. + # + # No relation towards the company: a conversation carries its account as a + # whole object, so the name is already on the row, and Intercom filters no + # company field on this endpoint -- the relation would be navigable and + # not filterable, where the column is already readable. + # + # The contact is a relation as of lot 4, and a filterable one: the + # endpoint matches a conversation against one of its contact ids + # (measured), so `contact:email` resolves against `/contacts/search` and + # is rewritten onto the key. A group conversation names its first contact + # here, like the column does; every contact of it is a hop away, through + # that contact's own conversations. + def define_relations + add_many_to_one('admin_assignee', foreign_collection: 'IntercomAdmin', foreign_key: 'admin_assignee_id') + add_many_to_one('team_assignee', foreign_collection: 'IntercomTeam', foreign_key: 'team_assignee_id') + add_many_to_one('closed_by', foreign_collection: 'IntercomAdmin', foreign_key: 'closed_by_id') + add_many_to_one('contact', foreign_collection: 'IntercomContact', foreign_key: 'contact_id') + end + + # Who the conversation sits with, and which account it belongs to. The + # conversation carries its company as a whole object, so the account name + # is free here -- unlike on a ticket, which carries the id alone. + def define_assignment_columns + add_column('admin_assignee_id', 'String') + add_column('team_assignee_id', 'String') + add_column('company_id', 'String') + add_column('company_name', 'String') + end + + # The message that opened the conversation lives in `source`, not in the + # parts. A timeline built from the parts alone loses it, which is the one + # message nobody opens a conversation without wanting to read. + def define_source_columns + add_column('source_type', 'String') + add_column('source_subject', 'String') + add_column('source_body', 'String') + add_column('source_author_name', 'String') + add_column('source_author_email', 'String') + add_column('source_delivered_as', 'String') + end + + # Intercom keeps the lifecycle of a conversation in `statistics`, which is + # where the closure date and the reply timestamps come from. Flattened onto + # the row: they cost nothing, they are exact, and they are what an ops lead + # reads a queue for. + def define_statistics_columns + add_column('closed_at', 'Date') + add_column('first_closed_at', 'Date') + add_column('closed_by_id', 'String') + add_column('first_contact_reply_at', 'Date') + add_column('last_contact_reply_at', 'Date') + add_column('last_admin_reply_at', 'Date') + add_column('reopen_count', 'Number') + add_column('part_count', 'Number') + end + + # A record read through the record endpoint already carries its parts, so + # its timeline is free; one read from the listing does not, and pays a + # request. Rows past the cap keep the nil the projection put there. + def embed_timeline(records, rows, projection) + return unless column_asked?(projection, 'timeline') + + budget = MAX_TIMELINE_READS + missing = 0 + + records.each_with_index do |record, index| + if parts_of(record) + rows[index]['timeline'] = build_timeline(record) + elsif budget.positive? + budget -= 1 + detail = read_detail(record['id']) + rows[index]['timeline'] = detail && build_timeline(detail) + else + missing += 1 + end + end + + warn_truncated_timelines(missing) if missing.positive? + end + + def read_detail(id) + client.fetch_record(list_endpoint, id, params: read_params) + rescue APIError => e + raise unless e.status == 404 + + nil + end + + def warn_truncated_timelines(missing) + ForestAdminDatasourceIntercom.logger.warn( + "[forest_admin_datasource_intercom] #{name} left the timeline of #{missing} row(s) unread: Intercom " \ + 'returns the parts only when retrieving one conversation, so a list view pays a request per row and ' \ + "this reads at most #{MAX_TIMELINE_READS}. Those rows show no timeline rather than an empty one." + ) + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/conversation/serializer.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/conversation/serializer.rb new file mode 100644 index 000000000..e241b168c --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/conversation/serializer.rb @@ -0,0 +1,75 @@ +module ForestAdminDatasourceIntercom + module Collections + class Conversation < CursorCollection + # One Intercom conversation flattened into the row the schema declares. + # Nothing here reads a sub-resource: every value comes from the payload the + # listing already returned. + module Serializer + protected + + def serialize(conversation) + attrs = conversation.is_a?(Hash) ? conversation : {} + + native(attrs) + .merge(contact_columns_for(attrs)) + .merge(source_of(attrs['source'])) + .merge(statistics_of(attrs['statistics'])) + end + + private + + def native(attrs) + company = attrs['company'].is_a?(Hash) ? attrs['company'] : {} + + { + 'id' => stringify_id(attrs['id']), + 'title' => attrs['title'], + 'state' => attrs['state'], + 'priority' => attrs['priority'], + 'open' => attrs['open'], + 'read' => attrs['read'], + 'created_at' => stamp(attrs['created_at']), + 'updated_at' => stamp(attrs['updated_at']), + 'waiting_since' => stamp(attrs['waiting_since']), + 'snoozed_until' => stamp(attrs['snoozed_until']), + 'admin_assignee_id' => stringify_id(attrs['admin_assignee_id']), + 'team_assignee_id' => stringify_id(attrs['team_assignee_id']), + 'company_id' => stringify_id(company['id']), + 'company_name' => company['name'], + 'tag_names' => nested_list(attrs['tags'], 'tags').filter_map { |tag| tag['name'] if tag.is_a?(Hash) }, + 'ai_agent_participated' => attrs['ai_agent_participated'] + } + end + + def source_of(source) + attrs = source.is_a?(Hash) ? source : {} + author = attrs['author'].is_a?(Hash) ? attrs['author'] : {} + + { 'source_type' => attrs['type'], + 'source_subject' => attrs['subject'], + # Plaintext, because `display_as=plaintext` rides on every read: the + # bodies are HTML written by end customers. + 'source_body' => attrs['body'], + 'source_author_name' => author['name'], + 'source_author_email' => author['email'], + 'source_delivered_as' => attrs['delivered_as'] } + end + + # `statistics` is null on a conversation Intercom has computed nothing + # for yet; every column then reads as absent rather than as zero. + def statistics_of(statistics) + attrs = statistics.is_a?(Hash) ? statistics : {} + + { 'closed_at' => stamp(attrs['last_close_at']), + 'first_closed_at' => stamp(attrs['first_close_at']), + 'closed_by_id' => stringify_id(attrs['last_closed_by_id']), + 'first_contact_reply_at' => stamp(attrs['first_contact_reply_at']), + 'last_contact_reply_at' => stamp(attrs['last_contact_reply_at']), + 'last_admin_reply_at' => stamp(attrs['last_admin_reply_at']), + 'reopen_count' => attrs['count_reopens'], + 'part_count' => attrs['count_conversation_parts'] } + end + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/conversation/timeline.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/conversation/timeline.rb new file mode 100644 index 000000000..bb7fe62cd --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/conversation/timeline.rb @@ -0,0 +1,40 @@ +module ForestAdminDatasourceIntercom + module Collections + class Conversation < CursorCollection + # What a conversation adds to the shared thread: where its parts live, and + # the entry that opens it. + # + # The opening message lives in `source`, not in the parts -- a timeline + # built from the parts alone opens on the first reply and loses what the + # customer actually asked. + module Timeline + # The pseudo type of the opening entry. Not an Intercom part type: it is + # the source, and calling it `comment` would make it indistinguishable + # from the replies that follow. + SOURCE_PART_TYPE = 'conversation_started'.freeze + + private + + # nil rather than an empty list when the payload carries no parts at all: + # a listing response has none, and reading that as "this conversation is + # empty" is exactly the answer that looks complete without being it. + def parts_of(conversation) + container = (conversation || {})['conversation_parts'] + return nil unless container.is_a?(Hash) + + parts = container['conversation_parts'] + parts.is_a?(Array) ? parts : nil + end + + def opening_entry(attrs) + source = attrs['source'] + return nil unless source.is_a?(Hash) + + entry(part_type: SOURCE_PART_TYPE, created_at: attrs['created_at'], author: source['author'], + body: source['body'], attachments: source['attachments']) + .merge('id' => stringify_id(source['id'])) + end + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/cursor_collection.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/cursor_collection.rb new file mode 100644 index 000000000..1f942c581 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/cursor_collection.rb @@ -0,0 +1,396 @@ +module ForestAdminDatasourceIntercom + module Collections + # Base for the collections Intercom paginates by cursor: conversations and + # tickets. The opposite tier of `FetchAllCollection` in every way -- what is + # in hand is a page of something far larger, so nothing may be filtered, + # sorted or counted in memory without answering a fraction as if it were the + # whole. + # + # Three routes, and no fourth: + # + # * no condition at all -- a list view -- walks the listing endpoint; + # * `id equals X` reads the record through its own endpoint, which is what a + # record detail is; + # * anything else is translated into Intercom's search DSL and walked + # through the search endpoint. What the translator will not express, it + # refuses by name: an unfiltered page served in answer to a filter is the + # one failure this datasource is built to avoid. + # + # A condition through a relation is resolved before any of that: the target + # collection is asked which of its records match, and what reaches Intercom + # is a condition on the foreign key. See `Relations`. + # + # Counting is the exception that costs nothing: `total_count` is exact on + # every response, filter included, so the record counter is one request. + # Long by line count only: half of it is the refusals, and a refusal that + # does not say what to do instead is a refusal an operator cannot act on. + class CursorCollection < BaseCollection # rubocop:disable Metrics/ClassLength + include RecordsById + + # Countable, and exactly: unlike the pages a walk collected, `total_count` + # is the whole dataset the filter names. + def initialize(datasource, name) + super + enable_count + end + + # One request per id here, so a relation pointing at this tier resolves a + # slice at a time and is refused past what a slice may hold -- see + # `Relations#target_rows`. The collection that reads its ids in bulk + # widens both; see `Contact`. + def ids_per_read = max_id_reads + def max_resolvable_ids = max_id_reads + + def list(caller, filter, projection) + records = fetch_records(caller, filter, server_sort(filter)) + # Serialized whole and projected afterwards rather than the other way + # round: a projection reaching through a relation names no foreign key, + # and the key is where the relation is read from. + serialized = records.map { |record| serialize(record) } + rows = serialized.map { |record| project(record, projection) } + + enrich(records, rows, projection) + embed_relations(caller, serialized, rows, projection) + rows + end + + # Count only, and never a group: Intercom exposes no aggregate endpoint, + # and grouping over the pages a walk happened to collect would look exact + # while answering a fraction. Refused here rather than through the + # contract's NotImplementedError, which reads as an oversight. + def aggregate(caller, filter, aggregation, _limit = nil) + refuse_unsupported_aggregation!(aggregation) + + [{ 'group' => {}, 'value' => count_records(caller, filter) }] + end + + protected + + # The listing endpoint, its record key, and the parameters every read of + # this collection carries. + def list_endpoint = raise(NotImplementedError, "#{self.class} did not implement list_endpoint") + def record_endpoint = list_endpoint + def list_key = 'data' + def read_params = {} + + # The row of the measured table this collection is filtered through: what + # its columns may advertise, and what the translator is allowed to write. + def searchable = raise(NotImplementedError, "#{self.class} did not implement searchable") + + def search_endpoint + @search_endpoint ||= Query::SearchFields.fetch(searchable) + end + + # The column a free-text search is answered on, for a collection whose + # endpoint has one. Nil elsewhere, and a search is then refused rather than + # answered by a page that ignored it. + def search_column = nil + + # One Intercom entity flattened into a record matching the schema. + def serialize(_entity) = raise(NotImplementedError, "#{self.class} did not implement serialize") + + # Hook for what a row needs beyond its own payload. Left empty here: what + # it costs is the collection's business, not this base's. + def enrich(_records, _rows, _projection); end + + # Bounded per collection rather than by the API maximum: Intercom offers no + # field selection, so a collection whose rows carry their whole timeline + # pays for it by the page. See Ticket. + def max_page_size = Client::MAX_PER_PAGE + + # One page of the collection. A listing for conversations, a search for + # tickets -- Intercom exposes no `GET /tickets` at all -- so the endpoint + # and its shape belong to the collection, while walking it does not. + def read_page(per_page:, cursor:, query: nil, sort: nil) + size = [per_page, max_page_size].min + # The listing endpoint neither filters nor sorts, and a collection whose + # records are only reachable through the search has no listing at all. + # Either way what routes the read to the search is a query, so the one + # that matches everything stands in for the condition there is none of. + query ||= match_all_query if sort || searchable_only? + + if query.nil? + client.list_page(list_endpoint, per_page: size, starting_after: cursor, + params: read_params, list_key: list_key) + else + client.search_page(search_endpoint.path, query: query, per_page: size, starting_after: cursor, + params: read_params, list_key: list_key, sort: sort) + end + end + + # A column advertises exactly the filters the search endpoint answers on + # it, taken from the measured table and derived by the operator table -- + # never written by hand here, so a column cannot offer a filter the + # translator would refuse. A column the table does not carry advertises + # none, which is how a refusal is spelled in a schema. + # + # The primary key is the exception, and it is not a filter: `id equals X` + # and `id in [...]` are answered by the record endpoint. + # + # A column is sortable only where the measured table says the endpoint + # sorts, which is `/contacts/search` and nowhere else: the other two + # accept a `sort` and ignore it without a word, so a sortable column there + # would promise an order that never happens. + def add_column(name, type, is_primary_key: false) + add_field(name, ColumnSchema.new(column_type: type, + filter_operators: column_operators(name, is_primary_key), + is_primary_key: is_primary_key, + is_read_only: true, + is_sortable: sortable_column?(name), + is_groupable: false)) + end + + def sortable_column?(name) + search_endpoint.field(name)&.sortable? == true + end + + # The Intercom query standing in for the condition a read has none of. + # Two collections need one and for different reasons: Intercom exposes no + # `GET /tickets` at all, and an order is applied by the search endpoint + # alone. Nil where the listing endpoint answers both, which is where a + # sort is reported as ignored rather than dropped. + def match_all_query = nil + + # Whether every read of this collection goes through the search, listing + # or not. + def searchable_only? = false + + # Bounded, unlike the tier read whole: one more than a group may hold is + # all it takes to know the fan-out will not fit, and reading further would + # walk a whole collection to refuse it afterwards. + def match_page + ForestAdminDatasourceToolkit::Components::Query::Page.new( + offset: 0, limit: Query::ConditionTreeTranslator::MAX_GROUP_SIZE + 1 + ) + end + + def walker + @walker ||= Pagination::CursorWalker.new + end + + # How many records a bulk read by id may fetch. A hook rather than the + # constant, since the collection that reads its ids in bulk answers far + # more of them for the same request count. + def max_id_reads = MAX_ID_READS + + private + + def column_operators(name, is_primary_key) + return [Operators::EQUAL, Operators::IN] if is_primary_key + + field = search_endpoint.field(name) + + field ? Query::OperatorTable.forest_operators(field) : [] + end + + def fetch_records(caller, filter, sort = nil) + ids = id_lookup(filter) + # The window is cut out of the ids rather than out of the records they + # read: Intercom reads them one request each, so paging after the read + # would pay for a whole page to hand back a slice of it -- and page 2 of + # a set larger than the cap would come back empty, the records it names + # having been dropped by the truncation before the window was applied. + if ids + # Which is also why an order cannot be honoured on this route: the + # window is cut in the order the ids were named, so sorting what comes + # back would order a slice picked by something else. Reported rather + # than dropped in silence, like every other order this tier cannot + # apply -- `server_sort` stayed quiet, having found a column Intercom + # does sort. + warn_unordered_ids(sort) if sort + return records_by_ids(page_window(ids, filter)) + end + + query = translate(caller, filter) + # A condition through a relation the target matched no record with names + # no row, and Intercom's DSL cannot say so: the read is skipped rather + # than sent as a filter that would come back with everything. + return [] if query == NOTHING + + listed_records(filter, query, sort) + end + + # The Intercom query a filter comes down to, or nil for a list view, which + # walks the listing endpoint instead. The free-text search is folded into + # the condition tree rather than added to the translated query: written as + # one tree, it is checked against the nesting Intercom allows like every + # other condition, instead of adding a level nothing counted. + def translate(caller, filter) + tree = rewrite_relation_conditions(caller, combined_tree(filter)) do |key, ids, leaf| + relation_group(key, ids, leaf) + end + return NOTHING if tree == NOTHING + + Query::ConditionTreeTranslator.call(tree, endpoint: search_endpoint, collection: name, + timezone: timezone_for(caller), + attribute_columns: attribute_column_names) + end + + # The ids the target matched, written as the filter Intercom does take on + # the foreign key: a group of equalities, its DSL offering no membership + # operator on these fields. + # + # That group counts against the fifteen conditions Intercom allows, so a + # relation condition matching more records than that is refused rather than + # sent -- and refused here, where the message can name the relation the + # operator filtered on rather than the key it resolved to. + # + # It also costs a level of nesting, which is why it is built as the group + # `Relations` can tell from one the operator wrote: inlined into a parent + # that aggregates the same way, it costs none. + def relation_group(key, ids, leaf) + refuse_fan_out!(leaf, key, ids) if ids.size > Query::ConditionTreeTranslator::MAX_GROUP_SIZE + return Leaf.new(key, Operators::EQUAL, ids.first) if ids.size == 1 + + RelationBranch.new('Or', ids.map { |id| Leaf.new(key, Operators::EQUAL, id) }, leaf.field) + end + + # Inlining a relation group into its parent trades a level of nesting for + # width, and Intercom bounds both. Past the conditions a group may hold, + # the nested form is the one it answers. + def absorb_relation_group?(size) + size <= Query::ConditionTreeTranslator::MAX_GROUP_SIZE + end + + # A relation this endpoint filters nothing through. It is navigable all the + # same -- the read costs nothing, the target being read whole -- and saying + # which of the two it is, is the whole point of the message. + def check_relation_filterable!(leaf, relation) + key = relation.foreign_key + refuse_unfilterable_key!(leaf, key) if search_endpoint.field(key).nil? + end + + def refuse_unfilterable_key!(leaf, key) + raise UnsupportedOperatorError, + "#{name} cannot filter #{leaf.field.inspect}: the relation resolves to #{key.inspect}, on " \ + "which #{search_endpoint.path} takes no filter. The relation is there to be read and " \ + "navigated; filter on one of: #{search_endpoint.filterable_columns.join(", ")}." + end + + def refuse_fan_out!(leaf, key, _ids) + # "more than", never a count: the target is read one record past what a + # group may hold, so what is known is that it does not fit -- printing + # 16 where a workspace holds three thousand would read as a number the + # operator could go and narrow by one. + raise UnsupportedOperatorError, + "#{name} cannot filter #{leaf.field.inspect}: it names more than " \ + "#{Query::ConditionTreeTranslator::MAX_GROUP_SIZE} records, " \ + "#{search_endpoint.path} answers #{key.inspect} one value at a time, and Intercom takes " \ + "#{Query::ConditionTreeTranslator::MAX_GROUP_SIZE} conditions per group. Narrow the condition " \ + "on the relation, or filter on #{key.inspect} itself." + end + + def combined_tree(filter) + conditions = [filter&.condition_tree, search_condition(filter)].compact + return conditions.first if conditions.size < 2 + + ForestAdminDatasourceToolkit::Components::Query::ConditionTree::ConditionTreeFactory.intersect(conditions) + end + + # A free-text search reaches Intercom as a condition on the one column its + # endpoint matches text on -- per word, not as a substring, which the + # README says rather than the interface implying otherwise. + def search_condition(filter) + return nil if blank_search?(filter) + + refuse_search! if search_column.nil? + + Leaf.new(search_column, Operators::CONTAINS, filter.search.to_s.strip) + end + + # The records of this tier carry bodies written by end customers, so every + # read of one asks Intercom for plain text (R10). `BaseCollection` reads + # this on the way to the record endpoint. + def record_read_params = read_params + + def listed_records(filter, query, sort = nil) + offset, limit = translate_page(filter&.page) + + walker.walk(offset: offset, limit: limit) do |per_page, cursor| + read_page(per_page: per_page, cursor: cursor, query: query, sort: sort) + end + end + + # A filter with no page asks for every record it matched; the walker reads + # that as the nil limit it bounds with its own caps. + def translate_page(page) + return [0, nil] if page.nil? + + limit = page.limit.to_i + [page.offset.to_i.clamp(0, nil), limit.positive? ? limit : nil] + end + + # Exact, and one request: `total_count` counts what the filter names, not + # what a page happened to hold. An id lookup counts the records it found, + # which is cheaper still. + def count_records(caller, filter) + ids = id_lookup(filter) + return count_by_ids(ids) if ids + + query = translate(caller, filter) + return 0 if query == NOTHING + + exact_count(read_page(per_page: 1, cursor: nil, query: query)) + end + + def refuse_search! + raise UnsupportedOperatorError, + "#{name} cannot answer a free-text search: #{search_endpoint.path} matches values field by field, " \ + 'and this collection exposes no text column it searches. Filter on a column instead of searching.' + end + + # The order Intercom will really apply, written the way the client sends + # it -- or nil, and the order the operator asked for is then reported + # rather than dropped in silence. + # + # The ascending primary-key sort the agent injects when a request names + # none is neither honoured nor reported: it is not an order anybody asked + # for, and `/contacts/search` does not sort on an id anyway. + def server_sort(filter) + clauses = Array(filter&.sort) + return nil if clauses.empty? || default_pk_sort?(clauses) + + honoured = honourable_sort(clauses) + warn_ignored_sort(clauses) if honoured.nil? + honoured + end + + # One clause, on a column the measured table says the endpoint sorts. + # Intercom takes a single `{ field, order }` and nothing composite, so a + # second clause is not half-honoured: honouring the first alone would + # order the page by something the operator did not ask for. + def honourable_sort(clauses) + return nil unless clauses.size == 1 + + clause = clauses.first + field = search_endpoint.field(sort_field(clause).to_s) + return nil unless field&.sortable? + + { field: field.field, ascending: ascending?(clause) } + end + + # Intercom accepts a `sort` on the other two endpoints and ignores it + # without a word -- measured -- so an order asked for and not applied has + # to be reported here or nowhere. + def warn_ignored_sort(clauses) + ForestAdminDatasourceIntercom.logger.warn( + "[forest_admin_datasource_intercom] #{name} was asked to sort on " \ + "#{clauses.map { |clause| sort_field(clause) }.join(", ")}, which Intercom does not sort this " \ + 'collection on -- and it ignores a sort it refuses without reporting it. The rows come back in the ' \ + 'order the API imposes.' + ) + end + + # `sort` here is the clause `server_sort` found the endpoint does honour, + # which is why nothing has reported it yet: it is this route, not the + # column, that cannot carry it. + def warn_unordered_ids(sort) + ForestAdminDatasourceIntercom.logger.warn( + "[forest_admin_datasource_intercom] #{name} was asked to sort on #{sort[:field].inspect} while reading " \ + 'records by id, which Intercom reads by id and in no order. The rows come back in the order the ids ' \ + 'were named.' + ) + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/custom_attributes.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/custom_attributes.rb new file mode 100644 index 000000000..771931a79 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/custom_attributes.rb @@ -0,0 +1,75 @@ +module ForestAdminDatasourceIntercom + module Collections + # The columns a workspace's own attributes become, and the two things that + # go wrong if they are published as they come. + # + # **A name can land on a column the collection already carries.** Measured + # on a real workspace: a custom contact attribute named `id`. Adding it + # would raise on the second declaration -- the toolkit refuses a field twice + # -- and, if it did not, the serializer would write the attribute where the + # operator expects the record's key. It is skipped, and the log names which + # one, since the fix is on Intercom's side. + # + # Relations count as taken names, which is why every collection here + # declares them **before** registering these columns. + # + # **A date arrives as epoch seconds**, like every other Intercom date, and a + # Date column that receives an integer renders as one. + module CustomAttributes + private + + def register_attribute_columns + @attribute_columns = @attributes.reject { |attribute| collides?(attribute) } + @attribute_columns.each { |attribute| add_column(attribute.column_name, attribute.column_type) } + end + + def attribute_columns = @attribute_columns || [] + + # What the translator needs of them: the names the schema published, so a + # filter reaching one is refused with the reason the table carries for + # the whole family rather than with the message for a column that is not + # in it. + def attribute_column_names = attribute_columns.map(&:column_name) + + # What the log calls these, which is the workspace's own vocabulary: a + # ticket attribute is declared per ticket type, a contact attribute per + # model. + def attribute_kind = raise(NotImplementedError, "#{self.class} did not implement attribute_kind") + + def collides?(attribute) + return false unless fields.key?(attribute.column_name) + + ForestAdminDatasourceIntercom.logger.warn( + "[forest_admin_datasource_intercom] #{name} skips the #{attribute_kind} attribute " \ + "#{attribute.name.inspect}: a native column or relation already carries the name " \ + "#{attribute.column_name.inspect}, and overwriting it would show the attribute where the operator " \ + 'expects the record field. Rename it in Intercom to publish it.' + ) + true + end + + # The value of each published attribute, read under the name the workspace + # gave it and written under the column name the schema publishes -- the + # two differ whenever the first could not travel through a Forest query + # string. + # + # Nil rather than absent for an attribute the record does not carry: the + # column exists on every row, and an absent key would read as a record + # missing it. + def attribute_values(values) + held = values.is_a?(Hash) ? values : {} + + attribute_columns.to_h do |attribute| + [attribute.column_name, coerce_attribute(held[attribute.name], attribute)] + end + end + + def coerce_attribute(value, attribute) + return nil if value.nil? + return stamp(value) if attribute.column_type == 'Date' && value.is_a?(Numeric) + + value + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/fetch_all_collection.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/fetch_all_collection.rb new file mode 100644 index 000000000..c3d39180f --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/fetch_all_collection.rb @@ -0,0 +1,223 @@ +module ForestAdminDatasourceIntercom + module Collections + # Base for the reference collections Intercom hands back whole in a single + # response: admins, teams, ticket types, ticket states. Their endpoints + # declare no pagination parameter, no filter and no sort. + # + # Paradoxically this is the most capable tier of the datasource. Filtering, + # sorting, paginating and counting that response in memory is *exact* rather + # than approximate, because the records in hand are every record Intercom + # holds: a window cut out of them carries the rows a server-side query would + # have returned. It is the one place where an in-memory pass does not risk + # the thing this datasource refuses everywhere else -- a result that looks + # filtered without being filtered -- which only arises when what one holds + # is a single page of something larger. The cost is bandwidth, not + # correctness. + # + # Each read re-reads the endpoint, so an operator sees what Intercom holds + # now rather than what it held when the process booted. One request per list + # against a 10 000-a-minute budget is not a figure any list view approaches. + class FetchAllCollection < BaseCollection # rubocop:disable Metrics/ClassLength + # The filters a column may advertise, per column type. Restricted to what + # the toolkit can evaluate in memory, since the in-memory pass is the only + # pass there is here: an operator with no equivalence makes `match` answer + # nil, which `apply` reads as "no match" and would empty the page instead + # of filtering it. + OPERATOR_CANDIDATES = { + 'String' => [Operators::EQUAL, Operators::NOT_EQUAL, Operators::IN, Operators::NOT_IN, + Operators::PRESENT, Operators::BLANK, Operators::CONTAINS, Operators::I_CONTAINS, + Operators::NOT_CONTAINS, Operators::STARTS_WITH, Operators::ENDS_WITH], + 'Boolean' => [Operators::EQUAL, Operators::NOT_EQUAL, Operators::IN, Operators::NOT_IN, + Operators::PRESENT, Operators::BLANK] + }.freeze + + # The operators `ConditionTreeLeaf#match` evaluates natively; anything else + # needs an equivalence for the column's type to be evaluable at all. + IN_MEMORY_OPERATORS = [Operators::IN, Operators::EQUAL, Operators::LESS_THAN, Operators::GREATER_THAN, + Operators::MATCH, Operators::STARTS_WITH, Operators::ENDS_WITH, + Operators::LONGER_THAN, Operators::SHORTER_THAN, Operators::INCLUDES_ALL, + Operators::NOT_IN, Operators::NOT_EQUAL, Operators::NOT_CONTAINS].freeze + + # Candidates are re-checked against the toolkit rather than trusted, so an + # equivalence it stops providing takes the filter out of the schema instead + # of turning every page using it into an empty one. + def self.operators_for(column_type) + Array(OPERATOR_CANDIDATES[column_type]).select do |operator| + Equivalent.equivalent_tree?(operator, IN_MEMORY_OPERATORS, column_type) + end + end + + # Countable, unlike the cursor-paginated collections: the count answered + # here is taken over every record Intercom holds rather than over the pages + # a walk happened to collect. + def initialize(datasource, name) + super + enable_count + end + + def list(caller, filter, projection) + records = sort_in_memory(filtered_records(caller, filter), filter&.sort) + window = page_window(records, filter) + rows = window.map { |record| project(record, projection) } + + enrich(window, rows, projection) + embed_relations(caller, window, rows, projection) + rows + end + + # Exact, like the filter and the sort above it, which is why these columns + # stay groupable. + # + # Rows come back keyed with strings because that is how the agent reads + # them, while `Aggregation#apply` hands them back keyed with symbols. + def aggregate(caller, filter, aggregation, limit = nil) + aggregation.apply(filtered_records(caller, filter), timezone_for(caller), limit) + .map { |row| { 'group' => row[:group], 'value' => row[:value] } } + end + + protected + + # Scalar columns are sortable and groupable, the in-memory pass honouring + # anything asked of them. A Json column is neither, nor filterable: it + # holds a list, and what a filter on it would mean has no in-memory + # counterpart. + # + # Every column is read-only: this lot writes nothing, and an editable + # column would offer a Save that reaches an `update` the collection does + # not implement. + def add_column(name, type, is_primary_key: false) + add_field(name, ColumnSchema.new(column_type: type, + filter_operators: self.class.operators_for(type), + is_primary_key: is_primary_key, + is_read_only: true, + is_sortable: type != 'Json', + is_groupable: type != 'Json')) + end + + # Hook for what a row needs beyond the endpoint this collection reads -- + # a name held on the other side of a membership, say. Called with the rows + # of the page only, and with the projection, so a column nobody asked for + # costs no request. + def enrich(_records, _rows, _projection); end + + # Every record of the collection, straight from its endpoint. + def fetch_all = raise(NotImplementedError, "#{self.class} did not implement fetch_all") + + # One Intercom entity flattened into a record matching the schema. + def serialize(_entity) = raise(NotImplementedError, "#{self.class} did not implement serialize") + + private + + # The complete dataset, serialized and narrowed to the rows the filter + # keeps: what `list` pages and what `aggregate` counts are the same rows. + def filtered_records(caller, filter) + records = fetch_all.map { |entity| serialize(entity) } + tree = filter&.condition_tree + return records if tree.nil? + + # A condition through a relation becomes a membership on the foreign key: + # this tier filters in memory, where a list of ids costs no more than one + # id and none of Intercom's group limits apply -- they bound its search + # DSL, which nothing here goes through. + tree = rewrite_relation_conditions(caller, tree) { |key, ids, _| Leaf.new(key, Operators::IN, ids) } + return [] if tree == NOTHING + + refuse_unevaluable!(tree) + tree.apply(records, self, timezone_for(caller)) + end + + # A condition this collection cannot evaluate is refused rather than + # applied. `match` answers nil for an operator with no in-memory + # equivalence and `apply` reads that as "no match", so an unevaluable + # condition would hand back an empty page that looks like a filter + # matching nothing -- indistinguishable, to the operator, from a real + # answer. The schema advertises no such operator; a scope, a segment or a + # customizer can still send one. + def refuse_unevaluable!(tree) + offender = nil + tree.some_leaf do |leaf| + offender = leaf unless evaluable?(leaf) + !offender.nil? + end + return if offender.nil? + + raise UnsupportedOperatorError, + "#{name} cannot filter '#{offender.field}' with '#{offender.operator}': it is read whole from " \ + 'Intercom and filtered in memory, which supports only the operators its columns advertise. ' \ + 'Change the condition, or the scope or segment carrying it.' + end + + def evaluable?(leaf) + schema = fields[leaf.field] + schema.is_a?(ColumnSchema) && schema.filter_operators.include?(leaf.operator) + end + + # Neither Ruby's `sort` nor the toolkit's `Sort#apply` can be used as is: + # `sort` is not stable, and `<=>` answers nil on a null, on two booleans + # and on mixed types, which leaves the comparator undefined and the order + # arbitrary. Ties therefore fall back to the position Intercom returned the + # record in. + # + # Every requested order is honoured, the ascending primary-key sort the + # agent injects when a request names none included, so there is no + # unsortable order to report here -- unlike the cursor collections, where + # Intercom ignores a sort without saying so. + def sort_in_memory(records, sort) + clauses = sort_clauses(sort) + return records if clauses.empty? + + records.each_with_index.sort do |(left, left_index), (right, right_index)| + compare_clauses(left, right, clauses).nonzero? || (left_index <=> right_index) + end.map(&:first) + end + + # A sort clause naming a field this collection does not carry is dropped: + # ordering by a column that is not there would compare nil to nil on every + # row and leave the order to the tie-break. Reported when it happens, + # rather than dropped in silence -- an order asked for and not honoured is + # reported everywhere else in this datasource, and there is one route that + # reaches this tier with a clause it cannot resolve: a related list through + # a many-to-many is served by the collection it travels through, and + # `Filter#nest` prefixes the condition tree without prefixing the sort, so + # the membership is handed the columns of the collection it reaches. + def sort_clauses(sort) + known, unknown = Array(sort).partition { |clause| fields.key?(sort_field(clause)) } + warn_unsortable(unknown) unless unknown.empty? + + known.map { |clause| [sort_field(clause), ascending?(clause) != false] } + end + + def warn_unsortable(clauses) + ForestAdminDatasourceIntercom.logger.warn( + "[forest_admin_datasource_intercom] #{name} was asked to sort on " \ + "#{clauses.map { |clause| sort_field(clause).inspect }.join(", ")}, which it does not carry; the rows " \ + 'come back in the order Intercom returned them. A related list through a many-to-many is ordered by the ' \ + 'columns of the collection it travels through, not by those of the collection it reaches.' + ) + end + + def compare_clauses(left, right, clauses) + clauses.each do |field, ascending| + comparison = compare_values(left[field], right[field]) + next if comparison.zero? + + return ascending ? comparison : -comparison + end + + 0 + end + + # Nulls sort last ascending and first descending, the way a database orders + # them; values `<=>` cannot compare -- two booleans, for one -- are + # compared through their string form rather than left undefined, which puts + # `false` before `true`, again like a database. + def compare_values(left, right) + return 0 if left.nil? && right.nil? + return 1 if left.nil? + return -1 if right.nil? + + (left <=> right) || (left.to_s <=> right.to_s) + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/offset_collection.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/offset_collection.rb new file mode 100644 index 000000000..4c7373ac5 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/offset_collection.rb @@ -0,0 +1,275 @@ +module ForestAdminDatasourceIntercom + module Collections + # The third pagination tier, and the only one that maps onto what Forest + # asks for without translating anything: Intercom paginates + # `POST /companies/list` by **offset**, so page 7 of a list view is one + # request rather than six pages walked to reach it. No cursor walker, no + # cap, and no truncation warning. + # + # What it pays for that is filtering. There is no search endpoint for + # companies at all -- `GET /companies/scroll` exists and is deliberately + # rejected, one open scroll per app expiring in a minute cannot serve + # concurrent list views -- so what a filter may say is four exact lookups + # and nothing else. Everything past them is **refused by name**, the rule + # the cursor tier already set: a page served in answer to a filter it + # ignored is the one failure this datasource is built to avoid. + # + # In memory it does nothing: no filter, no sort, no group. What is in hand + # is a page of something larger, exactly like the cursor tier, and the same + # reasoning applies. + # Long by line count only: half of it is the refusals, and a refusal that + # does not say what to do instead is one an operator cannot act on. + class OffsetCollection < BaseCollection # rubocop:disable Metrics/ClassLength + include RecordsById + + # What one page holds when the read names no window: a relation resolving + # its target, a segment, a customizer. A list view always names one. + UNBOUNDED_PAGE_SIZE = Client::MAX_PER_PAGE + + # And how many such pages are read before the answer is cut short. The + # figure only ever applies to a read with no window of its own. + MAX_COLLECTED_PAGES = 10 + + # What a read that *does* name a window may collect before it is cut + # short. A window is its own bound, so the page count above must not apply + # to it: a window needing eleven pages would come back one page short + # while the warning blamed it for naming no window -- which it did name. + # This is the backstop for a window nothing sane asked for, and it is the + # cursor walker's own record budget, that tier being bounded the same way. + MAX_COLLECTED_RECORDS = Pagination::CursorWalker::MAX_RECORDS + + # How many records of this collection one relation read may resolve. + # Higher than `MAX_ID_READS`, which bounds a single request batch, and + # deliberately above every page size a list view offers: a page naming one + # account per row resolves, and a read naming more -- an export, a + # segment resolved whole -- is refused by name rather than answered with + # a nil where an account exists. There is no bulk read for a company, so + # each one costs a request and the figure is what an operator waits for. + MAX_RELATION_READS = 100 + + def initialize(datasource, name) + super + enable_count + end + + def ids_per_read = MAX_ID_READS + def max_resolvable_ids = MAX_RELATION_READS + + def list(caller, filter, projection) + warn_ignored_sort(filter&.sort) + + records = fetch_records(filter) + serialized = records.map { |record| serialize(record) } + rows = serialized.map { |record| project(record, projection) } + + embed_relations(caller, serialized, rows, projection) + rows + end + + # Count only, and never a group: `total_count` is exact on every listing, + # while grouping the page in hand would answer a fraction as if it were + # the whole. + def aggregate(_caller, filter, aggregation, _limit = nil) + refuse_unsupported_aggregation!(aggregation) + + [{ 'group' => {}, 'value' => count_records(filter) }] + end + + protected + + # The endpoint that lists the collection by offset, the one that reads a + # record, and the lookups Intercom answers on the listing path. + def list_path = raise(NotImplementedError, "#{self.class} did not implement list_path") + def record_endpoint = raise(NotImplementedError, "#{self.class} did not implement record_endpoint") + def lookup_path = record_endpoint + def lookups = {} + + def serialize(_entity) = raise(NotImplementedError, "#{self.class} did not implement serialize") + + # A column advertises a filter only where Intercom looks the collection up + # by it, so a column cannot offer a filter this tier would then refuse. + # The primary key is the exception and it is not a filter: `id equals X` + # and `id in [...]` are answered by the record endpoint. + # + # Nothing is sortable: the listing takes no order and ordering a page in + # hand would order a fraction of the collection. + def add_column(name, type, is_primary_key: false) + add_field(name, ColumnSchema.new(column_type: type, + filter_operators: column_operators(name, is_primary_key), + is_primary_key: is_primary_key, + is_read_only: true, + is_sortable: false, + is_groupable: false)) + end + + private + + def column_operators(name, is_primary_key) + return [Operators::EQUAL, Operators::IN] if is_primary_key + + lookups.key?(name) ? [Operators::EQUAL] : [] + end + + def fetch_records(filter) + ids = id_lookup(filter) + return records_by_ids(page_window(ids, filter)) if ids + + lookup = lookup_condition(filter) + return page_window(looked_up_records(lookup), filter) if lookup + + refuse_condition!(filter.condition_tree) unless filter&.condition_tree.nil? + + listed_records(filter) + end + + def count_records(filter) + ids = id_lookup(filter) + return count_by_ids(ids) if ids + + lookup = lookup_condition(filter) + return looked_up_records(lookup).size if lookup + + refuse_condition!(filter.condition_tree) unless filter&.condition_tree.nil? + + exact_count(read_offset_page(page: 1, per_page: 1)) + end + + # The window a list view asked for, read as the page Intercom counts from + # 1. An offset that does not fall on a page boundary is served by reading + # the page it lands in and the ones after it until the window is filled -- + # exactly, rather than by rounding the offset to something the API likes. + def listed_records(filter) + offset, limit = window(filter&.page) + per_page = Client.bounded_per_page(limit || UNBOUNDED_PAGE_SIZE) + skip = offset % per_page + + collected = collect_pages(first_page: (offset / per_page) + 1, per_page: per_page, + wanted: limit && (skip + limit)) + + limit ? (collected[skip, limit] || []) : collected.drop(skip) + end + + def collect_pages(first_page:, per_page:, wanted:) + records = [] + page = first_page + read = 0 + + loop do + answer = read_offset_page(page: page, per_page: per_page) + records.concat(answer.records) + read += 1 + break if last_page?(answer, page) || (wanted && records.size >= wanted) + break if cap_reached?(read, records.size, wanted) + + page += 1 + end + + records + end + + def read_offset_page(page:, per_page:) + client.offset_page(list_path, page: page, per_page: per_page) + end + + def last_page?(answer, page) + answer.records.empty? || (answer.total_pages && page >= answer.total_pages) + end + + # Which of the two caps applies, and the reason that goes with it: a read + # with no window of its own is stopped by the page count, a read that + # named one by the record budget. Telling a list view it named no window + # is exactly the confusion these two keep apart. + def cap_reached?(read, collected, wanted) + if wanted + return false if collected < MAX_COLLECTED_RECORDS + + warn_capped("#{collected} record(s)", + 'this read named a window larger than one answer may hold') + else + return false if read < MAX_COLLECTED_PAGES + + warn_capped("#{read} page(s) / #{collected} record(s)", + 'this read named no window of its own, and a list view always does') + end + + true + end + + def warn_capped(reached, reason) + ForestAdminDatasourceIntercom.logger.warn( + "[forest_admin_datasource_intercom] Stopped reading #{name} after #{reached}; the rest is left out: " \ + "#{reason}." + ) + end + + # A filter with no page asks for every record it matched; nil is how the + # window says so. + def window(page) + return [0, nil] if page.nil? + + limit = page.limit.to_i + [page.offset.to_i.clamp(0, nil), limit.positive? ? limit : nil] + end + + def lookup_condition(filter) + tree = filter&.condition_tree + return nil unless tree.is_a?(Leaf) && tree.operator == Operators::EQUAL + + parameter = lookups[tree.field.to_s] + parameter && { parameter => tree.value.to_s } + end + + # An exact lookup answers few records -- one, for the keys this publishes + # -- so it is read as a single page. More than that page holds is reported + # rather than dropped in silence. + # + # A lookup naming no record is an empty page, not a failure: Intercom + # answers this route with a 404 where a search endpoint would answer an + # empty list, and a filter matching nothing is the most ordinary thing a + # list view does. Read the way a record read by its id already is. + def looked_up_records(params) + answer = client.lookup_page(lookup_path, params: params) + warn_truncated_lookup(params) if answer.next_cursor + + answer.records + rescue APIError => e + raise unless e.status == 404 + + [] + end + + def refuse_condition!(tree) + offender = nil + tree.some_leaf { |leaf| offender = leaf } + + raise UnsupportedOperatorError, + "#{name} cannot filter #{(offender&.field).inspect}: Intercom exposes no search endpoint for this " \ + "collection and looks a record up by #{lookups.keys.join(", ")} alone -- one exact value at a time, " \ + 'with no combination and no other operator. Filter on one of those, or reach the record from the ' \ + 'collection next door.' + end + + # Intercom takes no order on this listing at all -- there is no parameter + # for one -- so an order asked for and not applied is reported here or + # nowhere. The ascending primary-key sort the agent injects when a request + # names none is not one of those. + def warn_ignored_sort(sort) + clauses = Array(sort) + return if clauses.empty? || default_pk_sort?(clauses) + + ForestAdminDatasourceIntercom.logger.warn( + "[forest_admin_datasource_intercom] #{name} was asked to sort on " \ + "#{clauses.map { |clause| sort_field(clause) }.join(", ")}, and Intercom takes no order " \ + 'on this listing. The rows come back in the order the API imposes.' + ) + end + + def warn_truncated_lookup(params) + ForestAdminDatasourceIntercom.logger.warn( + "[forest_admin_datasource_intercom] #{name} looked up #{params.inspect} and Intercom advertised more " \ + 'records than one page holds; the rest is left out. This lookup is meant for a key that names one record.' + ) + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/records_by_id.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/records_by_id.rb new file mode 100644 index 000000000..23e56ed20 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/records_by_id.rb @@ -0,0 +1,126 @@ +module ForestAdminDatasourceIntercom + module Collections + # `id equals X` is a record detail and `id in [...]` a bulk read of related + # records, and both are answered by the record endpoint rather than by + # whatever the collection is filtered through. What the two paginated tiers + # share, and what the tier read whole has no use for -- it holds every + # record already, so an id is a lookup in memory rather than a request. + # + # Which is also why the counting lives here: a count over the pages a read + # collected would answer a fraction of the collection as if it were the + # whole of it, and that risk only exists where what is in hand is a page. + module RecordsById + Operators = ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Operators + Leaf = ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Nodes::ConditionTreeLeaf + Aggregation = ForestAdminDatasourceToolkit::Components::Query::Aggregation + + # How many records a bulk read by id may fetch. Intercom exposes no "read + # these records" endpoint, so a set of ids is one request each -- the + # fan-out is bounded rather than turned into a rate limit halfway through + # a page. The collection that reads its ids in bulk raises the figure. + MAX_ID_READS = 25 + + protected + + # The endpoint one record is read from. + def record_endpoint = raise(NotImplementedError, "#{self.class} did not implement record_endpoint") + + # What rides along on that read. `display_as=plaintext` for the tiers + # whose records carry bodies written by end customers; nothing elsewhere. + def record_read_params = {} + + # How many records a bulk read may fetch, as a hook rather than the + # constant: the collection that reads its ids in bulk answers far more of + # them for the same request count. + def max_id_reads = MAX_ID_READS + + # Only a bare leaf on the primary key takes this route: an `and` also + # carrying a scope names a narrower set than the ids do, and answering it + # with the ids alone would serve records the scope excludes. A free-text + # search alongside is the same problem -- the ids would answer a question + # nobody asked. + # + # Deduplicated: Intercom reads a record by id, so a value named twice + # would be fetched twice and handed back as two rows carrying one id -- + # where the `in` this comes from is a membership, matching a record once. + def id_lookup(filter) + tree = filter&.condition_tree + return nil unless tree.is_a?(Leaf) && tree.field.to_s == primary_key + return nil unless blank_search?(filter) + + case tree.operator + when Operators::EQUAL then [tree.value].compact.map(&:to_s).uniq + when Operators::IN then Array(tree.value).compact.map(&:to_s).uniq + end + end + + # A record the operator can no longer reach -- deleted, or outside the + # token's scope -- reads as "no record" rather than as a failed page. + def records_by_ids(ids) + wanted = ids.first(max_id_reads) + warn_truncated_ids(ids.size) if ids.size > wanted.size + + wanted.filter_map do |id| + client.fetch_record(record_endpoint, id, params: record_read_params) + rescue APIError => e + raise unless e.status == 404 + + nil + end + end + + # How many of a set of ids name a record, which means reading them: + # Intercom has no "how many of these exist" endpoint. Past what a bulk + # read may fetch the count is refused rather than answered with the number + # the truncation left -- both paginated tiers advertise an exact count, + # and "25" where the question named forty records is not one. + def count_by_ids(ids) + refuse_id_count!(ids.size) if ids.size > max_id_reads + + records_by_ids(ids).size + end + + # The count Intercom answered, or nothing at all. Counting the pages a + # read collected would answer a fraction of the collection as if it were + # the whole of it, which is the one thing the paginated tiers do not do. + def exact_count(page) + return page.total_count if page.total_count + + raise UnsupportedOperatorError, + "#{name} cannot be counted: Intercom answered this listing without a total_count, and counting the " \ + 'pages the agent read would answer a fraction of the collection as if it were the whole of it.' + end + + def refuse_unsupported_aggregation!(aggregation) + return if aggregation.is_a?(Aggregation) && aggregation.operation.to_s.casecmp('count').zero? && + Array(aggregation.groups).empty? && aggregation.field.nil? + + raise UnsupportedOperatorError, + "#{name} can only be counted: Intercom exposes no aggregate endpoint, and grouping or summing the " \ + 'pages the agent read would answer a fraction of the collection as if it were the whole of it. ' \ + 'Chart it on a collection read whole, or wait for the bounded group-by of the reporting lot.' + end + + def refuse_id_count!(asked) + raise UnsupportedOperatorError, + "#{name} cannot count #{asked} records by id: Intercom exposes no endpoint that counts a set of ids, " \ + "so counting them means reading them, and this reads #{max_id_reads} at most. Counting the ones it " \ + 'read would answer a number smaller than the question. Narrow the condition, or count the ' \ + 'collection with a filter it answers instead of a list of ids.' + end + + def warn_truncated_ids(asked) + ForestAdminDatasourceIntercom.logger.warn( + "[forest_admin_datasource_intercom] #{name} was asked for #{asked} records by id and read the first " \ + "#{max_id_reads}: Intercom reads them one request each. The result is truncated." + ) + end + + def blank_search?(filter) + search = filter.respond_to?(:search) ? filter.search : nil + + search.nil? || search.to_s.strip.empty? + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/relations.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/relations.rb new file mode 100644 index 000000000..48ba492cc --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/relations.rb @@ -0,0 +1,354 @@ +module ForestAdminDatasourceIntercom + module Collections + # How this datasource declares a relation, and how it answers one. + # + # Intercom joins nothing: a ticket carries an assignee id, and the teammate + # behind it is a second read of a second endpoint. What makes that affordable + # -- and exact -- is that every collection on the far end of these relations + # is read whole in one request, the tier `FetchAllCollection` serves. + # + # Declaring a relation obliges both halves below, and the second is not + # optional: a many-to-one is published filterable as soon as *any* column of + # its target is (`GeneratorField#build_many_to_one_schema`), so the interface + # offers `assignee:name` the moment the relation exists. Left unanswered, + # that filter reaches a translator refusing every traversing field -- the + # interface offering a filter the datasource then refuses, which is the one + # thing this package is built not to do. + # + # * a projection through a relation (`assignee:name`) is answered by reading + # the target once per page and nesting its row under the relation name; + # * a filter through a relation is answered by asking the target which of its + # records match, and filtering Intercom on the ids it names. + # + # The second is exact rather than approximate -- the target answers over + # every record it holds, not over a page -- with one semantic worth stating + # plainly: a row whose foreign key is null matches no relation filter, the + # way a join drops it, negated filters included. A ticket with no assignee is + # not "assigned to someone other than Marie". + module Relations # rubocop:disable Metrics/ModuleLength + ManyToOneSchema = ForestAdminDatasourceToolkit::Schema::Relations::ManyToOneSchema + ManyToManySchema = ForestAdminDatasourceToolkit::Schema::Relations::ManyToManySchema + OneToManySchema = ForestAdminDatasourceToolkit::Schema::Relations::OneToManySchema + Filter = ForestAdminDatasourceToolkit::Components::Query::Filter + Projection = ForestAdminDatasourceToolkit::Components::Query::Projection + Operators = ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Operators + Branch = ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Nodes::ConditionTreeBranch + Leaf = ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Nodes::ConditionTreeLeaf + + # What a relation condition comes to when the target matched no record: no + # row can satisfy it. It is a value rather than an empty condition because + # the two tiers spell "match nothing" differently -- in memory an `in []` + # says it, while Intercom's search DSL has no way to. + NOTHING = :matches_nothing + + # The group a relation condition expands into, told apart from a group the + # operator wrote. Two things hang on the difference: this one may be + # inlined into a parent aggregating the same way -- the level it adds is + # one nobody budgeted for -- and a tier refusing it can name the relation + # rather than a shape the operator never wrote. + class RelationBranch < Branch + attr_reader :relation_field + + def initialize(aggregator, conditions, relation_field) + @relation_field = relation_field + super(aggregator, conditions) + end + end + + protected + + # Every relation of this package points at a collection keyed by `id` and + # is read-only: nothing in this lot writes, and an editable relation would + # offer an association the collection cannot perform. + def add_many_to_one(name, foreign_collection:, foreign_key:) + add_field(name, ManyToOneSchema.new(foreign_collection: foreign_collection, + foreign_key: foreign_key, + foreign_key_target: 'id', + is_read_only: true)) + end + + # The other side of a many-to-one, and the whole point of the 360 degrees: + # a contact's conversations, a company's contacts. The agent serves it by + # listing the target on `origin_key equals `, so the + # target has to be able to answer that condition -- which is what makes + # this declarable here and not everywhere. + # + # Published unfilterable by the agent (`GeneratorField`), so unlike a + # many-to-one it carries no risk of offering a filter this datasource + # would then refuse. + def add_one_to_many(name, foreign_collection:, origin_key:) + add_field(name, OneToManySchema.new(foreign_collection: foreign_collection, + origin_key: origin_key, + origin_key_target: 'id', + is_read_only: true)) + end + + def add_many_to_many(name, foreign_collection:, through_collection:, origin_key:, foreign_key:) + add_field(name, ManyToManySchema.new(foreign_collection: foreign_collection, + through_collection: through_collection, + origin_key: origin_key, origin_key_target: 'id', + foreign_key: foreign_key, foreign_key_target: 'id', + is_read_only: true)) + end + + # The rows of one page, with the relations the projection named nested onto + # them. `records` are the same rows before projection: that is where the + # foreign keys are read, since a projection naming `assignee:name` does not + # have to name `admin_assignee_id`. + # + # One request per target *collection* per page -- never one per row, and + # never twice for two relations that point at the same collection: a + # ticket's `state` and `previous_state` are one read of `/ticket_states`, + # over the ids both of them name. + def embed_relations(caller, records, rows, projection) + asked = many_to_one_asked(projection) + return if asked.empty? + + indexed = indexed_targets(caller, records, asked) + + asked.each do |name, relation, wanted| + rows_of_target = indexed[relation.foreign_collection] + + # Nil rather than absent when an id names no record: a teammate who + # left the workspace reads as no teammate, not as a broken row. Sliced + # back to what this relation asked for, the read having been widened to + # the union of what every relation on that collection did. + records.each_with_index do |record, index| + target_row = rows_of_target[record[relation.foreign_key]] + rows[index][name] = target_row&.slice(*wanted) + end + end + end + + # The tree this tier can filter on, every `relation:field` leaf traded for + # a condition on the foreign key. The block is handed that key and the ids + # the target matched, and answers the node the tier wants: an `in` where + # the filtering is done in memory, a group of equalities where Intercom's + # DSL takes no membership operator. + # + # An `and` carrying a leaf that matches nothing matches nothing itself; an + # `or` drops that leaf and keeps its siblings. + def rewrite_relation_conditions(caller, node, &builder) + case node + when Branch then rewrite_branch(caller, node, &builder) + when Leaf then relation_leaf?(node) ? rewrite_relation_leaf(caller, node, &builder) : node + else node + end + end + + private + + def relation_leaf?(node) + node.is_a?(Leaf) && node.field.to_s.include?(':') + end + + def relations_asked(projection) + Projection.new(Array(projection).map(&:to_s)).relations + end + + # The many-to-one relations the projection named, each with the columns it + # asked of its target. The target's own key travels with them whether or + # not it was asked for: it is what the rows are indexed by here, and what + # makes the nested row a link rather than a label in the interface. + def many_to_one_asked(projection) + relations_asked(projection).filter_map do |relation_name, sub_projection| + relation = fields[relation_name] + next unless relation.is_a?(ManyToOneSchema) + + columns = Array(sub_projection).map(&:to_s) + refuse_two_hop_projection!(relation_name, relation, columns) + + [relation_name, relation, columns.union([relation.foreign_key_target])] + end + end + + # `admin_assignee:teams:name` is a projection Forest's own parser builds + # and this cannot answer: it nests one target row under the relation name, + # not a tree of them, so the second hop would be dropped by the target's + # projection and the column would come back missing from a row that looks + # complete. Refused by name, like the filter that reaches that deep -- + # `refuse_two_hops!` is its counterpart. + def refuse_two_hop_projection!(relation_name, relation, columns) + deep = columns.select { |column| column.include?(':') } + return if deep.empty? + + raise UnsupportedOperatorError, + "#{name} cannot read #{deep.map { |column| "#{relation_name}:#{column}".inspect }.join(", ")}: it " \ + 'reaches through two relations, and this datasource resolves one. Read the column from a record ' \ + "page of #{relation.foreign_collection}, one hop away." + end + + def indexed_targets(caller, records, asked) + asked.group_by { |_, relation, _| relation.foreign_collection } + .transform_values { |group| target_rows(caller, records, group) } + end + + # One read per target collection, over the ids every relation pointing at + # it names and the union of the columns they asked for. + # + # The ids are sliced by what the target resolves in one read, and the + # fan-out is refused past what it resolves at all. Neither is a detail: + # a bulk read by id is bounded on every tier -- Intercom offers no "read + # these records" endpoint, so one of them pays a request per id -- and + # handing the whole page's keys to that bound would resolve the first + # slice and leave the rest of the rows carrying a nil where a record + # exists. A relation answered for part of a page is the failure this + # datasource is built to avoid, so it is named instead. + def target_rows(caller, records, group) + relation = group.first[1] + ids = group.flat_map { |_, rel, _| records.filter_map { |record| record[rel.foreign_key] } }.uniq + return {} if ids.empty? + + collection = foreign_collection(relation) + refuse_relation_fan_out!(group, collection, ids.size) if beyond_reach?(collection, ids.size) + + target = relation.foreign_key_target + wanted = Projection.new(group.flat_map { |_, _, columns| columns }.uniq) + + read_targets(caller, collection, target, ids, wanted).to_h { |row| [row[target], row] } + end + + def beyond_reach?(collection, count) + limit = collection.max_resolvable_ids + + !limit.nil? && count > limit + end + + def read_targets(caller, collection, target, ids, wanted) + chunk = collection.ids_per_read + slices = chunk ? ids.each_slice(chunk).to_a : [ids] + + slices.flat_map do |slice| + collection.list(caller, Filter.new(condition_tree: Leaf.new(target, Operators::IN, slice)), wanted) + end + end + + # Names the relations rather than the key they resolve to: what the + # operator can act on is the column they put in the view or the export, + # and how many rows they asked for at once. + def refuse_relation_fan_out!(group, collection, count) + asked = group.map { |relation_name, _, _| relation_name }.join(', ') + + raise UnsupportedOperatorError, + "#{name} cannot resolve #{asked} over this read: it names #{count} distinct #{collection.name} " \ + 'records and Intercom has no bulk read for that collection, so this resolves at most ' \ + "#{collection.max_resolvable_ids} of them per read -- one request each. Read fewer rows at a time, " \ + 'or leave the relation out of the projection: a relation resolved for part of a page would show no ' \ + 'record where one exists.' + end + + def rewrite_branch(caller, branch, &builder) + rewritten = Array(branch.conditions).map { |node| rewrite_relation_conditions(caller, node, &builder) } + + if branch.aggregator.to_s.casecmp('or').zero? + kept = rewritten.reject { |node| node == NOTHING } + kept.empty? ? NOTHING : Branch.new(branch.aggregator, absorb_groups(branch.aggregator, kept)) + elsif rewritten.include?(NOTHING) + NOTHING + else + Branch.new(branch.aggregator, absorb_groups(branch.aggregator, rewritten)) + end + end + + # A relation group aggregating the way its parent does is inlined into it: + # `or(x, or(a, b))` is `or(x, a, b)` and one level shallower. The level it + # saves is one nothing budgeted for -- what a tier measured its nesting + # limit against is the tree the operator wrote, not the equalities a + # relation expands into afterwards. + # + # Only a relation group is inlined. A group the operator wrote is what + # their filter means, and flattening it would spend on one group the + # conditions two groups were holding. + def absorb_groups(aggregator, nodes) + absorbed = nodes.flat_map do |node| + inlinable_group?(node, aggregator) ? Array(node.conditions) : [node] + end + + absorb_relation_group?(absorbed.size) ? absorbed : nodes + end + + def inlinable_group?(node, aggregator) + node.is_a?(RelationBranch) && node.aggregator.to_s.casecmp(aggregator.to_s).zero? + end + + # Hook for a tier that bounds how many conditions a group may hold: past + # that, the nested form is the one that fits, and the level it costs is + # spent rather than the width. + def absorb_relation_group?(_size) = true + + def rewrite_relation_leaf(caller, leaf) + name, path = leaf.field.to_s.split(':', 2) + relation = fields[name] + refuse_unknown_relation!(leaf, name) unless relation.is_a?(ManyToOneSchema) + refuse_two_hops!(leaf) if path.include?(':') + + check_relation_filterable!(leaf, relation) + + ids = matching_ids(caller, relation, Leaf.new(path, leaf.operator, leaf.value)) + return NOTHING if ids.empty? + + yield(relation.foreign_key, ids, leaf) + end + + # Hook for a tier that cannot filter on every foreign key it declares a + # relation on. Answered before the target is read: a refusal that spends a + # request first costs exactly what it refuses to do. + def check_relation_filterable!(_leaf, _relation); end + + # Which records of the target the condition names, asked of the target + # itself: it owns what its columns can be filtered with, and it answers + # over every record Intercom holds rather than over a page of them. + def matching_ids(caller, relation, leaf) + target = relation.foreign_key_target + filter = Filter.new(condition_tree: leaf, page: match_page) + + foreign_collection(relation) + .list(caller, filter, Projection.new([target])) + .filter_map { |row| row[target] } + .uniq + end + + # How much of the target a relation condition may read. Unbounded here: + # the tier that answers one in memory holds every record already, and + # cutting the read short would drop ids its `in` can carry for free. + # + # The tier whose target is a page of something larger overrides it -- see + # `CursorCollection`. Resolving `contact:email contains "@"` against a + # workspace's whole contact list, to then refuse the fan-out it comes to, + # would spend a full cursor walk on a filter that was never going to be + # answered. + def match_page = nil + + # The target as the datasource holds it, undecorated -- so a permission + # scope or a segment defined on the target does not narrow what a relation + # resolves. That is how a native datasource behaves too: it joins the table + # without applying the scopes of the collection mapped to it. + def foreign_collection(relation) + datasource.get_collection(relation.foreign_collection) + end + + # Either a name this collection carries no relation under -- a condition + # from a scope or a segment written against another schema -- or a + # many-to-many, which is published unfilterable: resolving one would mean + # reading the collection it travels through once per value. + def refuse_unknown_relation!(leaf, relation_name) + filterable = fields.select { |_, field| field.is_a?(ManyToOneSchema) }.keys + + raise UnsupportedOperatorError, + "#{name} cannot filter #{leaf.field.inspect}: #{relation_name.inspect} is not a relation it filters " \ + "through. #{filterable.empty? ? "It has none." : "Those it does: #{filterable.join(", ")}."} Filter " \ + 'on a column of the collection next door instead.' + end + + # Two hops would mean resolving a relation of a relation, one read per + # level, and nothing in this datasource publishes a filter that deep -- a + # many-to-many is published unfilterable. It is refused by name rather than + # half-answered. + def refuse_two_hops!(leaf) + raise UnsupportedOperatorError, + "#{name} cannot filter #{leaf.field.inspect}: it reaches through two relations, and this datasource " \ + 'resolves one. Filter on a column of the collection next door instead.' + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/team.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/team.rb new file mode 100644 index 000000000..a0e111bcf --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/team.rb @@ -0,0 +1,68 @@ +module ForestAdminDatasourceIntercom + module Collections + # The inbox teams a conversation can be assigned to, rather than a single + # teammate. + class Team < FetchAllCollection + def initialize(datasource) + super(datasource, 'IntercomTeam') + end + + protected + + # Like `/admins`, `/teams` uses its own key instead of the `data` envelope. + def fetch_all + client.fetch_all('teams', list_key: 'teams') + end + + def serialize(team) + attrs = team.is_a?(Hash) ? team : {} + + { 'id' => stringify_id(attrs['id']), + 'name' => attrs['name'], + # Not a column: it is what the names below are read from, and what the + # membership collection turns into the relation. Intercom types these + # as numbers here and as strings on the admin itself, so they are + # stringified for both sides to carry the same id. + 'admin_ids' => Array(attrs['admin_ids']).map { |id| stringify_id(id) }, + 'admin_names' => nil } + end + + # The teammates of the team, by name, and only when a projection asked for + # them: one read of `/admins` for the whole page, never one per team. A + # token that cannot read the teammates costs the column and nothing else. + def enrich(records, rows, projection) + return unless column_asked?(projection, 'admin_names') + + names = admin_names + records.each_with_index do |record, index| + rows[index]['admin_names'] = Array(record['admin_ids']).filter_map { |id| names[id] } + end + end + + private + + def admin_names + client.fetch_all('admins', list_key: 'admins') + .to_h { |admin| [stringify_id(admin['id']), admin['name']] } + rescue APIError => e + ForestAdminDatasourceIntercom.logger.warn( + "[forest_admin_datasource_intercom] #{name} could not read the teammates of the workspace (HTTP " \ + "#{e.status || "-"}); the names are left empty. The relation to IntercomAdmin is unaffected." + ) + {} + end + + def define_schema + add_column('id', 'String', is_primary_key: true) + add_column('name', 'String') + # A list, so neither filterable nor sortable -- as the array of ids it + # replaces was. It reads the team without a join; the relation below is + # what navigates it, and filtering happens on the Admins collection. + add_column('admin_names', 'Json') + add_many_to_many('admins', foreign_collection: 'IntercomAdmin', + through_collection: 'IntercomTeamMembership', + origin_key: 'team_id', foreign_key: 'admin_id') + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/team_membership.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/team_membership.rb new file mode 100644 index 000000000..9a1d662b0 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/team_membership.rb @@ -0,0 +1,61 @@ +module ForestAdminDatasourceIntercom + module Collections + # The membership of the inbox teams: one record per team-and-teammate pair. + # + # It exists because Intercom's does not. The workspace carries the membership + # on the team (`admin_ids`) and on the teammate (`team_ids`) both, and offers + # no resource for the pair -- while a many-to-many needs a collection to + # travel through, whose two many-to-one relations are what + # `Utils::Collection.get_through_target` looks for. Declared here, Teams and + # Admins navigate to each other in both directions; left undeclared, both + # sides read as an array of ids. + # + # Read from `/teams` alone. The admin side names the same pairs, so reading + # it too would spend a request confirming what the first answer already said. + # Read-only, like the relations it carries: Intercom exposes no endpoint that + # writes a membership, and an editable relation would offer an association + # that could only fail. + class TeamMembership < FetchAllCollection + def initialize(datasource) + super(datasource, 'IntercomTeamMembership') + end + + protected + + def fetch_all + client.fetch_all('teams', list_key: 'teams').flat_map { |team| pairs_of(team) } + end + + # Keyed by both ids rather than by a counter: a related list is read over + # two requests, and a key that changed between them would move the rows + # under the operator. + def serialize(pair) + { 'id' => "#{pair["team_id"]}:#{pair["admin_id"]}", + 'team_id' => pair['team_id'], + 'admin_id' => pair['admin_id'] } + end + + private + + def pairs_of(team) + return [] unless team.is_a?(Hash) + + team_id = stringify_id(team['id']) + return [] if team_id.nil? + + Array(team['admin_ids']).filter_map do |admin_id| + id = stringify_id(admin_id) + { 'team_id' => team_id, 'admin_id' => id } unless id.nil? + end + end + + def define_schema + add_column('id', 'String', is_primary_key: true) + add_column('team_id', 'String') + add_column('admin_id', 'String') + add_many_to_one('team', foreign_collection: 'IntercomTeam', foreign_key: 'team_id') + add_many_to_one('admin', foreign_collection: 'IntercomAdmin', foreign_key: 'admin_id') + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/ticket.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/ticket.rb new file mode 100644 index 000000000..3afc81497 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/ticket.rb @@ -0,0 +1,164 @@ +module ForestAdminDatasourceIntercom + module Collections + # The tickets of the workspace. + # + # Read through `POST /tickets/search`: Intercom exposes no `GET /tickets` at + # all, so even an unfiltered list view goes through the search endpoint with + # a predicate that matches everything. Its records come back under `tickets` + # rather than under the `data` envelope -- measured. + # + # The response carries the whole timeline of every ticket, and there is no + # way to ask it not to: Intercom offers no field selection. Measured, one + # ticket carried 155 parts, so a page of 150 would move some 23 000 part + # objects, customer message bodies included. Two consequences run through + # this class: the page size is bounded far below what the API accepts, and + # everything derived from those parts is free, since they are paid for + # whether or not anything asks. + class Ticket < CursorCollection + include ContactIdentity + include Ticket::Serializer + include Ticket::DerivedColumns + include CustomAttributes + # The same thread a conversation publishes, and free here: the parts are + # in the response whether or not anything asks for them. + include Timeline + + # Intercom accepts 150. This is not that: it is what keeps one page of + # tickets, timelines included, a response an agent can hold and an operator + # can wait for. Provisional until measured against real response sizes on + # the customer's workspace. + MAX_TICKETS_PER_PAGE = 25 + + # `/tickets/search` demands a query, so a list view sends the least noisy + # predicate that matches everything. Every ticket has a creation date, and + # a bound at the epoch keeps whatever the day-granular truncation does to + # it harmless. + MATCH_EVERY_TICKET = { 'field' => 'created_at', 'operator' => '>', 'value' => '0' }.freeze + + def initialize(datasource, attributes: []) + @attributes = attributes + super(datasource, 'IntercomTicket') + end + + protected + + def record_endpoint = 'tickets' + def list_key = 'tickets' + def searchable = 'tickets' + def max_page_size = MAX_TICKETS_PER_PAGE + + # The part bodies are HTML written by end customers, and rendering + # third-party HTML inside Forest is neither safe nor useful (R10). Sent on + # the search, where Intercom does not document it: a parameter it ignores + # costs a query string, while the one it honours saves every row of the + # thread from coming back as markup. + def read_params = { 'display_as' => 'plaintext' } + + # Intercom exposes no `GET /tickets`, so a list view searches too: with the + # filter it was given, or with the predicate that matches everything when + # it was given none. + def searchable_only? = true + def match_all_query = MATCH_EVERY_TICKET + + def enrich(records, rows, projection) + embed_contact_identity(records, rows, projection) + embed_derived_columns(records, rows, projection) + embed_timeline(records, rows, projection) + end + + private + + def define_schema + add_column('id', 'String', is_primary_key: true) + # The number the support team says out loud, next to the id the API + # answers by. + add_column('ticket_id', 'String') + # `request` / `task` / `tracker` on the wire, never the labels the + # Intercom interface shows -- the same mismatch a filter on it will have + # to respect. + add_column('category', 'String') + add_column('open', 'Boolean') + add_column('is_shared', 'Boolean') + add_column('created_at', 'Date') + add_column('updated_at', 'Date') + add_column('admin_assignee_id', 'String') + add_column('team_assignee_id', 'String') + # The ticket carries its company as an id alone, unlike a conversation + # which carries the whole object: the account name would cost a request + # per row, so it is not offered here. Measured: the id is Intercom's own, + # not the customer's external one, which is what a relation will have to + # target in lot 4. + add_column('company_id', 'String') + define_state_columns + define_type_columns + define_contact_columns + define_derived_columns + add_column('part_count', 'Number') + add_column('timeline', 'Json') + # Before the attribute columns rather than after: a workspace attribute + # whose name lands on a relation is then skipped with a warning, the way + # one landing on a column already is. Declared after, it would collide + # and take the boot with it. + define_relations + register_attribute_columns + end + + # The state arrives embedded as a whole object, so its label costs nothing: + # a queue reads without a join. One label and not three -- the category and + # the customer-facing label are read through the `state` relation, which is + # where every field of a state lives. Neither of the two was ever + # filterable, so nothing that could be saved in a segment or a scope + # depended on them. + def define_state_columns + add_column('state_id', 'String') + add_column('state_label', 'String') + add_column('previous_state_id', 'String') + end + + def define_type_columns + add_column('ticket_type_id', 'String') + add_column('ticket_type_name', 'String') + end + + # The reference collections a ticket points at, and the contact who opened + # it. Every reference target is read whole in one request, so those + # relations resolve for a page at the price of a single read; the contact + # is read from `/contacts/search`, one request for the page as well. + # + # Which of them can be filtered *through* is the measured table's + # business, not this method's: `/tickets/search` takes a filter on + # `admin_assignee_id`, `team_assignee_id` and `ticket_type_id`, none on a + # state id, and `contact_ids` is a `spec` row the probe has yet to + # confirm. Where the endpoint filters nothing, the traversal is refused by + # name rather than left for the interface to offer and the API to drop. + def define_relations + add_many_to_one('admin_assignee', foreign_collection: 'IntercomAdmin', foreign_key: 'admin_assignee_id') + add_many_to_one('team_assignee', foreign_collection: 'IntercomTeam', foreign_key: 'team_assignee_id') + add_many_to_one('state', foreign_collection: 'IntercomTicketState', foreign_key: 'state_id') + add_many_to_one('previous_state', foreign_collection: 'IntercomTicketState', + foreign_key: 'previous_state_id') + add_many_to_one('ticket_type', foreign_collection: 'IntercomTicketType', foreign_key: 'ticket_type_id') + add_many_to_one('contact', foreign_collection: 'IntercomContact', foreign_key: 'contact_id') + end + + # Costs no request, unlike a conversation's: Intercom returns the parts of + # a ticket in the search response and offers no way to ask it not to, so + # the page pays for them whatever the projection says. Building the thread + # out of them is what is guarded here. + # + # An empty list means an empty thread, and says so -- where a conversation + # read from a listing carries no parts at all and its timeline stays nil, + # which reads as unknown. + def embed_timeline(records, rows, projection) + return unless column_asked?(projection, 'timeline') + + records.each_with_index { |record, index| rows[index]['timeline'] = build_timeline(record) } + end + + # The attribute columns of every ticket type, in union. Read at boot by + # `TicketAttributesIntrospector`, which is also where a workspace's own + # name is turned into one a Forest query string can carry. + def attribute_kind = 'ticket' + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/ticket/derived_columns.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/ticket/derived_columns.rb new file mode 100644 index 000000000..db18cb873 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/ticket/derived_columns.rb @@ -0,0 +1,131 @@ +module ForestAdminDatasourceIntercom + module Collections + class Ticket < CursorCollection + # The two columns a support queue is read for and that Intercom does not + # carry: when the ticket was closed, and who spoke last. + # + # A ticket has no `statistics` block -- measured against a workspace of 81 + # 142 tickets, confirming the specification -- so neither exists as a + # field. Both are derived from the parts, and the parts arrive with the + # search response whether or not anything asks for them, so both cost + # nothing: this is the one place where deriving a column is cheaper than + # reading one. + # + # Display only, and that is not a temporary state: `/tickets/search` + # filters on neither and ignores a sort without reporting it, so a column + # advertising either would put in the interface what the read cannot + # honour. + module DerivedColumns + # A ticket is not "closed" on Intercom, it enters a state whose category + # is resolved. + RESOLVED = 'resolved'.freeze + + # Matched on the prefix, never on the full `ticket_state_updated_by_admin` + # the sample showed: a workspace running workflows closes tickets through + # other variants of the same event, and a closure nobody can see is worse + # than a column nobody offers. + STATE_CHANGE_PREFIX = 'ticket_state_updated'.freeze + + # A reply to the customer. A `note` is an internal touch, not an answer: + # counting it would name as "last responder" someone who never wrote to + # the person waiting. + REPLY_PART = 'comment'.freeze + + # The columns a truncated timeline leaves unknown rather than empty, and + # therefore the ones whose presence in a projection is worth a warning. + CLOSURE_COLUMNS = %w[closed_at closed_by_name].freeze + + private + + def define_derived_columns + add_column('closed_at', 'Date') + add_column('closed_by_name', 'String') + add_column('last_reply_at', 'Date') + add_column('last_responder_name', 'String') + # `admin` or `contact`: whether the last word came from the team or + # from the customer is what tells a queue who owes the next one. + add_column('last_responder_type', 'String') + end + + def derived_columns_for(attrs) + parts = parts_of(attrs) + closure = last_closure(parts) + reply = last_reply(parts) + + { 'closed_at' => stamp(closure&.dig('created_at')), + 'closed_by_name' => author_of(closure)['name'], + 'last_reply_at' => stamp(reply&.dig('created_at')), + 'last_responder_name' => author_of(reply)['name'], + 'last_responder_type' => author_of(reply)['type'] } + end + + # The hook of the base's `enrich`: nothing to read here, since + # `serialize` already derived everything. What is left is telling the + # operator when a value is missing because the timeline was truncated + # rather than because the event never happened. + def embed_derived_columns(records, _rows, projection) + return unless any_column_asked?(projection, CLOSURE_COLUMNS) + + unknown = records.count { |record| closure_unknown?(record) } + warn_unknown_closures(unknown) if unknown.positive? + end + + # A resolved ticket with no closure in hand, on a timeline Intercom + # truncated: the date is *unknown*, not absent. A Date column cannot say + # that, so the log does. + def closure_unknown?(record) + state = record['ticket_state'].is_a?(Hash) ? record['ticket_state'] : {} + return false unless state['category'] == RESOLVED + + last_closure(parts_of(record)).nil? && truncated?(record) + end + + # Intercom keeps the 500 most recent parts of a ticket. Past that, the + # transition that closed it may have fallen out of the window. + def truncated?(record) + total = parts_total(record) + total ? total > parts_of(record).size : false + end + + def last_closure(parts) + closures = parts.select { |part| state_change?(part) && part['ticket_state'] == RESOLVED } + + closures.max_by { |part| part['created_at'].to_i } + end + + # A part can record a transition to the state the ticket was already in + # -- measured -- and that is not an event. + def state_change?(part) + part['part_type'].to_s.start_with?(STATE_CHANGE_PREFIX) && + part['ticket_state'] != part['previous_ticket_state'] + end + + def last_reply(parts) + parts.select { |part| part['part_type'] == REPLY_PART }.max_by { |part| part['created_at'].to_i } + end + + def author_of(part) + author = (part || {})['author'] + author.is_a?(Hash) ? author : {} + end + + def parts_of(record) + nested_list((record || {})['ticket_parts'], 'ticket_parts') + end + + def parts_total(record) + container = (record || {})['ticket_parts'] + container.is_a?(Hash) ? container['total_count'] : nil + end + + def warn_unknown_closures(unknown) + ForestAdminDatasourceIntercom.logger.warn( + "[forest_admin_datasource_intercom] #{name}: #{unknown} resolved ticket(s) of this page show no " \ + 'closure date because Intercom truncated their timeline at 500 parts, not because they were never ' \ + 'closed. The column is unknown for those rows.' + ) + end + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/ticket/serializer.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/ticket/serializer.rb new file mode 100644 index 000000000..a094bbf20 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/ticket/serializer.rb @@ -0,0 +1,60 @@ +module ForestAdminDatasourceIntercom + module Collections + class Ticket < CursorCollection + # One Intercom ticket flattened into the row the schema declares. Nothing + # here reads a sub-resource: the state, the type and the attributes all + # travel with the ticket. + module Serializer + protected + + # Intercom keys the attribute values by **name**, which is what lets a + # single collection display the union of every ticket type's -- and what + # stops it from filtering on them, the filter being written by an id + # that differs from one type to the next. A ticket of another type + # simply does not carry the key, and the column reads as empty. + def serialize(ticket) + attrs = ticket.is_a?(Hash) ? ticket : {} + + native(attrs) + .merge(state_of(attrs)) + .merge(type_of(attrs['ticket_type'])) + .merge(contact_columns_for(attrs)) + .merge(attribute_values(attrs['ticket_attributes'])) + .merge(derived_columns_for(attrs)) + end + + private + + def native(attrs) + { 'id' => stringify_id(attrs['id']), + 'ticket_id' => stringify_id(attrs['ticket_id']), + 'category' => attrs['category'], + 'open' => attrs['open'], + 'is_shared' => attrs['is_shared'], + 'created_at' => stamp(attrs['created_at']), + 'updated_at' => stamp(attrs['updated_at']), + 'admin_assignee_id' => stringify_id(attrs['admin_assignee_id']), + 'team_assignee_id' => stringify_id(attrs['team_assignee_id']), + 'company_id' => stringify_id(attrs['company_id']), + 'part_count' => parts_total(attrs) } + end + + def state_of(attrs) + state = attrs['ticket_state'].is_a?(Hash) ? attrs['ticket_state'] : {} + + # `internal_label` is what the support team reads. The category and the + # customer-facing label are a hop away, on the `state` relation. + { 'state_id' => stringify_id(state['id']), + 'state_label' => state['internal_label'], + 'previous_state_id' => stringify_id(attrs['previous_ticket_state_id']) } + end + + def type_of(ticket_type) + attrs = ticket_type.is_a?(Hash) ? ticket_type : {} + + { 'ticket_type_id' => stringify_id(attrs['id']), 'ticket_type_name' => attrs['name'] } + end + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/ticket_state.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/ticket_state.rb new file mode 100644 index 000000000..f9845d7e7 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/ticket_state.rb @@ -0,0 +1,40 @@ +module ForestAdminDatasourceIntercom + module Collections + # The ticket states of the workspace. A ticket carries its state as an id, so + # without this collection a support queue reads as a column of numbers. + class TicketState < FetchAllCollection + def initialize(datasource) + super(datasource, 'IntercomTicketState') + end + + protected + + def fetch_all + client.fetch_all('ticket_states') + end + + # Two labels rather than one: `internal_label` is what the support team + # sees, `external_label` what the customer is shown. An operator reading a + # queue needs the first, and needs to know what the second says. + def serialize(ticket_state) + attrs = ticket_state.is_a?(Hash) ? ticket_state : {} + + { 'id' => stringify_id(attrs['id']), + 'category' => attrs['category'], + 'internal_label' => attrs['internal_label'], + 'external_label' => attrs['external_label'], + 'archived' => attrs['archived'] } + end + + private + + def define_schema + add_column('id', 'String', is_primary_key: true) + add_column('category', 'String') + add_column('internal_label', 'String') + add_column('external_label', 'String') + add_column('archived', 'Boolean') + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/ticket_type.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/ticket_type.rb new file mode 100644 index 000000000..7b6367198 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/ticket_type.rb @@ -0,0 +1,47 @@ +module ForestAdminDatasourceIntercom + module Collections + # The ticket types the workspace defines. They are what makes a ticket's type + # readable, and they are also where the ticket attributes are declared -- + # which is what the ticket collection reads them for. + class TicketType < FetchAllCollection + def initialize(datasource) + super(datasource, 'IntercomTicketType') + end + + protected + + def fetch_all + client.fetch_all('ticket_types') + end + + # `ticket_type_attributes` is deliberately left out: it is a nested list of + # attribute definitions, useful to the ticket collection and meaningless as + # a column. An attribute of the same name carries a different id from one + # type to the next (measured), which is exactly why the ticket collection + # has to read the definitions rather than assume them. + def serialize(ticket_type) + attrs = ticket_type.is_a?(Hash) ? ticket_type : {} + + { 'id' => stringify_id(attrs['id']), + 'name' => attrs['name'], + 'description' => attrs['description'], + 'category' => attrs['category'], + 'icon' => attrs['icon'], + 'archived' => attrs['archived'] } + end + + private + + def define_schema + add_column('id', 'String', is_primary_key: true) + add_column('name', 'String') + add_column('description', 'String') + # `request` / `task` / `tracker` on the wire, not the labels the Intercom + # interface shows -- the same mismatch the ticket filter has to respect. + add_column('category', 'String') + add_column('icon', 'String') + add_column('archived', 'Boolean') + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/timeline.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/timeline.rb new file mode 100644 index 000000000..8cb3736d2 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/collections/timeline.rb @@ -0,0 +1,53 @@ +module ForestAdminDatasourceIntercom + module Collections + # The thread of a conversation or of a ticket, as a structured list the + # record view can render: who said what, when, and through which kind of + # event. + # + # `part_type` is kept on every entry: an assignment, an internal note and a + # reply are not the same event, and a thread that flattens them reads as an + # exchange that never happened the way it did. Which also means **the + # internal notes of the team are in there**, alongside what the customer + # was told -- that is what a thread is on Intercom, and hiding half of it + # would be the more surprising answer. + # + # Intercom keeps the 500 most recent parts of either resource, so the entry + # count is what is in hand, never necessarily what exists. The two + # collections differ in where they read those parts and in whether anything + # opens the thread before them -- both are hooks. + module Timeline + private + + def build_timeline(entity) + attrs = entity.is_a?(Hash) ? entity : {} + + [opening_entry(attrs)].compact + (parts_of(attrs) || []).map { |part| part_entry(part) } + end + + # What comes before the parts. A conversation opens on its `source`, which + # is not a part at all; a ticket opens on its first part like any other + # event, so there is nothing to prepend. + def opening_entry(_attrs) = nil + + def part_entry(part) + attrs = part.is_a?(Hash) ? part : {} + + entry(part_type: attrs['part_type'], created_at: attrs['created_at'], author: attrs['author'], + body: attrs['body'], attachments: attrs['attachments']) + .merge('id' => stringify_id(attrs['id']), 'redacted' => attrs['redacted']) + end + + def entry(part_type:, created_at:, author:, body:, attachments:) + writer = author.is_a?(Hash) ? author : {} + + { 'part_type' => part_type, + 'created_at' => stamp(created_at), + 'author_type' => writer['type'], + 'author_name' => writer['name'], + 'author_email' => writer['email'], + 'body' => body, + 'attachment_count' => Array(attachments).size } + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/configuration.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/configuration.rb new file mode 100644 index 000000000..fa7c771b4 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/configuration.rb @@ -0,0 +1,146 @@ +module ForestAdminDatasourceIntercom + class Configuration + # A workspace is hosted in one region and answers in that region only. The + # host is therefore a configuration parameter rather than a constant: + # `api.intercom.io` does route to the right region, but a customer under + # GDPR wants its requests to reach the European host and nothing else. + REGION_HOSTS = { + us: 'https://api.intercom.io', + eu: 'https://api.eu.intercom.io', + au: 'https://api.au.intercom.io' + }.freeze + + DEFAULT_REGION = :us + + # Where a cleartext base_url reaches nobody else's network, so the bearer + # header crossing it in clear is not worth a word. + LOOPBACK_HOSTS = %w[localhost 127.0.0.1 ::1 [::1] 0.0.0.0].freeze + + # Without an explicit version a request follows the workspace's own default, + # which an operator can change on Intercom's side -- and the payloads change + # shape under us. Pinned to what the spike ran against; 2.14 and 2.16 both + # answered, and the response echoes the version back, so `Client#me` + # verifies at boot that the pin was honoured. + DEFAULT_API_VERSION = '2.16'.freeze + + attr_reader :access_token, :region, :base_url, :api_version, :open_timeout, :timeout, + :retry_policy, :rate_limiter, :boot_open_timeout, :boot_timeout, :boot_retry_policy + + # `rate_limiter: nil` takes the pacing out of the stack, leaving the 429 + # retry as the only rate-limit handling. For a deployment that meters on its + # own side, or one that would rather see the 429. + # + # The `boot_` trio governs what the datasource reads while it is being + # constructed -- the custom-attribute introspection above all -- where the + # wait is a Rails boot the operator sits through rather than a request that + # has already returned a page. + def initialize(access_token:, region: nil, base_url: nil, api_version: DEFAULT_API_VERSION, + open_timeout: 5, timeout: 30, retry_policy: RetryPolicy.new, + rate_limiter: RateLimiter.new, boot_open_timeout: 3, boot_timeout: 10, + boot_retry_policy: RetryPolicy.boot) + @access_token = access_token + @region = (region || DEFAULT_REGION).to_s.downcase.to_sym + @base_url = base_url + @api_version = api_version.to_s + @open_timeout = open_timeout + @timeout = timeout + @retry_policy = retry_policy + @rate_limiter = rate_limiter + @boot_open_timeout = boot_open_timeout + @boot_timeout = boot_timeout + @boot_retry_policy = boot_retry_policy + validate! + end + + # An explicit `base_url` wins over the region: it is what points the client + # at a mock server or an egress proxy, neither of which is a region. + def url + @url ||= (@base_url || REGION_HOSTS.fetch(@region)).chomp('/') + end + + # Whatever precedes the endpoint in the path, for a base url mounted under a + # subpath. Empty against the API itself. + def base_path + @base_path ||= URI.parse(url).path + end + + # The url with any credentials in it masked. `URI` accepts them -- + # `https://user:pass@proxy/...` -- and a credentialed egress proxy is one of + # the two reasons to set a `base_url` at all, so printing the url verbatim + # would put a second secret exactly where this class keeps the first one + # from going. Read by `Client#inspect` too, which prints the same url. + def redacted_url + @redacted_url ||= url.sub(%r{\A([a-zA-Z][\w+.-]*://)[^/@]*@}, '\1[FILTERED]@') + end + + # `access_token` is a bearer credential, and nothing prints a Configuration + # on purpose: what reaches an `inspect` is a Rails error page, or a + # `logger.debug` of something holding one. The default would put the token + # in clear there. `Client` and `Datasource` mask their own for the same + # reason -- together they cut every path from an object this package hands + # out to the credential. + def inspect + "#<#{self.class.name} url=#{redacted_url.inspect} api_version=#{@api_version.inspect} " \ + 'access_token=[FILTERED]>' + end + + private + + def validate! + raise ConfigurationError, 'ForestAdminDatasourceIntercom missing required config: access_token' if + blank?(@access_token) + + validate_region! + validate_base_url! + raise ConfigurationError, 'ForestAdminDatasourceIntercom api_version cannot be empty' if blank?(@api_version) + end + + def validate_region! + return if @base_url || REGION_HOSTS.key?(@region) + + raise ConfigurationError, + "ForestAdminDatasourceIntercom unknown region #{@region.inspect}: " \ + "expected one of #{REGION_HOSTS.keys.map(&:inspect).join(", ")}, or an explicit base_url." + end + + # A base url that is not absolute makes Faraday resolve every path against + # the process's working directory instead of Intercom, which surfaces much + # later as a connection failure naming nothing. + def validate_base_url! + return if @base_url.nil? + + uri = URI.parse(@base_url) + return warn_cleartext!(uri) if uri.is_a?(URI::HTTP) && !blank?(uri.host) + + raise ConfigurationError, + "ForestAdminDatasourceIntercom base_url must be an absolute http(s) url, got #{@base_url.inspect}" + rescue URI::InvalidURIError + raise ConfigurationError, + "ForestAdminDatasourceIntercom base_url is not a valid url: #{@base_url.inspect}" + end + + # `URI::HTTPS` is a `URI::HTTP`, so plain http passes above -- and it is + # worth passing: a mock server in a test suite is one, and that is half of + # what `base_url` is for. What it costs is `Authorization: Bearer ` + # travelling in clear to whatever sits at the other end, and that token + # reads the whole workspace. Named rather than allowed in silence; a + # loopback host is nobody else's network, so it says nothing there. + def warn_cleartext!(uri) + return if uri.scheme == 'https' || loopback?(uri.host) + + ForestAdminDatasourceIntercom.logger.warn( + "[forest_admin_datasource_intercom] base_url #{redacted_url.inspect} is not https, and every request " \ + 'carries the Intercom access token as a bearer header. Anything on the path can read it and use it ' \ + 'against the workspace. Use https, or terminate TLS before the network this crosses.' + ) + end + + def loopback?(host) + LOOPBACK_HOSTS.include?(host.to_s.downcase) || host.to_s.downcase.end_with?('.localhost') + end + + def blank?(value) + value.nil? || value.to_s.strip.empty? + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/datasource.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/datasource.rb new file mode 100644 index 000000000..704ea1383 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/datasource.rb @@ -0,0 +1,86 @@ +module ForestAdminDatasourceIntercom + class Datasource < ForestAdminDatasourceToolkit::Datasource + attr_reader :client, :configuration + + def initialize(access_token:, **options) + super() + @configuration = Configuration.new(access_token: access_token, **options) + @client = Client.new(@configuration) + + verify_api_version + register_collections + end + + # The datasource is what a Rails error page or a `logger.debug` is likeliest + # to print, and it holds the client whose connections carry the access token. + # Every collection will reach that token the same way, through the + # `@datasource` the toolkit's Collection keeps, so cutting the chain here + # covers them too -- and spares the recursive dump the default `inspect` + # walks into, a datasource and its collections pointing at each other. + def inspect + "#<#{self.class.name} collections=#{collections.keys.inspect}>" + end + + private + + # The reference collections first: they are what turns an assignee id into a + # teammate and a state id into a label. No request is made here -- each + # collection reads its endpoint when it is listed, so a datasource boots + # whatever Intercom is doing, and a workspace the token cannot read costs + # rows rather than the agent. + def register_collections + add_collection(Collections::Admin.new(self)) + add_collection(Collections::Team.new(self)) + # The join Intercom does not expose: without it the membership of a team is + # an array of ids on either side, since a many-to-many needs a collection + # to travel through. + add_collection(Collections::TeamMembership.new(self)) + add_collection(Collections::TicketType.new(self)) + add_collection(Collections::TicketState.new(self)) + # Contacts and Companies before the collections that point at them, so a + # relation is declared next to a target the datasource already holds. + # Each carries the custom attributes its workspace defines, read at boot: + # they cannot be discovered from a payload, a contact carrying the values + # of the attributes it happens to have been given. + add_collection(Collections::Contact.new(self, attributes: model_attributes('contact'))) + add_collection(Collections::Company.new(self, attributes: model_attributes('company'))) + add_collection(Collections::Conversation.new(self)) + # The attributes a workspace defines on its ticket types, which are + # columns of the Tickets collection and cannot be discovered from a ticket + # payload either -- a ticket carries the values of its own type only. + add_collection(Collections::Ticket.new(self, attributes: ticket_attributes)) + end + + # The one boot read that is not about a column, and the only place the + # pinned API version can be checked: Intercom serves the workspace's own + # default when the pin is not honoured, echoes what it served in a response + # header, and the payload shapes differ between versions -- the silent + # drift this package refuses everywhere else. `Client#me` reads that echo + # and reports a mismatch; nothing else in the datasource calls it, so + # leaving it uncalled left the check as code that never ran. + # + # Degrades like the three attribute reads below: a token that cannot reach + # `/me` costs the check, never the boot. + def verify_api_version + @client.me(boot: true) + rescue APIError => e + ForestAdminDatasourceIntercom.logger.warn( + "[forest_admin_datasource_intercom] could not read /me at boot (HTTP #{e.status || "-"}); the API " \ + 'version Intercom serves was not checked against the pinned one, and a workspace serving another one ' \ + 'answers payloads of another shape.' + ) + end + + # The three attribute reads of the boot, alongside the version check above. + # Each degrades to no attribute column rather than to a failed boot: a + # token missing a permission costs the columns it could not read, never + # the agent. + def ticket_attributes + Schema::TicketAttributesIntrospector.new(@client).attributes + end + + def model_attributes(model) + Schema::DataAttributesIntrospector.new(@client, model: model).attributes + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/pagination/cursor_walker.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/pagination/cursor_walker.rb new file mode 100644 index 000000000..e701c3bc4 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/pagination/cursor_walker.rb @@ -0,0 +1,157 @@ +module ForestAdminDatasourceIntercom + module Pagination + # Forest asks for an offset/limit window; Intercom only knows how to hand + # out the page after a cursor, and documents that jumping to page N is not + # supported. Bridging the two means walking pages until the window is + # covered, then slicing it out. Reaching page 20 therefore costs 20 + # requests -- sequential ones, a cursor only being known once the page + # before it came back. + # + # The walk is capped for that reason rather than for the quota's: 10 000 + # requests a minute is generous enough that the caps below are about what an + # operator is willing to wait for, and about the fact that page 200 of a + # list view answers no real question (R9). Every truncation is logged -- + # never silent, since a page that looks like the whole answer and is not is + # the failure this datasource exists to avoid. + class CursorWalker + # 50 pages of 150 records. Intercom's quota lets these be generous: the + # walk is bounded by patience, and by the point past which a list view is + # not being read but scraped. + MAX_PAGES = 50 + MAX_RECORDS = 7_500 + + def initialize(max_pages: MAX_PAGES, max_records: MAX_RECORDS) + @max_pages = max_pages + @max_records = max_records + end + + # Yields `(per_page, cursor)` and expects a Client::Page back. + # + # A nil limit asks for every record past the offset: the walk then runs + # until Intercom says there is no page left, or until a cap stops it. That + # distinction is the whole point of accepting nil rather than a huge limit + # standing in for "everything": a walk told to collect a thousand records + # stops at a thousand having covered the window it was given, and reports + # nothing, while a walk told to collect everything and stopped by a cap + # knows it is handing back less than it was asked for, and says so. + def walk(offset:, limit:, &page_source) + offset = offset.to_i.clamp(0, nil) + limit = limit&.to_i + return [] if limit && !limit.positive? + + records = collect(offset, limit, &page_source) + + limit ? (records[offset, limit] || []) : records.drop(offset) + end + + private + + # The walk itself: pages are collected until the window is covered, the + # source says there is nothing left, or a cap stops it. The slicing is + # `walk`'s; this only decides how far to go. + def collect(offset, limit) + needed = limit && (offset + limit) + records = [] + cursor = nil + seen_ids = Set.new + seen_cursors = Set.new + pages = 0 + + loop do + page = yield(batch_size(needed, records.size), cursor) + records.concat(fresh(page.records, seen_ids)) + pages += 1 + + # The only silent exits: Intercom said there is no page left, or the + # window asked for is covered. Everything below hands back less than + # was asked for, and says so. + break if page.next_cursor.nil? + break if needed && records.size >= needed + break if cut_short?(page, seen_cursors, pages, records: records.size, window: window_of(offset, limit)) + + cursor = page.next_cursor + end + + records + end + + # Whether the walk is handing back less than it was asked for, and which + # of the two reasons it is. Both are logged here rather than by the + # caller: a page that looks like the whole answer and is not is the + # failure this datasource exists to avoid, so there is no route out of + # this walk that is short and quiet. + def cut_short?(page, seen_cursors, pages, records:, window:) + if stalled?(page, seen_cursors) + log_stalled(window: window, pages: pages, collected: records) + elsif capped?(pages, records) + log_truncation(window: window, pages: pages, collected: records) + else + return false + end + + true + end + + # Intercom documents that "if items are modified between paginated + # requests it is possible to see duplicate or missed records" -- and + # conversations move constantly, so a deep walk over them will see the + # same record twice. A duplicate is dropped here rather than being served + # as two rows carrying one id, which is what a list view would render as + # two identical lines and a record count that never adds up. The missing + # counterpart is inherent to cursor pagination and is documented instead. + # + # A record with no id is kept: it is not this walk's business to decide + # that a payload it does not recognise is not a record. + def fresh(records, seen_ids) + records.select { |record| record['id'].nil? || seen_ids.add?(record['id']) } + end + + # A page that advertises a next cursor and holds nothing, and a cursor + # already followed. Intercom does neither today -- `pages.next` is simply + # absent on the last page -- but a walk driven by a remote value stops on + # its own terms rather than on the caps only: a cycle wider than one page + # would otherwise collect the same pages until a cap cut it short. + # + # Which is exactly why it is reported. Stopping here means Intercom said + # there was more and this could not follow it, so the answer is short of + # what was asked -- a truncation like the caps below, and the same rule + # applies: never silent. + def stalled?(page, seen_cursors) + page.records.empty? || !seen_cursors.add?(page.next_cursor) + end + + def capped?(pages, collected) + pages >= @max_pages || collected >= @max_records + end + + # Bounded by the record budget left, and by the window still missing when + # there is one, so the walk never collects past @max_records. `Client` + # bounds it again to what Intercom accepts. + def batch_size(needed, collected) + budget = @max_records - collected + budget = [needed - collected, budget].min if needed + Client.bounded_per_page(budget) + end + + def log_truncation(window:, pages:, collected:) + ForestAdminDatasourceIntercom.logger.warn( + "[forest_admin_datasource_intercom] Stopped paginating after #{pages} page(s) / " \ + "#{collected} record(s) while fetching #{window}; results are truncated. " \ + 'Narrow the filter to reach records past this point.' + ) + end + + def log_stalled(window:, pages:, collected:) + ForestAdminDatasourceIntercom.logger.warn( + "[forest_admin_datasource_intercom] Stopped paginating after #{pages} page(s) / " \ + "#{collected} record(s) while fetching #{window}: Intercom advertised a next page this could not " \ + 'follow -- an empty page, or a cursor already read. Results are truncated.' + ) + end + + def window_of(offset, limit) + limit ? "offset=#{offset} limit=#{limit}" : "every record past offset=#{offset}" + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/query/caller_zone.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/query/caller_zone.rb new file mode 100644 index 000000000..f06eac6f8 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/query/caller_zone.rb @@ -0,0 +1,56 @@ +module ForestAdminDatasourceIntercom + module Query + # The timezone a filter was written in, and what reading a value in it + # costs when the caller names one nothing knows. + # + # Split from FilterValue for the reason DayBounds was: that class knows + # what shape a field expects on the wire, and whether a value carries the + # caller's wall clock or the server's is a different question. + class CallerZone + # Kept stripped rather than only checked stripped: as it came, a + # `" Europe/Paris "` passes the blank guard and then fails the zone + # lookup, and a day boundary lands an offset away from where the filter + # meant it. + attr_reader :name + + def initialize(identifier) + stripped = identifier.to_s.strip + @name = stripped.empty? ? 'UTC' : stripped + end + + # A day with no time of day is the caller's day, not the server's: it is + # the timezone the filter was written in that says when that day starts. + def start_of_day(date) + read('the day boundary') { Time.zone.local(date.year, date.month, date.day).to_i } + end + + # A timestamp carrying no offset is the caller's wall clock. `FilterFactory` + # writes the bounds of a previous period as `%Y-%m-%d %H:%M:%S`, so a chart + # comparing to the previous month sent a midnight read in whatever timezone + # the process happened to run in -- and a midnight moved by any offset at + # all lands on another UTC day once truncated, which is a whole day of rows + # beside the ones asked for. + # + # A timestamp carrying an offset is untouched, which is every operator the + # toolkit rewrites into a pair of bounds: those come through as UTC ISO8601 + # and read the same in any zone. + def timestamp(value) + read('the timestamp') { Time.zone.parse(value).to_i } + end + + private + + # Falling back to UTC silently would move a boundary by the offset, which + # is the failure a timezone is read for in the first place. + def read(what, &block) + Time.use_zone(@name, &block) + rescue ArgumentError + ForestAdminDatasourceIntercom.logger.warn( + "[forest_admin_datasource_intercom] unknown timezone #{@name.inspect}, reading #{what} of a date " \ + 'filter in UTC instead.' + ) + Time.use_zone('UTC', &block) + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/query/condition_tree_translator.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/query/condition_tree_translator.rb new file mode 100644 index 000000000..4556015fd --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/query/condition_tree_translator.rb @@ -0,0 +1,181 @@ +module ForestAdminDatasourceIntercom + module Query + # Turns a Forest condition tree into the query `POST /conversations/search` + # and `POST /tickets/search` take: + # + # leaf -> { 'field' => ..., 'operator' => ..., 'value' => ... } + # branch -> { 'operator' => 'AND' | 'OR', 'value' => [...] } + # + # What it will not translate, it refuses. A condition dropped on the way to + # Intercom comes back as a page of unfiltered records that looks filtered, + # and every refusal below therefore names the field, the operator, or the + # thing to change -- an operator reads that message and nothing else. + # + # Which field the endpoint filters, and with which operator, is not decided + # here: it is `search_fields.yml`, measured against a real workspace. This + # walks the tree and formats what the table allows. + class ConditionTreeTranslator + Branch = ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Nodes::ConditionTreeBranch + Leaf = ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Nodes::ConditionTreeLeaf + + AGGREGATORS = { 'and' => 'AND', 'or' => 'OR' }.freeze + + # Intercom nests a search two levels deep and takes fifteen conditions per + # group. Both are checked here rather than left to the API: over either, + # Intercom answers a 400 whose body names neither the limit nor the part of + # the filter that reached it, and an operator reading it has no way of + # knowing that their segment plus their scope plus their own filter is what + # went over. + MAX_DEPTH = 2 + MAX_GROUP_SIZE = 15 + + def self.call(condition_tree, endpoint:, collection:, timezone: nil, attribute_columns: []) + return nil if condition_tree.nil? + + new(endpoint: endpoint, collection: collection, timezone: timezone, + attribute_columns: attribute_columns).translate(condition_tree) + end + + # `attribute_columns` are the columns a workspace's own attributes became, + # which no row of the table can name: they are discovered at boot. They + # share one refusal, carried by the endpoint, and passing them here is + # what lets it be read instead of the generic message. + def initialize(endpoint:, collection:, timezone: nil, attribute_columns: []) + @endpoint = endpoint + @collection = collection + @attribute_columns = Array(attribute_columns).map(&:to_s) + @value = FilterValue.new(collection: collection, timezone: timezone) + end + + def translate(node, depth = 1) + case node + when Branch then translate_branch(node, depth) + when Leaf then translate_leaf(node) + else raise UnsupportedOperatorError, "#{@collection} cannot read #{node.class} as a condition." + end + end + + private + + def translate_branch(branch, depth) + conditions = Array(branch.conditions) + refuse_empty_branch!(branch) if conditions.empty? + + # Read before the unwrap below, so a branch is refused on the aggregator + # it carries rather than on how many conditions it holds. + operator = aggregator(branch) + + # A branch holding one condition needs no group of its own. The agent + # builds a tree one branch at a time -- a scope, then a segment, then the + # operator's own filter -- and the nesting Intercom allows is shallow + # enough that a wrapper around nothing is a level worth not spending. + return translate(conditions.first, depth) if conditions.size == 1 + + refuse_too_deep!(branch, depth) if depth > MAX_DEPTH + refuse_too_wide!(branch, conditions.size) if conditions.size > MAX_GROUP_SIZE + + { 'operator' => operator, 'value' => conditions.map { |condition| translate(condition, depth + 1) } } + end + + # What reaches this depth is a group inside a group inside a group. The + # message names the shape rather than a number, since the tree an operator + # can act on is the segment and the scope they wrote, not the one the agent + # assembled out of them. + def refuse_too_deep!(branch, depth) + raise UnsupportedOperatorError, + "#{@collection} cannot answer this filter: Intercom nests a search #{MAX_DEPTH} levels deep and this " \ + "one reaches #{depth}. #{deepening_cause(branch)}" + end + + # A group the operator wrote is theirs to flatten. A group a relation + # expanded into is not: it is the several records the relation matched, + # written one equality each because Intercom takes no membership operator + # on a foreign key -- and telling them to flatten a nesting they never + # wrote is a refusal they cannot act on. + def deepening_cause(branch) + field = branch.respond_to?(:relation_field) ? branch.relation_field : nil + + if field.nil? + 'A group inside a group inside a group is one level too many -- flatten the segment, the scope or the ' \ + 'filter carrying the innermost one.' + else + "The innermost group is what #{field.inspect} expanded into, one equality per record it matched: narrow " \ + 'that condition until it names a single record, or lift it out of the groups nesting it.' + end + end + + # Fifteen is reached without trying: a scope, a segment and a filter add up, + # and a condition naming several values is expanded into one condition per + # value on the way here, Intercom taking no membership operator on these + # fields. + def refuse_too_wide!(branch, size) + raise UnsupportedOperatorError, + "#{@collection} cannot answer this filter: Intercom takes #{MAX_GROUP_SIZE} conditions per group and " \ + "this #{branch.aggregator} carries #{size}. A filter naming several values counts one condition per " \ + 'value here, so narrowing the list, the segment or the scope is what brings it back under the limit.' + end + + def aggregator(branch) + AGGREGATORS[branch.aggregator.to_s.downcase] || + raise(UnsupportedOperatorError, + "#{@collection} cannot read #{branch.aggregator.inspect} as a condition tree aggregator; " \ + "expected 'And' or 'Or'.") + end + + def translate_leaf(leaf) + field = @endpoint.field(leaf.field.to_s) || refuse_unfilterable!(leaf.field.to_s) + spelling = OperatorTable.intercom_operator(field, leaf.operator) || refuse_operator!(leaf, field) + + { 'field' => field.field, 'operator' => spelling, 'value' => @value.call(leaf, field, spelling) } + end + + # A column the endpoint does not filter, and the reason it does not, taken + # from the table when it carries one: those reasons are the difference + # between "no" and a message an operator can do something with. + def refuse_unfilterable!(column) + raise UnsupportedOperatorError, "#{@collection} cannot filter #{column.inspect}: #{unfilterable_reason(column)}" + end + + def unfilterable_reason(column) + return relation_reason(column) if column.include?(':') + + refusal = @endpoint.refusal(column) || attribute_refusal(column) + return refusal.reason if refusal + + "#{@endpoint.path} takes no filter on it. Filter on one of: #{@endpoint.filterable_columns.join(", ")}." + end + + # A relation reaches the translator as `relation:field`, and it should not: + # `Relations#rewrite_relation_conditions` trades every one of them for a + # condition on the foreign key before a tree gets this far, or refuses it + # by name -- with the relations the collection does declare. So this is + # the message for a caller that skipped that pass, and it names what this + # endpoint filters rather than a relation nobody resolved. + def relation_reason(_column) + "#{@endpoint.path} filters columns, not paths through a relation -- those are resolved against the " \ + 'collection they point at before a condition reaches here. Filter on one of: ' \ + "#{@endpoint.filterable_columns.join(", ")}." + end + + # A column the table cannot carry a row for, the workspace having named it: + # the endpoint's one refusal for the whole family answers for it. + def attribute_refusal(column) + @attribute_columns.include?(column) ? @endpoint.attribute_refusal : nil + end + + def refuse_operator!(leaf, field) + supported = OperatorTable.forest_operators(field) + + raise UnsupportedOperatorError, + "#{@collection} cannot filter #{leaf.field.inspect} with #{leaf.operator.inspect}: " \ + "#{@endpoint.path} answers #{supported.join(", ")} on #{field.field.inspect} and nothing else." + end + + def refuse_empty_branch!(branch) + raise UnsupportedOperatorError, + "#{@collection} was given a #{branch.aggregator} branch carrying no condition, which names no record " \ + 'and no filter.' + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/query/day_bounds.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/query/day_bounds.rb new file mode 100644 index 000000000..c5df547db --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/query/day_bounds.rb @@ -0,0 +1,69 @@ +module ForestAdminDatasourceIntercom + module Query + # Where a date bound lands once Intercom's day truncation is accounted for. + # + # Intercom truncates a date search to the day, at the **UTC** boundary -- + # measured, and it contradicts the documentation, which promises the + # workspace's timezone. `> V` answers from the start of the day *after* V, + # and `< V` answers before the start of V's own day. + # + # Sent as they come, the two bounds of an interval cancel each other out: + # `today` reaches here as `> 00:00` and `< 23:59` of one day, which Intercom + # reads as "from tomorrow" and "before today" -- an empty answer to the most + # ordinary filter there is. So each bound is moved to the day boundary that + # makes Intercom answer the day the filter named: + # + # `>` V -> the day before V's day, so the answer starts at V's day; + # `<` V -> the day after V's day, so the answer runs through V's day, + # except when V already sits on a boundary, where V's day is + # exactly what was asked to be left out. + # + # The window is therefore day-granular: a bound naming a time of day matches + # from the start of that day, or through the end of it. That is the + # granularity the Intercom interface filters on, and the README says so. + # + # Split from FilterValue, which knows how to read a date out of whatever a + # condition carries but has no business knowing that Intercom answers a + # different question from the one it was asked. + class DayBounds + UTC_DAY = 86_400 + + def initialize(collection:, timezone:) + @collection = collection + @timezone = timezone + end + + def call(seconds, spelling) + day = seconds - (seconds % UTC_DAY) + report_utc_day(seconds, day) + + return day - UTC_DAY if spelling == '>' + + seconds == day ? seconds : day + UTC_DAY + end + + private + + # A window written in another timezone is answered on the UTC days it + # overlaps, which is up to a day wider at each end. Reported once per + # filter rather than per bound: what an operator needs to know is that the + # day boundary is not theirs, not how many bounds crossed it. + def report_utc_day(seconds, day) + return if @reported || same_day_locally?(seconds, day) + + @reported = true + ForestAdminDatasourceIntercom.logger.warn( + "[forest_admin_datasource_intercom] #{@collection} was filtered on a date written in #{@timezone}, and " \ + 'Intercom truncates a date search to the UTC day whatever the workspace timezone says. The rows come ' \ + 'back for the UTC days the window overlaps, which is up to a day wider at each end.' + ) + end + + def same_day_locally?(seconds, day) + Time.use_zone(@timezone) { Time.zone.at(seconds).to_date } == Time.at(day).utc.to_date + rescue ArgumentError + true + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/query/filter_value.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/query/filter_value.rb new file mode 100644 index 000000000..9940489b6 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/query/filter_value.rb @@ -0,0 +1,174 @@ +module ForestAdminDatasourceIntercom + module Query + # How the value of a Forest condition reaches Intercom's search DSL, and + # every way it can fail to. Split from the translator, which knows the shape + # of the tree but not what a field expects on the wire. + # + # Intercom types its filter values: a date is epoch seconds, a number is a + # number, a flag is a boolean. A value of the wrong shape is refused with + # `data_invalid`, the same code an unsupported operator returns, so the + # conversion belongs here rather than in a rescue reading error codes. + class FilterValue + # What the frontend sends for a Dateonly column, and what a segment written + # in Ruby carries: a day with no time of day, which is midnight in the + # timezone of whoever wrote the filter rather than in the server's. + DATE_ONLY = /\A\d{4}-\d{2}-\d{2}\z/ + + # What `Date._parse` has to have found for a string to name a day at all. + DATE_PARTS = %i[year mon mday].freeze + + INTEGER = /\A-?\d+\z/ + + def initialize(collection:, timezone: nil) + @collection = collection + @zone = CallerZone.new(timezone) + @day_bounds = DayBounds.new(collection: collection, timezone: @zone.name) + end + + def call(leaf, field, spelling) + return list(leaf, field) if OperatorTable.list_operator?(spelling) + + refuse_absence!(leaf, field) if blank?(leaf.value) + + value = scalar(leaf.value, leaf, field) + field.type == 'date' ? @day_bounds.call(value, spelling) : value + end + + private + + # Dropping the blanks would answer a different question: `not_in [nil, + # 'open']` was asked to exclude the records carrying neither and would come + # back including them. An empty list is as bad the other way round, being a + # filter that matches everything. + def list(leaf, field) + values = Array(leaf.value) + refuse_empty_list!(leaf) if values.empty? + refuse_absence!(leaf, field) if values.any? { |value| blank?(value) } + + values.map { |value| scalar(value, leaf, field) } + end + + def scalar(value, leaf, field) + case field.type + when 'date' then epoch(value, leaf) + when 'number' then number(value, leaf) + when 'boolean' then boolean(value, leaf) + else value.to_s + end + end + + # Intercom stores and compares its dates as epoch seconds. What arrives + # here is an ISO8601 string most of the time -- the frontend sends one, and + # so does every interval operator the toolkit rewrites into a pair of + # bounds -- but a scope or a segment written in Ruby carries a Time or a + # Date, and neither has a timezone of its own. + def epoch(value, leaf) + case value + when DateTime then value.to_time.to_i + when Date then @zone.start_of_day(value) + when Time then value.to_i + when Numeric then seconds(value, leaf) + when String then parse(value, leaf) + else refuse_value!(leaf, value, 'a date') + end + end + + # An Infinity or a NaN -- a cast that overflowed above this datasource -- + # makes `Integer()` raise a FloatDomainError naming a float where the + # operator asked for a date, and a Complex a RangeError, which is the + # same class one level up. Refused like every other value the field + # cannot take, for the reason `number` refuses them too. + def seconds(value, leaf) + Integer(value) + rescue RangeError, TypeError + refuse_value!(leaf, value, 'a date') + end + + # What the string names is read before it is trusted, rather than being + # handed to a parser that completes what is missing. `Time.zone.parse` + # answers today for a string naming no date at all -- a filter on `last + # tuesday` coming back as a filter on today is the silent wrong answer + # this datasource exists not to give -- and `Time.parse`, which raises on + # that one, *accepts* a time of day on its own and fills the date in from + # the server's clock: `"12:00"` would have travelled as a bound on + # whichever day the request happened to run. + # + # A date is a year, a month and a day. Anything short of the three is + # refused rather than completed, `"Jan 2026"` included -- what it means is + # the caller's to say, not this class's to guess. + def parse(value, leaf) + return @zone.start_of_day(Date.parse(value)) if DATE_ONLY.match?(value) + + refuse_value!(leaf, value, 'a date') unless dated?(value) + + @zone.timestamp(value) + rescue ArgumentError, TypeError + refuse_value!(leaf, value, 'a date') + end + + def dated?(value) + parts = Date._parse(value) + + DATE_PARTS.all? { |part| parts.key?(part) } + end + + # The agent casts every Number column with `to_f`, so an integer field + # would be filtered with `42.0` -- a form none of its values carry. A float + # with nothing after the point travels as the integer it is. + # + # A cast that overflowed to Infinity, or a NaN, is refused rather than + # passed on: the JSON encoder raises on both, a step later, as a 500 naming + # nothing the operator can act on. + def number(value, leaf) + case value + when Integer then value + when Float then finite(value, leaf) + when String then value.match?(INTEGER) ? value.to_i : finite(Float(value, exception: false), leaf) + else refuse_value!(leaf, value, 'a number') + end + end + + def finite(value, leaf) + refuse_value!(leaf, value, 'a number') unless value.is_a?(Float) && value.finite? + + value == value.to_i ? value.to_i : value + end + + def boolean(value, leaf) + case value + when true, false then value + when 'true' then true + when 'false' then false + else refuse_value!(leaf, value, 'a true or a false') + end + end + + def blank?(value) = value.nil? || value.to_s.empty? + + # `present`, `blank` and `missing` are derived by the agent from an + # equality, above this datasource, and rewritten into a comparison with an + # empty value. Intercom's search matches values and has no spelling for the + # absence of one, so the rewritten condition is refused here rather than + # sent as a comparison against the empty string -- which Intercom would + # answer as if it were a value of its own. + def refuse_absence!(leaf, field) + raise UnsupportedOperatorError, + "#{@collection} cannot filter #{leaf.field.inspect} for absence: Intercom's search matches values " \ + "and has no operator for the lack of one, so a #{leaf.operator} condition on " \ + "#{field.field.inspect} cannot be translated. Filter on a value instead." + end + + def refuse_empty_list!(leaf) + raise UnsupportedOperatorError, + "#{@collection} was asked to filter #{leaf.field.inspect} with #{leaf.operator} and an empty list, " \ + 'which names no record and no filter. Pass at least one value.' + end + + def refuse_value!(leaf, value, expected) + raise UnsupportedOperatorError, + "#{@collection} cannot filter #{leaf.field.inspect} with #{value.inspect}: Intercom expects " \ + "#{expected} on this field." + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/query/operator_table.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/query/operator_table.rb new file mode 100644 index 000000000..e08f14f74 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/query/operator_table.rb @@ -0,0 +1,79 @@ +module ForestAdminDatasourceIntercom + module Query + # How a Forest operator is spelled in Intercom's search DSL, per kind of + # field. Two things read this table and they must never disagree: the schema, + # which publishes a column's `filter_operators`, and the translator, which + # writes the filter. Both go through `forest_operators` and + # `intercom_operator`, so a column cannot advertise a filter the translator + # would then refuse. + # + # What an endpoint really accepts on a given field is not here -- that is + # `search_fields.yml`, measured. This is only the spelling, and the set is + # narrowed by the table before anything is published. + module OperatorTable + Operators = ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Operators + + EQUALITY = { Operators::EQUAL => '=', Operators::NOT_EQUAL => '!=', + Operators::IN => 'IN', Operators::NOT_IN => 'NIN' }.freeze + + # `contains` and `i_contains` both land on `~`: Intercom documents one + # substring operator and no case semantics for it, and the frontend sends + # either spelling depending on the column. Declaring one alone would leave + # the other refused at read time on a field Intercom does filter. + # + # `not_i_contains` is deliberately absent although Intercom would answer it + # the same way as `not_contains`: the toolkit's own `Rules` does not allow + # it on a String column, so publishing it would put a filter in the + # interface that the agent rejects before this datasource is ever reached + # -- the failure PRD-989 describes. + SUBSTRING = { Operators::CONTAINS => '~', Operators::I_CONTAINS => '~', + Operators::NOT_CONTAINS => '!~', + Operators::STARTS_WITH => '^', Operators::ENDS_WITH => '$' }.freeze + + BOUNDS = { Operators::GREATER_THAN => '>', Operators::LESS_THAN => '<' }.freeze + + # A date field carries the two bounds and nothing else, even where the + # endpoint accepts `=`, `!=`, `>=` and `<=` -- measured, it does on both + # search endpoints. The reason is on the agent's side: declaring `equal` on + # a Date column makes the toolkit republish `in`, which its own validator + # then refuses (PRD-989), so the interface would offer a date filter + # answered by a 400 having nothing to do with Intercom. + # + # Nothing is lost that an operator can see. From the two bounds the toolkit + # derives `before`, `after`, `today`, `yesterday`, `past`, `future` and the + # whole `previous_*` family -- twenty operators, all rewritten into a pair + # of bounds before they reach here. What stays out is an equality on an + # instant, which day-granular filtering could not honour anyway. + MAPS = { + 'string' => EQUALITY, + 'boolean' => EQUALITY, + 'number' => EQUALITY.merge(BOUNDS), + 'text' => EQUALITY.merge(SUBSTRING), + 'date' => BOUNDS + }.freeze + + # The operators that take a list rather than a value, on the wire. + LIST_OPERATORS = %w[IN NIN].freeze + + class << self + def types = MAPS.keys + + # What the column publishes: the Forest operators whose Intercom spelling + # this endpoint accepts on this field, and no other. + def forest_operators(field) + MAPS.fetch(field.type).select { |_, spelling| field.operators.include?(spelling) }.keys + end + + # nil when the endpoint does not accept that operator on that field, + # which the translator turns into a refusal naming what it does accept. + def intercom_operator(field, forest_operator) + spelling = MAPS.fetch(field.type)[forest_operator] + + spelling if spelling && field.operators.include?(spelling) + end + + def list_operator?(spelling) = LIST_OPERATORS.include?(spelling) + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/query/search_fields.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/query/search_fields.rb new file mode 100644 index 000000000..2f3f44a4a --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/query/search_fields.rb @@ -0,0 +1,243 @@ +module ForestAdminDatasourceIntercom + module Query + # Reads `search_fields.yml`, the table of what Intercom's search endpoints + # filter, and hands it to the schema and to the translator as objects rather + # than as nested hashes. + # + # The table is data rather than code for one reason: `forest_admin_intercom_probe` + # rewrites it from a real workspace. Anything derived from it -- which + # columns are filterable, with which Forest operators, and what an operator + # is told about the ones that are not -- therefore follows a measurement + # instead of a hand-written list that drifts from the endpoint. + # + # Every row is validated on load. The file ships with the gem and is written + # by a script, so a typo in an operator, or a column filed as both filterable + # and refused, is a defect of this package: it fails at boot rather than + # producing a schema nobody can explain. + module SearchFields + PATH = File.expand_path('search_fields.yml', __dir__) + + # The operators Intercom's search DSL spells, and nothing else: `=`, `!=`, + # `>`, `<`, `>=`, `<=`, the substring pair `~` / `!~`, the anchors `^` / + # `$`, and the membership pair. Which of them an endpoint honours on a + # given field is the table's business; this is only the alphabet. + KNOWN_OPERATORS = ['=', '!=', '>', '<', '>=', '<=', '~', '!~', '^', '$', 'IN', 'NIN'].freeze + # Read off the operator table rather than listed again here: a type with + # no spelling of its own would pass this validation and raise when the + # schema asked what to publish on it. + KNOWN_TYPES = OperatorTable.types + KNOWN_SOURCES = %w[measured spec].freeze + + # `source` says where a row comes from, and `measured?` is what the boot + # report and the README section read: a row taken from the documentation is + # a candidate the probe has not confirmed. + # `sortable` is the one thing here Intercom answers on a single endpoint: + # `/contacts/search` takes a `sort`, the other two take one and ignore it + # without a word. Absent reads as false, so a table saying nothing leaves + # a column unsortable rather than promising an order. + Field = Struct.new(:column, :field, :type, :operators, :source, :sortable, keyword_init: true) do + def measured? = source == 'measured' + def sortable? = sortable == true + end + + # A column that stays unfilterable, and why. The reason travels into the + # refusal the operator reads, so it names what to filter on instead + # wherever there is something to name. + Refusal = Struct.new(:column, :reason, :source, keyword_init: true) do + def measured? = source == 'measured' + end + + Endpoint = Struct.new(:name, :path, :measured_at, :fields, :refused, :candidates, :attribute_refusal, + keyword_init: true) do + # Whether the probe has run against a real workspace for this endpoint. + # False means every `spec` row is still a candidate. + def measured? = !measured_at.nil? + + def field(column) = fields[column] + def refusal(column) = refused[column] + def filterable_columns = fields.keys + def sortable_columns = fields.values.select(&:sortable?).map(&:column) + def unmeasured_fields = fields.values.reject(&:measured?) + end + + # Long by line count only: it is one parse method and one validation per + # thing the file can get wrong, and a validation that does not say what + # is wrong is one nobody can act on. + class << self # rubocop:disable Metrics/ClassLength + def fetch(name) + table[name.to_s] || + raise(ConfigurationError, "Unknown Intercom search endpoint #{name.inspect}; " \ + "the table declares #{table.keys.join(", ")}.") + end + + def endpoints = table.keys + + def table + @table ||= build(YAML.safe_load_file(PATH)) + end + + # Public so a spec can feed it a table of its own: this validation is the + # reason the file can be rewritten by a script without the package + # trusting whatever comes back. + def build(raw) + raw.fetch('endpoints').to_h { |name, definition| [name, endpoint(name, definition)] }.freeze + end + + private + + def endpoint(name, definition) + filterable = fields(name, definition['fields']) + refused = refusals(name, definition['refused']) + validate_no_overlap!(name, filterable, refused) + + Endpoint.new( + name: name, + path: definition.fetch('path'), + measured_at: definition['measured_at'], + fields: filterable, + refused: refused, + candidates: Array(definition['candidates']).freeze, + attribute_refusal: attribute_refusal(name, definition) + ).freeze + end + + # The refusal that covers a whole family of columns rather than one: + # the attributes a workspace defines, whose names are unknown until the + # datasource boots and which therefore cannot have a row each. Spelled + # `ticket_attributes` on the endpoint that carries them per ticket type + # and `custom_attributes` on the ones that carry them per model, since + # that is the workspace's own vocabulary and what an operator goes and + # renames. + # + # Read by the translator, which is the point: without it a filter on + # such a column falls back on the generic "takes no filter on it", + # where this says why -- and the reason is the arbitration, not an + # oversight. + def attribute_refusal(endpoint, definition) + column = %w[ticket_attributes custom_attributes].find { |key| definition.key?(key) } + return nil if column.nil? + + row = definition.fetch(column) + refusal = Refusal.new(column: column, reason: squish(row.fetch('reason')), source: row.fetch('source')) + validate_source!(endpoint, column, refusal) + validate_attributes_unfilterable!(endpoint, column, row) + + refusal.freeze + end + + # `filterable: true` is a state nothing here implements: the whole + # block is a refusal, and a rewrite flipping the flag would publish + # nothing new while making the file say the opposite of what it does. + def validate_attributes_unfilterable!(endpoint, column, row) + return if row.fetch('filterable') == false + + malformed!(endpoint, column, + "filterable #{row["filterable"].inspect} is not something this reads; the attribute columns " \ + 'are published for display only, and the block exists to say why') + end + + # A column filed as both filterable and refused. `Endpoint#field` is + # consulted first, so the filterable row would win and the refusal -- + # with the reason an operator reads -- would be ignored in silence. The + # file is rewritten by a script, so this is the shape a bad rewrite + # takes, and it fails at load rather than producing a schema nobody can + # explain. + def validate_no_overlap!(endpoint, filterable, refused) + both = filterable.keys & refused.keys + return if both.empty? + + malformed!(endpoint, both.join(', '), + 'it is declared filterable and refused at once; the refusal would be ignored') + end + + def fields(endpoint, declared) + (declared || {}).to_h do |column, row| + field = Field.new(column: column, field: row.fetch('field'), type: row.fetch('type'), + operators: Array(row['operators']).freeze, source: row.fetch('source'), + sortable: row.fetch('sortable', false)).freeze + validate_field!(endpoint, field) + + [column, field] + end.freeze + end + + def refusals(endpoint, declared) + (declared || {}).to_h do |column, row| + refusal = Refusal.new(column: column, reason: squish(row.fetch('reason')), + source: row.fetch('source')).freeze + validate_source!(endpoint, column, refusal) + + [column, refusal] + end.freeze + end + + def validate_field!(endpoint, field) + validate_source!(endpoint, field.column, field) + validate_type!(endpoint, field) + validate_operators!(endpoint, field) + validate_publishable!(endpoint, field) + validate_sortable!(endpoint, field) + end + + # Operators the DSL spells, none of which this column's *type* can carry: + # a `date` row declaring `~` passes the alphabet above and publishes + # nothing, `OperatorTable` mapping no Forest operator onto it. The result + # is a column the table calls filterable that no filter can reach, and + # the translator refusing every request on it -- the same + # advertise-then-refuse this package exists to prevent, one layer lower. + def validate_publishable!(endpoint, field) + return unless OperatorTable.forest_operators(field).empty? + + malformed!(endpoint, field.column, + "none of #{field.operators.join(", ")} is an operator Intercom answers on a " \ + "#{field.type}, so the column would publish no filter at all") + end + + # Anything but a boolean, `"true"` above all: YAML reads it as a string, + # which is truthy in Ruby and would publish a sortable column out of a + # typo the file cannot otherwise show. + def validate_sortable!(endpoint, field) + return if [true, false].include?(field.sortable) + + malformed!(endpoint, field.column, "sortable #{field.sortable.inspect} is neither true nor false") + end + + def validate_type!(endpoint, field) + return if KNOWN_TYPES.include?(field.type) + + malformed!(endpoint, field.column, "type #{field.type.inspect} is not one of #{KNOWN_TYPES.join(", ")}") + end + + # An empty operator list is how a table stops short of saying anything: + # it would publish a filterable column no operator can reach. A column + # Intercom does not filter belongs in the refused table, where it comes + # with the reason an operator reads. + def validate_operators!(endpoint, field) + if field.operators.empty? + malformed!(endpoint, field.column, + 'it declares no operator; a column Intercom cannot filter belongs in the refused table') + end + + unknown = field.operators - KNOWN_OPERATORS + return if unknown.empty? + + malformed!(endpoint, field.column, "Intercom's search DSL has no operator #{unknown.join(", ")}") + end + + def validate_source!(endpoint, column, row) + return if KNOWN_SOURCES.include?(row.source) + + malformed!(endpoint, column, "source #{row.source.inspect} is neither #{KNOWN_SOURCES.join(" nor ")}") + end + + def malformed!(endpoint, column, detail) + raise ConfigurationError, "#{File.basename(PATH)} is malformed at #{endpoint}.#{column}: #{detail}." + end + + # A YAML folded block keeps the newlines the file needs to stay readable; + # the reason travels into a one-line message. + def squish(text) = text.to_s.split.join(' ') + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/query/search_fields.yml b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/query/search_fields.yml new file mode 100644 index 000000000..a97b5b610 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/query/search_fields.yml @@ -0,0 +1,657 @@ +# The table of (endpoint x field x operator): what Intercom's search endpoints +# really filter, and with which operators. It is the single source of truth of +# this datasource's filtering -- the schema derives every column's +# filter_operators from it, so no collection can advertise a filter the +# translator would then refuse. +# +# `source` is the provenance of a row, and it is not decoration: +# +# measured -- observed against a real workspace, by `forest_admin_intercom_probe` +# or during a spike, and recorded here with the date; +# spec -- read off Intercom's documentation and nothing else. +# +# The distinction exists because the two disagree. Measured during lot 1: +# `company_id` is documented on a ticket and refused by `/tickets/search` with +# `invalid_field`. A `spec` row is therefore a candidate, never a promise, and +# the probe is what turns it into one. Run it against the customer's workspace +# and commit what it writes. +# +# `type` decides how a column's Forest operators are derived from the Intercom +# ones, and how a value reaches the wire: +# +# string -- an exact-match field +# text -- a field Intercom also matches per word (`~`) +# date -- epoch seconds on the wire, truncated to the UTC day on read +# boolean, number +version: 1 + +endpoints: + conversations: + path: conversations/search + # Nothing has been probed yet: fill this in with the probe's output. + measured_at: null + fields: + # The primary key, and the one column whose Forest operators are not + # derived from this table: every collection publishes `equal` and `in` on + # its key whatever is written here, the toolkit refusing a collection + # whose key carries neither. It has to be declared all the same, or the + # schema advertises a filter the translator has no row to write -- which + # is what a record detail becomes the moment a permission scope turns + # `id equals X` into an `and` the record endpoint cannot answer alone. + # + # A bare `id equals X` still reads the record endpoint, one request + # instead of a search. This row is for the compound case. + id: + field: id + type: string + operators: ['=', 'IN'] + source: spec + state: + field: state + type: string + operators: ['=', '!='] + source: spec + open: + field: open + type: boolean + operators: ['='] + source: spec + read: + field: read + type: boolean + operators: ['='] + source: spec + priority: + field: priority + type: string + operators: ['=', '!='] + source: spec + title: + field: title + type: text + operators: ['=', '!=', '~', '!~', '^', '$'] + source: spec + admin_assignee_id: + field: admin_assignee_id + type: string + operators: ['=', '!='] + source: spec + team_assignee_id: + field: team_assignee_id + type: string + operators: ['=', '!='] + source: spec + source_type: + field: source.type + type: string + operators: ['=', '!='] + source: spec + source_delivered_as: + field: source.delivered_as + type: string + operators: ['=', '!='] + source: spec + source_subject: + field: source.subject + type: text + operators: ['=', '!=', '~', '!~', '^', '$'] + source: spec + # The full-text search of the collection: Intercom matches `~` per word, + # not as a substring, which the README states rather than the interface + # implying otherwise. + source_body: + field: source.body + type: text + operators: ['~', '!~'] + source: spec + source_author_email: + field: source.author.email + type: text + operators: ['=', '!=', '~', '!~', '^', '$'] + source: spec + # The date operators are measured, and they are not the same on every + # endpoint: `/contacts/search` refuses `>=`, `<=` and `!=` where this one + # accepts them (25 August 2026, API 2.16). Whatever this table allows, a + # Date column publishes the two bounds alone -- see the translator. + created_at: + field: created_at + type: date + operators: ['>', '<', '>=', '<=', '=', '!='] + source: measured + updated_at: + field: updated_at + type: date + operators: ['>', '<', '>=', '<=', '=', '!='] + source: measured + waiting_since: + field: waiting_since + type: date + operators: ['>', '<', '>=', '<=', '=', '!='] + source: spec + snoozed_until: + field: snoozed_until + type: date + operators: ['>', '<', '>=', '<=', '=', '!='] + source: spec + closed_at: + field: statistics.last_close_at + type: date + operators: ['>', '<', '>=', '<=', '=', '!='] + source: spec + first_closed_at: + field: statistics.first_close_at + type: date + operators: ['>', '<', '>=', '<=', '=', '!='] + source: spec + closed_by_id: + field: statistics.last_closed_by_id + type: string + operators: ['=', '!='] + source: spec + first_contact_reply_at: + field: statistics.first_contact_reply_at + type: date + operators: ['>', '<', '>=', '<=', '=', '!='] + source: spec + last_contact_reply_at: + field: statistics.last_contact_reply_at + type: date + operators: ['>', '<', '>=', '<=', '=', '!='] + source: spec + last_admin_reply_at: + field: statistics.last_admin_reply_at + type: date + operators: ['>', '<', '>=', '<=', '=', '!='] + source: spec + reopen_count: + field: statistics.count_reopens + type: number + operators: ['=', '!=', '>', '<'] + source: spec + part_count: + field: statistics.count_conversation_parts + type: number + operators: ['=', '!=', '>', '<'] + source: spec + ai_agent_participated: + field: ai_agent_participated + type: boolean + operators: ['='] + source: spec + # The foreign key of the `contact` relation, and the reason lot 4 could + # turn the denormalized contact columns into one: Intercom matches a + # conversation against one of the contacts it carries. The column is + # singular and the wire field plural on purpose -- the row names its first + # contact, the endpoint asks "is this contact one of yours". + # + # `!=` is deliberately absent: on a field holding a list, "not that + # contact" is a question Intercom's DSL does not clearly answer, and the + # one thing this table may not do is publish a filter whose meaning is a + # guess. + contact_id: + field: contact_ids + type: string + operators: ['='] + source: measured + # Columns this collection publishes and will not filter, with the reason an + # operator reads when they try. A column absent from both tables is refused + # by name too -- these are the ones whose refusal is permanent and has an + # explanation worth giving. + refused: + tag_names: + reason: >- + Intercom filters conversations by tag id, and this column holds the + tag names. Filtering it would mean resolving a name to an id per + request, and a name the workspace renamed would silently match + nothing. + source: spec + company_name: + reason: >- + The conversation carries its company as an object read from the + payload; the search endpoint filters no company field. + source: spec + company_id: + reason: >- + The search endpoint filters no company field. Measured on + `/tickets/search`, which refuses `company_id` with `invalid_field` + although a ticket carries one; the same is assumed here until probed. + source: spec + timeline: + reason: >- + Built by the agent from the parts of a conversation, which the search + endpoint does not read. + source: measured + contact_name: + reason: >- + Read from the Contacts endpoint, not from the conversation. Filter + through the `contact` relation, which resolves against + `/contacts/search` and rewrites onto the contact id -- or on + `source_author_email`, which is on the conversation itself. + source: spec + contact_count: + reason: Counted by the agent from the contacts the payload carries. + source: measured + source_author_name: + reason: >- + The endpoint matches the author of the opening message by e-mail + alone -- `source.author.email` is a field, `source.author.name` is + not. Filter on `source_author_email`. + source: spec + # What `forest_admin_intercom_probe` enumerates on top of the fields above: + # names the documentation mentions, or that an ops team would plausibly + # search on. A candidate that turns out to be filterable becomes a field + # above -- with a column to expose it on, or with none, in which case it is + # a column the next lot may add. + candidates: + - source.id + - source.author.id + - source.author.type + - source.author.name + - statistics.time_to_assignment + - statistics.time_to_admin_reply + - statistics.time_to_first_close + - statistics.median_time_to_reply + - statistics.first_assignment_at + - statistics.first_admin_reply_at + - statistics.last_assignment_at + - statistics.count_assignments + - conversation_rating.score + - conversation_rating.remark + - conversation_rating.contact_id + - ai_agent.resolution_state + - ai_agent.last_answer_type + - ai_agent.rating + - channel_initiated + - tag_ids + - teammate_ids + - company_id + - topics + + tickets: + path: tickets/search + measured_at: null + fields: + # The primary key, declared here for the same reason as on conversations: + # the schema publishes `equal` and `in` on it whatever this table says, + # so a row has to exist for the translator to write. `GET /tickets/{id}` + # answers the key on its own; the search answers it in an `and`. + id: + field: id + type: string + operators: ['=', 'IN'] + source: spec + open: + field: open + type: boolean + operators: ['='] + source: spec + category: + field: category + type: string + operators: ['=', '!='] + source: spec + ticket_type_id: + field: ticket_type_id + type: string + operators: ['=', '!='] + source: spec + admin_assignee_id: + field: admin_assignee_id + type: string + operators: ['=', '!='] + source: spec + team_assignee_id: + field: team_assignee_id + type: string + operators: ['=', '!='] + source: spec + created_at: + field: created_at + type: date + operators: ['>', '<', '>=', '<=', '=', '!='] + source: measured + updated_at: + field: updated_at + type: date + operators: ['>', '<', '>=', '<=', '=', '!='] + source: measured + # The foreign key of the `contact` relation. `spec` rather than + # `measured`, and the difference is visible to an operator: it is what + # decides whether the contacts of a ticket can be filtered at all, and the + # probe has not run on this endpoint. If Intercom refuses it, this row + # moves to the refused table and the relation stays navigable without + # being filterable -- the way `state` already is. + contact_id: + field: contact_ids + type: string + operators: ['='] + source: spec + refused: + company_id: + reason: >- + Measured during lot 1: `/tickets/search` refuses `company_id` with + `invalid_field`, although a ticket carries one. Filtering tickets by + account is not something this endpoint does. + source: measured + closed_at: + reason: >- + Derived by the agent from the parts of the ticket. The search endpoint + filters nothing of the sort, and ignores a sort on it without a word. + source: measured + closed_by_name: + reason: Derived by the agent from the parts of the ticket. + source: measured + last_reply_at: + reason: Derived by the agent from the parts of the ticket. + source: measured + last_responder_name: + reason: Derived by the agent from the parts of the ticket. + source: measured + last_responder_type: + reason: Derived by the agent from the parts of the ticket. + source: measured + state_label: + reason: >- + Read off the state object the ticket embeds. Whether the endpoint + filters a ticket state at all, and under which name, is one of the + probe's questions. + source: spec + ticket_type_name: + reason: >- + Read off the type object the ticket embeds. Filter on + `ticket_type_id`, which the endpoint does take. + source: spec + ticket_id: + reason: >- + The number the support team says out loud, which is not the id the + API answers by. The search endpoint carries no field for it. Filter + on `id`, which takes the value the url and the record page use. + source: spec + is_shared: + reason: >- + Whether the ticket is visible to the customer. On the payload and not + in the search DSL; the probe lists it as a candidate. + source: spec + state_id: + reason: >- + Read off the state object the ticket embeds. The endpoint takes no + filter on a ticket state under any name this table knows, which is + what keeps the `state` relation navigable without being filterable. + Whether it filters one at all is the probe's second question. Filter + on `open` or on `category` meanwhile. + source: spec + previous_state_id: + reason: >- + The state the ticket left, read off its payload. Filterable no more + than the state it is in, and for the same reason. + source: spec + contact_name: + reason: >- + Read from the Contacts endpoint, not from the ticket. Filter through + the `contact` relation, which resolves against `/contacts/search` and + rewrites onto `contact_ids`. + source: spec + contact_count: + reason: Counted by the agent from the contacts the payload carries. + source: spec + part_count: + reason: Counted by the agent from the parts the payload carries. + source: measured + timeline: + reason: >- + Built by the agent from the parts of the ticket, which the search + endpoint does not read. Filter on `last_reply_at` or on the state + instead. + source: measured + # R7, and it is a product decision rather than an implementation choice: a + # ticket attribute is filtered as `ticket_attribute.{id}`, and a same-named + # attribute carries a different id per ticket type (measured: + # `_default_title_` is 14162161 on one type and 14162165 on another). One + # union column cannot know which id a row's type uses, so filtering it + # server-side would take one collection per ticket type -- more collections + # in the interface, and a schema that changes shape when the customer adds a + # type. Until the customer says that trade is worth it, the attributes stay + # display-only, as lot 1 published them. + ticket_attributes: + filterable: false + reason: >- + An Intercom ticket attribute is filtered through an id that differs from + one ticket type to the next, so a column showing the attribute of every + type at once cannot say which id to filter on. Filter on the ticket type + and a native field instead. + source: measured + + candidates: + - ticket_id + - state + - ticket_state.category + - ticket_state.id + - state_id + - is_shared + - previous_state_id + - company_ids + - source.author.email + - source.subject + - source.body + - title + - description + + # The one endpoint of the whole API that sorts. `sortable: true` is what a + # column reads to publish `isSortable`, and it appears nowhere else in this + # file on purpose: `/conversations/search` accepts a `sort` and ignores it + # without a word (measured), so a row that stayed silent there would be a + # column promising an order Intercom never applies. + # + # The set is deliberately narrower than what the documentation implies. It + # says the results may be sorted by any attribute; nothing has been measured, + # and a sort Intercom refuses is a list view that fails rather than one that + # comes back unordered. These are the fields an ops team orders a contact list + # by, and the probe is what may widen the set. + contacts: + path: contacts/search + measured_at: null + fields: + # Measured rather than taken from the documentation: lot 1 reads the + # contacts of a page through `id IN [...]` on this endpoint, which is the + # membership operator every other field refuses. + id: + field: id + type: string + operators: ['=', 'IN'] + source: measured + role: + field: role + type: string + operators: ['=', '!='] + source: spec + name: + field: name + type: text + operators: ['=', '!=', '~', '!~', '^', '$'] + source: spec + sortable: true + email: + field: email + type: text + operators: ['=', '!=', '~', '!~', '^', '$'] + source: spec + sortable: true + # Derived by the agent from the address, and filtered by Intercom all the + # same: the endpoint carries a field of its own for it, which is what + # makes "everyone at this customer" a filter rather than a wildcard. + email_domain: + field: email_domain + type: string + operators: ['=', '!='] + source: spec + phone: + field: phone + type: text + operators: ['=', '!=', '~', '!~', '^', '$'] + source: spec + external_id: + field: external_id + type: string + operators: ['=', '!='] + source: spec + owner_id: + field: owner_id + type: string + operators: ['=', '!='] + source: spec + # Measured on 25 August 2026, API 2.16, and this is the asymmetry the + # per-endpoint table exists for: `/conversations/search` and + # `/tickets/search` accept `>=`, `<=` and `!=` on a date where this + # endpoint answers `data_invalid`. `IN` is refused on a date everywhere. + # + # Nothing is lost that an operator can see: a Date column publishes the + # two bounds alone whatever this table allows -- see the operator table -- + # and the toolkit derives `before`, `after`, `today` and the whole + # `previous_*` family from them. + created_at: + field: created_at + type: date + operators: ['>', '<'] + source: measured + sortable: true + updated_at: + field: updated_at + type: date + operators: ['>', '<'] + source: measured + sortable: true + signed_up_at: + field: signed_up_at + type: date + operators: ['>', '<'] + source: spec + sortable: true + last_seen_at: + field: last_seen_at + type: date + operators: ['>', '<'] + source: spec + sortable: true + last_contacted_at: + field: last_contacted_at + type: date + operators: ['>', '<'] + source: spec + sortable: true + last_replied_at: + field: last_replied_at + type: date + operators: ['>', '<'] + source: spec + sortable: true + last_email_opened_at: + field: last_email_opened_at + type: date + operators: ['>', '<'] + source: spec + last_email_clicked_at: + field: last_email_clicked_at + type: date + operators: ['>', '<'] + source: spec + unsubscribed_from_emails: + field: unsubscribed_from_emails + type: boolean + operators: ['='] + source: spec + has_hard_bounced: + field: has_hard_bounced + type: boolean + operators: ['='] + source: spec + marked_email_as_spam: + field: marked_email_as_spam + type: boolean + operators: ['='] + source: spec + language_override: + field: language_override + type: string + operators: ['=', '!='] + source: spec + browser: + field: browser + type: string + operators: ['=', '!='] + source: spec + browser_language: + field: browser_language + type: string + operators: ['=', '!='] + source: spec + os: + field: os + type: string + operators: ['=', '!='] + source: spec + location_country: + field: location.country + type: string + operators: ['=', '!='] + source: spec + location_region: + field: location.region + type: string + operators: ['=', '!='] + source: spec + location_city: + field: location.city + type: string + operators: ['=', '!='] + source: spec + refused: + avatar: + reason: >- + The url of the contact's picture. Nothing filters an image, and + Intercom's search does not carry the field. + source: spec + company_id: + reason: >- + The first of the companies the contact belongs to, read off the + payload. `/contacts/search` filters no company field, so a company + list is reached from the company side -- open the company and read + its contacts. + source: spec + company_count: + reason: Counted by the agent from the companies the payload carries. + source: spec + session_count: + reason: >- + On the contact payload and not in the search DSL. Filter on + `last_seen_at` instead, which the endpoint does take. + source: spec + # Contact custom attributes are filtered as `custom_attributes.{name}` -- + # by name, unlike a ticket attribute, which carries a different id per + # ticket type. So the obstacle is not R7 here, it is measurement: the + # operator set Intercom answers on a custom attribute of each data type has + # not been probed, and this package does not publish a filter it has not + # seen work. They ship typed and display-only, and the probe is what turns + # that around. + custom_attributes: + filterable: false + reason: >- + Contact custom attributes are published for reading only: which + operators Intercom answers on `custom_attributes.{name}` has not been + measured on a real workspace, and this datasource does not offer a + filter it has not seen work. Filter on a native field instead. + source: spec + candidates: + - tag_ids + - segment_id + - company_id + - companies.id + - companies.name + - android_app_name + - android_last_seen_at + - ios_app_name + - ios_last_seen_at + - utm_source + - utm_campaign + - utm_medium + - referrer + - session_count diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/rate_limiter.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/rate_limiter.rb new file mode 100644 index 000000000..82dcbd12f --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/rate_limiter.rb @@ -0,0 +1,182 @@ +module ForestAdminDatasourceIntercom + # Paces requests on what Intercom says is left of the current window, so the + # budget is spent rather than exceeded. + # + # Intercom meters the app and, above it, the whole workspace -- 25 000 + # requests a minute shared with every other private app the customer runs -- + # and it allocates that budget in 10-second windows: the measured + # `x-ratelimit-limit` is 1667, not 10 000. A burst of 3 000 requests in two + # seconds therefore takes a 429 while the minute's budget is barely touched, + # which is why what matters here is the instantaneous rate and not a volume + # per minute. + # + # Unlike an API that only documents its budgets, Intercom reports the state of + # the window on every response, so this is driven by those headers rather than + # by a table: what is left, and when it refills. This sits in front of the 429 + # retry rather than replacing it -- the retry stays the backstop for the part + # of the workspace budget spent by traffic this process never sees. + # + # One limiter per Configuration, hence per token, since that is what Intercom + # meters. + # + # **What it does not do**, and deliberately: it paces, it does not queue. A + # caller that finds the window spent computes the delay to the reset and + # sleeps it, and it reserves nothing while it waits -- so a hundred callers + # arriving in a spent window all compute the same delay and all wake at the + # reset, and past that instant `remaining` is still zero while the wait + # computes as elapsed, which lets them through unmetered until the first + # response reports the new window. Two things make that acceptable rather + # than a bug to work around: the burst is bounded by the callers that were + # already blocked, and the 429 retry behind this is what absorbs it -- this + # sits in front of that retry, it does not replace it. A real reservation + # would mean guessing the boundaries of a window Intercom has not reported + # yet, and a wrong guess suppresses the figures it later sends. + class RateLimiter + # Intercom's allocation window. Only used as the ceiling below: the reset + # instant itself always comes from the response. + WINDOW = 10.0 + + # How far past the reset a request may be held. A little over one window, so + # a full window can be waited out, and no more: past this the wait is not + # Intercom's window emptying but a clock disagreeing. + DEFAULT_MAX_WAIT = 12.0 + + attr_reader :max_wait + + def initialize(max_wait: DEFAULT_MAX_WAIT, now: nil, sleeper: nil) + @max_wait = max_wait.to_f + # Wall clock rather than monotonic on purpose: `X-RateLimit-Reset` is an + # absolute epoch second on Intercom's clock, so the two have to be + # comparable. `clamp_wait` is what keeps a skewed clock from turning that + # comparison into a long sleep. + @now = now || -> { Time.now.to_f } + @sleeper = sleeper || ->(seconds) { sleep(seconds) } + @mutex = Mutex.new + @remaining = nil + @reset_at = nil + @limit = nil + @warned_at = nil + end + + # Blocks until the current window has room, then returns. Called once per + # attempt, retries included: a replayed request spends the budget a first + # one did. + def acquire + wait, declined = @mutex.synchronize { plan_wait } + + warn_saturated(declined) if declined + return if wait <= 0 + + @sleeper.call(wait) + end + + # What a response says about the window it was answered in. Called on every + # response, the 429 included -- that one carries the most useful reset of + # all. + def observe(headers) + limit = integer_header(headers, 'x-ratelimit-limit') + remaining = integer_header(headers, 'x-ratelimit-remaining') + reset_at = integer_header(headers, 'x-ratelimit-reset') + return if remaining.nil? && reset_at.nil? + + @mutex.synchronize { record(limit, remaining, reset_at) } + end + + private + + # A local decrement per request on top of what the headers report: several + # requests can be in flight before any of them comes back, and a `remaining` + # that only ever moves on a response lets all of them through on the same + # stale figure. + # + # Returns the wait the caller owes and, when the reset is too far out to be + # waited for, the wait that was declined -- nil otherwise. Both are settled + # here, the second reading shared state like the first: it is nil on every + # bypass but the first of a window, so the log line is not repeated. + def plan_wait + @remaining -= 1 if @remaining + return [0, nil] unless exhausted? + + wait = clamp_wait(@reset_at - @now.call) + return [0, nil] if wait <= 0 + return [0, first_warning? ? wait : nil] if wait > @max_wait + + [wait, nil] + end + + # Nothing left in a window that has not refilled yet. An unknown state -- + # before the first response -- is not exhaustion: the first request is what + # discovers the budget. + def exhausted? + !@remaining.nil? && @remaining <= 0 && !@reset_at.nil? + end + + # Intercom's reset is a timestamp from its clock, and the two clocks can + # disagree by more than the window is long. A wait longer than a window plus + # its own slack is that disagreement rather than a window emptying, so it is + # cut back to something a caller can afford to wait. + def clamp_wait(seconds) + return 0.0 if seconds <= 0 + + [seconds, WINDOW + @max_wait].min + end + + # A window is adopted whole. A response answered in an older window than the + # one already recorded is ignored: replies come back out of order, and one + # from the previous window would otherwise resurrect a budget already spent. + # + # Within the same window the smaller `remaining` wins, so the local + # decrements of in-flight requests are not undone by a response that left + # Intercom before they were made. + def record(limit, remaining, reset_at) + return if reset_at && @reset_at && reset_at < @reset_at + + new_window = reset_at && (@reset_at.nil? || reset_at > @reset_at) + @reset_at = reset_at if reset_at + @limit = limit if limit + return if remaining.nil? + + @remaining = new_window || @remaining.nil? ? remaining : [remaining, @remaining].min + end + + # One line per window. What the warning reports is a saturation that lasts, + # so a line per request puts one on every request it describes -- hundreds + # of them, burying the first, which is the only one the operator needed. + def first_warning? + now = @now.call + return false if @warned_at && now - @warned_at < WINDOW + + @warned_at = now + true + end + + def warn_saturated(wait) + ForestAdminDatasourceIntercom.logger.warn( + "[forest_admin_datasource_intercom] the Intercom rate-limit window is spent (limit #{@limit || "unknown"} " \ + "per #{WINDOW.round}s) and its reset is #{wait.round(1)}s out, past the #{@max_wait.round(1)}s this waits. " \ + 'Letting the request through -- Intercom may answer 429, which the client retries. A reset this far out ' \ + "usually means this host's clock disagrees with Intercom's. Reported once per #{WINDOW.round}s." + ) + end + + def integer_header(headers, name) + value = header_value(headers, name) + return nil if value.nil? || value.to_s.strip.empty? + + Integer(value.to_s.strip, exception: false) + end + + # Faraday hands over headers that look themselves up case-insensitively, but + # what reaches here is whatever the middleware was given -- a plain hash + # included -- and HTTP header names are case-insensitive on the wire. + def header_value(headers, name) + return nil if headers.nil? + + direct = headers[name] + return direct unless direct.nil? + return nil unless headers.respond_to?(:find) + + headers.find { |key, _value| key.to_s.downcase == name }&.last + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/retry_policy.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/retry_policy.rb new file mode 100644 index 000000000..249b74eb2 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/retry_policy.rb @@ -0,0 +1,72 @@ +module ForestAdminDatasourceIntercom + # Everything governing how the client reacts to a failed request, in one + # place: which statuses and exceptions are worth another attempt, on which + # verbs, and how long to wait. + class RetryPolicy + # Intercom allocates its quota in 10-second windows, so a 429 is recovered + # from within one of them -- unlike an API metering by the minute. The cap + # still has to cover a whole window: faraday-retry gives up outright when + # Retry-After exceeds max_interval, which would turn the 429 retry into an + # immediate give-up exactly when it matters. + DEFAULT_MAX_INTERVAL = 12 + + STATUSES = [429, 500, 502, 503, 504].freeze + + # faraday-retry's defaults plus ConnectionFailed: a dropped connection is + # exactly the transient failure a resilient client should absorb, and it is + # not retried out of the box. + EXCEPTIONS = [ + Errno::ETIMEDOUT, 'Timeout::Error', Faraday::TimeoutError, + Faraday::RetriableResponse, Faraday::ConnectionFailed + ].freeze + + # The verbs that change nothing, so any transient failure is worth another + # attempt. Narrower than faraday-retry's idempotent default: a 502 or a + # dropped connection on the way back from a POST Intercom did perform would + # be replayed into a second reply on the conversation, or a second ticket. + # + # A 429 stays safe to retry on any verb, Intercom having rejected the + # request before processing it, and travels through retry_if rather than + # through this list: faraday-retry ORs the two, so `methods` can only widen + # the set, never restrict it. + RETRYABLE_METHODS = %i[get head options].freeze + RETRY_IF = ->(env, _exception) { env[:status] == 429 } + + # The cap for a call that must not hold the boot, deliberately below a + # rate-limit window where DEFAULT_MAX_INTERVAL sits above it: a Retry-After + # past the cap makes faraday-retry abandon outright, which is what turns a + # 429 at boot into an immediate give-up rather than a window of waiting per + # attempt. + BOOT_MAX_INTERVAL = 2 + + BACKOFF_FACTOR = 2 + + attr_reader :max_retries, :interval, :max_interval + + # One retry rather than none, for what is read once and never revisited: a + # transient failure there costs its result for the whole life of the + # process, and half a second absorbs the hiccup without waiting a 429 out. + def self.boot + new(max_retries: 1, interval: 0.5, max_interval: BOOT_MAX_INTERVAL) + end + + def initialize(max_retries: 3, interval: 0.5, max_interval: DEFAULT_MAX_INTERVAL) + @max_retries = max_retries + @interval = interval + @max_interval = max_interval + end + + def to_faraday_options + { + max: @max_retries, + interval: @interval, + max_interval: @max_interval, + backoff_factor: BACKOFF_FACTOR, + retry_statuses: STATUSES, + exceptions: EXCEPTIONS, + methods: RETRYABLE_METHODS, + retry_if: RETRY_IF + } + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/schema/attribute_naming.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/schema/attribute_naming.rb new file mode 100644 index 000000000..baa3e8f28 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/schema/attribute_naming.rb @@ -0,0 +1,52 @@ +module ForestAdminDatasourceIntercom + module Schema + # What the two introspectors do identically: turn the name a workspace gave + # an attribute into a name a Forest schema can carry, and say which + # attribute was left out when two of them land on one column. + # + # They differ in where the attributes come from -- `/ticket_types` per + # ticket type, `/data_attributes` per model -- and in the data types they + # map, which is why `COLUMN_TYPES` stays with each of them. + module AttributeNaming + # What a column name may not contain, and it has nothing to do with + # Intercom: Forest lists the fields of a request in a **comma-separated** + # query parameter, and uses a colon to name a field through a relation. + # A workspace names its attributes in free text -- measured, one is called + # `ID de l'objet en question (immo, facture, user)` -- and a comma in + # there splits the projection into fields no collection has, which the + # agent rejects as a 400 before the page is ever read. + UNSAFE_IN_A_COLUMN_NAME = /[,:]/ + + private + + # Intercom hands these back HTML-escaped -- `Ce que j'ai vérifié` -- + # which is an artefact of where they were typed, not part of the name. + def column_name_for(name) + CGI.unescapeHTML(name).gsub(UNSAFE_IN_A_COLUMN_NAME, ' ').squeeze(' ').strip + end + + # An unknown data type reads as a string rather than being dropped: + # showing the value Intercom sent beats hiding a column because its type + # is new. + def column_type_for(definition) + self.class::COLUMN_TYPES.fetch(definition['data_type'].to_s, self.class::DEFAULT_COLUMN_TYPE) + end + + # Two attributes reading as one column: the second is left out rather than + # sharing the first's entry, which would show its values under the first's + # name -- wrong values rather than missing ones, and worse. + # + # `attribute_kind` is the workspace's own vocabulary for these, which is + # what the operator has to go and rename. + def warn_collision(name, kept, column) + ForestAdminDatasourceIntercom.logger.warn( + "[forest_admin_datasource_intercom] the #{attribute_kind} attribute #{name.inspect} is left out: it " \ + "reads as the column #{column.inspect}, which #{kept.inspect} already carries. Rename one of them in " \ + 'Intercom to publish both.' + ) + end + + def attribute_kind = raise(NotImplementedError, "#{self.class} did not implement attribute_kind") + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/schema/data_attributes_introspector.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/schema/data_attributes_introspector.rb new file mode 100644 index 000000000..9a1c9cc6b --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/schema/data_attributes_introspector.rb @@ -0,0 +1,98 @@ +module ForestAdminDatasourceIntercom + module Schema + # The attributes a workspace defines on its contacts and on its companies, + # read once while the datasource is being constructed. + # + # Unlike a ticket attribute, one of these is declared **per model** rather + # than per ticket type, so the union is the whole set and a column maps onto + # exactly one Intercom attribute -- the ambiguity that keeps ticket + # attributes display-only (R7) does not arise here. What keeps these + # display-only is narrower and temporary: which operators Intercom answers + # on `custom_attributes.{name}` has not been measured, and this package + # publishes no filter it has not seen work. + # + # `api_writable` is read and carried although every column of this lot is + # published read-only. It costs nothing now and it is exactly what lot 4b + # needs to tell an attribute it may write from one Intercom fills in + # itself -- re-reading it later would be a second boot-time round trip. + class DataAttributesIntrospector + include AttributeNaming + + # Intercom's attribute data types, mapped onto what Forest can render. + COLUMN_TYPES = { + 'string' => 'String', 'integer' => 'Number', 'float' => 'Number', 'decimal' => 'Number', + 'boolean' => 'Boolean', 'date' => 'Date', 'datetime' => 'Date' + }.freeze + + DEFAULT_COLUMN_TYPE = 'String'.freeze + + # `name` is the key `custom_attributes` uses, `column_name` the one the + # schema publishes; they differ whenever the workspace's own name cannot + # travel through Forest's query string. + Attribute = Struct.new(:name, :column_name, :column_type, :data_type, :api_writable, keyword_init: true) + + def initialize(client, model:) + @client = client + @model = model + end + + # Degrades to nothing rather than to a failure: a token without the + # permission on this model costs the custom-attribute columns, never the + # boot of the agent. + def attributes + @attributes ||= build + rescue APIError => e + ForestAdminDatasourceIntercom.logger.warn( + "[forest_admin_datasource_intercom] could not read the #{@model} attributes (HTTP #{e.status || "-"}); " \ + 'the collection boots without its custom-attribute columns.' + ) + @attributes = [] + end + + private + + def build + # Read on the boot connection: this happens while Rails is starting, and + # a slow Intercom must not turn that into minutes the operator sits + # through. + @client.fetch_all('data_attributes', params: { 'model' => @model }, boot: true) + .each_with_object({}) { |definition, union| collect(definition, union) } + .values + end + + # The standard attributes are left out: they are columns this datasource + # declares by hand, with the filters the search table measured, and + # publishing them a second time under their `custom_attributes` name would + # show one fact twice -- the unfilterable copy winning nothing. + def collect(definition, union) + return unless definition.is_a?(Hash) && definition['custom'] && !definition['archived'] + + name = definition['name'].to_s + return if name.empty? + + column = column_name_for(name) + return if column.empty? + + add(union, name, column, definition) + end + + def add(union, name, column, definition) + entry = union[column] + return union[column] = attribute_from(name, column, definition) if entry.nil? + return if entry.name == name + + warn_collision(name, entry.name, column) + end + + def attribute_from(name, column, definition) + Attribute.new(name: name, column_name: column, column_type: column_type_for(definition), + data_type: definition['data_type'], api_writable: definition['api_writable'] == true) + end + + # These are declared per model, so that is what the log calls them: it is + # the workspace's own vocabulary, and where the operator goes to rename + # the one that was left out. + def attribute_kind = @model + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/schema/ticket_attributes_introspector.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/schema/ticket_attributes_introspector.rb new file mode 100644 index 000000000..933004c1b --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/schema/ticket_attributes_introspector.rb @@ -0,0 +1,120 @@ +module ForestAdminDatasourceIntercom + module Schema + # The attributes a workspace defines on its ticket types, read once while the + # datasource is being constructed. + # + # They are declared **per ticket type**, so a single Tickets collection can + # only carry their union -- and that union is for display. Measured on a real + # workspace: two types share the names `_default_title_` and + # `_default_description_` while carrying different attribute ids (14162161 + # against 14162165), and Intercom filters an attribute by id + # (`ticket_attribute.{id}`), never by name. A union column therefore has no + # single id to translate to unless the type of the row is known, which is why + # these ship unfilterable and why filtering on one means a collection per + # ticket type (R7). + # + # The ids are kept per type all the same: they are exactly what the filter + # translation of the next lot will need, and reading them again would cost a + # second boot-time round trip. + class TicketAttributesIntrospector + include AttributeNaming + + # Intercom's attribute data types, mapped onto what Forest can render. A + # `list` is a single choice among values the workspace defined, so it reads + # as a string rather than as a Json blob; `files` is a list of attachments + # and has no scalar form at all. + COLUMN_TYPES = { + 'string' => 'String', 'list' => 'String', 'integer' => 'Number', 'decimal' => 'Number', + 'boolean' => 'Boolean', 'datetime' => 'Date', 'date' => 'Date', 'files' => 'Json' + }.freeze + + DEFAULT_COLUMN_TYPE = 'String'.freeze + + # `name` is the key the payload uses, `column_name` the one the schema + # publishes; they differ whenever the workspace's own name cannot travel + # through Forest's query string. + Attribute = Struct.new(:name, :column_name, :column_type, :data_type, :ids_by_ticket_type, + keyword_init: true) + + def initialize(client) + @client = client + end + + # The union, one entry per attribute name. Degrades to nothing rather than + # to a failure: a token without the ticket-types permission costs the + # attribute columns, never the boot of the agent. + def attributes + @attributes ||= build + rescue APIError => e + ForestAdminDatasourceIntercom.logger.warn( + "[forest_admin_datasource_intercom] could not read the ticket types (HTTP #{e.status || "-"}); " \ + 'the Tickets collection boots without its attribute columns.' + ) + @attributes = [] + end + + private + + def build + # Read on the boot connection: this happens while Rails is starting, and + # a slow Intercom must not turn that into minutes the operator sits + # through. + @client.fetch_all('ticket_types', boot: true) + .each_with_object({}) { |ticket_type, union| collect(ticket_type, union) } + .values + end + + def collect(ticket_type, union) + type_id = ticket_type['id'].to_s + definitions(ticket_type).each do |definition| + entry = entry_for(definition, union) + next if entry.nil? + + entry.ids_by_ticket_type[type_id] = definition['id'].to_s + end + end + + # The union is keyed by column name rather than by the workspace's own, + # since that is what has to be unique in a schema. Two different attributes + # landing on one column would otherwise share an entry, and the second's + # values would be read under the first's name -- wrong values rather than + # missing ones, which is worse. + def entry_for(definition, union) + name = definition['name'].to_s + # An archived attribute is not offered any more, and a nameless one has + # nothing to be a column of. + return nil if name.empty? || definition['archived'] + + column = column_name_for(name) + return nil if column.empty? + + entry = union[column] + return union[column] = attribute_from(name, column, definition) if entry.nil? + return entry if entry.name == name + + warn_collision(name, entry.name, column) + nil + end + + def attribute_from(name, column, definition) + Attribute.new(name: name, column_name: column, column_type: column_type_for(definition), + data_type: definition['data_type'], ids_by_ticket_type: {}) + end + + def definitions(ticket_type) + return [] unless ticket_type.is_a?(Hash) + + container = ticket_type['ticket_type_attributes'] + return [] unless container.is_a?(Hash) + + list = container['data'] + list.is_a?(Array) ? list : [] + end + + # These are declared per ticket type, and that is what the log calls + # them: the workspace's own vocabulary, and where the operator goes to + # rename the one that was left out. + def attribute_kind = 'ticket' + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/throttle.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/throttle.rb new file mode 100644 index 000000000..fa8fdad3b --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/throttle.rb @@ -0,0 +1,19 @@ +module ForestAdminDatasourceIntercom + # Holds a request until the rate-limit window has room, and feeds the window + # back what the response says about it. A middleware rather than a call in + # each client method: there is one code path for every request here, where the + # client has one per endpoint, and this one also covers the requests the client + # never issues itself -- the replays `retry` performs. + class Throttle < Faraday::Middleware + def initialize(app, limiter:) + super(app) + @limiter = limiter + end + + def call(env) + @limiter.acquire + + @app.call(env).on_complete { |response_env| @limiter.observe(response_env[:response_headers]) } + end + end +end diff --git a/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/version.rb b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/version.rb new file mode 100644 index 000000000..bcf5c5876 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/lib/forest_admin_datasource_intercom/version.rb @@ -0,0 +1,3 @@ +module ForestAdminDatasourceIntercom + VERSION = "0.1.0" +end diff --git a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/client_spec.rb b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/client_spec.rb new file mode 100644 index 000000000..65a9c9d56 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/client_spec.rb @@ -0,0 +1,487 @@ +module ForestAdminDatasourceIntercom + RSpec.describe Client do + subject(:client) { described_class.new(configuration) } + + let(:retry_policy) { RetryPolicy.new(max_retries: 2, interval: 0) } + let(:configuration) { Configuration.new(access_token: 's3cr3t', retry_policy: retry_policy, rate_limiter: nil) } + let(:base) { configuration.url } + + def json(payload, status = 200, headers = {}) + { status: status, + body: payload.is_a?(String) ? payload : payload.to_json, + headers: { 'Content-Type' => 'application/json' }.merge(headers) } + end + + describe 'authentication and version pinning' do + before { stub_request(:get, "#{base}/me").to_return(json({ 'type' => 'admin' })) } + + it 'sends the access token as a bearer token' do + client.me + + expect(WebMock).to have_requested(:get, "#{base}/me") + .with(headers: { 'Authorization' => 'Bearer s3cr3t', 'Accept' => 'application/json' }) + end + + # Without the header the request follows the workspace's own default + # version, which an operator can change on Intercom's side. + it 'pins the API version on every request' do + client.me + + expect(WebMock).to have_requested(:get, "#{base}/me").with(headers: { 'Intercom-Version' => '2.16' }) + end + + it 'advertises a versioned user agent' do + client.me + + expect(WebMock).to have_requested(:get, "#{base}/me") + .with(headers: { 'User-Agent' => "forest_admin_datasource_intercom/#{VERSION}" }) + end + end + + describe '#me' do + it 'returns the admin the token belongs to' do + stub_request(:get, "#{base}/me").to_return(json('type' => 'admin', 'id' => '1', 'email' => 'a@b.test')) + + expect(client.me).to include('id' => '1', 'email' => 'a@b.test') + end + + it 'reaches the regional host it was configured for' do + eu = described_class.new(Configuration.new(access_token: 's3cr3t', region: :eu, rate_limiter: nil)) + stub_request(:get, 'https://api.eu.intercom.io/me').to_return(json('type' => 'admin')) + + eu.me + + expect(WebMock).to have_requested(:get, 'https://api.eu.intercom.io/me') + end + end + + describe 'the version Intercom actually served' do + # Intercom echoes the version it served. A mismatch means the payloads may + # not be the ones this datasource expects, which is worth saying out loud + # -- and worth saying rather than raising: running against a version we + # did not ask for beats not running. + it 'warns when it differs from the pinned one' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + stub_request(:get, "#{base}/me").to_return(json({ 'type' => 'admin' }, 200, 'intercom-version' => '2.14')) + + client.me + + expect(ForestAdminDatasourceIntercom.logger).to have_received(:warn).with(/asked.*2\.16.*served 2\.14/m) + end + + it 'stays quiet when the pin was honoured' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + stub_request(:get, "#{base}/me").to_return(json({ 'type' => 'admin' }, 200, 'intercom-version' => '2.16')) + + client.me + + expect(ForestAdminDatasourceIntercom.logger).not_to have_received(:warn) + end + + it 'stays quiet when Intercom echoes nothing' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + stub_request(:get, "#{base}/me").to_return(json('type' => 'admin')) + + client.me + + expect(ForestAdminDatasourceIntercom.logger).not_to have_received(:warn) + end + end + + describe 'failures' do + it "carries Intercom's status, parsed body and error text" do + body = { 'type' => 'error.list', 'request_id' => 'req_1', + 'errors' => [{ 'code' => 'unauthorized', 'message' => 'Access Token Invalid' }] } + stub_request(:get, "#{base}/me").to_return(json(body, 401)) + + expect { client.me }.to raise_error(APIError) { |error| + expect(error.message).to eq('Intercom API call failed: me: HTTP 401 unauthorized: ' \ + 'Access Token Invalid (request_id: req_1)') + expect(error.status).to eq(401) + expect(error.body).to eq(body) + } + end + + it 'joins the several errors one response can carry' do + body = { 'errors' => [{ 'code' => 'parameter_invalid', 'message' => 'per_page' }, + { 'code' => 'parameter_invalid', 'message' => 'starting_after' }] } + stub_request(:get, "#{base}/me").to_return(json(body, 400)) + + expect { client.me }.to raise_error(APIError, /per_page; parameter_invalid: starting_after/) + end + + it 'falls back to the whole body when the shape is not the documented one' do + stub_request(:get, "#{base}/me").to_return(json({ 'oops' => true }, 500)) + + expect { client.me }.to raise_error(APIError, /\{"oops":true\}/) + end + + it 'keeps a body that is not JSON at all, which is what a gateway answers' do + stub_request(:get, "#{base}/me").to_return(status: 502, body: 'bad gateway') + + expect { client.me }.to raise_error(APIError) { |error| + expect(error.status).to eq(502) + expect(error.body).to eq('bad gateway') + } + end + + # No status to report: the request never reached Intercom, so there is + # nothing of its to surface. + it 'reports a dropped connection without a status' do + stub_request(:get, "#{base}/me").to_raise(Faraday::ConnectionFailed.new('closed')) + + expect { client.me }.to raise_error(APIError) { |error| + expect(error.message).to include('Faraday::ConnectionFailed') + expect(error.status).to be_nil + } + end + + it 'replays a 429 rather than surfacing it' do + stub_request(:get, "#{base}/me") + .to_return(json({ 'errors' => [{ 'code' => 'rate_limit_exceeded' }] }, 429)) + .then.to_return(json('type' => 'admin')) + + expect(client.me).to eq('type' => 'admin') + end + + it 'gives up on a 429 that outlasts the retries, saying which endpoint' do + stub_request(:get, "#{base}/me").to_return(json({ 'errors' => [{ 'code' => 'rate_limit_exceeded' }] }, 429)) + + expect { client.me }.to raise_error(APIError, /me: HTTP 429 rate_limit_exceeded/) + end + + # A body that failed to parse is a payload, not an error: on a 200 it is + # customer content, and this message is shown in the interface and + # collected by whatever watches the agent (R10). + it 'names a body it could not read rather than quoting it' do + stub_request(:get, "#{base}/me") + .to_return(json('Bonjour, voici mon RIB FR76 3000 4000 0500 0012 3456 789')) + + expect { client.me }.to raise_error(APIError) { |error| + expect(error.message).to eq('Intercom API call failed: me: the response could not be read as JSON') + # Faraday hands a parsing error an unfinished response, so there is no + # body to keep here -- which suits this one: the point is that the + # payload does not travel with the error. + expect(error.body).to be_nil + } + end + + # The same guard, one level down: a parser raising on its own would + # otherwise reach the catch-all, whose message is the exception's -- and + # a JSON parser quotes what it choked on. + it 'says as little when a parser raises outside Faraday' do + stub_request(:get, "#{base}/me").to_raise(JSON::ParserError.new("unexpected token 'mon RIB FR76'")) + + expect { client.me }.to raise_error(APIError) { |error| + expect(error.message).to include('could not be read as JSON') + expect(error.message).not_to include('RIB') + } + end + + # Whatever else goes wrong on the way, a caller of this client only ever + # has to rescue APIError -- and the message names the operation, since a + # failure with no endpoint in it is a failure nobody can place. + it 'still names the operation when the failure is not one it expected' do + stub_request(:get, "#{base}/me").to_raise(ArgumentError.new('unexpected')) + + expect { client.me }.to raise_error(APIError, /me: ArgumentError: unexpected/) + end + end + + describe '#list_page' do + def list_body(data, next_page: nil, total: 2) + body = { 'type' => 'list', 'data' => data, 'total_count' => total, + 'pages' => { 'type' => 'pages', 'page' => 1, 'per_page' => 50 } } + body['pages']['next'] = next_page unless next_page.nil? + body + end + + it 'reads the records, the next cursor and the exact count off one response' do + body = list_body([{ 'id' => '1' }], next_page: { 'starting_after' => 'cursor_2' }) + stub_request(:get, "#{base}/conversations").with(query: { 'per_page' => '150' }).to_return(json(body)) + + page = client.list_page('conversations', per_page: 150) + + expect(page.records).to eq([{ 'id' => '1' }]) + expect(page.next_cursor).to eq('cursor_2') + expect(page.total_count).to eq(2) + end + + it 'sends the cursor the previous page advertised' do + stub_request(:get, "#{base}/conversations") + .with(query: { 'per_page' => '50', 'starting_after' => 'cursor_2' }) + .to_return(json(list_body([]))) + + client.list_page('conversations', per_page: 50, starting_after: 'cursor_2') + + expect(WebMock).to have_requested(:get, "#{base}/conversations") + .with(query: { 'per_page' => '50', 'starting_after' => 'cursor_2' }) + end + + it 'bounds the page size before sending it, Intercom refusing rather than clamping' do + stub_request(:get, "#{base}/conversations").with(query: { 'per_page' => '150' }) + .to_return(json(list_body([]))) + + client.list_page('conversations', per_page: 500) + + expect(WebMock).to have_requested(:get, "#{base}/conversations").with(query: { 'per_page' => '150' }) + end + + it 'carries the parameters an endpoint of its own needs' do + stub_request(:get, "#{base}/conversations") + .with(query: { 'per_page' => '150', 'display_as' => 'plaintext' }).to_return(json(list_body([]))) + + client.list_page('conversations', per_page: 150, params: { 'display_as' => 'plaintext' }) + + expect(WebMock).to have_requested(:get, "#{base}/conversations") + .with(query: hash_including('display_as' => 'plaintext')) + end + + # The last page simply carries no `pages.next`, which is what stops a walk. + it 'reports no next cursor on the last page' do + stub_request(:get, "#{base}/conversations").with(query: hash_including({})) + .to_return(json(list_body([{ 'id' => '1' }]))) + + expect(client.list_page('conversations', per_page: 150).next_cursor).to be_nil + end + + # An older API version spells `pages.next` as a url, and one can be served + # despite the pin -- reading the cursor out of it beats taking the page for + # the last one and truncating the answer. + it 'reads the cursor out of a next page spelled as a url' do + url = "#{base}/conversations?per_page=50&starting_after=cursor_9" + stub_request(:get, "#{base}/conversations").with(query: hash_including({})) + .to_return(json(list_body([], next_page: url))) + + expect(client.list_page('conversations', per_page: 50).next_cursor).to eq('cursor_9') + end + + # An advertised page taken for the last one is a silently truncated + # answer, so every unreadable shape is refused rather than dropped. + it 'refuses a next-page url carrying no cursor' do + body = list_body([], next_page: "#{base}/conversations?per_page=50") + stub_request(:get, "#{base}/conversations").with(query: hash_including({})).to_return(json(body)) + + expect { client.list_page('conversations', per_page: 50) } + .to raise_error(APIError, /pages\.next' carries no cursor/) + end + + it 'refuses a next-page url it cannot parse' do + stub_request(:get, "#{base}/conversations").with(query: hash_including({})) + .to_return(json(list_body([], next_page: 'http://[bad'))) + + expect { client.list_page('conversations', per_page: 50) } + .to raise_error(APIError, /pages\.next' carries no cursor/) + end + + it 'refuses a next page it can read neither way, rather than truncating silently' do + stub_request(:get, "#{base}/conversations").with(query: hash_including({})) + .to_return(json(list_body([], next_page: 42))) + + expect { client.list_page('conversations', per_page: 50) } + .to raise_error(APIError, /unexpected response shape.*pages\.next/m) + end + + # `Array()` on the envelope would hand the collection rows built out of + # [key, value] pairs: a page that looks answered and holds nothing. + it 'refuses a response whose data is not a list' do + stub_request(:get, "#{base}/conversations").with(query: hash_including({})) + .to_return(json({ 'type' => 'list', 'data' => { 'id' => '1' } })) + + expect { client.list_page('conversations', per_page: 50) } + .to raise_error(APIError, /unexpected response shape.*'data' is not a list/m) + end + + it 'refuses a response carrying no data at all' do + stub_request(:get, "#{base}/conversations").with(query: hash_including({})) + .to_return(json({ 'type' => 'list', 'total_count' => 0 })) + + expect { client.list_page('conversations', per_page: 50) }.to raise_error(APIError, /'data' is not a list/) + end + + it 'serves an empty page as an empty page, zero being an answer' do + stub_request(:get, "#{base}/conversations").with(query: hash_including({})) + .to_return(json(list_body([], total: 0))) + + expect(client.list_page('conversations', per_page: 50)) + .to have_attributes(records: [], next_cursor: nil, total_count: 0) + end + + # nil rather than 0: zero is an answer, and this is the absence of one. + it 'reports no count when Intercom sends none' do + stub_request(:get, "#{base}/conversations").with(query: hash_including({})) + .to_return(json({ 'type' => 'list', 'data' => [] })) + + expect(client.list_page('conversations', per_page: 50).total_count).to be_nil + end + + it 'reads a page through the boot connection when asked to' do + stub_request(:get, "#{base}/ticket_types").with(query: hash_including({})) + .to_return(json(list_body([{ 'id' => '1' }]))) + + expect(client.list_page('ticket_types', per_page: 50, boot: true).records.size).to eq(1) + end + + it 'names the endpoint when the read fails' do + stub_request(:get, "#{base}/conversations").with(query: hash_including({})) + .to_return(json({ 'errors' => [{ 'code' => 'not_found' }] }, 404)) + + expect { client.list_page('conversations', per_page: 50) } + .to raise_error(APIError, /conversations: HTTP 404 not_found/) + end + end + + describe '#fetch_all' do + it 'reads the records under the key the endpoint uses' do + stub_request(:get, "#{base}/admins") + .to_return(json('type' => 'admin.list', 'admins' => [{ 'id' => '1' }, { 'id' => '2' }])) + + expect(client.fetch_all('admins', list_key: 'admins').size).to eq(2) + end + + # Intercom is not consistent about it: /admins and /teams use their own + # key, /ticket_types the `data` envelope every paginated listing uses. + it 'falls back to the data envelope' do + stub_request(:get, "#{base}/ticket_types").to_return(json('type' => 'list', 'data' => [{ 'id' => '1' }])) + + expect(client.fetch_all('ticket_types')).to eq([{ 'id' => '1' }]) + end + + it 'asks for no page: these endpoints answer whole' do + stub_request(:get, "#{base}/teams").to_return(json('teams' => [])) + + client.fetch_all('teams', list_key: 'teams') + + expect(WebMock).to have_requested(:get, "#{base}/teams").with(query: {}) + end + + # A reference collection read as empty is a state column with no values and + # an assignee shown as a raw id -- worse than a failure naming the shape. + it 'refuses a response holding neither key' do + stub_request(:get, "#{base}/admins").to_return(json('type' => 'admin.list', 'admins' => { 'id' => '1' })) + + expect { client.fetch_all('admins', list_key: 'admins') } + .to raise_error(APIError, /neither 'admins' nor 'data' is a list/) + end + + it 'reads an empty body as no record' do + stub_request(:get, "#{base}/admins").to_return(status: 200, body: '') + + expect(client.fetch_all('admins', list_key: 'admins')).to eq([]) + end + + # No pagination parameter in the specification is not a promise that a + # large workspace answers in one response, and a truncated reference + # collection would show an operator a state list missing its last states. + it 'follows a cursor if one is advertised anyway' do + stub_request(:get, "#{base}/tags").with(query: {}) + .to_return(json('data' => [{ 'id' => '1' }], + 'pages' => { 'next' => { 'starting_after' => 'c2' } })) + stub_request(:get, "#{base}/tags").with(query: { 'starting_after' => 'c2' }) + .to_return(json('data' => [{ 'id' => '2' }])) + + expect(client.fetch_all('tags').map { |tag| tag['id'] }).to eq(%w[1 2]) + end + + it 'stops at its page cap and says what it left out' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + stub_request(:get, "#{base}/tags").with(query: hash_including({})) + .to_return(json('data' => [{ 'id' => '1' }], + 'pages' => { 'next' => { 'starting_after' => 'c' } })) + + client.fetch_all('tags') + + expect(WebMock).to have_requested(:get, "#{base}/tags") + .with(query: hash_including({})).times(described_class::MAX_COLLECTED_PAGES) + expect(ForestAdminDatasourceIntercom.logger).to have_received(:warn).with(/Stopped reading tags/) + end + + it 'reads through the boot connection when asked to' do + stub_request(:get, "#{base}/ticket_types").to_return(json('data' => [])) + + expect(client.fetch_all('ticket_types', boot: true)).to eq([]) + end + + it 'names the endpoint when the read fails' do + stub_request(:get, "#{base}/admins").to_return(json({ 'errors' => [{ 'code' => 'forbidden' }] }, 403)) + + expect { client.fetch_all('admins', list_key: 'admins') } + .to raise_error(APIError, /admins: HTTP 403 forbidden/) + end + end + + describe '.bounded_per_page' do + # Intercom answers `invalid_per_page` past 150 instead of clamping, so a + # page size is bounded before it is sent or the list view breaks. + it 'caps a page size at what Intercom accepts' do + expect(described_class.bounded_per_page(200)).to eq(150) + end + + it 'leaves an acceptable size alone' do + expect(described_class.bounded_per_page(50)).to eq(50) + end + + it 'asks for one record rather than none, an empty page being no answer' do + expect([described_class.bounded_per_page(0), described_class.bounded_per_page(-5)]).to eq([1, 1]) + end + end + + describe 'the boot connection' do + # What is read while the datasource is being constructed waits far less + # than a request that already has a page on screen: the wait there is + # minutes of Rails boot the operator sits through. + it 'honours the configured boot timeouts' do + booted = described_class.new(Configuration.new(access_token: 's3cr3t', boot_open_timeout: 1, boot_timeout: 2)) + conn = booted.send(:boot_connection) + + expect(conn.options).to have_attributes(open_timeout: 1, timeout: 2) + end + + it 'keeps the patience of a regular request on the regular connection' do + expect(client.send(:connection).options).to have_attributes(open_timeout: 5, timeout: 30) + end + + it 'reads through it when asked to' do + stub_request(:get, "#{base}/me").to_return(json('type' => 'admin')) + + expect(client.me(boot: true)).to eq('type' => 'admin') + end + end + + describe 'pacing' do + let(:limiter) { instance_double(RateLimiter, acquire: nil, observe: nil) } + let(:paced) do + described_class.new(Configuration.new(access_token: 's3cr3t', retry_policy: retry_policy, + rate_limiter: limiter)) + end + + it 'asks the limiter for room, and feeds it the window back' do + stub_request(:get, "#{base}/me").to_return(json({ 'type' => 'admin' }, 200, + 'x-ratelimit-remaining' => '1666')) + + paced.me + + expect(limiter).to have_received(:acquire) + expect(limiter).to have_received(:observe).with(hash_including('x-ratelimit-remaining' => '1666')) + end + + # The throttle sits inside the retry, so a replay waits for the window + # like a first attempt rather than going out on a budget already spent. + it 'asks again for every replay, not once per call' do + stub_request(:get, "#{base}/me") + .to_return(json({}, 429)).then.to_return(json('type' => 'admin')) + + paced.me + + expect(limiter).to have_received(:acquire).twice + end + end + + describe '#inspect' do + it 'never prints the token its connections carry' do + expect(client.inspect).to include(base) + expect(client.inspect).not_to include('s3cr3t') + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/admin_spec.rb b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/admin_spec.rb new file mode 100644 index 000000000..9ef97e218 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/admin_spec.rb @@ -0,0 +1,119 @@ +module ForestAdminDatasourceIntercom + RSpec.describe Collections::Admin do + subject(:collection) { described_class.new(datasource) } + + let(:datasource) { Datasource.new(access_token: 's3cr3t', rate_limiter: nil) } + let(:base) { datasource.configuration.url } + + def filter + ForestAdminDatasourceToolkit::Components::Query::Filter.new + end + + def json(payload, status = 200) + { status: status, body: payload.to_json, headers: { 'Content-Type' => 'application/json' } } + end + + def stub_admins(*admins) + stub_request(:get, "#{base}/admins").to_return(json('type' => 'admin.list', 'admins' => admins)) + end + + def stub_teams(*teams) + stub_request(:get, "#{base}/teams").to_return(json('type' => 'team.list', 'teams' => teams)) + end + + it 'is named IntercomAdmin' do + expect(collection.name).to eq('IntercomAdmin') + end + + it 'exposes the columns an ops lead reads before assigning anything' do + expect(collection.fields.keys) + .to eq(%w[id name email job_title away_mode_enabled away_mode_reassign has_inbox_seat team_names teams]) + end + + it 'declares id as the primary key' do + expect(collection.fields['id']).to have_attributes(is_primary_key: true, column_type: 'String') + end + + # `/admins` puts its records under `admins` rather than under the `data` + # envelope the paginated listings use. + it 'reads the endpoint and flattens the teammate' do + stub_admins('type' => 'admin', 'id' => '1', 'name' => 'Alice', 'email' => 'alice@acme.test', + 'job_title' => 'Support', 'away_mode_enabled' => true, 'away_mode_reassign' => false, + 'has_inbox_seat' => true, 'team_ids' => [814_865]) + stub_teams('id' => '814865', 'name' => 'Support') + + expect(collection.list(nil, filter, nil)) + .to eq([{ 'id' => '1', 'name' => 'Alice', 'email' => 'alice@acme.test', 'job_title' => 'Support', + 'away_mode_enabled' => true, 'away_mode_reassign' => false, 'has_inbox_seat' => true, + 'team_names' => ['Support'] }]) + end + + # A projection naming nothing asks for every declared column, which is how + # `project` reads it -- so the enrichment has to read it that way too, or the + # one column the projection publishes comes back nil. + it 'fills the derived names when the projection names no column at all' do + stub_admins('id' => '1', 'name' => 'Alice', 'team_ids' => [814_865]) + stub_teams('id' => '814865', 'name' => 'Support') + + expect(collection.list(nil, filter, nil).first['team_names']).to eq(['Support']) + end + + # Intercom types a team id as a number here and as a string on the team + # itself; a filter value from Forest always arrives as a string. Both sides + # of the membership therefore carry the same id, without which the relation + # would resolve to nothing rather than to an error. + it 'stringifies the ids so both sides of the membership match' do + stub_admins('id' => 493_881, 'team_ids' => [814_865]) + stub_teams('id' => '814865', 'name' => 'Support') + + expect(collection.list(nil, filter, %w[id team_names]).first) + .to eq({ 'id' => '493881', 'team_names' => ['Support'] }) + end + + describe 'the teams of a teammate' do + it 'names them for the whole page in one read, rather than one read per row' do + stub_admins({ 'id' => '1', 'team_ids' => [814_865] }, { 'id' => '2', 'team_ids' => [814_865, 814_866] }) + stub_teams({ 'id' => '814865', 'name' => 'Support' }, { 'id' => '814866', 'name' => 'Billing' }) + + expect(collection.list(nil, filter, %w[id team_names])) + .to eq([{ 'id' => '1', 'team_names' => ['Support'] }, + { 'id' => '2', 'team_names' => %w[Support Billing] }]) + expect(WebMock).to have_requested(:get, "#{base}/teams").once + end + + # A column nobody asked for costs no request. + it 'reads nothing when no projection asks for the names' do + stub_admins('id' => '1', 'team_ids' => [814_865]) + + collection.list(nil, filter, %w[id name]) + + expect(WebMock).not_to have_requested(:get, "#{base}/teams") + end + + # A missing permission costs the column, never the page -- and never the + # relation, which reads the teams from the other side. + it 'leaves the names empty when the teams cannot be read' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + stub_admins('id' => '1', 'team_ids' => [814_865]) + stub_request(:get, "#{base}/teams").to_return(json({ 'type' => 'error.list' }, 403)) + + expect(collection.list(nil, filter, %w[id team_names]).first) + .to eq({ 'id' => '1', 'team_names' => [] }) + end + + it 'reads a teammate with no team as one with no team, not as one with a null' do + stub_admins('id' => '1', 'team_ids' => nil) + stub_teams('id' => '814865', 'name' => 'Support') + + expect(collection.list(nil, filter, %w[team_names]).first['team_names']).to eq([]) + end + + it 'is a many-to-many through the membership collection, read-only' do + expect(collection.fields['teams']) + .to have_attributes(type: 'ManyToMany', foreign_collection: 'IntercomTeam', + through_collection: 'IntercomTeamMembership', + origin_key: 'admin_id', foreign_key: 'team_id', is_read_only: true) + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/company_spec.rb b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/company_spec.rb new file mode 100644 index 000000000..dec029a7f --- /dev/null +++ b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/company_spec.rb @@ -0,0 +1,490 @@ +module ForestAdminDatasourceIntercom + # The offset tier is exercised through Companies, the one collection Intercom + # paginates that way -- and the only one it does not search at all. + RSpec.describe Collections::Company do + subject(:collection) { datasource.get_collection('IntercomCompany') } + + let(:datasource) { Datasource.new(access_token: 's3cr3t', rate_limiter: nil) } + let(:base) { datasource.configuration.url } + let(:operators) { ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Operators } + + def json(payload, status = 200) + { status: status, body: payload.to_json, headers: { 'Content-Type' => 'application/json' } } + end + + def leaf(field, operator, value = nil) + ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Nodes::ConditionTreeLeaf + .new(field, operator, value) + end + + def branch(aggregator, *conditions) + ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Nodes::ConditionTreeBranch + .new(aggregator, conditions) + end + + def filter(condition_tree: nil, page: nil, sort: nil) + ForestAdminDatasourceToolkit::Components::Query::Filter.new(condition_tree: condition_tree, page: page, + sort: sort) + end + + def page(offset, limit) + ForestAdminDatasourceToolkit::Components::Query::Page.new(offset: offset, limit: limit) + end + + def sort(*clauses) + ForestAdminDatasourceToolkit::Components::Query::Sort.new(clauses) + end + + def count(condition_tree: nil) + collection.aggregate(nil, filter(condition_tree: condition_tree), + ForestAdminDatasourceToolkit::Components::Query::Aggregation.new(operation: 'Count')) + end + + def company(id, overrides = {}) + { 'type' => 'company', 'id' => id, 'company_id' => "erp-#{id}", 'name' => "Company #{id}", + 'plan' => { 'type' => 'plan', 'id' => '9', 'name' => 'Paid' }, 'size' => 85, + 'industry' => 'Manufacturing', 'website' => 'https://acme.test', 'monthly_spend' => 49, + 'session_count' => 26, 'user_count' => 10, 'created_at' => 1_700_000_000, + 'updated_at' => 1_700_000_600, 'last_request_at' => 1_700_000_900, + 'remote_created_at' => 1_394_531_169, 'custom_attributes' => {} }.merge(overrides) + end + + def stub_page(*companies, page: 1, per_page: 15, total_pages: 1, total: nil) + stub_request(:post, "#{base}/companies/list") + .with(query: { 'page' => page.to_s, 'per_page' => per_page.to_s }) + .to_return(json('type' => 'list', 'data' => companies, 'total_count' => total || companies.size, + 'pages' => { 'type' => 'pages', 'page' => page, 'per_page' => per_page, + 'total_pages' => total_pages })) + end + + def rows(projection = %w[id], **options) + collection.list(nil, filter(**options), projection) + end + + describe 'schema' do + it 'is named IntercomCompany' do + expect(collection.name).to eq('IntercomCompany') + end + + it 'publishes the columns of an account' do + expect(collection.fields.keys) + .to include('id', 'company_id', 'name', 'plan_name', 'size', 'industry', 'website', + 'monthly_spend', 'user_count', 'session_count', 'created_at', 'remote_created_at') + end + + it 'publishes every column read-only and unsortable' do + columns = collection.fields.values.grep(ForestAdminDatasourceToolkit::Schema::ColumnSchema) + + expect(columns.map(&:is_read_only).uniq).to eq([true]) + expect(columns.map(&:is_sortable).uniq).to eq([false]) + end + + # Four lookups is what `GET /companies` answers, and two of them name a + # column of this collection. A tag and a segment are collections of their + # own, and filtering by them belongs with the lot that adds them. + it 'offers a filter on the two keys Intercom looks a company up by' do + expect(collection.fields['name'].filter_operators).to eq(%w[equal]) + expect(collection.fields['company_id'].filter_operators).to eq(%w[equal]) + expect(collection.fields['id'].filter_operators).to eq(%w[equal in]) + end + + it 'offers no filter on anything else' do + %w[plan_name size industry website monthly_spend user_count created_at].each do |column| + expect(collection.fields[column].filter_operators).to be_empty, "#{column} advertises a filter" + end + end + + it 'declares the contacts of the account' do + expect(collection.fields['contacts'].origin_key).to eq('company_id') + end + + it 'is countable, total_count being exact on every answer' do + expect(collection.is_countable?).to be(true) + end + end + + describe 'the custom attributes' do + before do + stub_data_attributes('company', + { 'name' => 'arr', 'data_type' => 'float', 'custom' => true, + 'api_writable' => true, 'archived' => false }) + end + + it 'publishes one column per attribute, typed and display-only' do + expect(collection.fields['arr'].column_type).to eq('Number') + expect(collection.fields['arr'].filter_operators).to be_empty + end + + it 'skips an attribute whose name a native column already carries, and says which' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + stub_data_attributes('company', { 'name' => 'name', 'data_type' => 'string', 'custom' => true, + 'api_writable' => true, 'archived' => false }) + stub_page(company('co1', 'custom_attributes' => { 'name' => 'Not the account name' }), + page: 1, per_page: 15) + + expect(rows(nil, page: page(0, 15)).first['name']).to eq('Company co1') + expect(ForestAdminDatasourceIntercom.logger) + .to have_received(:warn).with(/skips the company attribute "name"/) + end + + it 'reads a date attribute as a date' do + stub_data_attributes('company', { 'name' => 'renewal', 'data_type' => 'date', 'custom' => true, + 'api_writable' => true, 'archived' => false }) + stub_page(company('co1', 'custom_attributes' => { 'renewal' => 1_700_000_000 }), page: 1, per_page: 15) + + expect(rows(%w[id renewal], page: page(0, 15)).first['renewal']).to eq('2023-11-14T22:13:20Z') + end + + it 'reads its value off the payload' do + stub_page(company('co1', 'custom_attributes' => { 'arr' => 12_000 })) + + expect(rows(%w[id arr], page: page(0, 15))).to eq([{ 'id' => 'co1', 'arr' => 12_000 }]) + end + end + + describe 'pagination by offset' do + # The one place R1 does not apply: Intercom counts pages, which is what a + # list view asks for. No cursor walked, no page read to be thrown away. + it 'asks for the page the window names, in one request' do + stub_page(company('co1'), page: 3, per_page: 15, total_pages: 3) + + expect(rows(%w[id], page: page(30, 15))).to eq([{ 'id' => 'co1' }]) + expect(WebMock).to have_requested(:post, "#{base}/companies/list") + .with(query: { 'page' => '3', 'per_page' => '15' }).once + end + + # An offset that does not fall on a page boundary is served exactly, by + # reading the page it lands in and the next -- never by rounding the + # window to something the API likes better. + it 'reads across two pages when the window straddles them' do + stub_page(company('a'), company('b'), page: 1, per_page: 2, total_pages: 3) + stub_page(company('c'), company('d'), page: 2, per_page: 2, total_pages: 3) + + expect(rows(%w[id], page: page(1, 2)).map { |row| row['id'] }).to eq(%w[b c]) + end + + it 'stops at the last page rather than asking for one past it' do + stub_page(company('a'), page: 1, per_page: 15, total_pages: 1) + + expect(rows(%w[id], page: page(0, 15)).size).to eq(1) + expect(WebMock).not_to have_requested(:post, "#{base}/companies/list") + .with(query: { 'page' => '2', 'per_page' => '15' }) + end + + it 'stops on a page Intercom answers empty' do + stub_page(page: 1, per_page: 2, total_pages: 9) + + expect(rows(%w[id], page: page(0, 2))).to be_empty + end + + # A read naming no window -- a segment, a customizer -- is the only one + # that can run long, and it is the only one this bounds. + it 'reads full pages when the read names no window' do + stub_page(company('a'), page: 1, per_page: 150, total_pages: 1) + + expect(rows(%w[id]).map { |row| row['id'] }).to eq(%w[a]) + end + + it 'stops such a read after the pages it allows, and says so' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + (1..10).each { |number| stub_page(company("c#{number}"), page: number, per_page: 150, total_pages: 99) } + + expect(rows(%w[id]).size).to eq(10) + expect(ForestAdminDatasourceIntercom.logger) + .to have_received(:warn).with(/Stopped reading IntercomCompany.*named no window of its own/) + end + + # A window is its own bound, so the page cap must not apply to it: a + # window needing eleven pages would come back one page short while the + # warning blamed it for naming no window -- which it did name. The record + # budget is what bounds a window nothing sane asked for, and it says so + # in its own words. + it 'reads past the page cap for a window that needs it' do + (1..12).each { |number| stub_page(company("c#{number}"), page: number, per_page: 150, total_pages: 12) } + + expect(rows(%w[id], page: page(0, 1800)).size).to eq(12) + end + + it 'stops a window on the record budget, and says which cap it hit' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + stub_const('ForestAdminDatasourceIntercom::Collections::OffsetCollection::MAX_COLLECTED_RECORDS', 2) + (1..4).each { |number| stub_page(company("c#{number}"), page: number, per_page: 150, total_pages: 99) } + + expect(rows(%w[id], page: page(0, 1800)).size).to eq(2) + expect(ForestAdminDatasourceIntercom.logger) + .to have_received(:warn).with(/named a window larger than one answer may hold/) + end + + it 'counts what Intercom counted, in one request' do + stub_page(company('a'), page: 1, per_page: 1, total_pages: 40, total: 597) + + expect(count).to eq([{ 'group' => {}, 'value' => 597 }]) + end + + # Counting the pages read would answer a fraction of the collection as if + # it were the whole of it. + it 'refuses to count a listing Intercom answered without a total' do + stub_request(:post, "#{base}/companies/list").with(query: hash_including({})) + .to_return(json('type' => 'list', 'data' => [company('a')], + 'pages' => {})) + + expect { count }.to raise_error(UnsupportedOperatorError, /cannot be counted/) + end + end + + describe 'the four lookups, and everything past them' do + it 'reads a record detail through its own endpoint' do + stub_request(:get, "#{base}/companies/co1").to_return(json(company('co1'))) + + expect(rows(%w[id], condition_tree: leaf('id', operators::EQUAL, 'co1'))).to eq([{ 'id' => 'co1' }]) + end + + it 'reads a set of ids one request each' do + stub_request(:get, "#{base}/companies/co1").to_return(json(company('co1'))) + stub_request(:get, "#{base}/companies/co2").to_return(json(company('co2'))) + + expect(rows(%w[id], condition_tree: leaf('id', operators::IN, %w[co1 co2])).map { |row| row['id'] }) + .to eq(%w[co1 co2]) + end + + it 'reads a company the token can no longer reach as no record' do + stub_request(:get, "#{base}/companies/co1").to_return(json({ 'type' => 'error.list' }, 404)) + + expect(rows(%w[id], condition_tree: leaf('id', operators::EQUAL, 'co1'))).to be_empty + end + + it 'raises on a failure that is not a missing record' do + stub_request(:get, "#{base}/companies/co1").to_return(json({ 'type' => 'error.list' }, 500)) + + expect { rows(%w[id], condition_tree: leaf('id', operators::EQUAL, 'co1')) }.to raise_error(APIError) + end + + it 'truncates a set of ids larger than it will read, and says so' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + stub_request(:get, %r{#{base}/companies/co\d+}).to_return(json(company('co1'))) + + rows(%w[id], condition_tree: leaf('id', operators::IN, (1..30).map { |n| "co#{n}" })) + + expect(ForestAdminDatasourceIntercom.logger).to have_received(:warn).with(/asked for 30 records by id/) + end + + it 'counts the records the ids named' do + stub_request(:get, "#{base}/companies/co1").to_return(json(company('co1'))) + + expect(count(condition_tree: leaf('id', operators::EQUAL, 'co1'))).to eq([{ 'group' => {}, 'value' => 1 }]) + end + + # Counting a set of ids means reading it, so past what a bulk read fetches + # the count is refused rather than answered with the number the truncation + # left: this collection advertises an exact count, and 25 where the + # question named forty records is not one. + it 'refuses to count more ids than it will read' do + expect { count(condition_tree: leaf('id', operators::IN, (1..40).map { |n| "co#{n}" })) } + .to raise_error(UnsupportedOperatorError, /cannot count 40 records by id/) + end + + # The `in` this comes from is a membership: it matches a record once, + # where a read by id would fetch a repeated value twice and hand it back + # as two rows carrying one id. + it 'reads a value named twice once' do + stub_request(:get, "#{base}/companies/co1").to_return(json(company('co1'))) + + expect(rows(%w[id], condition_tree: leaf('id', operators::IN, %w[co1 co1]))).to eq([{ 'id' => 'co1' }]) + expect(WebMock).to have_requested(:get, "#{base}/companies/co1").once + end + end + + # A relation pointing at this collection resolves it by id, and Intercom has + # no bulk read for a company: the fan-out is sliced by what one batch holds + # and refused past what the tier resolves at all, rather than answered with a + # nil where an account exists. + describe 'as the target of a relation' do + subject(:contacts) { datasource.get_collection('IntercomContact') } + + # An unfiltered, unsorted contact list reads the listing endpoint; the + # search is for the routes that filter. + def stub_contact_list(*records) + stub_request(:get, "#{base}/contacts") + .with(query: hash_including({})) + .to_return(json('type' => 'list', 'data' => records, 'total_count' => records.size, 'pages' => {})) + end + + def contact_of(company_id) + { 'type' => 'contact', 'id' => "c#{company_id}", + 'companies' => { 'type' => 'list', 'data' => [{ 'type' => 'company', 'id' => company_id }], + 'total_count' => 1 } } + end + + it 'slices the fan-out into batches rather than truncating it' do + stub_contact_list(*(1..30).map { |n| contact_of("co#{n}") }) + stub_request(:get, %r{#{base}/companies/co\d+}).to_return do |request| + json(company(request.uri.path.split('/').last)) + end + + rows = contacts.list(nil, filter, %w[id company:name]) + + expect(rows.filter_map { |row| row['company'] }.size).to eq(30) + end + + it 'refuses a fan-out past what it resolves, naming the relation' do + stub_contact_list(*(1..120).map { |n| contact_of("co#{n}") }) + + expect { contacts.list(nil, filter, %w[id company:name]) } + .to raise_error(UnsupportedOperatorError, /cannot resolve company over this read: it names 120/) + end + + # `GET /companies?name=` answers the company itself where a listing would + # answer an envelope: a record is read as a page of one rather than as a + # shape every caller has to test for. + it 'looks a company up by name, and reads the record Intercom answers' do + stub_request(:get, "#{base}/companies").with(query: { 'name' => 'Acme' }) + .to_return(json(company('co1'))) + + expect(rows(%w[id], condition_tree: leaf('name', operators::EQUAL, 'Acme'))).to eq([{ 'id' => 'co1' }]) + end + + it 'looks one up by the identifier the workspace gave it' do + stub_request(:get, "#{base}/companies").with(query: { 'company_id' => 'erp-co1' }) + .to_return(json('type' => 'list', 'data' => [company('co1')], + 'total_count' => 1, 'pages' => {})) + + expect(rows(%w[id], condition_tree: leaf('company_id', operators::EQUAL, 'erp-co1'))) + .to eq([{ 'id' => 'co1' }]) + end + + it 'counts what a lookup found' do + stub_request(:get, "#{base}/companies").with(query: { 'name' => 'Acme' }) + .to_return(json(company('co1'))) + + expect(count(condition_tree: leaf('name', operators::EQUAL, 'Acme'))) + .to eq([{ 'group' => {}, 'value' => 1 }]) + end + + # Intercom answers this route with a 404 where a search endpoint would + # answer an empty list, and a filter matching nothing is the most + # ordinary thing a list view does: it reads as no record rather than as a + # failure the operator sees as "Unexpected error". + it 'answers a lookup naming no record with an empty page' do + stub_request(:get, "#{base}/companies").with(query: { 'name' => 'Nope' }) + .to_return(json({ 'type' => 'error.list', + 'errors' => [{ 'code' => 'company_not_found', + 'message' => 'Company Not Found' }] }, + 404)) + + expect(rows(%w[id], condition_tree: leaf('name', operators::EQUAL, 'Nope'))).to eq([]) + end + + it 'counts a lookup naming no record as none' do + stub_request(:get, "#{base}/companies").with(query: { 'name' => 'Nope' }) + .to_return(json({ 'type' => 'error.list', + 'errors' => [{ 'code' => 'company_not_found', + 'message' => 'Company Not Found' }] }, + 404)) + + expect(count(condition_tree: leaf('name', operators::EQUAL, 'Nope'))) + .to eq([{ 'group' => {}, 'value' => 0 }]) + end + + # A 404 is the only status read as an absence: anything else is a failure + # this must not answer an empty page to. + it 'raises on a lookup Intercom refused for another reason' do + stub_request(:get, "#{base}/companies").with(query: { 'name' => 'Acme' }) + .to_return(json({ 'type' => 'error.list' }, 500)) + + expect { rows(%w[id], condition_tree: leaf('name', operators::EQUAL, 'Acme')) } + .to raise_error(APIError, /HTTP 500/) + end + + it 'reports a lookup Intercom answered with more than one page' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + stub_request(:get, "#{base}/companies").with(query: { 'name' => 'Acme' }) + .to_return(json('type' => 'list', 'data' => [company('co1')], + 'pages' => { 'next' => { 'starting_after' => 'zzz' } })) + + rows(%w[id], condition_tree: leaf('name', operators::EQUAL, 'Acme')) + + expect(ForestAdminDatasourceIntercom.logger).to have_received(:warn).with(/advertised more records/) + end + + # A filter this collection cannot look up is refused by name. A page + # served in answer to a filter it ignored is the one failure this + # datasource is built to avoid. + it 'refuses a filter on a column Intercom does not look up' do + expect { rows(%w[id], condition_tree: leaf('industry', operators::EQUAL, 'Manufacturing')) } + .to raise_error(UnsupportedOperatorError, /cannot filter "industry".*name, company_id alone/m) + end + + it 'refuses an operator the lookup has no equivalent for' do + expect { rows(%w[id], condition_tree: leaf('name', operators::CONTAINS, 'Acm')) } + .to raise_error(UnsupportedOperatorError, /cannot filter "name"/) + end + + it 'refuses a combination, the lookup answering one value at a time' do + expect do + rows(%w[id], condition_tree: branch('And', leaf('name', operators::EQUAL, 'Acme'), + leaf('company_id', operators::EQUAL, 'erp-co1'))) + end.to raise_error(UnsupportedOperatorError, /one exact value at a time/) + end + + it 'refuses to count what it refuses to list' do + expect { count(condition_tree: leaf('industry', operators::EQUAL, 'Manufacturing')) } + .to raise_error(UnsupportedOperatorError, /cannot filter "industry"/) + end + end + + describe 'what it will not do' do + it 'refuses to group, Intercom exposing no aggregate endpoint' do + expect do + collection.aggregate(nil, filter, ForestAdminDatasourceToolkit::Components::Query::Aggregation + .new(operation: 'Count', groups: [{ field: 'industry' }])) + end.to raise_error(UnsupportedOperatorError, /can only be counted/) + end + + # There is no order parameter on this listing at all, so an order asked + # for and not applied is reported here or nowhere. + it 'reports an order it cannot apply' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + stub_page(company('a'), page: 1, per_page: 15) + + rows(%w[id], page: page(0, 15), sort: sort({ field: 'name', ascending: true })) + + expect(ForestAdminDatasourceIntercom.logger).to have_received(:warn).with(/takes no order on this listing/) + end + + it 'says nothing of the ascending primary-key order the agent injects' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + stub_page(company('a'), page: 1, per_page: 15) + + rows(%w[id], page: page(0, 15), sort: sort({ field: 'id', ascending: true })) + + expect(ForestAdminDatasourceIntercom.logger).not_to have_received(:warn) + end + + it 'reports an explicit descending order on the key, which it cannot apply either' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + stub_page(company('a'), page: 1, per_page: 15) + + rows(%w[id], page: page(0, 15), sort: sort({ field: 'id', ascending: false })) + + expect(ForestAdminDatasourceIntercom.logger).to have_received(:warn).with(/sort on id/) + end + end + + describe 'the row' do + it 'flattens the payload, the plan read off the object that carries it' do + stub_page(company('co1'), page: 1, per_page: 15) + + expect(rows(nil, page: page(0, 15)).first) + .to include('id' => 'co1', 'company_id' => 'erp-co1', 'name' => 'Company co1', 'plan_name' => 'Paid', + 'size' => 85, 'monthly_spend' => 49, 'user_count' => 10, + 'created_at' => '2023-11-14T22:13:20Z', 'remote_created_at' => '2014-03-11T09:46:09Z') + end + + it 'reads a company carrying no plan without failing' do + stub_page(company('co1', 'plan' => nil), page: 1, per_page: 15) + + expect(rows(nil, page: page(0, 15)).first).to include('plan_name' => nil) + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/contact_spec.rb b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/contact_spec.rb new file mode 100644 index 000000000..fc9926424 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/contact_spec.rb @@ -0,0 +1,565 @@ +module ForestAdminDatasourceIntercom + RSpec.describe Collections::Contact do + subject(:collection) { datasource.get_collection('IntercomContact') } + + let(:datasource) { Datasource.new(access_token: 's3cr3t', rate_limiter: nil) } + let(:base) { datasource.configuration.url } + let(:operators) { ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Operators } + + def json(payload, status = 200) + { status: status, body: payload.to_json, headers: { 'Content-Type' => 'application/json' } } + end + + def leaf(field, operator, value = nil) + ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Nodes::ConditionTreeLeaf + .new(field, operator, value) + end + + def branch(aggregator, *conditions) + ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Nodes::ConditionTreeBranch + .new(aggregator, conditions) + end + + def filter(condition_tree: nil, page: nil, sort: nil, search: nil) + ForestAdminDatasourceToolkit::Components::Query::Filter.new(condition_tree: condition_tree, page: page, + sort: sort, search: search) + end + + def page(offset, limit) + ForestAdminDatasourceToolkit::Components::Query::Page.new(offset: offset, limit: limit) + end + + def sort(*clauses) + ForestAdminDatasourceToolkit::Components::Query::Sort.new(clauses) + end + + def contact(id, overrides = {}) + { 'type' => 'contact', 'id' => id, 'role' => 'user', 'name' => "Contact #{id}", + 'email' => "#{id}@acme.test", 'phone' => nil, 'external_id' => "ext-#{id}", 'owner_id' => 493_881, + 'created_at' => 1_700_000_000, 'updated_at' => 1_700_000_600, 'session_count' => 3, + 'unsubscribed_from_emails' => false, 'has_hard_bounced' => false, 'marked_email_as_spam' => false, + 'browser' => 'chrome', 'browser_language' => 'fr', 'os' => 'OS X', 'language_override' => nil, + 'location' => { 'type' => 'location', 'country' => 'France', 'region' => 'IdF', 'city' => 'Paris' }, + 'companies' => { 'type' => 'list', 'data' => [{ 'type' => 'company', 'id' => 'co1' }], + 'total_count' => 2 }, + 'custom_attributes' => {} }.merge(overrides) + end + + def stub_list(*contacts, cursor: nil) + pages = cursor ? { 'next' => { 'starting_after' => cursor } } : {} + stub_request(:get, "#{base}/contacts").with(query: hash_including({})) + .to_return(json('type' => 'list', 'data' => contacts, + 'total_count' => contacts.size, 'pages' => pages)) + end + + def stub_search(*contacts, total: nil) + stub_request(:post, "#{base}/contacts/search") + .to_return(json('type' => 'list', 'data' => contacts, 'total_count' => total || contacts.size, + 'pages' => {})) + end + + def rows(projection = %w[id], **options) + collection.list(nil, filter(**options), projection) + end + + describe 'schema' do + it 'is named IntercomContact' do + expect(collection.name).to eq('IntercomContact') + end + + it 'publishes the columns of a contact, the account it belongs to included' do + expect(collection.fields.keys) + .to include('id', 'role', 'name', 'email', 'email_domain', 'phone', 'external_id', 'avatar', + 'owner_id', 'company_id', 'company_count', 'session_count', 'created_at', + 'last_seen_at', 'unsubscribed_from_emails', 'location_country') + end + + it 'publishes every column read-only, this lot writing nothing' do + columns = collection.fields.values.grep(ForestAdminDatasourceToolkit::Schema::ColumnSchema) + + expect(columns.map(&:is_read_only).uniq).to eq([true]) + end + + # The measured asymmetry: `/contacts/search` refuses `>=`, `<=`, `!=` and + # `IN` on a date where the other two endpoints take them. A Date column + # publishes the two bounds alone anyway -- declaring `equal` would make + # the toolkit republish `in`, which its own validator then refuses + # (PRD-989) -- so what the restriction really has to guarantee is that + # nothing wider reaches the wire. + it 'offers the two bounds on a date, and nothing the endpoint refuses' do + expect(collection.fields['created_at'].filter_operators).to eq(%w[greater_than less_than]) + expect(collection.fields['last_seen_at'].filter_operators).to eq(%w[greater_than less_than]) + end + + it 'offers on a text column exactly what the table measured' do + expect(collection.fields['email'].filter_operators) + .to eq(%w[equal not_equal contains i_contains not_contains starts_with ends_with]) + expect(collection.fields['role'].filter_operators).to eq(%w[equal not_equal]) + end + + # A column the table does not carry advertises nothing, which is how a + # refusal is spelled in a schema. + it 'advertises no filter on a column the endpoint does not filter' do + %w[avatar company_id company_count session_count].each do |column| + expect(collection.fields[column].filter_operators).to be_empty, "#{column} advertises a filter" + end + end + + # The one collection of the whole API Intercom sorts, and the reason this + # tier reads a `sortable` flag off the measured table at all. + it 'is sortable on the columns the table measured, and on no others' do + sortable = collection.fields.select { |_, f| f.respond_to?(:is_sortable) && f.is_sortable }.keys + + expect(sortable).to contain_exactly('name', 'email', 'created_at', 'updated_at', 'signed_up_at', + 'last_seen_at', 'last_contacted_at', 'last_replied_at') + end + + it 'declares the relations the 360 degrees is walked through' do + expect(collection.fields['owner']).to be_a(ForestAdminDatasourceToolkit::Schema::Relations::ManyToOneSchema) + expect(collection.fields['company']).to be_a(ForestAdminDatasourceToolkit::Schema::Relations::ManyToOneSchema) + expect(collection.fields['conversations']) + .to be_a(ForestAdminDatasourceToolkit::Schema::Relations::OneToManySchema) + expect(collection.fields['tickets'].origin_key).to eq('contact_id') + end + + it 'is countable, total_count being exact on every answer' do + expect(collection.is_countable?).to be(true) + end + + it 'is searchable on the address an ops team types' do + expect(collection.is_searchable?).to be(true) + end + end + + describe 'the custom attributes' do + subject(:collection) { datasource.get_collection('IntercomContact') } + + before do + stub_data_attributes('contact', + { 'name' => 'paid_subscriber', 'data_type' => 'boolean', 'custom' => true, + 'api_writable' => true, 'archived' => false }, + { 'name' => 'email', 'data_type' => 'string', 'custom' => false, + 'api_writable' => false, 'archived' => false }) + end + + it 'publishes one column per custom attribute, typed from the introspection' do + expect(collection.fields['paid_subscriber'].column_type).to eq('Boolean') + end + + # Display-only: which operators Intercom answers on + # `custom_attributes.{name}` has not been measured, and this package + # publishes no filter it has not seen work. + it 'publishes it unfilterable and unsortable' do + expect(collection.fields['paid_subscriber'].filter_operators).to be_empty + expect(collection.fields['paid_subscriber'].is_sortable).to be(false) + end + + # Measured on a real workspace, and it took the agent's boot with it: the + # toolkit refuses a field declared twice, so an attribute landing on a + # native column has to be skipped -- and skipped rather than renamed, + # since the operator expects the record's own key under `id`. + it 'skips an attribute whose name a native column already carries, and says which' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + stub_data_attributes('contact', { 'name' => 'id', 'data_type' => 'string', 'custom' => true, + 'api_writable' => true, 'archived' => false }) + stub_list(contact('1', 'custom_attributes' => { 'id' => 'erp-42' })) + + expect { collection }.not_to raise_error + expect(rows(nil).first['id']).to eq('1') + expect(ForestAdminDatasourceIntercom.logger) + .to have_received(:warn).with(/skips the contact attribute "id"/) + end + + # Relations are declared before these columns for exactly this reason. + it 'skips one landing on a relation name too' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + stub_data_attributes('contact', { 'name' => 'company', 'data_type' => 'string', 'custom' => true, + 'api_writable' => true, 'archived' => false }) + + expect(collection.fields['company']) + .to be_a(ForestAdminDatasourceToolkit::Schema::Relations::ManyToOneSchema) + end + + # A custom date arrives as epoch seconds like every other Intercom date, + # and a Date column handed an integer renders as one. + it 'reads a date attribute as a date' do + stub_data_attributes('contact', { 'name' => 'renewal', 'data_type' => 'date', 'custom' => true, + 'api_writable' => true, 'archived' => false }) + stub_list(contact('1', 'custom_attributes' => { 'renewal' => 1_700_000_000 })) + + expect(rows(%w[id renewal]).first['renewal']).to eq('2023-11-14T22:13:20Z') + end + + it 'reads its value off the payload, nil where the contact carries none' do + stub_list(contact('1', 'custom_attributes' => { 'paid_subscriber' => true }), contact('2')) + + expect(rows(%w[id paid_subscriber])) + .to eq([{ 'id' => '1', 'paid_subscriber' => true }, { 'id' => '2', 'paid_subscriber' => nil }]) + end + end + + describe '#list' do + it 'walks the listing endpoint when nothing is filtered, sorted or searched' do + stub_list(contact('1'), contact('2')) + + expect(rows).to eq([{ 'id' => '1' }, { 'id' => '2' }]) + expect(WebMock).not_to have_requested(:post, "#{base}/contacts/search") + end + + it 'flattens the payload onto the row' do + stub_list(contact('1')) + + expect(rows(nil).first) + .to include('id' => '1', 'role' => 'user', 'email' => '1@acme.test', 'email_domain' => 'acme.test', + 'owner_id' => '493881', 'session_count' => 3, 'created_at' => '2023-11-14T22:13:20Z', + 'location_country' => 'France', 'location_city' => 'Paris') + end + + # A contact belongs to several accounts: the row names the first and says + # how many there are, which Intercom counts itself -- the nested list is + # capped and a contact of twelve accounts must not read as one of ten. + it 'names the first account and takes the count from Intercom' do + stub_list(contact('1')) + + expect(rows(nil).first).to include('company_id' => 'co1', 'company_count' => 2) + end + + it 'counts the accounts it can see when Intercom sends no count' do + stub_list(contact('1', 'companies' => { 'type' => 'list', 'data' => [{ 'id' => 'co1' }] })) + + expect(rows(nil).first).to include('company_count' => 1) + end + + it 'leaves the account columns empty on a contact belonging to none' do + stub_list(contact('1', 'companies' => nil)) + + expect(rows(nil).first).to include('company_id' => nil, 'company_count' => 0) + end + + it 'reads no domain out of an address that has none' do + stub_list(contact('1', 'email' => nil)) + + expect(rows(nil).first).to include('email_domain' => nil) + end + + it 'translates a filter into the search DSL' do + stub_search(contact('1')) + + expect(rows(%w[id], condition_tree: leaf('role', operators::EQUAL, 'lead'))).to eq([{ 'id' => '1' }]) + expect(WebMock).to have_requested(:post, "#{base}/contacts/search") + .with(body: hash_including('query' => { 'field' => 'role', 'operator' => '=', 'value' => 'lead' })) + end + + it 'answers a free-text search on the address, per word' do + stub_search(contact('1')) + + rows(%w[id], search: 'acme.test') + + expect(WebMock).to have_requested(:post, "#{base}/contacts/search") + .with(body: hash_including('query' => { 'field' => 'email', 'operator' => '~', 'value' => 'acme.test' })) + end + end + + describe 'the one collection Intercom sorts' do + # A list view asking for an order has no condition to send, and the + # listing endpoint does not sort: the order is what routes the read + # through the search, with the predicate that matches everything. + it 'sends the order to the search endpoint, with the match-all predicate' do + stub_search(contact('1')) + + rows(%w[id], sort: sort({ field: 'last_seen_at', ascending: false })) + + expect(WebMock).to have_requested(:post, "#{base}/contacts/search") + .with(body: hash_including('query' => described_class::MATCH_EVERY_CONTACT, + 'sort' => { 'field' => 'last_seen_at', 'order' => 'descending' })) + end + + it 'sends it alongside the filter when there is one' do + stub_search(contact('1')) + + rows(%w[id], condition_tree: leaf('role', operators::EQUAL, 'user'), + sort: sort({ field: 'name', ascending: true })) + + expect(WebMock).to have_requested(:post, "#{base}/contacts/search") + .with(body: hash_including('query' => { 'field' => 'role', 'operator' => '=', 'value' => 'user' }, + 'sort' => { 'field' => 'name', 'order' => 'ascending' })) + end + + # The ascending primary-key sort the agent injects when a request names + # none is not an order anybody asked for, and this endpoint does not sort + # on an id anyway. + it 'keeps the listing route for the default primary-key order, and says nothing' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + stub_list(contact('1')) + + rows(%w[id], sort: sort({ field: 'id', ascending: true })) + + expect(WebMock).to have_requested(:get, "#{base}/contacts").with(query: hash_including({})) + expect(ForestAdminDatasourceIntercom.logger).not_to have_received(:warn) + end + + it 'reports an order on a column Intercom does not sort, rather than dropping it' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + stub_list(contact('1')) + + rows(%w[id], sort: sort({ field: 'browser', ascending: true })) + + expect(ForestAdminDatasourceIntercom.logger).to have_received(:warn).with(/sort on browser/) + expect(WebMock).not_to have_requested(:post, "#{base}/contacts/search") + end + + # Intercom takes a single `{ field, order }`: honouring the first clause + # alone would order the page by something the operator did not ask for. + it 'reports a composite order rather than honouring half of it' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + stub_list(contact('1')) + + rows(%w[id], sort: sort({ field: 'name', ascending: true }, { field: 'email', ascending: false })) + + expect(ForestAdminDatasourceIntercom.logger).to have_received(:warn).with(/sort on name, email/) + end + end + + describe 'reading records by id' do + # One request per hundred ids rather than one per id: this endpoint + # answers `id IN [...]`, which is what makes a related list of contacts + # affordable at all. + it 'reads a set of ids in one request through the search' do + stub_search(contact('1'), contact('2')) + + expect(rows(%w[id], condition_tree: leaf('id', operators::IN, %w[1 2]))) + .to eq([{ 'id' => '1' }, { 'id' => '2' }]) + expect(WebMock).to have_requested(:post, "#{base}/contacts/search") + .with(body: hash_including('query' => { 'field' => 'id', 'operator' => 'IN', 'value' => %w[1 2] })).once + end + + it 'reads a record detail the same way' do + stub_search(contact('1')) + + expect(rows(%w[id], condition_tree: leaf('id', operators::EQUAL, '1'))).to eq([{ 'id' => '1' }]) + end + + # A contact merged into another disappears from the search: the row reads + # as gone rather than as an error, which is what a merge means. + it 'answers no row for a contact that was merged away' do + stub_search + + expect(rows(%w[id], condition_tree: leaf('id', operators::EQUAL, 'merged'))).to be_empty + end + + it 'counts what the ids named' do + stub_search(contact('1')) + + expect(collection.aggregate(nil, filter(condition_tree: leaf('id', operators::EQUAL, '1')), + ForestAdminDatasourceToolkit::Components::Query::Aggregation + .new(operation: 'Count'))) + .to eq([{ 'group' => {}, 'value' => 1 }]) + end + + it 'truncates a set larger than it will read, and says so' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + stub_search(contact('1')) + + rows(%w[id], condition_tree: leaf('id', operators::IN, (1..400).map(&:to_s))) + + expect(ForestAdminDatasourceIntercom.logger).to have_received(:warn).with(/asked for 400 records by id/) + expect(WebMock).to have_requested(:post, "#{base}/contacts/search").times(3) + end + + # The window is cut in the order the ids were named, so ordering what + # comes back would order a slice picked by something else. This is the one + # route of the collection Intercom does sort where an order cannot be + # honoured, so `server_sort` stayed quiet and this has to speak. + it 'reports an order this route cannot apply, on a column Intercom does sort' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + stub_search(contact('1')) + + rows(%w[id], condition_tree: leaf('id', operators::IN, %w[1 2]), + sort: sort({ field: 'name', ascending: true })) + + expect(ForestAdminDatasourceIntercom.logger).to have_received(:warn).with(/sort on "name" while reading/) + end + + # The `in` this comes from is a membership: it matches a record once. + it 'reads a value named twice once' do + stub_search(contact('1')) + + rows(%w[id], condition_tree: leaf('id', operators::IN, %w[1 1])) + + expect(WebMock).to have_requested(:post, "#{base}/contacts/search") + .with(body: hash_including('query' => { 'field' => 'id', 'operator' => 'IN', 'value' => %w[1] })) + end + + # Counting a set of ids means reading it, so past what a bulk read fetches + # the count is refused rather than answered with the number the truncation + # left -- this collection advertises an exact count. + it 'refuses to count more ids than it will read' do + expect do + collection.aggregate(nil, filter(condition_tree: leaf('id', operators::IN, (1..400).map(&:to_s))), + ForestAdminDatasourceToolkit::Components::Query::Aggregation + .new(operation: 'Count')) + end.to raise_error(UnsupportedOperatorError, /cannot count 400 records by id/) + end + end + + describe 'the contacts of an account' do + # `/contacts/search` filters no company field, and `GET + # /companies/{id}/contacts` is what answers the one relation an ops team + # walks the most. Without this route it would be a refusal. + it 'reads them from the company endpoint rather than from the search' do + stub_request(:get, "#{base}/companies/co1/contacts").with(query: hash_including({})) + .to_return(json('type' => 'list', 'data' => [contact('1')], + 'total_count' => 1, 'pages' => {})) + + expect(rows(%w[id], condition_tree: leaf('company_id', operators::EQUAL, 'co1'))) + .to eq([{ 'id' => '1' }]) + expect(WebMock).not_to have_requested(:post, "#{base}/contacts/search") + end + + it 'counts them from the same endpoint' do + stub_request(:get, "#{base}/companies/co1/contacts").with(query: hash_including({})) + .to_return(json('type' => 'list', 'data' => [contact('1')], + 'total_count' => 42, 'pages' => {})) + + expect(collection.aggregate(nil, filter(condition_tree: leaf('company_id', operators::EQUAL, 'co1')), + ForestAdminDatasourceToolkit::Components::Query::Aggregation + .new(operation: 'Count'))) + .to eq([{ 'group' => {}, 'value' => 42 }]) + end + + it 'reports an order this route cannot apply' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + stub_request(:get, "#{base}/companies/co1/contacts").with(query: hash_including({})) + .to_return(json('type' => 'list', 'data' => [contact('1')], + 'total_count' => 1, 'pages' => {})) + + rows(%w[id], condition_tree: leaf('company_id', operators::EQUAL, 'co1'), + sort: sort({ field: 'name', ascending: true })) + + # Named for what it is: this collection *is* the one Intercom sorts, so + # the reason is the route, not the column. + expect(ForestAdminDatasourceIntercom.logger) + .to have_received(:warn).with(/sort on name while reading the contacts of an account/) + expect(ForestAdminDatasourceIntercom.logger) + .to have_received(:warn).with(/a route that takes no order, unlike the search/) + end + + # An account Intercom no longer answers for -- deleted, or moved out of + # the token's reach between the moment the row was rendered and the + # moment its related list was opened -- reads as an account with no + # contact rather than as a failed page. + it 'reads an account Intercom no longer answers for as one with no contact' do + stub_request(:get, "#{base}/companies/gone/contacts").with(query: hash_including({})) + .to_return(json({ 'type' => 'error.list' }, 404)) + + expect(rows(%w[id], condition_tree: leaf('company_id', operators::EQUAL, 'gone'))).to eq([]) + expect(collection.aggregate(nil, filter(condition_tree: leaf('company_id', operators::EQUAL, 'gone')), + ForestAdminDatasourceToolkit::Components::Query::Aggregation + .new(operation: 'Count'))) + .to eq([{ 'group' => {}, 'value' => 0 }]) + end + + it 'raises where Intercom refused the account for another reason' do + stub_request(:get, "#{base}/companies/co1/contacts").with(query: hash_including({})) + .to_return(json({ 'type' => 'error.list' }, 500)) + + expect { rows(%w[id], condition_tree: leaf('company_id', operators::EQUAL, 'co1')) } + .to raise_error(APIError, /HTTP 500/) + end + + # An `and` also carrying a scope names a narrower set than the account + # does, and answering it with the account alone would serve contacts the + # scope excludes. There is no request that answers both halves either: + # the account endpoint narrows nothing and the search filters no company + # field, which is what a related list runs into the moment a scope or a + # segment exists on this collection. + it 'refuses to take the route for anything but a bare equality' do + expect do + rows(%w[id], condition_tree: branch('And', leaf('company_id', operators::EQUAL, 'co1'), + leaf('role', operators::EQUAL, 'user'))) + end.to raise_error(UnsupportedOperatorError, %r{GET /companies/\{id\}/contacts}) + end + + # The refusal names the half that does not fit rather than telling the + # operator to read the account's contacts -- which is what they asked for. + it 'names the condition filtered alongside the account' do + expect do + rows(%w[id], condition_tree: branch('And', leaf('company_id', operators::EQUAL, 'co1'), + leaf('role', operators::EQUAL, 'user'))) + end.to raise_error(UnsupportedOperatorError, /condition on role/) + end + + it 'refuses the count of a narrowed account the same way' do + expect do + collection.aggregate(nil, + filter(condition_tree: branch('And', leaf('company_id', operators::EQUAL, 'co1'), + leaf('role', operators::EQUAL, 'user'))), + ForestAdminDatasourceToolkit::Components::Query::Aggregation.new(operation: 'Count')) + end.to raise_error(UnsupportedOperatorError, %r{GET /companies/\{id\}/contacts}) + end + end + + describe 'a condition through a relation' do + def stub_admins(*admins) + stub_request(:get, "#{base}/admins").to_return(json('type' => 'admin.list', 'admins' => admins)) + end + + # The owner is a teammate, read whole in one request, and + # `/contacts/search` filters on the key: readable, navigable and + # filterable alike. + it 'resolves the owner against the teammates and filters on the key' do + stub_admins({ 'id' => '493881', 'name' => 'Marie' }) + stub_search(contact('1')) + + rows(%w[id], condition_tree: leaf('owner:name', operators::EQUAL, 'Marie')) + + expect(WebMock).to have_requested(:post, "#{base}/contacts/search") + .with(body: hash_including('query' => { 'field' => 'owner_id', 'operator' => '=', 'value' => '493881' })) + end + + it 'nests the owner under the relation when the projection names it' do + stub_admins({ 'id' => '493881', 'name' => 'Marie' }) + stub_list(contact('1')) + + expect(rows(%w[id owner:name]).first).to eq('id' => '1', + 'owner' => { 'id' => '493881', 'name' => 'Marie' }) + end + + # The endpoint filters no company field, so the relation is there to be + # read and navigated. Refused by name, before the target is read: a + # refusal that spends a request costs exactly what it refuses to do. + it 'refuses a condition through the company, and says what to filter instead' do + expect { rows(%w[id], condition_tree: leaf('company:name', operators::EQUAL, 'Acme')) } + .to raise_error(UnsupportedOperatorError, + %r{resolves to "company_id", on which contacts/search takes no filter}) + expect(WebMock).not_to have_requested(:post, /companies/) + end + end + + describe 'what it will not do' do + it 'refuses to group, Intercom exposing no aggregate endpoint' do + expect do + collection.aggregate(nil, filter, ForestAdminDatasourceToolkit::Components::Query::Aggregation + .new(operation: 'Count', groups: [{ field: 'role' }])) + end.to raise_error(UnsupportedOperatorError, /can only be counted/) + end + + it 'counts what the filter names in one request' do + stub_search(contact('1'), total: 812) + + expect(collection.aggregate(nil, filter(condition_tree: leaf('role', operators::EQUAL, 'user')), + ForestAdminDatasourceToolkit::Components::Query::Aggregation + .new(operation: 'Count'))) + .to eq([{ 'group' => {}, 'value' => 812 }]) + end + end + + describe 'paging' do + it 'asks Intercom for the window the list view named' do + stub_list(contact('1'), contact('2'), contact('3')) + + expect(rows(%w[id], page: page(1, 2)).map { |row| row['id'] }).to eq(%w[2 3]) + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/conversation_spec.rb b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/conversation_spec.rb new file mode 100644 index 000000000..d671f7824 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/conversation_spec.rb @@ -0,0 +1,758 @@ +module ForestAdminDatasourceIntercom + RSpec.describe Collections::Conversation do + subject(:collection) { datasource.get_collection('IntercomConversation') } + + let(:datasource) { Datasource.new(access_token: 's3cr3t', rate_limiter: nil) } + let(:base) { datasource.configuration.url } + let(:operators) { ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Operators } + + def filter(condition_tree: nil, page: nil, sort: nil) + ForestAdminDatasourceToolkit::Components::Query::Filter.new(condition_tree: condition_tree, page: page, + sort: sort) + end + + def leaf(field, operator, value = nil) + ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Nodes::ConditionTreeLeaf + .new(field, operator, value) + end + + def branch(aggregator, *conditions) + ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Nodes::ConditionTreeBranch + .new(aggregator, conditions) + end + + def json(payload, status = 200) + { status: status, body: payload.to_json, headers: { 'Content-Type' => 'application/json' } } + end + + # The columns alone: a relation carries neither operators nor an order, and + # what it may be filtered through is asserted on its own below. + def columns + collection.fields.select { |_, field| field.type == 'Column' } + end + + # Hand-written from the OpenAPI 2.16 spec, never captured from a workspace: + # a conversation body is personal data. + def conversation(id, overrides = {}) + { + 'type' => 'conversation', 'id' => id, 'title' => "Facture #{id}", 'state' => 'closed', + 'priority' => 'priority', 'open' => false, 'read' => true, + 'created_at' => 1_700_000_000, 'updated_at' => 1_700_003_600, + 'waiting_since' => nil, 'snoozed_until' => nil, + 'admin_assignee_id' => 493_881, 'team_assignee_id' => 814_865, + 'company' => { 'type' => 'company', 'id' => '696dd52099f73812610d9c7b', 'name' => 'Acme' }, + 'contacts' => { 'type' => 'contact.list', + 'contacts' => [{ 'type' => 'contact', 'id' => 'c1' }, { 'type' => 'contact', 'id' => 'c2' }] }, + 'tags' => { 'type' => 'tag.list', 'tags' => [{ 'id' => 't1', 'name' => 'billing' }] }, + 'ai_agent_participated' => true, + 'source' => { 'type' => 'conversation', 'id' => 's1', 'delivered_as' => 'customer_initiated', + 'subject' => 'Ma facture', 'body' => 'Bonjour, ou est ma facture ?', + 'author' => { 'type' => 'user', 'id' => 'c1', 'name' => 'Camille', + 'email' => 'camille@acme.test' }, + 'attachments' => [] }, + 'statistics' => { 'type' => 'conversation_statistics', 'first_close_at' => 1_700_002_000, + 'last_close_at' => 1_700_003_000, 'last_closed_by_id' => '493881', + 'first_contact_reply_at' => 1_700_000_050, 'last_contact_reply_at' => 1_700_001_000, + 'last_admin_reply_at' => 1_700_002_500, 'count_reopens' => 1, + 'count_conversation_parts' => 4 } + }.merge(overrides) + end + + def parts(*entries) + { 'conversation_parts' => { 'type' => 'conversation_part.list', 'conversation_parts' => entries } } + end + + def part(part_type, overrides = {}) + { 'type' => 'conversation_part', 'id' => 'p1', 'part_type' => part_type, 'body' => 'Je regarde.', + 'created_at' => 1_700_002_500, 'redacted' => false, 'attachments' => [], + 'author' => { 'type' => 'admin', 'id' => '493881', 'name' => 'Alice', + 'email' => 'alice@acme.test' } }.merge(overrides) + end + + # Intercom puts the records under `conversations`, not under the `data` + # envelope -- measured on `/tickets/search`, and the listings follow the same + # habit. + def stub_list(*records, next_cursor: nil, total: nil, query: hash_including({})) + body = { 'type' => 'conversation.list', 'conversations' => records, + 'total_count' => total || records.size, 'pages' => { 'type' => 'pages', 'page' => 1 } } + body['pages']['next'] = { 'starting_after' => next_cursor } if next_cursor + + stub_request(:get, "#{base}/conversations").with(query: query).to_return(json(body)) + end + + def stub_search(*records, total: nil, next_cursor: nil) + body = { 'type' => 'conversation.list', 'conversations' => records, + 'total_count' => total || records.size, 'pages' => { 'type' => 'pages', 'page' => 1 } } + body['pages']['next'] = { 'starting_after' => next_cursor } if next_cursor + + stub_request(:post, "#{base}/conversations/search").with(query: hash_including({})).to_return(json(body)) + end + + def stub_record(id, payload, status = 200) + stub_request(:get, "#{base}/conversations/#{id}").with(query: hash_including({})).to_return(json(payload, status)) + end + + def ids(rows) + rows.map { |row| row['id'] } + end + + describe 'schema' do + it 'is named IntercomConversation' do + expect(collection.name).to eq('IntercomConversation') + end + + # Neither search endpoint takes a sort, and Intercom ignores the one it is + # sent without a word, so no column of this tier may advertise one. + it 'declares every column unsortable' do + expect(columns.values.map(&:is_sortable).uniq).to eq([false]) + end + + # Derived from the measured table, never written by hand: a column + # advertises exactly what the search endpoint answers on it. + it 'advertises the filters the search endpoint answers, and only those' do + expect(collection.fields['state'].filter_operators).to eq(%w[equal not_equal]) + expect(collection.fields['created_at'].filter_operators).to eq(%w[greater_than less_than]) + expect(collection.fields['source_body'].filter_operators).to eq(%w[contains i_contains not_contains]) + end + + # A column the table does not carry advertises nothing, which is how a + # refusal is spelled in a schema: the tag names, the company, the timeline + # and the contact identity are all read from somewhere the endpoint does + # not filter. + it 'advertises no filter on a column the endpoint does not filter' do + %w[tag_names company_name contact_name timeline contact_count].each do |column| + expect(collection.fields[column].filter_operators).to be_empty, "#{column} advertises a filter" + end + end + + it 'is searchable, Intercom matching text on the body of the first message' do + expect(collection.is_searchable?).to be(true) + end + + # The record detail is `id equals X`, answered by the record endpoint + # rather than by a filter. + it 'answers the primary key with equal and in' do + expect(collection.fields['id'].filter_operators).to eq(%w[equal in]) + end + + it 'is countable, since total_count is exact' do + expect(collection.is_countable?).to be(true) + end + + # No aggregate endpoint, so no group-by may be offered. + it 'declares no column groupable' do + expect(columns.values.map(&:is_groupable).uniq).to eq([false]) + end + + # This lot writes nothing: an editable column would offer a Save that + # reaches an `update` the collection does not implement. + it 'declares every column read-only' do + expect(collection.fields.values.map(&:is_read_only).uniq).to eq([true]) + end + end + + # All three targets are read whole in one request, and + # `/conversations/search` takes a filter on each of the three keys -- so + # these relations can be read, navigated and filtered through alike, unlike + # the state of a ticket. + describe 'the relations to the reference collections' do + it 'points every id at the collection that reads it' do + expect(collection.fields['admin_assignee']) + .to have_attributes(type: 'ManyToOne', foreign_collection: 'IntercomAdmin', + foreign_key: 'admin_assignee_id', foreign_key_target: 'id', is_read_only: true) + expect(collection.fields['team_assignee']) + .to have_attributes(foreign_collection: 'IntercomTeam', foreign_key: 'team_assignee_id') + expect(collection.fields['closed_by']) + .to have_attributes(foreign_collection: 'IntercomAdmin', foreign_key: 'closed_by_id') + end + + # The account is on the payload as a whole object, so the name is already a + # column; the Companies collection arrives with lot 4, and a relation whose + # target is missing is a schema the agent refuses to boot on. + it 'declares no relation towards the account' do + expect(collection.fields.keys).not_to include('company') + end + + it 'nests the teammate who closed it, reading the teammates once for the page' do + stub_list(conversation('1'), conversation('2')) + stub_request(:get, "#{base}/admins") + .to_return(json('type' => 'admin.list', 'admins' => [{ 'id' => '493881', 'name' => 'Alice' }])) + + rows = collection.list(nil, filter, ['id', 'closed_by:name']) + + expect(rows).to eq([{ 'id' => '1', 'closed_by' => { 'name' => 'Alice', 'id' => '493881' } }, + { 'id' => '2', 'closed_by' => { 'name' => 'Alice', 'id' => '493881' } }]) + expect(WebMock).to have_requested(:get, "#{base}/admins").once + end + + it 'filters on the foreign key the target resolved to' do + stub_request(:get, "#{base}/admins") + .to_return(json('type' => 'admin.list', 'admins' => [{ 'id' => '493881', 'name' => 'Alice' }])) + stub_search(conversation('1')) + + collection.list(nil, filter(condition_tree: leaf('admin_assignee:name', operators::EQUAL, 'Alice')), %w[id]) + + expect(WebMock).to have_requested(:post, "#{base}/conversations/search") + .with(query: hash_including({}), + body: hash_including('query' => { 'field' => 'admin_assignee_id', 'operator' => '=', + 'value' => '493881' })) + end + end + + # The other half of the 360 degrees: `IntercomContact#conversations` is a + # one-to-many, and the agent serves it by listing this collection on the key + # -- so what makes that list possible is this endpoint filtering on a + # contact id at all. The column is singular and the wire field plural: the + # row names its first contact, the endpoint asks whether a contact is one of + # the conversation's. + describe 'the conversations of a contact' do + it 'filters on the contact id the relation resolves to' do + stub_search(conversation('1')) + + collection.list(nil, filter(condition_tree: leaf('contact_id', operators::EQUAL, 'c1')), %w[id]) + + expect(WebMock).to have_requested(:post, "#{base}/conversations/search") + .with(query: hash_including({}), + body: hash_including('query' => { 'field' => 'contact_ids', 'operator' => '=', 'value' => 'c1' })) + end + + it 'counts them in one request' do + stub_search(conversation('1'), total: 37) + + expect(collection.aggregate(nil, filter(condition_tree: leaf('contact_id', operators::EQUAL, 'c1')), + ForestAdminDatasourceToolkit::Components::Query::Aggregation + .new(operation: 'Count'))) + .to eq([{ 'group' => {}, 'value' => 37 }]) + end + end + + describe '#list' do + # No projection at all asks for every declared column, `contact_name` and + # `timeline` included, so an all-columns read pays for both enrichments -- + # the identity read of the page, and the detail read the parts are only + # returned by. Guarding them on a column being *named* would leave nil the + # very columns the row publishes. + before do + stub_request(:post, "#{base}/contacts/search") + .to_return(json('type' => 'list', 'data' => [{ 'id' => 'c1', 'name' => 'Camille' }])) + stub_record('1', conversation('1')) + end + + it 'reads the listing endpoint as plain text and pages by cursor' do + stub_list(conversation('1')) + + collection.list(nil, filter, nil) + + expect(WebMock).to have_requested(:get, "#{base}/conversations") + .with(query: hash_including('display_as' => 'plaintext')) + end + + it 'flattens the payload into the row the schema declares' do + stub_list(conversation('1')) + + row = collection.list(nil, filter, nil).first + + expect(row).to include('id' => '1', 'title' => 'Facture 1', 'state' => 'closed', 'open' => false, + 'company_id' => '696dd52099f73812610d9c7b', 'company_name' => 'Acme', + 'admin_assignee_id' => '493881', 'team_assignee_id' => '814865', + 'tag_names' => %w[billing], 'ai_agent_participated' => true) + end + + # Epoch seconds are what Intercom sends; a Date column and a date filter + # both read ISO8601, and UTC is where Intercom truncates. + it 'reads the dates as ISO8601 in UTC' do + row = (stub_list(conversation('1')) && collection.list(nil, filter, nil)).first + + expect(row['created_at']).to eq('2023-11-14T22:13:20Z') + end + + it 'flattens the lifecycle Intercom keeps in statistics' do + stub_list(conversation('1')) + + expect(collection.list(nil, filter, nil).first) + .to include('closed_at' => '2023-11-14T23:03:20Z', 'closed_by_id' => '493881', + 'last_admin_reply_at' => '2023-11-14T22:55:00Z', 'reopen_count' => 1, 'part_count' => 4) + end + + # A conversation Intercom has computed nothing for yet answers a null + # statistics; the columns then read as absent rather than as zero. + it 'reads a missing statistics block as absent, not as zero' do + stub_list(conversation('1', 'statistics' => nil)) + + expect(collection.list(nil, filter, nil).first) + .to include('closed_at' => nil, 'reopen_count' => nil) + end + + # A group conversation has several contacts: the row names the first, says + # how many there are, and the relation resolves that same first one -- the + # column and the relation cannot disagree. + it 'names the first contact and counts them' do + stub_list(conversation('1')) + + expect(collection.list(nil, filter, nil).first) + .to include('contact_id' => 'c1', 'contact_count' => 2) + end + + it 'narrows the row to the projection' do + stub_list(conversation('1')) + + expect(collection.list(nil, filter, %w[id state])).to eq([{ 'id' => '1', 'state' => 'closed' }]) + end + + # The counterpart of the guard above: a row that publishes a column has + # that column filled, rather than carrying the nil the projection put + # there. + it 'fills the enriched columns of an all-columns read' do + stub_list(conversation('1')) + + row = collection.list(nil, filter, nil).first + + expect(row['contact_name']).to eq('Camille') + expect(row['timeline']).to be_an(Array) + end + + it 'walks the cursor until the window is covered' do + first = { 'conversations' => [conversation('1'), conversation('2')], + 'pages' => { 'next' => { 'starting_after' => 'c2' } } } + stub_request(:get, "#{base}/conversations").with(query: hash_including('per_page' => '3')) + .to_return(json(first)) + stub_request(:get, "#{base}/conversations").with(query: hash_including('starting_after' => 'c2')) + .to_return(json('conversations' => [conversation('3')])) + + page = ForestAdminDatasourceToolkit::Components::Query::Page.new(offset: 2, limit: 1) + + expect(ids(collection.list(nil, filter(page: page), %w[id]))).to eq(%w[3]) + end + end + + describe '#list of one record' do + # What a record detail is. It goes to the record endpoint rather than to + # the listing, which is also what brings the parts along. + it 'reads id equals X through the record endpoint' do + stub_record('1', conversation('1')) + + expect(ids(collection.list(nil, filter(condition_tree: leaf('id', operators::EQUAL, '1')), %w[id]))) + .to eq(%w[1]) + end + + # A stale link, or a record outside the token's scope: no record, not a + # failed page. + it 'reads a 404 as no record' do + stub_record('gone', { 'errors' => [{ 'code' => 'not_found' }] }, 404) + + expect(collection.list(nil, filter(condition_tree: leaf('id', operators::EQUAL, 'gone')), %w[id])).to eq([]) + end + + it 'still raises on a failure that is not a missing record' do + stub_record('1', { 'errors' => [{ 'code' => 'forbidden' }] }, 403) + + expect { collection.list(nil, filter(condition_tree: leaf('id', operators::EQUAL, '1')), %w[id]) } + .to raise_error(APIError) + end + + # A permission scope turns the record detail into `id equals X and `, which the record endpoint cannot answer: the ids name a wider + # set than the scope does, and reading them alone would serve a record the + # scope excludes. It goes to the search, where the key is a field like any + # other -- and the whole condition travels, or none of it does. + it 'reads the key through the search once a scope is filtered alongside it' do + search = stub_search(conversation('1')) + tree = branch('And', leaf('id', operators::EQUAL, '1'), leaf('state', operators::EQUAL, 'closed')) + + expect(ids(collection.list(nil, filter(condition_tree: tree), %w[id]))).to eq(%w[1]) + expect(search).to have_been_requested + end + + it 'sends the scope and the key as the one query, neither dropped' do + stub_search(conversation('1')) + tree = branch('And', leaf('id', operators::IN, %w[1 2]), leaf('open', operators::EQUAL, false)) + + collection.list(nil, filter(condition_tree: tree), %w[id]) + + expected = { 'operator' => 'AND', + 'value' => [{ 'field' => 'id', 'operator' => 'IN', 'value' => %w[1 2] }, + { 'field' => 'open', 'operator' => '=', 'value' => false }] } + + expect(a_request(:post, "#{base}/conversations/search") + .with(query: hash_including({}), body: hash_including('query' => expected))).to have_been_made + end + end + + describe '#list of several records by id' do + # What a pointing collection asks for when it reads related records in + # bulk. Intercom has no "read these records" endpoint, so it is one + # request per id -- and therefore bounded. + it 'reads each id through the record endpoint' do + %w[1 2].each do |id| + stub_record(id, conversation(id)) + end + + rows = collection.list(nil, filter(condition_tree: leaf('id', operators::IN, %w[1 2])), %w[id]) + + expect(ids(rows)).to eq(%w[1 2]) + end + + # A pointing collection pages through the records it named, and every page + # must name different ones: cut out of the records instead of out of the + # ids, the window would render the same rows on page 1 and page 2 -- and + # a page past the cap would come back empty, its ids having been dropped + # by the truncation before the window was applied. + it 'reads only the ids the page window names' do + stub_record('2', conversation('2')) + page = ForestAdminDatasourceToolkit::Components::Query::Page.new(offset: 1, limit: 1) + tree = leaf('id', operators::IN, %w[1 2 3]) + + expect(ids(collection.list(nil, filter(condition_tree: tree, page: page), %w[id]))).to eq(%w[2]) + end + + it 'reads the first of too many and says the result is truncated' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + asked = (1..(Collections::CursorCollection::MAX_ID_READS + 3)).map(&:to_s) + stub_request(:get, %r{/conversations/\d+}).to_return(json(conversation('1'))) + + rows = collection.list(nil, filter(condition_tree: leaf('id', operators::IN, asked)), %w[id]) + + expect(rows.size).to eq(Collections::CursorCollection::MAX_ID_READS) + expect(ForestAdminDatasourceIntercom.logger).to have_received(:warn).with(/read the first 25/) + end + end + + describe 'a filter Intercom answers' do + # A condition switches the read from the listing to the search endpoint, + # which is the only one that takes a filter. + it 'searches instead of listing, with the query the translator wrote' do + search = stub_search + tree = leaf('state', operators::EQUAL, 'open') + + collection.list(nil, filter(condition_tree: tree), %w[id]) + + expect(search.with(body: hash_including('query' => { 'field' => 'state', 'operator' => '=', + 'value' => 'open' }))).to have_been_made + end + + # The bodies are HTML written by end customers (R10), and a filtered read + # must not come back as markup where an unfiltered one comes back as text. + it 'asks the search for plain text too' do + search = stub_search + + collection.list(nil, filter(condition_tree: leaf('open', operators::EQUAL, true)), %w[id]) + + expect(search.with(query: hash_including('display_as' => 'plaintext'))).to have_been_made + end + + it 'walks the search cursor for the window a list view asked for' do + stub_request(:post, "#{base}/conversations/search") + .with(query: hash_including({})) + .to_return(json({ 'conversations' => [conversation('1'), conversation('2')], + 'pages' => { 'next' => { 'starting_after' => 'c2' } } })) + stub_request(:post, "#{base}/conversations/search") + .with(query: hash_including({}), + body: hash_including('pagination' => hash_including('starting_after' => 'c2'))) + .to_return(json('conversations' => [conversation('3')])) + page = ForestAdminDatasourceToolkit::Components::Query::Page.new(offset: 2, limit: 1) + + rows = collection.list(nil, filter(condition_tree: leaf('open', operators::EQUAL, true), page: page), %w[id]) + + expect(ids(rows)).to eq(%w[3]) + end + + # Per word rather than as a substring, which the README says out loud. + it 'answers a free-text search on the body of the message that opened the conversation' do + search = stub_search + searched = ForestAdminDatasourceToolkit::Components::Query::Filter.new(search: ' facture ') + + collection.list(nil, searched, %w[id]) + + expect(search.with(body: hash_including('query' => { 'field' => 'source.body', 'operator' => '~', + 'value' => 'facture' }))).to have_been_made + end + + # Written as one tree rather than added to the translated query: the + # nesting Intercom allows is then checked over the whole of it. + it 'ands a free-text search with the condition it came with' do + search = stub_search + searched = ForestAdminDatasourceToolkit::Components::Query::Filter.new( + search: 'facture', condition_tree: leaf('open', operators::EQUAL, true) + ) + + collection.list(nil, searched, %w[id]) + + expect(search.with { |request| JSON.parse(request.body)['query']['operator'] == 'AND' }).to have_been_made + end + + # The date bounds Intercom answers are the ones the day rule moved, which + # is what makes an interval answer the day it names. + it 'sends a date bound on the UTC day boundary that answers the day asked for' do + search = stub_search + tree = leaf('created_at', operators::GREATER_THAN, '2026-09-01T08:30:00Z') + + collection.list(nil, filter(condition_tree: tree), %w[id]) + + expect(search.with(body: hash_including('query' => hash_including('value' => Time.utc(2026, 8, + 31).to_i)))) + .to have_been_made + end + end + + describe 'a filter it cannot honour' do + # A condition dropped on the way to Intercom comes back as an unfiltered + # page that looks filtered, which is the one answer this datasource must + # not give. + it 'refuses a condition on a column the endpoint does not filter' do + expect { collection.list(nil, filter(condition_tree: leaf('tag_names', operators::EQUAL, 'billing')), %w[id]) } + .to raise_error(UnsupportedOperatorError, /cannot filter "tag_names"/) + end + + it 'refuses an operator the endpoint does not answer on that column' do + expect { collection.list(nil, filter(condition_tree: leaf('state', operators::CONTAINS, 'op')), %w[id]) } + .to raise_error(UnsupportedOperatorError, /answers equal, not_equal on "state"/) + end + + it 'makes no request at all when it refuses' do + expect { collection.list(nil, filter(condition_tree: leaf('tag_names', operators::EQUAL, 'x')), %w[id]) } + .to raise_error(UnsupportedOperatorError) + expect(a_request(:post, "#{base}/conversations/search")).not_to have_been_made + end + end + + describe 'a sort Intercom ignores' do + before { allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) } + + # Measured: a sort sent to this endpoint raises nothing and changes + # nothing, so an order the operator asked for and did not get can only be + # reported here. + it 'reports the order it did not get' do + stub_list(conversation('1')) + sort = ForestAdminDatasourceToolkit::Components::Query::Sort.new([{ field: 'created_at', ascending: false }]) + + collection.list(nil, filter(sort: sort), %w[id]) + + expect(ForestAdminDatasourceIntercom.logger).to have_received(:warn).with(/ignores a sort/) + end + + # `?sort=-id` is an order the operator asked for, not the ascending default + # the agent injects when a request names none. + it 'reports an explicit descending order on the primary key' do + stub_list(conversation('1')) + sort = ForestAdminDatasourceToolkit::Components::Query::Sort.new([{ field: 'id', ascending: false }]) + + collection.list(nil, filter(sort: sort), %w[id]) + + expect(ForestAdminDatasourceIntercom.logger).to have_received(:warn).with(/ignores a sort/) + end + + it 'stays quiet on the primary-key order the agent injects by default' do + stub_list(conversation('1')) + sort = ForestAdminDatasourceToolkit::Components::Query::Sort.new([{ field: 'id', ascending: true }]) + + collection.list(nil, filter(sort: sort), %w[id]) + + expect(ForestAdminDatasourceIntercom.logger).not_to have_received(:warn) + end + end + + describe '#aggregate' do + def aggregation(operation, field: nil, groups: []) + ForestAdminDatasourceToolkit::Components::Query::Aggregation.new(operation: operation, field: field, + groups: groups) + end + + # One request, and exact on the whole collection rather than on the page + # the walk happened to read. + it 'counts through total_count' do + stub_list(conversation('1'), total: 81_142, query: hash_including('per_page' => '1')) + + expect(collection.aggregate(nil, filter, aggregation('Count'))) + .to eq([{ 'group' => {}, 'value' => 81_142 }]) + end + + it 'counts the records an id lookup found' do + stub_record('1', conversation('1')) + + expect(collection.aggregate(nil, filter(condition_tree: leaf('id', operators::EQUAL, '1')), + aggregation('Count')).first['value']).to eq(1) + end + + # Grouping over the pages a walk collected would look exact while + # answering a fraction. + it 'refuses a group-by' do + expect { collection.aggregate(nil, filter, aggregation('Count', groups: [{ field: 'state' }])) } + .to raise_error(UnsupportedOperatorError, /can only be counted/) + end + + it 'refuses a sum' do + expect { collection.aggregate(nil, filter, aggregation('Sum', field: 'reopen_count')) } + .to raise_error(UnsupportedOperatorError, /can only be counted/) + end + + # `total_count` is exact on a search too, so a filtered count is one + # request over the whole filtered set rather than over a page of it. + it 'counts a filtered collection through the search, in one request' do + stub_search(total: 1_234) + + value = collection.aggregate(nil, filter(condition_tree: leaf('state', operators::EQUAL, 'open')), + aggregation('Count')).first['value'] + + expect(value).to eq(1_234) + end + + it 'refuses to count what it refuses to list' do + expect do + collection.aggregate(nil, filter(condition_tree: leaf('tag_names', operators::EQUAL, 'billing')), + aggregation('Count')) + end.to raise_error(UnsupportedOperatorError, /cannot filter "tag_names"/) + end + + # Counting the pages a walk collected would answer a fraction as if it + # were the whole, so a listing with no total_count is a listing this + # cannot count. + it 'refuses to count a listing Intercom answered without a total_count' do + stub_request(:get, "#{base}/conversations").with(query: hash_including({})) + .to_return(json('conversations' => [], + 'pages' => { 'type' => 'pages' })) + + expect { collection.aggregate(nil, filter, aggregation('Count')) } + .to raise_error(UnsupportedOperatorError, /without a total_count/) + end + end + + describe 'the contact identity' do + before do + stub_list(conversation('1')) + stub_request(:post, "#{base}/contacts/search") + .to_return(json('type' => 'list', + 'data' => [{ 'id' => 'c1', 'name' => 'Camille', 'email' => 'camille@acme.test' }])) + end + + # One label on the row plus the relation to navigate, which is the rule + # lot 2.5 set for the ticket labels: `contact_email` is gone, it is a hop + # away on `contact:email`. + it 'reads the identity of the page in one request and puts it on the row' do + row = collection.list(nil, filter, %w[id contact_name]).first + + expect(row).to include('contact_name' => 'Camille') + expect(WebMock).to have_requested(:post, "#{base}/contacts/search").once + end + + it 'no longer publishes the e-mail the relation carries' do + expect(collection.fields.keys).not_to include('contact_email', 'contact_ids') + end + + it 'asks for the contacts of the page by id' do + collection.list(nil, filter, %w[id contact_name]) + + expect(WebMock).to have_requested(:post, "#{base}/contacts/search") + .with(body: hash_including('query' => { 'field' => 'id', 'operator' => 'IN', 'value' => %w[c1] })) + end + + # A page that never asked for the identity must not pay for it. + it 'reads nothing when the projection does not name it' do + collection.list(nil, filter, %w[id state]) + + expect(WebMock).not_to have_requested(:post, "#{base}/contacts/search") + end + + # An identity that could not be read is not a page that could not be + # served: it costs the two columns. + it 'degrades to empty columns when the read fails' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + stub_request(:post, "#{base}/contacts/search").to_return(json({ 'errors' => [] }, 403)) + + row = collection.list(nil, filter, %w[id contact_name]).first + + expect(row['contact_name']).to be_nil + expect(ForestAdminDatasourceIntercom.logger).to have_received(:warn).with(/could not read the contacts/) + end + end + + describe 'the timeline' do + let(:conversation_with_parts) do + conversation('1').merge(parts(part('assignment', 'id' => 'p1', 'body' => nil, 'created_at' => 1_700_000_100), + part('comment', 'id' => 'p2', 'created_at' => 1_700_002_500))) + end + + # The message that opened the conversation lives in `source`, not in the + # parts: a timeline built from the parts alone loses what the customer + # actually asked. + it 'opens on the source message' do + stub_record('1', conversation_with_parts) + + timeline = collection.list(nil, filter(condition_tree: leaf('id', operators::EQUAL, '1')), + %w[id timeline]).first['timeline'] + + expect(timeline.first).to include('part_type' => 'conversation_started', + 'body' => 'Bonjour, ou est ma facture ?', + 'author_name' => 'Camille', 'created_at' => '2023-11-14T22:13:20Z') + end + + # An assignment, a note and a reply are not the same event; a thread that + # flattens them reads as a conversation that never happened that way. + it 'keeps the part type of every entry' do + stub_record('1', conversation_with_parts) + + timeline = collection.list(nil, filter(condition_tree: leaf('id', operators::EQUAL, '1')), + %w[timeline]).first['timeline'] + + expect(timeline.map { |entry| entry['part_type'] }).to eq(%w[conversation_started assignment comment]) + end + + it 'costs no request on a record read, the parts riding along with it' do + stub_record('1', conversation_with_parts) + + collection.list(nil, filter(condition_tree: leaf('id', operators::EQUAL, '1')), %w[timeline]) + + expect(WebMock).to have_requested(:get, "#{base}/conversations/1") + .with(query: hash_including({})).once + end + + # Intercom returns the parts only when retrieving one conversation, so a + # list view pays a request per row. + it 'reads the record when a listed row has no parts' do + stub_list(conversation('1')) + stub_record('1', conversation_with_parts) + + rows = collection.list(nil, filter, %w[id timeline]) + + expect(rows.first['timeline'].size).to eq(3) + end + + it 'reads nothing when the projection does not name it' do + stub_list(conversation('1')) + + collection.list(nil, filter, %w[id state]) + + expect(WebMock).not_to have_requested(:get, "#{base}/conversations/1").with(query: hash_including({})) + end + + # A conversation deleted between the page and the read of its timeline: + # the row keeps a nil timeline rather than failing the whole page. + it 'leaves the timeline unread when the record has gone' do + stub_list(conversation('1')) + stub_record('1', { 'errors' => [{ 'code' => 'not_found' }] }, 404) + + expect(collection.list(nil, filter, %w[id timeline]).first['timeline']).to be_nil + end + + it 'still raises when the timeline read fails for another reason' do + stub_list(conversation('1')) + stub_record('1', { 'errors' => [{ 'code' => 'forbidden' }] }, 403) + + expect { collection.list(nil, filter, %w[id timeline]) }.to raise_error(APIError) + end + + # Rows past the cap keep a nil timeline -- unknown -- rather than an empty + # list, which would read as "this conversation has no message". + it 'bounds the fan-out and says what it left unread' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + listed = (1..(described_class::MAX_TIMELINE_READS + 2)).map { |index| conversation(index.to_s) } + stub_list(*listed) + stub_request(:get, %r{/conversations/\d+}).to_return(json(conversation_with_parts)) + + rows = collection.list(nil, filter, %w[id timeline]) + + expect(rows.count { |row| row['timeline'].nil? }).to eq(2) + expect(ForestAdminDatasourceIntercom.logger).to have_received(:warn).with(/left the timeline of 2 row/) + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/fetch_all_collection_spec.rb b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/fetch_all_collection_spec.rb new file mode 100644 index 000000000..94d03933c --- /dev/null +++ b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/fetch_all_collection_spec.rb @@ -0,0 +1,279 @@ +module ForestAdminDatasourceIntercom + # The in-memory tier is exercised through Admin, a real collection carrying one + # column of each kind it has to handle: strings, booleans and a list. + RSpec.describe Collections::FetchAllCollection do + subject(:collection) { Collections::Admin.new(datasource) } + + let(:datasource) { Datasource.new(access_token: 's3cr3t', rate_limiter: nil) } + let(:base) { datasource.configuration.url } + let(:operators) { ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Operators } + + def leaf(field, operator, value = nil) + ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Nodes::ConditionTreeLeaf + .new(field, operator, value) + end + + def filter(condition_tree: nil, page: nil, sort: nil) + ForestAdminDatasourceToolkit::Components::Query::Filter.new(condition_tree: condition_tree, page: page, + sort: sort) + end + + def page(offset, limit) + ForestAdminDatasourceToolkit::Components::Query::Page.new(offset: offset, limit: limit) + end + + def sort(*clauses) + ForestAdminDatasourceToolkit::Components::Query::Sort.new(clauses) + end + + def aggregation(operation, field: nil, groups: []) + ForestAdminDatasourceToolkit::Components::Query::Aggregation.new(operation: operation, field: field, + groups: groups) + end + + def admin(id, overrides = {}) + { 'type' => 'admin', 'id' => id, 'name' => "Admin #{id}", 'email' => "#{id}@acme.test", + 'away_mode_enabled' => false, 'has_inbox_seat' => true, 'team_ids' => [] }.merge(overrides) + end + + def stub_admins(*admins) + stub_request(:get, "#{base}/admins") + .to_return(status: 200, body: { 'type' => 'admin.list', 'admins' => admins }.to_json, + headers: { 'Content-Type' => 'application/json' }) + end + + def ids(records) + records.map { |record| record['id'] } + end + + # An operator is evaluable in memory when `ConditionTreeLeaf#match` handles + # it natively or the toolkit can rewrite it into operators that it does. + def evaluable?(operator, column_type) + ForestAdminDatasourceToolkit::Components::Query::ConditionTree::ConditionTreeEquivalent + .equivalent_tree?(operator, described_class::IN_MEMORY_OPERATORS, column_type) + end + + # A projection naming nothing asks for every declared column, `team_names` + # among them, so a list with no projection reads the teams as well. + before do + stub_request(:get, "#{base}/teams") + .to_return(status: 200, body: { 'type' => 'team.list', 'teams' => [] }.to_json, + headers: { 'Content-Type' => 'application/json' }) + end + + # The count and the group are taken over every record Intercom holds, not + # over a page of them, which is what makes them exact. + it 'is countable' do + expect(collection.is_countable?).to be(true) + end + + describe 'columns' do + it 'declares a scalar column filterable, sortable and groupable' do + expect(collection.fields['name']) + .to have_attributes(is_sortable: true, is_groupable: true) + expect(collection.fields['name'].filter_operators).to include(operators::EQUAL) + end + + # This lot writes nothing: an editable column would offer a Save that + # reaches an `update` the collection does not implement. + it 'declares every column read-only' do + expect(collection.fields.values.map(&:is_read_only).uniq).to eq([true]) + end + + # A list has no in-memory counterpart for any of the three. + it 'declares a Json column neither filterable nor sortable' do + expect(collection.fields['team_names']) + .to have_attributes(column_type: 'Json', is_sortable: false, is_groupable: false, filter_operators: []) + end + + # A filter the UI offers and the collection then answers by emptying the + # page is the failure this whole datasource is built to avoid. + it 'advertises only operators it can actually evaluate' do + advertised = collection.fields.select { |_, field| field.type == 'Column' }.flat_map do |_name, column| + column.filter_operators.map { |operator| [operator, column.column_type] } + end + + expect(advertised.reject { |operator, type| evaluable?(operator, type) }).to be_empty + end + end + + describe '#list' do + it 'reads every record of the endpoint and serializes it' do + stub_admins(admin('1'), admin('2')) + + expect(ids(collection.list(nil, filter, nil))).to eq(%w[1 2]) + end + + it 'narrows the record to the projection' do + stub_admins(admin('1')) + + expect(collection.list(nil, filter, %w[id email])).to eq([{ 'id' => '1', 'email' => '1@acme.test' }]) + end + + # Freshness over rate-limit thrift: nothing is kept from the previous list, + # so an operator sees the teammates the workspace has now. + it 'reads the endpoint again on the next list' do + stub_admins(admin('1')) + + 2.times { collection.list(nil, filter, nil) } + + expect(WebMock).to have_requested(:get, "#{base}/admins").twice + end + + it 'propagates a failure rather than answering with no record' do + stub_request(:get, "#{base}/admins").to_return(status: 500, body: '{}', + headers: { 'Content-Type' => 'application/json' }) + + expect { collection.list(nil, filter, nil) }.to raise_error(APIError) + end + end + + describe '#list with a filter' do + before { stub_admins(admin('1', 'name' => 'Alice'), admin('2', 'name' => 'Bob', 'has_inbox_seat' => false)) } + + def filtered(field, operator, value = nil) + ids(collection.list(nil, filter(condition_tree: leaf(field, operator, value)), nil)) + end + + it 'keeps the rows a string condition names' do + expect(filtered('name', operators::EQUAL, 'Alice')).to eq(%w[1]) + end + + it 'keeps the rows a boolean condition names' do + expect(filtered('has_inbox_seat', operators::EQUAL, false)).to eq(%w[2]) + end + + it 'answers an operator it advertises through an equivalence' do + expect(filtered('name', operators::I_CONTAINS, 'ali')).to eq(%w[1]) + end + + it 'answers nothing when nothing matches, rather than everything' do + expect(filtered('name', operators::EQUAL, 'Nobody')).to be_empty + end + + # `match` answers nil for an operator with no equivalence and `apply` reads + # that as "no match", so this would otherwise be an empty page an operator + # cannot tell from a real answer. The schema advertises no such operator; a + # scope or a segment can still send one. + it 'refuses a condition it cannot evaluate instead of emptying the page' do + expect { filtered('team_ids', operators::EQUAL, 'x') } + .to raise_error(UnsupportedOperatorError, /cannot filter 'team_ids' with 'equal'/) + end + + it 'refuses a condition on a column it does not carry' do + expect { filtered('unknown', operators::EQUAL, 'x') } + .to raise_error(UnsupportedOperatorError, /cannot filter 'unknown'/) + end + + it 'names a refusal after the operator, so the message says what to change' do + expect { filtered('name', operators::LESS_THAN, 'x') } + .to raise_error(UnsupportedOperatorError, /'less_than'/) + end + end + + describe '#list with a sort' do + before do + stub_admins(admin('2', 'name' => 'Bob'), admin('1', 'name' => 'Alice'), admin('3', 'name' => nil)) + end + + it 'orders on the column asked for' do + expect(ids(collection.list(nil, filter(sort: sort({ field: 'name', ascending: true })), nil))) + .to eq(%w[1 2 3]) + end + + # Nulls last ascending, first descending, the way a database orders them. + it 'puts a null first on a descending order' do + expect(ids(collection.list(nil, filter(sort: sort({ field: 'name', ascending: false })), nil))) + .to eq(%w[3 2 1]) + end + + it 'keeps the order Intercom returned for rows the sort cannot separate' do + rows = collection.list(nil, filter(sort: sort({ field: 'away_mode_enabled', ascending: true })), nil) + + expect(ids(rows)).to eq(%w[2 1 3]) + end + + # An order asked for and not honoured is reported everywhere else in this + # datasource, and a related list through a many-to-many reaches this tier + # with the columns of the collection it *reaches* rather than the one it + # travels through -- `Filter#nest` prefixes the condition tree and not the + # sort. Dropped in silence, that is the one order nothing would report. + it 'drops a clause naming a column it does not carry, and says so' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + + rows = collection.list(nil, filter(sort: sort({ field: 'unknown', ascending: true })), nil) + + expect(ids(rows)).to eq(%w[2 1 3]) + expect(ForestAdminDatasourceIntercom.logger) + .to have_received(:warn).with(/sort on "unknown", which it does not carry/) + end + + it 'keeps the clauses it does carry alongside the one it drops' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + + rows = collection.list( + nil, filter(sort: sort({ field: 'unknown', ascending: true }, { field: 'name', ascending: true })), nil + ) + + expect(ids(rows)).to eq(%w[1 2 3]) + end + end + + describe '#list with a page' do + before { stub_admins(admin('1'), admin('2'), admin('3')) } + + it 'cuts the window out of the records in hand' do + expect(ids(collection.list(nil, filter(page: page(1, 1)), nil))).to eq(%w[2]) + end + + it 'reads a page with no limit as every record past the offset' do + expect(ids(collection.list(nil, filter(page: page(1, 0)), nil))).to eq(%w[2 3]) + end + + it 'answers an offset past the end with no record' do + expect(collection.list(nil, filter(page: page(50, 10)), nil)).to be_empty + end + end + + describe '#aggregate' do + before { stub_admins(admin('1', 'name' => 'Alice'), admin('2', 'name' => 'Bob', 'has_inbox_seat' => false)) } + + it 'counts every record, exactly' do + expect(collection.aggregate(nil, filter, aggregation('Count'))) + .to eq([{ 'group' => {}, 'value' => 2 }]) + end + + it 'counts the rows a filter keeps' do + filtered = filter(condition_tree: leaf('has_inbox_seat', operators::EQUAL, true)) + + expect(collection.aggregate(nil, filtered, aggregation('Count')).first['value']).to eq(1) + end + + it 'groups on a column, which is exact for the same reason' do + rows = collection.aggregate(nil, filter, aggregation('Count', groups: [{ field: 'has_inbox_seat' }])) + + expect(rows.sum { |row| row['value'] }).to eq(2) + expect(rows.size).to eq(2) + end + end + + describe 'the hooks a collection has to implement' do + let(:incomplete) do + Class.new(described_class) do + def initialize(datasource) + super(datasource, 'Incomplete') + end + + def define_schema + add_column('id', 'String', is_primary_key: true) + end + end + end + + it 'says which one is missing rather than failing obscurely' do + expect { incomplete.new(datasource).list(nil, filter, nil) } + .to raise_error(NotImplementedError, /did not implement fetch_all/) + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/team_membership_spec.rb b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/team_membership_spec.rb new file mode 100644 index 000000000..656301c2a --- /dev/null +++ b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/team_membership_spec.rb @@ -0,0 +1,206 @@ +module ForestAdminDatasourceIntercom + RSpec.describe Collections::TeamMembership do + subject(:collection) { datasource.get_collection('IntercomTeamMembership') } + + let(:datasource) { Datasource.new(access_token: 's3cr3t', rate_limiter: nil) } + let(:base) { datasource.configuration.url } + let(:operators) { ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Operators } + + def filter(condition_tree: nil) + ForestAdminDatasourceToolkit::Components::Query::Filter.new(condition_tree: condition_tree) + end + + def leaf(field, operator, value = nil) + ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Nodes::ConditionTreeLeaf + .new(field, operator, value) + end + + def branch(aggregator, *conditions) + ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Nodes::ConditionTreeBranch + .new(aggregator, conditions) + end + + def json(payload, status = 200) + { status: status, body: payload.to_json, headers: { 'Content-Type' => 'application/json' } } + end + + def stub_teams(*teams) + stub_request(:get, "#{base}/teams").to_return(json('type' => 'team.list', 'teams' => teams)) + end + + def stub_admins(*admins) + stub_request(:get, "#{base}/admins").to_return(json('type' => 'admin.list', 'admins' => admins)) + end + + it 'is named IntercomTeamMembership' do + expect(collection.name).to eq('IntercomTeamMembership') + end + + it 'exposes the pair and a relation to either side of it' do + expect(collection.fields.keys).to eq(%w[id team_id admin_id team admin]) + end + + # One record per pair, keyed by both ids: a related list is read over two + # requests, and a key that changed in between would move the rows under the + # operator. + it 'reads one record per pair from the team side alone' do + stub_teams({ 'id' => '814865', 'name' => 'Support', 'admin_ids' => [493_881, 493_882] }, + { 'id' => '814866', 'name' => 'Billing', 'admin_ids' => [493_882] }) + + expect(collection.list(nil, filter, nil)) + .to eq([{ 'id' => '814865:493881', 'team_id' => '814865', 'admin_id' => '493881' }, + { 'id' => '814865:493882', 'team_id' => '814865', 'admin_id' => '493882' }, + { 'id' => '814866:493882', 'team_id' => '814866', 'admin_id' => '493882' }]) + expect(WebMock).not_to have_requested(:get, "#{base}/admins") + end + + it 'holds no record for a team nobody belongs to' do + stub_teams({ 'id' => '814865', 'admin_ids' => [] }, { 'id' => '814866', 'admin_ids' => nil }) + + expect(collection.list(nil, filter, nil)).to be_empty + end + + # Intercom types the id as a number inside `admin_ids` and as a string on the + # teammate itself: the pair carries the form the other side answers by, or + # the relation resolves to nothing. + it 'stringifies both ids' do + stub_teams('id' => 814_865, 'admin_ids' => [493_881]) + + expect(collection.list(nil, filter, nil).first) + .to eq({ 'id' => '814865:493881', 'team_id' => '814865', 'admin_id' => '493881' }) + end + + describe 'a projection through a relation' do + it 'nests the teammate under the relation, reading them once for the page' do + stub_teams('id' => '814865', 'admin_ids' => [493_881, 493_882]) + stub_admins({ 'id' => '493881', 'name' => 'Alice' }, { 'id' => '493882', 'name' => 'Bruno' }) + + expect(collection.list(nil, filter, ['id', 'admin:name'])) + .to eq([{ 'id' => '814865:493881', 'admin' => { 'name' => 'Alice', 'id' => '493881' } }, + { 'id' => '814865:493882', 'admin' => { 'name' => 'Bruno', 'id' => '493882' } }]) + expect(WebMock).to have_requested(:get, "#{base}/admins").once + end + + # A teammate who left the workspace is still named by the membership until + # Intercom drops the pair. The row reads as having no teammate rather than + # as a broken record. + it 'nests nothing when the id names no record' do + stub_teams('id' => '814865', 'admin_ids' => [493_881]) + stub_admins('id' => '493882', 'name' => 'Bruno') + + expect(collection.list(nil, filter, ['id', 'admin:name']).first) + .to eq({ 'id' => '814865:493881', 'admin' => nil }) + end + + it 'reads no relation the projection did not name' do + stub_teams('id' => '814865', 'admin_ids' => [493_881]) + + collection.list(nil, filter, %w[id team_id]) + + expect(WebMock).not_to have_requested(:get, "#{base}/admins") + end + end + + # This tier filters in memory, over every record Intercom holds, so a + # condition through a relation becomes a plain membership on the foreign key + # -- none of the limits of Intercom's search DSL apply here, nothing going + # through it. + describe 'a condition through a relation' do + it 'keeps the pairs whose teammate the target matched' do + stub_teams('id' => '814865', 'admin_ids' => [493_881, 493_882]) + stub_admins({ 'id' => '493881', 'name' => 'Alice' }, { 'id' => '493882', 'name' => 'Bruno' }) + + rows = collection.list(nil, filter(condition_tree: leaf('admin:name', operators::EQUAL, 'Alice')), %w[id]) + + expect(rows).to eq([{ 'id' => '814865:493881' }]) + end + + it 'answers nothing when the target matched no record, rather than everything' do + stub_teams('id' => '814865', 'admin_ids' => [493_881]) + stub_admins('id' => '493881', 'name' => 'Alice') + + rows = collection.list(nil, filter(condition_tree: leaf('admin:name', operators::EQUAL, 'Zoe')), %w[id]) + + expect(rows).to be_empty + end + + it 'counts the pairs a relation condition keeps, exactly' do + stub_teams('id' => '814865', 'admin_ids' => [493_881, 493_882]) + stub_admins({ 'id' => '493881', 'name' => 'Alice' }, { 'id' => '493882', 'name' => 'Bruno' }) + aggregation = ForestAdminDatasourceToolkit::Components::Query::Aggregation.new(operation: 'Count') + + rows = collection.aggregate(nil, filter(condition_tree: leaf('admin:name', operators::EQUAL, 'Alice')), + aggregation) + + expect(rows).to eq([{ 'group' => {}, 'value' => 1 }]) + end + + # The shape the agent really builds: a scope, then a segment, then the + # operator's own filter, one branch at a time. + it 'keeps a relation condition standing next to a condition on its own column' do + stub_teams({ 'id' => '814865', 'admin_ids' => [493_881] }, { 'id' => '814866', 'admin_ids' => [493_881] }) + stub_admins('id' => '493881', 'name' => 'Alice') + tree = branch('And', leaf('team_id', operators::EQUAL, '814866'), + leaf('admin:name', operators::EQUAL, 'Alice')) + + expect(collection.list(nil, filter(condition_tree: tree), %w[id])) + .to eq([{ 'id' => '814866:493881' }]) + end + + # An `and` carrying a condition nothing can satisfy matches nothing itself. + it 'answers nothing when a relation condition inside an and matches nothing' do + stub_teams('id' => '814865', 'admin_ids' => [493_881]) + stub_admins('id' => '493881', 'name' => 'Alice') + tree = branch('And', leaf('team_id', operators::EQUAL, '814865'), + leaf('admin:name', operators::EQUAL, 'Zoe')) + + expect(collection.list(nil, filter(condition_tree: tree), %w[id])).to be_empty + end + + # An `or` drops it and keeps its siblings: what the others name is still + # named. + it 'keeps the siblings of a relation condition inside an or' do + stub_teams({ 'id' => '814865', 'admin_ids' => [493_881] }, { 'id' => '814866', 'admin_ids' => [493_882] }) + stub_admins({ 'id' => '493881', 'name' => 'Alice' }, { 'id' => '493882', 'name' => 'Bruno' }) + tree = branch('Or', leaf('admin:name', operators::EQUAL, 'Zoe'), + leaf('team_id', operators::EQUAL, '814866')) + + expect(collection.list(nil, filter(condition_tree: tree), %w[id])) + .to eq([{ 'id' => '814866:493882' }]) + end + + it 'answers nothing when every branch of an or matches nothing' do + stub_teams('id' => '814865', 'admin_ids' => [493_881]) + stub_admins('id' => '493881', 'name' => 'Alice') + tree = branch('Or', leaf('admin:name', operators::EQUAL, 'Zoe'), + leaf('admin:name', operators::EQUAL, 'Yann')) + + expect(collection.list(nil, filter(condition_tree: tree), %w[id])).to be_empty + end + + # The operators a relation condition may carry are the target's own: it is + # the one that evaluates them, and the one whose refusal is worth reading. + it 'refuses an operator the target cannot evaluate, naming it' do + stub_teams('id' => '814865', 'admin_ids' => [493_881]) + stub_admins('id' => '493881', 'name' => 'Alice') + + expect { collection.list(nil, filter(condition_tree: leaf('admin:team_names', operators::EQUAL, 'x')), nil) } + .to raise_error(UnsupportedOperatorError, /IntercomAdmin cannot filter 'team_names'/) + end + + it 'refuses a relation it does not declare, naming the ones it has' do + stub_teams('id' => '814865', 'admin_ids' => [493_881]) + + expect { collection.list(nil, filter(condition_tree: leaf('owner:name', operators::EQUAL, 'x')), nil) } + .to raise_error(UnsupportedOperatorError, /"owner" is not a relation it filters through.*team, admin/m) + end + + it 'refuses a path reaching through two relations' do + stub_teams('id' => '814865', 'admin_ids' => [493_881]) + + expect { collection.list(nil, filter(condition_tree: leaf('admin:teams:name', operators::EQUAL, 'x')), nil) } + .to raise_error(UnsupportedOperatorError, /reaches through two relations/) + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/team_spec.rb b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/team_spec.rb new file mode 100644 index 000000000..54c01b75a --- /dev/null +++ b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/team_spec.rb @@ -0,0 +1,172 @@ +module ForestAdminDatasourceIntercom + RSpec.describe Collections::Team do + subject(:collection) { datasource.get_collection('IntercomTeam') } + + let(:datasource) { Datasource.new(access_token: 's3cr3t', rate_limiter: nil) } + let(:base) { datasource.configuration.url } + + def filter(condition_tree: nil) + ForestAdminDatasourceToolkit::Components::Query::Filter.new(condition_tree: condition_tree) + end + + def admins_named(name) + operators = ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Operators + ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Nodes::ConditionTreeLeaf + .new('admins:name', operators::EQUAL, name) + end + + def json(payload, status = 200) + { status: status, body: payload.to_json, headers: { 'Content-Type' => 'application/json' } } + end + + def stub_teams(*teams) + stub_request(:get, "#{base}/teams").to_return(json('type' => 'team.list', 'teams' => teams)) + end + + def stub_admins(*admins) + stub_request(:get, "#{base}/admins").to_return(json('type' => 'admin.list', 'admins' => admins)) + end + + it 'is named IntercomTeam' do + expect(collection.name).to eq('IntercomTeam') + end + + it 'exposes the team, its teammates by name, and the relation to them' do + expect(collection.fields.keys).to eq(%w[id name admin_names admins]) + end + + it 'reads the endpoint under its own key and stringifies the ids' do + stub_teams('type' => 'team', 'id' => '814865', 'name' => 'Support', 'admin_ids' => [493_881]) + + expect(collection.list(nil, filter, %w[id name])).to eq([{ 'id' => '814865', 'name' => 'Support' }]) + end + + describe 'the teammates of a team' do + it 'names them for the whole page in one read, rather than one read per row' do + stub_teams({ 'id' => '814865', 'name' => 'Support', 'admin_ids' => [493_881, 493_882] }, + { 'id' => '814866', 'name' => 'Billing', 'admin_ids' => [493_882] }) + stub_admins({ 'id' => '493881', 'name' => 'Alice' }, { 'id' => '493882', 'name' => 'Bruno' }) + + expect(collection.list(nil, filter, %w[id admin_names])) + .to eq([{ 'id' => '814865', 'admin_names' => %w[Alice Bruno] }, + { 'id' => '814866', 'admin_names' => %w[Bruno] }]) + expect(WebMock).to have_requested(:get, "#{base}/admins").once + end + + it 'names them when the projection names no column at all' do + stub_teams('id' => '814865', 'name' => 'Support', 'admin_ids' => [493_881]) + stub_admins('id' => '493881', 'name' => 'Alice') + + expect(collection.list(nil, filter, nil).first['admin_names']).to eq(['Alice']) + end + + it 'reads nothing when no projection asks for the names' do + stub_teams('id' => '814865', 'name' => 'Support', 'admin_ids' => [493_881]) + + collection.list(nil, filter, %w[id name]) + + expect(WebMock).not_to have_requested(:get, "#{base}/admins") + end + + # A missing permission costs the column, never the page -- and never the + # relation, which reads the teammates from the other side. + it 'leaves the names empty when the teammates cannot be read' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + stub_teams('id' => '814865', 'admin_ids' => [493_881]) + stub_request(:get, "#{base}/admins").to_return(json({ 'type' => 'error.list' }, 403)) + + expect(collection.list(nil, filter, %w[id admin_names]).first) + .to eq({ 'id' => '814865', 'admin_names' => [] }) + end + end + + # Intercom carries the membership on the team and on the teammate both and + # exposes no resource for the pair, so the many-to-many travels through the + # membership collection this datasource synthesizes. + describe 'the relation to the teammates' do + it 'is a many-to-many through the membership collection' do + expect(collection.fields['admins']) + .to have_attributes(type: 'ManyToMany', foreign_collection: 'IntercomAdmin', + through_collection: 'IntercomTeamMembership', + origin_key: 'team_id', foreign_key: 'admin_id') + end + + # Intercom exposes no endpoint that writes a membership, and an editable + # relation would offer an association that could only fail. + it 'is read-only' do + expect(collection.fields['admins'].is_read_only).to be(true) + end + + # What the toolkit looks for on the collection a many-to-many travels + # through. Without the two, a related list falls back to a filter on the + # foreign collection that nothing there can answer. + it 'is reachable from both ends of the membership' do + utils = ForestAdminDatasourceToolkit::Utils::Collection + + expect(utils.get_through_target(collection, 'admins')).to eq('admin') + expect(utils.get_through_origin(collection, 'admins')).to eq('team') + end + + # A many-to-many is published unfilterable, so nothing the interface offers + # reaches here -- a scope or a segment still can, and it is refused rather + # than resolved by reading the membership once per value. + it 'refuses a condition through the membership' do + stub_teams('id' => '814865', 'admin_ids' => [493_881]) + + expect { collection.list(nil, filter(condition_tree: admins_named('Alice')), nil) } + .to raise_error(UnsupportedOperatorError, /"admins" is not a relation it filters through. It has none/) + end + + # The path a related list really takes: through the membership, whose own + # relation towards the teammate is what carries the rows back. + it 'lists the teammates of one team, through the membership' do + stub_teams({ 'id' => '814865', 'admin_ids' => [493_881] }, { 'id' => '814866', 'admin_ids' => [493_882] }) + stub_admins({ 'id' => '493881', 'name' => 'Alice' }, { 'id' => '493882', 'name' => 'Bruno' }) + query = ForestAdminDatasourceToolkit::Components::Query + + rows = ForestAdminDatasourceToolkit::Utils::Collection.list_relation( + collection, %w[814865], 'admins', nil, query::Filter.new, query::Projection.new(%w[id name]) + ) + + expect(rows).to eq([{ 'id' => '493881', 'name' => 'Alice' }]) + end + + # `admin_ids` names a teammate `/admins` does not answer -- one who left + # the workspace, one outside the token's reach, one past the page cap of a + # large workspace. The membership nests a nil under the relation, which is + # right on a row an operator reads; unwrapped into a related list it would + # be a row holding nothing, and the serializer reads a row by key. + it 'drops a teammate the workspace no longer answers, rather than listing an empty row' do + stub_teams('id' => '814865', 'admin_ids' => [493_881, 999_999]) + stub_admins('id' => '493881', 'name' => 'Alice') + query = ForestAdminDatasourceToolkit::Components::Query + + rows = ForestAdminDatasourceToolkit::Utils::Collection.list_relation( + collection, %w[814865], 'admins', nil, query::Filter.new, query::Projection.new(%w[id name]) + ) + + expect(rows).to eq([{ 'id' => '493881', 'name' => 'Alice' }]) + end + + # The membership is handed the columns of the collection the relation + # reaches, `Filter#nest` prefixing the condition tree and not the sort. It + # cannot order on them, and says so rather than answering an order it did + # not honour. + it 'reports the order it cannot honour on a related list' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + stub_teams('id' => '814865', 'admin_ids' => [493_881, 493_882]) + stub_admins({ 'id' => '493881', 'name' => 'Zoe' }, { 'id' => '493882', 'name' => 'Alice' }) + query = ForestAdminDatasourceToolkit::Components::Query + + ForestAdminDatasourceToolkit::Utils::Collection.list_relation( + collection, %w[814865], 'admins', nil, + query::Filter.new(sort: query::Sort.new([{ field: 'name', ascending: true }])), + query::Projection.new(%w[id name]) + ) + + expect(ForestAdminDatasourceIntercom.logger) + .to have_received(:warn).with(/sort on "name", which it does not carry/) + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/ticket_spec.rb b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/ticket_spec.rb new file mode 100644 index 000000000..89139ab1d --- /dev/null +++ b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/ticket_spec.rb @@ -0,0 +1,742 @@ +module ForestAdminDatasourceIntercom + RSpec.describe Collections::Ticket do + subject(:collection) { described_class.new(datasource, attributes: attributes) } + + let(:datasource) { Datasource.new(access_token: 's3cr3t', rate_limiter: nil) } + # Not read off the datasource: that would build it, and boot the ticket-type + # introspection before the stub of it exists. + let(:base) { Configuration::REGION_HOSTS[:us] } + let(:operators) { ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Operators } + let(:attributes) { [attribute('_default_title_'), attribute('Due', column_type: 'Date')] } + + # `column_name` is what the schema publishes and `name` the key the payload + # uses; they differ when the workspace's own name cannot travel through a + # Forest query string. + def attribute(name, column_name: nil, column_type: 'String') + Schema::TicketAttributesIntrospector::Attribute.new(name: name, column_name: column_name || name, + column_type: column_type, data_type: 'string', + ids_by_ticket_type: { '1' => '9001' }) + end + + def json(payload, status = 200) + { status: status, body: payload.to_json, headers: { 'Content-Type' => 'application/json' } } + end + + def filter(condition_tree: nil, page: nil) + ForestAdminDatasourceToolkit::Components::Query::Filter.new(condition_tree: condition_tree, page: page) + end + + def leaf(field, operator, value = nil) + ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Nodes::ConditionTreeLeaf + .new(field, operator, value) + end + + def branch(aggregator, *conditions) + ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Nodes::ConditionTreeBranch + .new(aggregator, conditions) + end + + # Hand-written from the shape measured on a real workspace: the state comes + # embedded, the company as a bare id, and the parts ride along. + def ticket(id, overrides = {}) + { 'type' => 'ticket', 'id' => id, 'ticket_id' => "1#{id}", 'category' => 'request', + 'open' => true, 'is_shared' => false, 'created_at' => 1_700_000_000, 'updated_at' => 1_700_003_600, + 'admin_assignee_id' => 493_881, 'team_assignee_id' => 0, + 'company_id' => '696dd52099f73812610d9c7b', + 'ticket_state' => { 'type' => 'ticket_state', 'id' => '19', 'category' => 'in_progress', + 'internal_label' => 'En cours Tech', 'external_label' => 'Investigation en cours' }, + 'previous_ticket_state_id' => '14', + 'ticket_type' => { 'type' => 'ticket_type', 'id' => '1', 'name' => 'Bug' }, + 'contacts' => { 'type' => 'contact.list', 'contacts' => [{ 'type' => 'contact', 'id' => 'c1' }] }, + 'ticket_attributes' => { '_default_title_' => 'Facture manquante' }, + 'ticket_parts' => { 'type' => 'ticket_part.list', 'total_count' => 0, 'ticket_parts' => [] } } + .merge(overrides) + end + + def parts(*entries, total: nil) + { 'ticket_parts' => { 'type' => 'ticket_part.list', 'total_count' => total || entries.size, + 'ticket_parts' => entries } } + end + + def state_change(to, from: 'in_progress', at: 1_700_002_000, by: 'Alice', part_type: nil) + { 'type' => 'ticket_part', 'id' => "s#{at}", 'part_type' => part_type || 'ticket_state_updated_by_admin', + 'ticket_state' => to, 'previous_ticket_state' => from, 'created_at' => at, + 'author' => { 'type' => 'admin', 'id' => '1', 'name' => by, 'email' => 'alice@acme.test' } } + end + + def comment(at:, by: 'Alice', type: 'admin', part_type: 'comment') + { 'type' => 'ticket_part', 'id' => "c#{at}", 'part_type' => part_type, 'body' => 'Je regarde.', + 'created_at' => at, 'author' => { 'type' => type, 'id' => '1', 'name' => by } } + end + + def stub_admins(*admins) + stub_request(:get, "#{base}/admins").to_return(json('type' => 'admin.list', 'admins' => admins)) + end + + def stub_ticket_states(*states) + stub_request(:get, "#{base}/ticket_states").to_return(json('type' => 'list', 'data' => states)) + end + + def stub_search(*records, total: nil, body: nil) + answer = { 'type' => 'ticket.list', 'tickets' => records, 'total_count' => total || records.size, + 'pages' => { 'type' => 'pages', 'page' => 1 } } + + request = stub_request(:post, "#{base}/tickets/search").with(query: hash_including({})) + request = request.with(body: hash_including(body)) if body + request.to_return(json(answer)) + end + + def rows(projection = nil, **options) + collection.list(nil, filter(**options), projection) + end + + # The columns alone: a relation carries neither operators nor an order. + def columns + collection.fields.select { |_, field| field.type == 'Column' } + end + + # No projection at all asks for every declared column, `contact_name` + # included, so the reads below pay the identity read of the page -- guarding + # it on the column being *named* would leave nil the very column the row + # publishes. The blocks that assert on the identity itself stub over this. + before do + stub_request(:post, "#{base}/contacts/search") + .to_return(json('type' => 'list', 'data' => [{ 'id' => 'c1', 'name' => 'Camille' }])) + end + + describe 'schema' do + it 'is named IntercomTicket' do + expect(collection.name).to eq('IntercomTicket') + end + + # The state travels embedded, so its label costs nothing and the row does + # not depend on IntercomTicketState to be readable. One label and not + # three: the category and the customer-facing label are a hop away, on the + # relation, and neither was ever filterable -- so no segment or scope could + # rest on them. + it 'flattens the embedded state into one label, the rest being a hop away' do + expect(collection.fields.keys).to include('state_id', 'state_label', 'previous_state_id') + expect(collection.fields.keys).not_to include('state_category', 'state_external_label') + end + + it 'carries the attributes of every ticket type in union' do + expect(collection.fields.keys).to include('_default_title_', 'Due') + expect(collection.fields['Due'].column_type).to eq('Date') + end + + # `/tickets/search` ignores a sort without saying so, on every column. + it 'declares every column unsortable' do + expect(columns.values.map(&:is_sortable).uniq).to eq([false]) + end + + it 'advertises the filters the search endpoint answers, and only those' do + expect(collection.fields['category'].filter_operators).to eq(%w[equal not_equal]) + expect(collection.fields['created_at'].filter_operators).to eq(%w[greater_than less_than]) + end + + # Measured during lot 1: `/tickets/search` refuses `company_id` with + # `invalid_field` although a ticket carries one. Filtering tickets by + # account is not something this endpoint does. + it 'advertises no filter on the account, which the endpoint refuses' do + expect(collection.fields['company_id'].filter_operators).to be_empty + end + + # Derived by the agent from the parts of the ticket. A column advertising a + # filter the read cannot honour is what this lot exists to prevent. + it 'advertises no filter on the columns derived from the parts' do + %w[closed_at closed_by_name last_reply_at last_responder_name last_responder_type].each do |column| + expect(collection.fields[column].filter_operators).to be_empty, "#{column} advertises a filter" + end + end + + # R7: an attribute is filtered through an id that differs from one ticket + # type to the next, so the union column cannot say which id to use. + it 'advertises no filter on a ticket attribute while the arbitration stands' do + expect(collection.fields['Due'].filter_operators).to be_empty + expect(collection.fields['_default_title_'].filter_operators).to be_empty + end + + # The schema advertises no filter, but a scope, a segment or a customizer + # can still send one -- and the reason it is refused with is the + # arbitration itself rather than "this column is not in the table", which + # is what an attribute column would otherwise get: its name is the + # workspace's, so no row can carry it. + it 'refuses a filter on a ticket attribute with the arbitration as its reason' do + expect { rows(%w[id], condition_tree: leaf('Due', operators::EQUAL, 'x')) } + .to raise_error(UnsupportedOperatorError, /differs from one ticket type to the next/) + end + + # Intercom matches text field by field, and this endpoint exposes none + # this collection carries. + it 'is not searchable' do + expect(collection).not_to be_is_searchable + end + + # An attribute overwriting a native column would show the attribute where + # the operator expects the ticket field. + it 'skips an attribute whose name a native column already carries' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + + collection = described_class.new(datasource, attributes: [attribute('category')]) + + expect(collection.fields['category'].column_type).to eq('String') + expect(ForestAdminDatasourceIntercom.logger).to have_received(:warn).with(/skips the ticket attribute/) + end + end + + describe '#list' do + # There is no GET /tickets at all: even an unfiltered list view goes + # through the search endpoint with a predicate that matches everything. + it 'reads the search endpoint with a predicate matching every ticket' do + stub_search(ticket('1')) + + rows(%w[id]) + + expect(WebMock).to have_requested(:post, %r{#{base}/tickets/search}) + .with(body: hash_including('query' => { 'field' => 'created_at', 'operator' => '>', 'value' => '0' })) + end + + # Not the 150 the API accepts: a page carries every ticket's whole + # timeline, and there is no way to ask Intercom for less. + it 'asks for far fewer tickets than the API would allow' do + stub_search(ticket('1')) + + rows(%w[id]) + + pagination = hash_including('per_page' => described_class::MAX_TICKETS_PER_PAGE) + + expect(WebMock).to have_requested(:post, %r{#{base}/tickets/search}) + .with(body: hash_including('pagination' => pagination)) + end + + it 'reads the records under the tickets key' do + stub_search(ticket('1'), ticket('2')) + + expect(rows(%w[id]).map { |row| row['id'] }).to eq(%w[1 2]) + end + + it 'flattens the ticket into the row the schema declares' do + stub_search(ticket('1')) + + expect(rows.first) + .to include('id' => '1', 'ticket_id' => '11', 'category' => 'request', 'open' => true, + 'state_id' => '19', 'state_label' => 'En cours Tech', + 'previous_state_id' => '14', 'ticket_type_name' => 'Bug', + 'company_id' => '696dd52099f73812610d9c7b', 'admin_assignee_id' => '493881') + end + + # Intercom keys the values by attribute name, which is what lets one + # collection display the union -- and what stops it from filtering on them. + it 'reads an attribute value by its name' do + stub_search(ticket('1')) + + expect(rows.first['_default_title_']).to eq('Facture manquante') + end + + # Forest lists the fields of a request in a comma-separated query + # parameter, so a column name carrying one splits the projection into + # fields no collection has -- a 400 before the page is ever read. The + # introspector renames such an attribute; the value is still read under the + # name Intercom keys it by. + it 'reads a renamed attribute under the name the payload uses' do + renamed = attribute('ID de l\'objet (immo, facture)', column_name: "ID de l'objet (immo facture)") + collection = described_class.new(datasource, attributes: [renamed]) + stub_search(ticket('1', 'ticket_attributes' => { 'ID de l\'objet (immo, facture)' => 'immo_42' })) + + row = collection.list(nil, filter, nil).first + + expect(row["ID de l'objet (immo facture)"]).to eq('immo_42') + end + + # The invariant behind the rename, asserted on the whole schema rather than + # on one column. + it 'publishes no column name a Forest query string could not carry' do + collection = described_class.new(datasource, attributes: [attribute('Scope', column_name: 'Scope')]) + + expect(collection.fields.keys.grep(/[,:]/)).to be_empty + end + + it 'leaves an attribute of another ticket type absent rather than empty' do + stub_search(ticket('1')) + + expect(rows.first['Due']).to be_nil + end + + it 'reads a date attribute as ISO8601 like every other Intercom date' do + stub_search(ticket('1', 'ticket_attributes' => { 'Due' => 1_700_000_000 })) + + expect(rows.first['Due']).to eq('2023-11-14T22:13:20Z') + end + + # A record detail goes to its own endpoint, which is not the search one. + it 'reads one ticket through the record endpoint' do + stub_request(:get, "#{base}/tickets/1").with(query: hash_including({})).to_return(json(ticket('1'))) + + expect(rows(%w[id], condition_tree: leaf('id', operators::EQUAL, '1')).map { |row| row['id'] }).to eq(%w[1]) + end + + # The search carries the filter it was given, in place of the predicate + # that matches everything. + it 'searches with the query the translator wrote' do + search = stub_search(ticket('1'), body: { 'query' => { 'field' => 'category', 'operator' => '=', + 'value' => 'request' } }) + + rows(%w[id], condition_tree: leaf('category', operators::EQUAL, 'request')) + + expect(search).to have_been_made + end + + it 'refuses a condition on a column the endpoint does not filter, by name' do + expect { rows(%w[id], condition_tree: leaf('state_category', operators::EQUAL, 'resolved')) } + .to raise_error(UnsupportedOperatorError, /cannot filter "state_category"/) + end + + # The account, measured as refused by the endpoint itself. + it 'refuses a condition on the account with the measurement as its reason' do + expect { rows(%w[id], condition_tree: leaf('company_id', operators::EQUAL, '696dd')) } + .to raise_error(UnsupportedOperatorError, /invalid_field/) + end + + it 'refuses a free-text search, having no text column the endpoint matches' do + searched = ForestAdminDatasourceToolkit::Components::Query::Filter.new(search: 'facture') + + expect { collection.list(nil, searched, %w[id]) } + .to raise_error(UnsupportedOperatorError, /cannot answer a free-text search/) + end + end + + # Every target is read whole in one request, so a relation resolves for a + # whole page at the price of a single read -- which is what makes eight of + # them affordable at all. + describe 'the relations to the reference collections' do + it 'points every id at the collection that reads it' do + expect(collection.fields['admin_assignee']) + .to have_attributes(type: 'ManyToOne', foreign_collection: 'IntercomAdmin', + foreign_key: 'admin_assignee_id', foreign_key_target: 'id', is_read_only: true) + expect(collection.fields['team_assignee']) + .to have_attributes(foreign_collection: 'IntercomTeam', foreign_key: 'team_assignee_id') + expect(collection.fields['state']) + .to have_attributes(foreign_collection: 'IntercomTicketState', foreign_key: 'state_id') + expect(collection.fields['previous_state']) + .to have_attributes(foreign_collection: 'IntercomTicketState', foreign_key: 'previous_state_id') + expect(collection.fields['ticket_type']) + .to have_attributes(foreign_collection: 'IntercomTicketType', foreign_key: 'ticket_type_id') + end + + # Declared before the attribute columns, so a workspace attribute whose + # name lands on a relation is skipped with a warning the way one landing on + # a column is. Declared after, it would collide and take the boot with it. + it 'keeps the relation when a ticket attribute carries its name' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + + collection = described_class.new(datasource, attributes: [attribute('state')]) + + expect(collection.fields['state'].type).to eq('ManyToOne') + end + + it 'nests the assignee under the relation, reading the teammates once for the page' do + stub_search(ticket('1'), ticket('2')) + stub_admins('id' => '493881', 'name' => 'Alice', 'email' => 'alice@acme.test') + + rows = collection.list(nil, filter, ['id', 'admin_assignee:name']) + + expect(rows).to eq([{ 'id' => '1', 'admin_assignee' => { 'name' => 'Alice', 'id' => '493881' } }, + { 'id' => '2', 'admin_assignee' => { 'name' => 'Alice', 'id' => '493881' } }]) + expect(WebMock).to have_requested(:get, "#{base}/admins").once + end + + # A teammate who left the workspace: the row reads as unassigned rather + # than as a page that could not be served. + it 'nests nothing when the id names no teammate' do + stub_search(ticket('1')) + stub_admins('id' => '493882', 'name' => 'Bruno') + + expect(collection.list(nil, filter, ['id', 'admin_assignee:name']).first) + .to eq({ 'id' => '1', 'admin_assignee' => nil }) + end + + it 'reads no relation the projection did not name' do + stub_search(ticket('1')) + + collection.list(nil, filter, %w[id state_label]) + + expect(WebMock).not_to have_requested(:get, "#{base}/admins") + end + + # Forest's own parser builds a two-hop path, and this nests one target row + # under the relation name rather than a tree of them: the second hop would + # be dropped by the target's projection and the column would come back + # missing from a row that looks complete. Refused by name, like the filter + # that reaches that deep. + it 'refuses a projection reaching through two relations' do + stub_search(ticket('1')) + + expect { collection.list(nil, filter, ['id', 'admin_assignee:teams:name']) } + .to raise_error(UnsupportedOperatorError, /reaches through two relations/) + end + + # The price of a relation is one read per target *collection*, not one per + # relation: `state` and `previous_state` name the same endpoint, and it is + # read once, over the ids both of them point at. + it 'reads a collection two relations point at once, not once per relation' do + stub_search(ticket('1')) + stub_ticket_states({ 'id' => '19', 'internal_label' => 'En cours Tech' }, + { 'id' => '14', 'internal_label' => 'Recu' }) + + rows = collection.list(nil, filter, ['id', 'state:internal_label', 'previous_state:internal_label']) + + expect(rows.first).to eq({ 'id' => '1', + 'state' => { 'internal_label' => 'En cours Tech', 'id' => '19' }, + 'previous_state' => { 'internal_label' => 'Recu', 'id' => '14' } }) + expect(WebMock).to have_requested(:get, "#{base}/ticket_states").once + end + + # One read for two relations means one projection for two, and each of them + # still gets the columns it asked for rather than the union. + it 'nests under each relation only the columns that relation asked for' do + stub_search(ticket('1')) + stub_ticket_states({ 'id' => '19', 'category' => 'in_progress', 'internal_label' => 'En cours Tech' }, + { 'id' => '14', 'category' => 'submitted', 'internal_label' => 'Recu' }) + + rows = collection.list(nil, filter, ['id', 'state:category', 'previous_state:internal_label']) + + expect(rows.first).to eq({ 'id' => '1', + 'state' => { 'category' => 'in_progress', 'id' => '19' }, + 'previous_state' => { 'internal_label' => 'Recu', 'id' => '14' } }) + end + end + + # A relation is published filterable as soon as any column of its target is, + # so the interface offers `admin_assignee:name` the moment the relation + # exists. What Intercom is really filtered on is the foreign key: the target + # says which of its records match -- over every record it holds, not over a + # page -- and the ids it names are what the search carries. + # `IntercomContact#tickets` is a one-to-many the agent serves by listing this + # collection on the key. The row it rests on is `spec`, not `measured`: + # whether `/tickets/search` filters on a contact id at all is one of the + # probe's questions, and the answer moves the row into the table or into the + # refusals. + describe 'the tickets of a contact' do + it 'filters on the contact id the relation resolves to' do + stub_search(ticket('1')) + + collection.list(nil, filter(condition_tree: leaf('contact_id', operators::EQUAL, 'c1')), %w[id]) + + expect(WebMock).to have_requested(:post, %r{#{base}/tickets/search}) + .with(body: hash_including('query' => { 'field' => 'contact_ids', 'operator' => '=', + 'value' => 'c1' })) + end + end + + # The exchange itself, which lot 1 paid for and did not publish: the parts + # ride along in every ticket response, so the thread costs no request. + describe 'the thread' do + it 'names who said what, when, and through which kind of event' do + stub_search(ticket('1', parts(comment(at: 1_700_001_000, by: 'Camille', type: 'contact'), + comment(at: 1_700_002_000, by: 'Alice')))) + + expect(rows(%w[id timeline]).first['timeline']) + .to eq([{ 'id' => 'c1700001000', 'part_type' => 'comment', 'created_at' => '2023-11-14T22:30:00Z', + 'author_type' => 'contact', 'author_name' => 'Camille', 'author_email' => nil, + 'body' => 'Je regarde.', 'attachment_count' => 0, 'redacted' => nil }, + { 'id' => 'c1700002000', 'part_type' => 'comment', 'created_at' => '2023-11-14T22:46:40Z', + 'author_type' => 'admin', 'author_name' => 'Alice', 'author_email' => nil, + 'body' => 'Je regarde.', 'attachment_count' => 0, 'redacted' => nil }]) + end + + # An assignment, an internal note and a reply are not the same event, and + # a thread that flattens them reads as an exchange that never happened the + # way it did. Which also means the team's internal notes are in there. + it 'keeps the internal notes and the state changes, each under its own type' do + stub_search(ticket('1', parts(comment(at: 1_700_001_000, part_type: 'note'), + state_change('resolved', at: 1_700_002_000)))) + + expect(rows(%w[id timeline]).first['timeline'].map { |entry| entry['part_type'] }) + .to eq(%w[note ticket_state_updated_by_admin]) + end + + # Empty means empty here, and says so: the parts are in every response, + # unlike a conversation read from a listing, whose timeline stays nil + # because nothing is known. + it 'answers an empty thread on a ticket nothing happened to' do + stub_search(ticket('1')) + + expect(rows(%w[id timeline]).first['timeline']).to eq([]) + end + + it 'builds nothing when the projection does not name it' do + stub_search(ticket('1', parts(comment(at: 1_700_001_000)))) + + expect(rows(%w[id]).first).to eq('id' => '1') + end + + # The bodies are HTML written by end customers, and rendering third-party + # HTML inside Forest is neither safe nor useful (R10). + it 'asks Intercom for plain text rather than markup' do + stub_search(ticket('1')) + + rows(%w[id timeline]) + + expect(WebMock).to have_requested(:post, %r{#{base}/tickets/search}) + .with(query: { 'display_as' => 'plaintext' }) + end + end + + describe 'a condition through a relation' do + it 'filters on the foreign key the target resolved to' do + stub_admins('id' => '493881', 'name' => 'Alice') + stub_search(ticket('1')) + + rows(%w[id], condition_tree: leaf('admin_assignee:name', operators::EQUAL, 'Alice')) + + expect(WebMock).to have_requested(:post, %r{#{base}/tickets/search}) + .with(body: hash_including('query' => { 'field' => 'admin_assignee_id', 'operator' => '=', + 'value' => '493881' })) + end + + # Intercom takes no membership operator on these fields, so several matches + # become several conditions -- which is also why the group has a ceiling. + it 'writes a group of equalities when the target matched several records' do + stub_admins({ 'id' => '493881', 'name' => 'Alice' }, { 'id' => '493882', 'name' => 'Alice' }) + stub_search(ticket('1')) + + rows(%w[id], condition_tree: leaf('admin_assignee:name', operators::EQUAL, 'Alice')) + + expect(WebMock).to have_requested(:post, %r{#{base}/tickets/search}) + .with(body: hash_including('query' => { + 'operator' => 'OR', + 'value' => [{ 'field' => 'admin_assignee_id', 'operator' => '=', + 'value' => '493881' }, + { 'field' => 'admin_assignee_id', 'operator' => '=', + 'value' => '493882' }] + })) + end + + # No row can satisfy it, and Intercom's DSL has no way of saying so: the + # search is skipped rather than sent as a filter that would come back with + # every ticket. + it 'answers nothing, and reads nothing, when the target matched no record' do + stub_admins('id' => '493881', 'name' => 'Alice') + + expect(rows(%w[id], condition_tree: leaf('admin_assignee:name', operators::EQUAL, 'Zoe'))).to be_empty + expect(WebMock).not_to have_requested(:post, %r{#{base}/tickets/search}) + end + + it 'counts none of them either, without a request' do + stub_admins('id' => '493881', 'name' => 'Alice') + aggregation = ForestAdminDatasourceToolkit::Components::Query::Aggregation.new(operation: 'Count') + + counted = collection.aggregate(nil, filter(condition_tree: leaf('admin_assignee:name', + operators::EQUAL, 'Zoe')), aggregation) + + expect(counted).to eq([{ 'group' => {}, 'value' => 0 }]) + expect(WebMock).not_to have_requested(:post, %r{#{base}/tickets/search}) + end + + # A relation group nested inside the tree the agent assembled: a scope, a + # segment and the operator's own filter, and the group counts as one of + # them. + it 'nests the group inside the condition it was written next to' do + stub_admins({ 'id' => '493881', 'name' => 'Alice' }, { 'id' => '493882', 'name' => 'Alice' }) + stub_search(ticket('1')) + tree = branch('And', leaf('category', operators::EQUAL, 'request'), + leaf('admin_assignee:name', operators::EQUAL, 'Alice')) + + collection.list(nil, filter(condition_tree: tree), %w[id]) + + expect(WebMock).to have_requested(:post, %r{#{base}/tickets/search}) + .with(body: hash_including('query' => { + 'operator' => 'AND', + 'value' => [{ 'field' => 'category', 'operator' => '=', + 'value' => 'request' }, + { 'operator' => 'OR', + 'value' => [{ 'field' => 'admin_assignee_id', + 'operator' => '=', 'value' => '493881' }, + { 'field' => 'admin_assignee_id', + 'operator' => '=', 'value' => '493882' }] }] + })) + end + + # Intercom nests a search two levels deep, and the group a relation expands + # into is a level nobody wrote. Inlined into a parent aggregating the same + # way, it costs none -- without which a scope plus a "match any" filter + # carrying one relation condition would be refused for a nesting the + # operator cannot find in their own filter. + it 'inlines the group into a parent that aggregates the same way' do + stub_admins({ 'id' => '493881', 'name' => 'Alice' }, { 'id' => '493882', 'name' => 'Alice' }) + stub_search(ticket('1')) + tree = branch('And', leaf('category', operators::EQUAL, 'request'), + branch('Or', leaf('open', operators::EQUAL, true), + leaf('admin_assignee:name', operators::EQUAL, 'Alice'))) + + collection.list(nil, filter(condition_tree: tree), %w[id]) + + expect(WebMock).to have_requested(:post, %r{#{base}/tickets/search}) + .with(body: hash_including('query' => { + 'operator' => 'AND', + 'value' => [{ 'field' => 'category', 'operator' => '=', + 'value' => 'request' }, + { 'operator' => 'OR', + 'value' => [{ 'field' => 'open', 'operator' => '=', + 'value' => true }, + { 'field' => 'admin_assignee_id', + 'operator' => '=', 'value' => '493881' }, + { 'field' => 'admin_assignee_id', + 'operator' => '=', 'value' => '493882' }] }] + })) + end + + # Inlining trades a level of nesting for width, and Intercom bounds both. + # Past fifteen conditions the nested form is the one that fits, so the + # group stays where it was rather than emptying the budget it was spared. + it 'leaves the group nested when inlining it would pass fifteen conditions' do + stub_admins(*(1..3).map { |index| { 'id' => index.to_s, 'name' => 'Alice' } }) + stub_search(ticket('1')) + others = (1..14).map { |index| leaf('category', operators::NOT_EQUAL, "c#{index}") } + tree = branch('Or', *others, leaf('admin_assignee:name', operators::EQUAL, 'Alice')) + + collection.list(nil, filter(condition_tree: tree), %w[id]) + + expect(WebMock).to(have_requested(:post, %r{#{base}/tickets/search}).with do |request| + query = JSON.parse(request.body)['query'] + query['value'].size == 15 && query['value'].last['operator'] == 'OR' && + query['value'].last['value'].size == 3 + end) + end + + it 'reads nothing when a relation condition inside an and matches nothing' do + stub_admins('id' => '493881', 'name' => 'Alice') + tree = branch('And', leaf('category', operators::EQUAL, 'request'), + leaf('admin_assignee:name', operators::EQUAL, 'Zoe')) + + expect(collection.list(nil, filter(condition_tree: tree), %w[id])).to be_empty + expect(WebMock).not_to have_requested(:post, %r{#{base}/tickets/search}) + end + + # Fifteen conditions per group is Intercom's limit, and a relation reaches + # it without trying. Refused by name rather than sent and answered with a + # 400 naming neither the limit nor the filter that hit it. + # + # "more than fifteen" rather than a count: the target is read one record + # past what a group holds and no further, so what is known is that it does + # not fit -- see `match_page`. + it 'refuses a relation condition matching more records than a group holds' do + stub_admins(*(1..20).map { |index| { 'id' => index.to_s, 'name' => 'Alice' } }) + + expect { rows(%w[id], condition_tree: leaf('admin_assignee:name', operators::EQUAL, 'Alice')) } + .to raise_error(UnsupportedOperatorError, /names more than 15 records.*15 conditions per group/m) + end + + # `/tickets/search` filters no state id -- the table carries none -- so the + # state relation is there to be read and navigated, and the message says + # which of the two it is rather than naming a column the operator never + # wrote. Refused before the target is read, a refusal that spends a request + # costing exactly what it refuses. + it 'refuses a condition through a relation the endpoint filters nothing on' do + expect { rows(%w[id], condition_tree: leaf('state:category', operators::EQUAL, 'in_progress')) } + .to raise_error(UnsupportedOperatorError, %r{resolves to "state_id", on which tickets/search takes no}) + expect(WebMock).not_to have_requested(:get, /ticket_states/) + end + end + + describe '#aggregate' do + it 'counts a filtered collection through the total_count of its search' do + stub_search(total: 12, body: { 'query' => { 'field' => 'open', 'operator' => '=', 'value' => true } }) + aggregation = ForestAdminDatasourceToolkit::Components::Query::Aggregation.new(operation: 'Count') + + expect(collection.aggregate(nil, filter(condition_tree: leaf('open', operators::EQUAL, true)), + aggregation).first['value']).to eq(12) + end + + it 'counts through the total_count of the search, exactly' do + stub_search(ticket('1'), total: 81_142) + aggregation = ForestAdminDatasourceToolkit::Components::Query::Aggregation.new(operation: 'Count') + + expect(collection.aggregate(nil, filter, aggregation)).to eq([{ 'group' => {}, 'value' => 81_142 }]) + end + end + + describe 'the derived columns' do + # A ticket has no statistics block -- measured on 81 142 tickets -- so the + # closure date exists nowhere but in the parts, which ride along anyway. + it 'reads the closure from the last transition into a resolved state' do + stub_search(ticket('1', **parts(state_change('in_progress', from: 'submitted', at: 1_700_001_000), + state_change('resolved', at: 1_700_002_000, by: 'Alice')))) + + expect(rows.first).to include('closed_at' => '2023-11-14T22:46:40Z', 'closed_by_name' => 'Alice') + end + + # This workspace runs workflows: a closure done by automation carries + # another variant of the same event, and matching the admin one in full + # would make it invisible. + it 'reads a closure whatever the variant of the state-change event' do + stub_search(ticket('1', **parts(state_change('resolved', at: 1_700_002_000, + part_type: 'ticket_state_updated_by_workflow')))) + + expect(rows.first['closed_at']).to eq('2023-11-14T22:46:40Z') + end + + # Measured: a part can record a transition to the state the ticket was + # already in, and that is not an event. + it 'ignores a transition that changed nothing' do + stub_search(ticket('1', **parts(state_change('resolved', from: 'resolved', at: 1_700_002_000)))) + + expect(rows.first['closed_at']).to be_nil + end + + it 'keeps the last closure of a ticket that was reopened' do + stub_search(ticket('1', **parts(state_change('resolved', at: 1_700_001_000), + state_change('in_progress', from: 'resolved', at: 1_700_001_500), + state_change('resolved', at: 1_700_002_000)))) + + expect(rows.first['closed_at']).to eq('2023-11-14T22:46:40Z') + end + + it 'names the last responder and which side they are on' do + stub_search(ticket('1', **parts(comment(at: 1_700_001_000, by: 'Alice'), + comment(at: 1_700_002_000, by: 'Camille', type: 'contact')))) + + expect(rows.first) + .to include('last_reply_at' => '2023-11-14T22:46:40Z', 'last_responder_name' => 'Camille', + 'last_responder_type' => 'contact') + end + + # An internal note is a touch, not an answer: it would name as last + # responder someone who never wrote to the person waiting. + it 'ignores an internal note' do + stub_search(ticket('1', **parts(comment(at: 1_700_001_000, by: 'Alice'), + comment(at: 1_700_002_000, by: 'Bob', part_type: 'note')))) + + expect(rows.first['last_responder_name']).to eq('Alice') + end + + it 'leaves both columns empty on a ticket nothing happened to' do + stub_search(ticket('1')) + + expect(rows.first).to include('closed_at' => nil, 'last_responder_name' => nil) + end + + # Intercom keeps the 500 most recent parts. A resolved ticket whose + # transition fell out of that window has an *unknown* closure date, not an + # absent one -- a Date column cannot say it, so the log does. + it 'reports a resolved ticket whose timeline was truncated' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + resolved = { 'ticket_state' => { 'id' => '20', 'category' => 'resolved', 'internal_label' => 'Resolu' } } + stub_search(ticket('1', **resolved, **parts(comment(at: 1_700_001_000), total: 500))) + + rows(%w[id closed_at]) + + expect(ForestAdminDatasourceIntercom.logger).to have_received(:warn).with(/truncated their timeline/) + end + + it 'stays quiet when the closure is simply absent from a complete timeline' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + stub_search(ticket('1', **parts(comment(at: 1_700_001_000)))) + + rows(%w[id closed_at]) + + expect(ForestAdminDatasourceIntercom.logger).not_to have_received(:warn) + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/ticket_state_spec.rb b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/ticket_state_spec.rb new file mode 100644 index 000000000..88ab24ded --- /dev/null +++ b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/ticket_state_spec.rb @@ -0,0 +1,38 @@ +module ForestAdminDatasourceIntercom + RSpec.describe Collections::TicketState do + subject(:collection) { described_class.new(datasource) } + + let(:datasource) { Datasource.new(access_token: 's3cr3t', rate_limiter: nil) } + let(:base) { datasource.configuration.url } + + def filter + ForestAdminDatasourceToolkit::Components::Query::Filter.new + end + + def stub_ticket_states(*states) + stub_request(:get, "#{base}/ticket_states") + .to_return(status: 200, body: { 'type' => 'list', 'data' => states }.to_json, + headers: { 'Content-Type' => 'application/json' }) + end + + it 'is named IntercomTicketState' do + expect(collection.name).to eq('IntercomTicketState') + end + + # Two labels rather than one: what the support team sees, and what the + # customer is shown. + it 'exposes both labels of a state' do + expect(collection.fields.keys).to eq(%w[id category internal_label external_label archived]) + end + + it 'reads the endpoint and serializes a state' do + stub_ticket_states('type' => 'ticket_state', 'id' => '3', 'category' => 'submitted', + 'internal_label' => 'Waiting on triage', 'external_label' => 'We are on it', + 'archived' => false) + + expect(collection.list(nil, filter, nil)) + .to eq([{ 'id' => '3', 'category' => 'submitted', 'internal_label' => 'Waiting on triage', + 'external_label' => 'We are on it', 'archived' => false }]) + end + end +end diff --git a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/ticket_type_spec.rb b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/ticket_type_spec.rb new file mode 100644 index 000000000..38d1f490d --- /dev/null +++ b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/collections/ticket_type_spec.rb @@ -0,0 +1,45 @@ +module ForestAdminDatasourceIntercom + RSpec.describe Collections::TicketType do + subject(:collection) { described_class.new(datasource) } + + let(:datasource) { Datasource.new(access_token: 's3cr3t', rate_limiter: nil) } + let(:base) { datasource.configuration.url } + + def filter + ForestAdminDatasourceToolkit::Components::Query::Filter.new + end + + def stub_types(*types) + stub_request(:get, "#{base}/ticket_types") + .to_return(status: 200, body: { 'type' => 'list', 'data' => types }.to_json, + headers: { 'Content-Type' => 'application/json' }) + end + + it 'is named IntercomTicketType' do + expect(collection.name).to eq('IntercomTicketType') + end + + it 'exposes what makes a ticket type readable' do + expect(collection.fields.keys).to eq(%w[id name description category icon archived]) + end + + # This endpoint uses the `data` envelope, unlike /admins and /teams. + it 'reads the endpoint through the data envelope' do + stub_types('type' => 'ticket_type', 'id' => '1', 'name' => 'Bug', 'description' => 'A bug', + 'category' => 'request', 'icon' => '🐛', 'archived' => false) + + expect(collection.list(nil, filter, nil)) + .to eq([{ 'id' => '1', 'name' => 'Bug', 'description' => 'A bug', 'category' => 'request', + 'icon' => '🐛', 'archived' => false }]) + end + + # The attribute definitions nested here are what the ticket collection reads + # to build its columns -- an attribute of the same name carries a different + # id from one type to the next -- and they are meaningless as a column. + it 'leaves the nested attribute definitions out of the schema' do + stub_types('id' => '1', 'ticket_type_attributes' => { 'type' => 'list', 'data' => [{ 'id' => '9' }] }) + + expect(collection.list(nil, filter, nil).first.keys).not_to include('ticket_type_attributes') + end + end +end diff --git a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/configuration_spec.rb b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/configuration_spec.rb new file mode 100644 index 000000000..957f6ffff --- /dev/null +++ b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/configuration_spec.rb @@ -0,0 +1,136 @@ +module ForestAdminDatasourceIntercom + RSpec.describe Configuration do + subject(:configuration) { described_class.new(access_token: 's3cr3t') } + + it 'defaults to the US host, since that is where a workspace lands unasked' do + expect(configuration.url).to eq('https://api.intercom.io') + end + + it 'pins the API version the spike ran against' do + expect(configuration.api_version).to eq('2.16') + end + + it 'points at the regional host it is given' do + expect(described_class.new(access_token: 's3cr3t', region: :eu).url).to eq('https://api.eu.intercom.io') + end + + it 'takes the region as a string too' do + expect(described_class.new(access_token: 's3cr3t', region: 'AU').url).to eq('https://api.au.intercom.io') + end + + it 'lets an explicit base_url win over the region, for a proxy or a mock server' do + configured = described_class.new(access_token: 's3cr3t', region: :eu, base_url: 'https://intercom.test/api/') + + expect(configured.url).to eq('https://intercom.test/api') + end + + it 'reports the subpath a base_url is mounted under' do + configured = described_class.new(access_token: 's3cr3t', base_url: 'https://intercom.test/api') + + expect(configured.base_path).to eq('/api') + end + + it 'reports no subpath against the API itself' do + expect(configuration.base_path).to eq('') + end + + describe 'validation' do + it 'refuses a missing access token' do + expect { described_class.new(access_token: nil) } + .to raise_error(ConfigurationError, /missing required config: access_token/) + end + + it 'refuses a blank access token' do + expect { described_class.new(access_token: ' ') } + .to raise_error(ConfigurationError, /access_token/) + end + + it 'names the regions it knows when handed one it does not' do + expect { described_class.new(access_token: 's3cr3t', region: :moon) } + .to raise_error(ConfigurationError, /unknown region :moon.*:us, :eu, :au/m) + end + + # A relative base_url makes Faraday resolve paths against the working + # directory, which surfaces much later as a failure naming nothing. + it 'refuses a base_url that is not absolute' do + expect { described_class.new(access_token: 's3cr3t', base_url: 'api.intercom.io') } + .to raise_error(ConfigurationError, /must be an absolute http\(s\) url/) + end + + it 'refuses a base_url that is not a url at all' do + expect { described_class.new(access_token: 's3cr3t', base_url: 'http://[bad') } + .to raise_error(ConfigurationError, /not a valid url/) + end + + it 'refuses an empty api_version, which would let the workspace default decide' do + expect { described_class.new(access_token: 's3cr3t', api_version: '') } + .to raise_error(ConfigurationError, /api_version cannot be empty/) + end + end + + describe 'defaults' do + it 'paces requests and retries unless told otherwise' do + expect(configuration).to have_attributes(rate_limiter: an_instance_of(RateLimiter), + retry_policy: an_instance_of(RetryPolicy)) + end + + it 'is patient on a request and impatient on the boot' do + expect(configuration).to have_attributes(timeout: 30, open_timeout: 5, boot_timeout: 10, + boot_open_timeout: 3) + end + + it 'takes the pacing out of the stack when handed no limiter' do + expect(described_class.new(access_token: 's3cr3t', rate_limiter: nil).rate_limiter).to be_nil + end + end + + # `URI::HTTPS` is a `URI::HTTP`, so a cleartext base_url is accepted -- a + # mock server is one, and that is half of what the parameter is for. What it + # costs is the bearer header crossing that network in clear, and the token + # reads the whole workspace. + describe 'a cleartext base_url' do + before { allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) } + + it 'is accepted, and says the access token travels in clear over it' do + described_class.new(access_token: 's3cr3t', base_url: 'http://proxy.internal/intercom') + + expect(ForestAdminDatasourceIntercom.logger) + .to have_received(:warn).with(/is not https, and every request carries the Intercom access token/) + end + + it 'says nothing about a loopback host, which is nobody else s network' do + described_class.new(access_token: 's3cr3t', base_url: 'http://localhost:4010') + described_class.new(access_token: 's3cr3t', base_url: 'http://intercom.localhost') + + expect(ForestAdminDatasourceIntercom.logger).not_to have_received(:warn) + end + + it 'says nothing about https' do + described_class.new(access_token: 's3cr3t', base_url: 'https://intercom.test') + + expect(ForestAdminDatasourceIntercom.logger).not_to have_received(:warn) + end + end + + describe '#inspect' do + it 'never prints the bearer token' do + expect(configuration.inspect).to include('[FILTERED]') + expect(configuration.inspect).not_to include('s3cr3t') + end + + it 'still names the host and version, which is what one inspects it for' do + expect(configuration.inspect).to include('https://api.intercom.io', '2.16') + end + + # A credentialed egress proxy is the other reason to set a base_url, and + # `URI` accepts its credentials in the url: printing it verbatim would put + # a second secret exactly where this keeps the first one from going. + it 'masks the credentials of a base_url that carries them' do + configured = described_class.new(access_token: 's3cr3t', base_url: 'https://bob:hunter2@proxy.test/api') + + expect(configured.inspect).not_to include('hunter2', 'bob') + expect(configured.inspect).to include('https://[FILTERED]@proxy.test/api') + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/datasource_spec.rb b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/datasource_spec.rb new file mode 100644 index 000000000..290bc4123 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/datasource_spec.rb @@ -0,0 +1,107 @@ +module ForestAdminDatasourceIntercom + RSpec.describe Datasource do + subject(:datasource) { described_class.new(access_token: 's3cr3t') } + + it 'boots without reaching Intercom' do + expect { datasource }.not_to raise_error + end + + # The reference collections come first: they are what turns an assignee id + # into a teammate and a state id into a label. Contacts and Companies + # follow, before the two collections whose relations point at them. + # The membership sits with the two collections it joins: a many-to-many + # needs a collection to travel through, and Intercom exposes none. + it 'publishes the collections of the lot' do + expect(datasource.collections.keys) + .to eq(%w[IntercomAdmin IntercomTeam IntercomTeamMembership IntercomTicketType IntercomTicketState + IntercomContact IntercomCompany IntercomConversation IntercomTicket]) + end + + # The four reads a boot performs, and no fifth: `/me`, which is where + # Intercom echoes the API version it served, and the attributes a workspace + # declares on its ticket types, on its contacts and on its companies -- + # columns of those collections, since a payload carries the values of the + # attributes that record happens to have been given, never their + # definitions. + it 'checks the version and introspects the workspace attributes, and reads nothing else' do + datasource + + expect(WebMock).to have_requested(:get, %r{/me}).once + expect(WebMock).to have_requested(:get, /ticket_types/).once + expect(WebMock).to have_requested(:get, /data_attributes/).with(query: { 'model' => 'contact' }).once + expect(WebMock).to have_requested(:get, /data_attributes/).with(query: { 'model' => 'company' }).once + expect(WebMock).not_to have_requested(:get, %r{conversations|admins|teams|companies/list}) + end + + # Intercom serves the workspace's own default when the pin is not honoured + # and the payload shapes differ between versions. The echo is the only + # place that shows, and reading it at boot is what turns the check from + # code that exists into code that runs. + it 'reports a workspace serving another API version than the pinned one' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + stub_me(version: '2.11') + + datasource + + expect(ForestAdminDatasourceIntercom.logger) + .to have_received(:warn).with(/asked Intercom for API version 2\.16 and it served 2\.11/) + end + + # Degrades like the attribute reads: a token that cannot reach `/me` costs + # the check, never the agent. + it 'boots without the check when the token cannot read /me' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + stub_request(:get, %r{/me}).to_return(status: 403, body: '{}', + headers: { 'Content-Type' => 'application/json' }) + + expect(datasource.collections.keys).to include('IntercomTicket') + expect(ForestAdminDatasourceIntercom.logger) + .to have_received(:warn).with(%r{could not read /me at boot \(HTTP 403\)}) + end + + # A token without that permission costs the attribute columns, never the + # agent. + it 'boots without the attribute columns when the introspection is refused' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + stub_request(:get, /ticket_types/).to_return(status: 403, body: '{}', + headers: { 'Content-Type' => 'application/json' }) + + expect(datasource.get_collection('IntercomTicket').fields.keys).not_to include('_default_title_') + end + + # The same guarantee on the other two models, and it is what the acceptance + # criterion of lot 4 asks for: a token missing a permission costs the + # columns it could not read, and the collection still boots. + it 'boots the contact and company collections when their introspection is refused' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + stub_request(:get, /data_attributes/).to_return(status: 403, body: '{}', + headers: { 'Content-Type' => 'application/json' }) + + expect(datasource.get_collection('IntercomContact').fields.keys).to include('email') + expect(datasource.get_collection('IntercomCompany').fields.keys).to include('name') + end + + it 'configures a client from the options it is handed' do + stub_me(base: 'https://api.eu.intercom.io') + stub_ticket_types(base: 'https://api.eu.intercom.io') + stub_data_attributes('contact', base: 'https://api.eu.intercom.io') + stub_data_attributes('company', base: 'https://api.eu.intercom.io') + configured = described_class.new(access_token: 's3cr3t', region: :eu, rate_limiter: nil) + + expect(configured.configuration.url).to eq('https://api.eu.intercom.io') + expect(configured.client).to be_a(Client) + end + + it 'refuses to boot on a configuration it cannot use' do + expect { described_class.new(access_token: nil) }.to raise_error(ConfigurationError) + end + + it 'names the collections it holds when printed' do + expect(datasource.inspect).to include('IntercomAdmin', 'IntercomTicketState') + end + + it 'never prints the token the client carries' do + expect(datasource.inspect).not_to include('s3cr3t') + end + end +end diff --git a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/pagination/cursor_walker_spec.rb b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/pagination/cursor_walker_spec.rb new file mode 100644 index 000000000..06df6b725 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/pagination/cursor_walker_spec.rb @@ -0,0 +1,193 @@ +module ForestAdminDatasourceIntercom + module Pagination + RSpec.describe CursorWalker do + subject(:walker) { described_class.new } + + let(:asked) { [] } + + def record(id) + { 'id' => id } + end + + def page(records, next_cursor: nil) + Client::Page.new(records: records, next_cursor: next_cursor, total_count: nil) + end + + # A page source Intercom's own shape: each page advertises the cursor of + # the next one, and the last advertises nothing. + def source(*pages) + queue = pages.dup + + lambda do |per_page, cursor| + asked << [per_page, cursor] + queue.shift || page([]) + end + end + + def ids(records) + records.map { |r| r['id'] } + end + + it 'serves a window one page already covers' do + records = walker.walk(offset: 0, limit: 2, &source(page([record('a'), record('b')], next_cursor: 'c1'))) + + expect(ids(records)).to eq(%w[a b]) + end + + it 'asks only for the records the window still needs' do + walker.walk(offset: 0, limit: 3, &source(page([record('a'), record('b'), record('c')]))) + + expect(asked).to eq([[3, nil]]) + end + + it 'walks pages until the window is covered, then slices the offset out' do + pages = source(page([record('a'), record('b')], next_cursor: 'c1'), + page([record('c'), record('d')], next_cursor: 'c2')) + + records = walker.walk(offset: 2, limit: 2, &pages) + + expect(ids(records)).to eq(%w[c d]) + expect(asked).to eq([[4, nil], [2, 'c1']]) + end + + it 'follows the cursor each page advertises' do + walker.walk(offset: 0, limit: 4, &source(page([record('a')], next_cursor: 'c1'), + page([record('b')], next_cursor: 'c2'), + page([record('c')]))) + + expect(asked.map(&:last)).to eq([nil, 'c1', 'c2']) + end + + it 'stops where Intercom stops advertising a next page' do + records = walker.walk(offset: 0, limit: 10, &source(page([record('a')]))) + + expect(ids(records)).to eq(%w[a]) + expect(asked.size).to eq(1) + end + + # Stopping here is a truncation like a cap: Intercom said there was a next + # page and this could not follow it, so the window is short of what was + # asked -- and a short answer is never silent. + it 'stops on an empty page, and says the result is truncated' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + + records = walker.walk(offset: 0, limit: 10, &source(page([], next_cursor: 'c1'))) + + expect(records).to be_empty + expect(ForestAdminDatasourceIntercom.logger) + .to have_received(:warn).with(/advertised a next page this could not follow/) + end + + # None of this happens against Intercom today, but a walk driven by a + # remote value stops on its own terms rather than on the caps only. + it 'stops on a cursor it has already followed, and says the result is truncated' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + pages = source(page([record('a')], next_cursor: 'loop'), + page([record('b')], next_cursor: 'loop'), + page([record('c')], next_cursor: 'loop')) + + walker.walk(offset: 0, limit: 10, &pages) + + expect(asked.size).to eq(2) + expect(ForestAdminDatasourceIntercom.logger) + .to have_received(:warn).with(/advertised a next page this could not follow/) + end + + # The window was covered: there is nothing short about the answer, so the + # cursor that did not move is not worth a word. + it 'stays quiet when a stalled cursor comes after the window was covered' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + + walker.walk(offset: 0, limit: 1, &source(page([record('a')], next_cursor: 'loop'))) + + expect(ForestAdminDatasourceIntercom.logger).not_to have_received(:warn) + end + + # Intercom documents duplicates on a dataset that moves between two + # paginated requests, and conversations move constantly. Two rows carrying + # one id is what a list view renders as two identical lines. + it 'drops a record a previous page already served' do + pages = source(page([record('a'), record('b')], next_cursor: 'c1'), + page([record('b'), record('c')])) + + records = walker.walk(offset: 0, limit: 10, &pages) + + expect(ids(records)).to eq(%w[a b c]) + end + + it 'keeps records carrying no id rather than deciding they are not records' do + anonymous = source(page([{ 'email' => 'a@b.test' }, { 'email' => 'c@d.test' }])) + records = walker.walk(offset: 0, limit: 10, &anonymous) + + expect(records.size).to eq(2) + end + + it 'returns nothing, and asks nothing, for a limit of zero' do + records = walker.walk(offset: 0, limit: 0, &source(page([record('a')]))) + + expect(records).to be_empty + expect(asked).to be_empty + end + + it 'reads an offset past the end as an empty window rather than an error' do + records = walker.walk(offset: 50, limit: 10, &source(page([record('a')]))) + + expect(records).to be_empty + end + + it 'treats a negative offset as the beginning' do + records = walker.walk(offset: -5, limit: 1, &source(page([record('a')]))) + + expect(ids(records)).to eq(%w[a]) + end + + describe 'a limit of nil, which asks for everything past the offset' do + it 'walks to the end' do + pages = source(page([record('a')], next_cursor: 'c1'), page([record('b')])) + + expect(ids(walker.walk(offset: 0, limit: nil, &pages))).to eq(%w[a b]) + end + + it 'asks for the largest page Intercom accepts' do + walker.walk(offset: 0, limit: nil, &source(page([record('a')]))) + + expect(asked.first.first).to eq(Client::MAX_PER_PAGE) + end + end + + describe 'caps' do + before { allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) } + + it 'stops after the page it is allowed, and says the result is truncated' do + capped = described_class.new(max_pages: 2) + pages = source(page([record('a')], next_cursor: 'c1'), + page([record('b')], next_cursor: 'c2'), + page([record('c')], next_cursor: 'c3')) + + capped.walk(offset: 0, limit: nil, &pages) + + expect(asked.size).to eq(2) + expect(ForestAdminDatasourceIntercom.logger).to have_received(:warn).with(/truncated/) + end + + it 'stops on the record budget, and never asks for more than it has left' do + capped = described_class.new(max_records: 3) + pages = source(page([record('a'), record('b')], next_cursor: 'c1'), + page([record('c'), record('d')], next_cursor: 'c2')) + + capped.walk(offset: 0, limit: nil, &pages) + + expect(asked).to eq([[3, nil], [1, 'c1']]) + end + + # A walk that covered the window it was given hands back exactly that, + # and has nothing to report -- unlike one a cap cut short. + it 'stays quiet when the window was covered' do + walker.walk(offset: 0, limit: 1, &source(page([record('a')], next_cursor: 'c1'))) + + expect(ForestAdminDatasourceIntercom.logger).not_to have_received(:warn) + end + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/query/condition_tree_translator_spec.rb b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/query/condition_tree_translator_spec.rb new file mode 100644 index 000000000..cd891d7de --- /dev/null +++ b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/query/condition_tree_translator_spec.rb @@ -0,0 +1,211 @@ +module ForestAdminDatasourceIntercom + RSpec.describe Query::ConditionTreeTranslator do + let(:operators) { ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Operators } + let(:nodes) { ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Nodes } + let(:endpoint) { Query::SearchFields.fetch('conversations') } + + def translate(tree, timezone: 'UTC') + described_class.call(tree, endpoint: endpoint, collection: 'IntercomConversation', timezone: timezone) + end + + def leaf(field, operator, value = nil) + nodes::ConditionTreeLeaf.new(field, operator, value) + end + + def branch(aggregator, *conditions) + nodes::ConditionTreeBranch.new(aggregator, conditions) + end + + describe 'a leaf' do + it 'writes the Intercom field, operator and value the endpoint takes' do + expect(translate(leaf('state', operators::EQUAL, 'open'))) + .to eq({ 'field' => 'state', 'operator' => '=', 'value' => 'open' }) + end + + # The column is the operator's name for it; the field is Intercom's. They + # are not the same on a statistic, which the column flattens onto the row. + it 'filters a flattened statistic through the field Intercom nests it in' do + expect(translate(leaf('closed_at', operators::LESS_THAN, '2026-09-01T00:00:00Z'))['field']) + .to eq('statistics.last_close_at') + end + + it 'answers nil for no condition at all, which is a list view' do + expect(translate(nil)).to be_nil + end + end + + # A bare `id equals X` never reaches here -- the collection reads the record + # endpoint instead, one request rather than a search. What does reach here + # is the key nested in an `and`: a permission scope, a segment, or a second + # filter alongside it, which the record endpoint cannot answer on its own. + describe 'the primary key, once something else is filtered alongside it' do + it 'writes the key as the search field it is' do + tree = branch('And', leaf('id', operators::EQUAL, '42'), leaf('state', operators::EQUAL, 'open')) + + expect(translate(tree)).to eq({ 'operator' => 'AND', + 'value' => [{ 'field' => 'id', 'operator' => '=', 'value' => '42' }, + { 'field' => 'state', 'operator' => '=', 'value' => 'open' }] }) + end + + it 'writes a membership on the key as the list Intercom takes' do + tree = branch('And', leaf('id', operators::IN, %w[1 2]), leaf('open', operators::EQUAL, true)) + + expect(translate(tree)['value'].first) + .to eq({ 'field' => 'id', 'operator' => 'IN', 'value' => %w[1 2] }) + end + end + + describe 'a branch' do + it 'groups its conditions under the aggregator Intercom spells in capitals' do + tree = branch('Or', leaf('state', operators::EQUAL, 'open'), leaf('state', operators::EQUAL, 'snoozed')) + + expect(translate(tree)).to eq({ 'operator' => 'OR', + 'value' => [{ 'field' => 'state', 'operator' => '=', 'value' => 'open' }, + { 'field' => 'state', 'operator' => '=', 'value' => 'snoozed' }] }) + end + + # The agent builds a tree one branch at a time -- a scope, then a segment, + # then the operator's filter -- and Intercom allows two levels of nesting. + # A wrapper around a single condition is a level worth not spending. + it 'unwraps a branch carrying one condition rather than spending a level on it' do + tree = branch('And', branch('And', leaf('open', operators::EQUAL, true))) + + expect(translate(tree)).to eq({ 'field' => 'open', 'operator' => '=', 'value' => true }) + end + + it 'nests a group inside a group' do + tree = branch('And', leaf('open', operators::EQUAL, true), + branch('Or', leaf('state', operators::EQUAL, 'open'), + leaf('state', operators::EQUAL, 'snoozed'))) + + expect(translate(tree)['value'].last['operator']).to eq('OR') + end + + it 'refuses an aggregator that is neither and nor or' do + expect { translate(branch('Xor', leaf('open', operators::EQUAL, true))) } + .to raise_error(UnsupportedOperatorError, /cannot read "Xor" as a condition tree aggregator/) + end + + # A branch with nothing in it names no record and no filter, so sending it + # would answer a filtered question with the whole collection. + it 'refuses a branch carrying no condition' do + expect { translate(branch('And')) } + .to raise_error(UnsupportedOperatorError, /carrying no condition/) + end + + it 'refuses a node that is neither a leaf nor a branch' do + expect { translate(Object.new) } + .to raise_error(UnsupportedOperatorError, /cannot read Object as a condition/) + end + end + + # The headline failure this lot had to avoid: Intercom truncates a date + # search to the UTC day, so the pair of bounds the toolkit rewrites `today` + # into reads as "from tomorrow" and "before today" -- an empty answer to the + # most ordinary filter there is -- unless each bound is moved to the day + # boundary that makes Intercom answer the day the filter named. + describe 'the two bounds an interval is rewritten into' do + it 'asks for the day the interval names rather than cancelling out' do + tree = branch('And', leaf('created_at', operators::GREATER_THAN, '2026-09-01T00:00:00Z'), + leaf('created_at', operators::LESS_THAN, '2026-09-01T23:59:59Z')) + + expect(translate(tree)['value'].map { |bound| Time.at(bound['value']).utc.iso8601 }) + .to eq(['2026-08-31T00:00:00Z', '2026-09-02T00:00:00Z']) + end + end + + # Over either limit Intercom answers a 400 whose body names neither the + # limit nor the part of the filter that reached it. + describe 'the limits of the search DSL, checked before the request leaves' do + def leaves(count) + Array.new(count) { |index| leaf('state', operators::EQUAL, "state-#{index}") } + end + + it 'takes a group nested one level inside another' do + tree = branch('And', leaf('open', operators::EQUAL, true), + branch('Or', *leaves(2))) + + expect(translate(tree)['value'].last['operator']).to eq('OR') + end + + it 'refuses a group inside a group inside a group, naming the shape' do + tree = branch('And', leaf('open', operators::EQUAL, true), + branch('Or', leaf('read', operators::EQUAL, true), + branch('And', *leaves(2)))) + + expect { translate(tree) } + .to raise_error(UnsupportedOperatorError, + /nests a search 2 levels deep and this one reaches 3.*flatten the segment/m) + end + + # The innermost group is sometimes not one the operator wrote: a relation + # expands into one equality per record it matched, and telling them to + # flatten a nesting they never wrote is a refusal they cannot act on. + it 'names the relation when the innermost group is what one expanded into' do + relation_group = Collections::Relations::RelationBranch.new('Or', leaves(2), 'admin_assignee:name') + tree = branch('Or', leaf('open', operators::EQUAL, true), + branch('And', leaf('read', operators::EQUAL, true), relation_group)) + + expect { translate(tree) } + .to raise_error(UnsupportedOperatorError, + /"admin_assignee:name" expanded into, one equality per record it matched/) + end + + # A branch carrying a single condition is unwrapped, so it spends no level: + # the agent wraps a scope and a segment one branch at a time. + it 'does not spend a level on the branches the agent wraps around one condition' do + tree = branch('And', branch('And', branch('And', leaf('open', operators::EQUAL, true)))) + + expect(translate(tree)).to eq({ 'field' => 'open', 'operator' => '=', 'value' => true }) + end + + it 'takes a group of fifteen conditions' do + expect(translate(branch('Or', *leaves(15)))['value'].size).to eq(15) + end + + # A scope, a segment and a filter add up; and a condition naming several + # values arrives expanded into one condition per value, Intercom taking no + # membership operator on these fields. + it 'refuses a group of sixteen, and says what brings it back under' do + expect { translate(branch('Or', *leaves(16))) } + .to raise_error(UnsupportedOperatorError, /takes 15 conditions per group and this Or carries 16/) + end + end + + describe 'what it will not translate' do + # The whole point of the lot: a condition dropped on the way to Intercom + # comes back as an unfiltered page that looks filtered. + it 'refuses a column the endpoint does not filter, and names what it does' do + expect { translate(leaf('contact_name', operators::EQUAL, 'Camille')) } + .to raise_error(UnsupportedOperatorError, /cannot filter "contact_name".*Contacts endpoint/m) + end + + it 'refuses a column nothing declares, listing the ones it takes' do + expect { translate(leaf('nope', operators::EQUAL, 'x')) } + .to raise_error(UnsupportedOperatorError, /takes no filter on it. Filter on one of: id, state, open/) + end + + # A `relation:field` is traded for a condition on the foreign key before a + # tree reaches this translation, or refused by the collection with the + # relations it does declare. Reaching here means that pass was skipped, so + # the message names what the endpoint filters rather than a relation + # nobody resolved. + it 'refuses a path through a relation, naming what it does filter' do + expect { translate(leaf('contact:email', operators::EQUAL, 'camille@acme.test')) } + .to raise_error(UnsupportedOperatorError, /filters columns, not paths through a relation/) + end + + it 'refuses an operator the endpoint does not answer on that field' do + expect { translate(leaf('state', operators::CONTAINS, 'op')) } + .to raise_error(UnsupportedOperatorError, /answers equal, not_equal on "state" and nothing else/) + end + + # Published on a date column by the toolkit and refused by this endpoint: + # the datasource cannot express an equality on a day it truncates. + it 'refuses an equality on a date, which is not one of the two bounds' do + expect { translate(leaf('created_at', operators::EQUAL, '2026-09-01T00:00:00Z')) } + .to raise_error(UnsupportedOperatorError, /answers greater_than, less_than/) + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/query/filter_value_spec.rb b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/query/filter_value_spec.rb new file mode 100644 index 000000000..30cd4f71f --- /dev/null +++ b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/query/filter_value_spec.rb @@ -0,0 +1,271 @@ +module ForestAdminDatasourceIntercom + RSpec.describe Query::FilterValue do + subject(:formatter) { described_class.new(collection: 'IntercomConversation', timezone: timezone) } + + let(:timezone) { 'Europe/Paris' } + let(:operators) { ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Operators } + let(:nodes) { ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Nodes } + + def field(type, operators = ['=']) + Query::SearchFields::Field.new(column: 'c', field: 'c', type: type, operators: operators, source: 'spec') + end + + def call(type, value, spelling: '=', operator: nil) + leaf = nodes::ConditionTreeLeaf.new('c', operator || operators::EQUAL, value) + + formatter.call(leaf, field(type), spelling) + end + + # Intercom truncates a date search to the UTC day -- measured, and against + # its own documentation, which promises the workspace timezone. `>` answers + # from the start of the day after the value, `<` before the start of the + # day of the value. Sent as they come, the two bounds of an interval cancel + # each other out and `today` answers nothing. + describe 'a date, and the UTC day Intercom truncates it to' do + def bound(value, spelling) + Time.at(call('date', value, spelling: spelling, + operator: spelling == '>' ? operators::GREATER_THAN : operators::LESS_THAN)) + .utc.iso8601 + end + + it 'moves a lower bound back a day, so Intercom answers from the day it names' do + expect(bound('2026-09-01T08:30:00Z', '>')).to eq('2026-08-31T00:00:00Z') + end + + it 'moves an upper bound forward a day, so Intercom answers through the day it names' do + expect(bound('2026-09-01T08:30:00Z', '<')).to eq('2026-09-02T00:00:00Z') + end + + # An upper bound already sitting on a day boundary names the day to leave + # out, which is what Intercom answers on its own. + it 'leaves an upper bound already on a UTC day boundary where it is' do + expect(bound('2026-09-01T00:00:00Z', '<')).to eq('2026-09-01T00:00:00Z') + end + + # The pair the toolkit rewrites `today` into, from a caller in UTC: what + # comes back is that day and nothing else. + it 'answers the day itself for the two bounds of an interval' do + expect(bound('2026-09-01T00:00:00Z', '>')).to eq('2026-08-31T00:00:00Z') + expect(bound('2026-09-01T23:59:59Z', '<')).to eq('2026-09-02T00:00:00Z') + end + + it 'keeps the offset an ISO8601 timestamp carries' do + expect(bound('2026-09-01T10:30:00+02:00', '<')).to eq('2026-09-02T00:00:00Z') + end + + # `FilterFactory` writes the bounds of a previous period with strftime + # and no offset at all, so a chart comparing to the previous month sends + # `2026-08-01 00:00:00` meaning the caller's midnight. Read in the + # process timezone it is a different instant, and a midnight moved by any + # offset lands on another UTC day once truncated -- a whole day of rows + # beside the ones the chart named. + it 'reads a timestamp carrying no offset in the timezone of the caller' do + # Paris midnight on 1 September is 2026-08-31T22:00:00Z, whose UTC day + # is the 31st, so `>` must answer from the 31st and sit on the 30th. + expect(bound('2026-09-01 00:00:00', '>')).to eq('2026-08-30T00:00:00Z') + end + + it 'reads a whole previous-period window in the timezone of the caller' do + expect(bound('2026-08-01 00:00:00', '>')).to eq('2026-07-30T00:00:00Z') + expect(bound('2026-09-01 00:00:00', '<')).to eq('2026-09-01T00:00:00Z') + end + + # A day with no time of day is the caller's day: it is the timezone the + # filter was written in that says when that day starts. + it 'reads a bare date as midnight in the timezone of the caller' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + + expect(bound('2026-09-01', '>')).to eq('2026-08-30T00:00:00Z') + end + + it 'reads a Ruby Date, a Time, a DateTime and epoch seconds the same way' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + + expect(bound(Date.new(2026, 9, 1), '>')).to eq('2026-08-30T00:00:00Z') + expect(bound(Time.utc(2026, 9, 1, 8, 30), '<')).to eq('2026-09-02T00:00:00Z') + expect(bound(DateTime.new(2026, 9, 1, 8, 30, 0), '<')).to eq('2026-09-02T00:00:00Z') + expect(bound(1_788_251_400, '<')).to eq('2026-09-02T00:00:00Z') + end + + # A window written in another timezone is answered on the UTC days it + # overlaps, and an operator has no way of guessing that from the rows. + it 'reports the UTC day boundary once, when it is not the day of the caller' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + + bound('2026-08-31T22:00:00Z', '>') + bound('2026-08-31T23:00:00Z', '>') + + expect(ForestAdminDatasourceIntercom.logger).to have_received(:warn).once.with(/truncates a date search/) + end + + it 'stays quiet when the window and the UTC day are the same day anyway' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + + bound('2026-09-01T08:30:00Z', '>') + + expect(ForestAdminDatasourceIntercom.logger).not_to have_received(:warn) + end + + context 'when the caller names a timezone nothing knows' do + let(:timezone) { 'Middle-Earth/Shire' } + + # Falling back to UTC silently would move a day boundary by the offset, + # which is the failure a timezone is read for in the first place. + it 'reads the day boundary in UTC and says so' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + + expect(bound('2026-09-01', '>')).to eq('2026-08-31T00:00:00Z') + expect(ForestAdminDatasourceIntercom.logger).to have_received(:warn).with(/unknown timezone/) + end + + it 'reads a timestamp carrying no offset in UTC and says so' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + + expect(bound('2026-09-01 00:00:00', '>')).to eq('2026-08-31T00:00:00Z') + expect(ForestAdminDatasourceIntercom.logger).to have_received(:warn).with(/unknown timezone/) + end + end + + context 'when the caller names no timezone at all' do + let(:timezone) { ' ' } + + it 'reads the day boundary in UTC' do + expect(bound('2026-09-01', '>')).to eq('2026-08-31T00:00:00Z') + end + end + + it 'refuses a string that is not a date' do + expect { call('date', 'last tuesday', spelling: '>') } + .to raise_error(UnsupportedOperatorError, /Intercom expects a date on this field/) + end + + it 'refuses a value that is not a date at all' do + expect { call('date', { 'day' => 1 }, spelling: '>') } + .to raise_error(UnsupportedOperatorError, /Intercom expects a date on this field/) + end + + # `Time.parse` accepts a time of day on its own and fills the date in from + # the server's clock, so this would have travelled as a bound on whichever + # day the request happened to run -- a filter whose answer changes at + # midnight and never says why. + it 'refuses a time of day rather than completing it from today' do + expect { call('date', '12:00', spelling: '>') } + .to raise_error(UnsupportedOperatorError, /Intercom expects a date on this field/) + end + + # A month names a range, and which end of it the operator meant is theirs + # to say rather than this class's to guess. + it 'refuses a date missing its day' do + expect { call('date', 'Jan 2026', spelling: '>') } + .to raise_error(UnsupportedOperatorError, /Intercom expects a date on this field/) + end + + # It carries the three parts a date needs and still names no day: the + # parser is what says so, and its complaint is not a bound to send on. + it 'refuses a date whose parts are out of range' do + expect { call('date', '2026-13-45', spelling: '>') } + .to raise_error(UnsupportedOperatorError, /Intercom expects a date on this field/) + end + + # Reading either as epoch seconds raises a FloatDomainError, which would + # leave the read with an error naming a float where the operator asked + # for a date. The number branch already refuses them; a date is no + # different. + it 'refuses a cast that overflowed to Infinity, and a NaN' do + expect { call('date', Float::INFINITY, spelling: '>') } + .to raise_error(UnsupportedOperatorError, /Intercom expects a date on this field/) + expect { call('date', Float::NAN, spelling: '>') } + .to raise_error(UnsupportedOperatorError, /Intercom expects a date on this field/) + end + + it 'reads a finite number as the epoch seconds Intercom stores' do + expect(bound(1_767_225_600, '>')).to eq('2025-12-31T00:00:00Z') + end + end + + describe 'a number' do + # The agent casts every Number column with `to_f`, so an integer field + # would otherwise be filtered with `42.0`, a form none of its values carry. + it 'sends a whole float as the integer it is' do + expect(call('number', 42.0)).to eq(42) + end + + it 'keeps a decimal, and an integer, as they are' do + expect(call('number', 42.5)).to eq(42.5) + expect(call('number', 42)).to eq(42) + end + + it 'reads an integer written as a string' do + expect(call('number', '42')).to eq(42) + end + + it 'reads a decimal written as a string' do + expect(call('number', '42.5')).to eq(42.5) + end + + # `to_i` raises on both, and so does the JSON encoder a step later, as a + # 500 naming nothing the operator can act on. + it 'refuses a cast that overflowed and a value that is not a number' do + expect { call('number', Float::INFINITY) }.to raise_error(UnsupportedOperatorError, /expects a number/) + expect { call('number', 'many') }.to raise_error(UnsupportedOperatorError, /expects a number/) + expect { call('number', []) }.to raise_error(UnsupportedOperatorError, /expects a number/) + end + end + + describe 'a boolean' do + it 'sends a flag as a boolean, whichever way the filter spelled it' do + expect(call('boolean', true)).to be(true) + expect(call('boolean', false)).to be(false) + expect(call('boolean', 'true')).to be(true) + expect(call('boolean', 'false')).to be(false) + end + + it 'refuses anything else, rather than reading it as truthy' do + expect { call('boolean', 'yes') }.to raise_error(UnsupportedOperatorError, /expects a true or a false/) + end + end + + describe 'a list' do + it 'formats every value of an IN the way the field takes it' do + expect(call('number', ['4', 5.0], spelling: 'IN', operator: operators::IN)).to eq([4, 5]) + end + + # A filter matching everything is not what an empty list was asked for. + it 'refuses an empty list' do + expect { call('string', [], spelling: 'IN', operator: operators::IN) } + .to raise_error(UnsupportedOperatorError, /empty list/) + end + + # `not_in [nil, 'open']` was asked to exclude the records carrying neither + # and would come back including the blank ones. + it 'refuses a list holding a blank, rather than dropping it' do + expect { call('string', [nil, 'open'], spelling: 'NIN', operator: operators::NOT_IN) } + .to raise_error(UnsupportedOperatorError, /cannot filter "c" for absence/) + end + end + + describe 'a condition on the absence of a value' do + # `present`, `blank` and `missing` are derived from an equality above this + # datasource and rewritten into a comparison with an empty value. Intercom + # would answer it as if the empty string were a value of its own. + it 'refuses the rewritten comparison and says to filter on a value' do + expect { call('string', nil) } + .to raise_error(UnsupportedOperatorError, /matches values and has no operator for the lack of one/) + end + + it 'refuses an empty string the same way' do + expect { call('string', '') }.to raise_error(UnsupportedOperatorError, /for absence/) + end + end + + describe 'a string' do + it 'sends it as it came' do + expect(call('string', 'open')).to eq('open') + end + + it 'sends a text the same way' do + expect(call('text', 'facture')).to eq('facture') + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/query/operator_table_spec.rb b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/query/operator_table_spec.rb new file mode 100644 index 000000000..42a998071 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/query/operator_table_spec.rb @@ -0,0 +1,110 @@ +module ForestAdminDatasourceIntercom + RSpec.describe Query::OperatorTable do + let(:operators) { ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Operators } + let(:rules) { ForestAdminDatasourceToolkit::Validations::Rules } + let(:equivalent) { ForestAdminDatasourceToolkit::Components::Query::ConditionTree::ConditionTreeEquivalent } + + def field(type, operators) + Query::SearchFields::Field.new(column: 'c', field: 'c', type: type, operators: operators, source: 'spec') + end + + describe '.forest_operators' do + it 'publishes only what the endpoint accepts on that field' do + expect(described_class.forest_operators(field('string', ['=']))).to eq([operators::EQUAL]) + expect(described_class.forest_operators(field('string', ['=', '!=']))) + .to eq([operators::EQUAL, operators::NOT_EQUAL]) + end + + # Intercom documents one substring operator and no case semantics for it, + # and the frontend sends either spelling depending on the column. + it 'reads both spellings of contains onto the one operator Intercom has' do + published = described_class.forest_operators(field('text', ['~'])) + + expect(published).to eq([operators::CONTAINS, operators::I_CONTAINS]) + end + + # The point of the whole table: a date column carries the two bounds even + # where the endpoint accepts more, because declaring an equality on a Date + # makes the toolkit republish `in`, which its own validator refuses. + it 'keeps a date column to the two bounds whatever the endpoint accepts' do + published = described_class.forest_operators(field('date', ['>', '<', '>=', '<=', '=', '!='])) + + expect(published).to eq([operators::GREATER_THAN, operators::LESS_THAN]) + end + + it 'publishes nothing for a field the endpoint answers no known operator on' do + expect(described_class.forest_operators(field('string', ['~']))).to be_empty + end + end + + describe '.intercom_operator' do + it 'spells a Forest operator the way the search DSL does' do + expect(described_class.intercom_operator(field('number', ['>']), operators::GREATER_THAN)).to eq('>') + end + + it 'answers nil for an operator the endpoint does not accept on that field' do + expect(described_class.intercom_operator(field('string', ['=']), operators::NOT_EQUAL)).to be_nil + end + + it 'answers nil for an operator no field of that type carries' do + expect(described_class.intercom_operator(field('boolean', ['=']), operators::CONTAINS)).to be_nil + end + end + + describe 'the operators every published column ends up with' do + # The invariant PRD-989 says nothing checks: everything the agent publishes + # from what a column declares must be an operator its own validator allows. + # A column advertising a filter the agent then rejects is a 400 in the + # interface for a reason that has nothing to do with Intercom. + it 'is a set the toolkit validator allows, for every column of every endpoint' do + column_types = { 'string' => 'String', 'text' => 'String', 'date' => 'Date', + 'boolean' => 'Boolean', 'number' => 'Number' } + + Query::SearchFields.endpoints.each do |name| + Query::SearchFields.fetch(name).fields.each_value do |searchable| + declared = described_class.forest_operators(searchable) + column_type = column_types.fetch(searchable.type) + published = operators.all.select { |o| equivalent.equivalent_tree?(o, declared, column_type) } + + expect(published - rules.get_allowed_operators_for_column_type(column_type)) + .to be_empty, "#{name}.#{searchable.column} publishes an operator Rules refuses" + end + end + end + + # Walked from the schema rather than from the table, which is the only way + # to see the primary key: `add_column` writes its operators by hand -- the + # toolkit refuses a collection whose key carries neither `equal` nor `in` + # -- so the loop above, reading the table, could never reach the one + # column no row of the table is derived from. A column the schema + # publishes and the table cannot spell is a filter the translator refuses + # at read time, which is what a record detail hits the moment a scope + # nests `id equals X` in an `and`. + it 'is a set the table can express, for every column the datasource publishes' do + cursor_collections.each do |collection| + endpoint = collection.send(:search_endpoint) + + collection.fields.each do |column, schema| + published = Array(schema.respond_to?(:filter_operators) ? schema.filter_operators : nil) + next if published.empty? + + searchable = endpoint.field(column) + expect(searchable).not_to be_nil, + "#{collection.name}.#{column} publishes #{published.join(", ")} and " \ + "#{endpoint.path} carries no row to translate it" + expect(published - described_class.forest_operators(searchable)) + .to be_empty, "#{collection.name}.#{column} publishes an operator the translator would refuse" + end + end + end + end + + # Only the two collections a search endpoint backs: the reference ones are + # read whole and filtered in memory, and publish no operator from a table. + def cursor_collections + datasource = Datasource.new(access_token: 's3cr3t', rate_limiter: nil) + + datasource.collections.each_value.grep(Collections::CursorCollection) + end + end +end diff --git a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/query/search_fields_spec.rb b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/query/search_fields_spec.rb new file mode 100644 index 000000000..e6edc9088 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/query/search_fields_spec.rb @@ -0,0 +1,279 @@ +module ForestAdminDatasourceIntercom + RSpec.describe Query::SearchFields do + def table(fields: {}, refused: {}, measured_at: nil, attributes: nil) + definition = { 'path' => 'tickets/search', 'measured_at' => measured_at, 'fields' => fields, + 'refused' => refused, 'candidates' => [] } + definition['ticket_attributes'] = attributes if attributes + + described_class.build('endpoints' => { 'tickets' => definition })['tickets'] + end + + def attributes_row(overrides = {}) + { 'filterable' => false, 'reason' => 'display only', 'source' => 'spec' }.merge(overrides) + end + + def field_row(overrides = {}) + { 'field' => 'created_at', 'type' => 'date', 'operators' => ['>', '<'], 'source' => 'spec' }.merge(overrides) + end + + describe 'the committed table' do + # The schema derives its filters from this file, so a malformed row is a + # boot failure rather than a column nobody can explain. Reading it here is + # what turns that guarantee into a test. + it 'reads the three search endpoints' do + expect(described_class.endpoints).to eq(%w[conversations tickets contacts]) + end + + # The asymmetry the per-endpoint table exists for, asserted on the file + # that ships rather than on a table a spec built: a date is bounded on + # every endpoint, and only two of them take the closed bounds and the + # inequality. + it 'carries the measured date restrictions of /contacts/search' do + expect(described_class.fetch('contacts').field('created_at').operators).to eq(['>', '<']) + expect(described_class.fetch('conversations').field('created_at').operators) + .to include('>=', '<=', '!=') + end + + # Intercom sorts one endpoint and ignores the sort it is sent on the other + # two, so a sortable column outside contacts would promise an order that + # never happens. + it 'declares a sortable column on the one endpoint that sorts' do + expect(described_class.fetch('contacts').sortable_columns).not_to be_empty + expect(described_class.fetch('conversations').sortable_columns).to be_empty + expect(described_class.fetch('tickets').sortable_columns).to be_empty + end + + it 'names the path each endpoint is searched through' do + expect(described_class.fetch('conversations').path).to eq('conversations/search') + expect(described_class.fetch('tickets').path).to eq('tickets/search') + end + + # The one thing lot 1 measured about the field lists: `/tickets/search` + # refuses `company_id`, which the specification lists on a ticket. + it 'refuses company_id on tickets, with the measurement as its reason' do + refusal = described_class.fetch('tickets').refusal('company_id') + + expect(refusal.reason).to include('invalid_field') + expect(refusal).to be_measured + end + + # The columns the agent derives from the parts of a ticket: Intercom + # filters none of them, and lot 1 published them without an operator. + it 'refuses every column derived from the parts of a ticket' do + refused = described_class.fetch('tickets').refused.keys + + expect(refused).to include('closed_at', 'closed_by_name', 'last_reply_at', 'last_responder_name', + 'last_responder_type') + end + + # The columns a workspace's own attributes become cannot have a row each + # -- their names are discovered at boot -- so the endpoint carries one + # refusal for the whole family, and the translator reads it in place of + # the message for a column nobody declared. + it 'keeps the ticket attributes unfilterable while the arbitration stands' do + refusal = described_class.fetch('tickets').attribute_refusal + + expect(refusal.reason).to include('differs from one ticket type to the next') + expect(refusal).to be_measured + end + + it 'carries the same refusal for the custom attributes of a contact' do + expect(described_class.fetch('contacts').attribute_refusal.reason) + .to include('custom_attributes.{name}') + end + + # `/conversations/search` has no attribute family of its own, and a nil + # here is what makes the translator fall back on the column message. + it 'carries none where the endpoint declares no attribute family' do + expect(described_class.fetch('conversations').attribute_refusal).to be_nil + end + + # Until the probe runs against the customer's workspace, the date rows are + # the only ones a measurement backs. + it 'reports which rows nothing has measured yet' do + conversations = described_class.fetch('conversations') + + expect(conversations).not_to be_measured + expect(conversations.unmeasured_fields.map(&:column)).not_to include('created_at', 'updated_at') + end + + it 'declares no column both filterable and refused' do + described_class.endpoints.each do |name| + endpoint = described_class.fetch(name) + + expect(endpoint.filterable_columns & endpoint.refused.keys).to be_empty + end + end + + # A candidate is what the probe enumerates on top of the table; one that + # is already declared would be probed twice and read as a discovery. + it 'names no candidate already declared as a field' do + described_class.endpoints.each do |name| + endpoint = described_class.fetch(name) + + expect(endpoint.candidates & endpoint.fields.values.map(&:field)).to be_empty + end + end + end + + # The table is checked in one direction by the schema itself: a column + # publishes exactly the operators its row allows, so it cannot advertise a + # filter the translator would refuse. This is the other direction, and it + # is what the refusal reasons are worth: a column the table says nothing + # about falls back on the generic "takes no filter on it", which is the one + # refusal an operator cannot act on -- and a row whose column was dropped + # survives its own collection, which is how `state_external_label` outlived + # the schema by two lots. + # + # The datasource boots here with no workspace attribute, those columns + # being named by the workspace rather than by the table; they are covered + # by `attribute_refusal` instead. + describe 'the columns of the collections it answers for' do + let(:datasource) { Datasource.new(access_token: 's3cr3t', rate_limiter: nil) } + + { 'IntercomConversation' => 'conversations', 'IntercomTicket' => 'tickets', + 'IntercomContact' => 'contacts' }.each do |collection_name, endpoint_name| + it "names every column of #{collection_name}, as filterable or as refused" do + endpoint = described_class.fetch(endpoint_name) + columns = datasource.get_collection(collection_name).fields + .select { |_, field| field.is_a?(ForestAdminDatasourceToolkit::Schema::ColumnSchema) } + .keys + named = endpoint.filterable_columns + endpoint.refused.keys + + expect(columns - named).to be_empty + expect(named - columns).to be_empty + end + end + end + + # The README is where an operator reads what they may filter on before the + # interface shows it to them, so it is checked against the table rather than + # left to drift from it. + describe 'the README section the table feeds' do + let(:filterable) do + File.read(File.expand_path('../../../README.md', __dir__), encoding: 'UTF-8')[ + /### What is filterable\n(.*?)\n### /m, 1 + ] + end + + it 'lists exactly the columns each endpoint filters' do + { 'IntercomConversation' => 'conversations', 'IntercomTicket' => 'tickets', + 'IntercomContact' => 'contacts' }.each do |collection, endpoint| + row = filterable.lines.find { |line| line.start_with?("| `#{collection}` |") } + listed = row.to_s.scan(/`([a-z_]+)`/).flatten + + expect(listed).to match_array(described_class.fetch(endpoint).filterable_columns) + end + end + + # How much of the table a measurement backs is the first thing the README + # says about it, and a figure typed by hand is a figure that drifts the + # next time a row moves. + it 'counts the measured rows the way the file does' do + rows = described_class.endpoints.flat_map do |name| + endpoint = described_class.fetch(name) + + endpoint.fields.values + endpoint.refused.values + end + readme = File.read(File.expand_path('../../../README.md', __dir__), encoding: 'UTF-8') + + expect(readme).to include("#{rows.count(&:measured?)} of #{rows.size} are measured") + end + end + + describe 'a table that cannot be trusted' do + it 'refuses an operator Intercom has no spelling for' do + expect { table(fields: { 'created_at' => field_row('operators' => ['~=']) }) } + .to raise_error(ConfigurationError, /has no operator ~=/) + end + + it 'refuses a type nothing knows how to send' do + expect { table(fields: { 'created_at' => field_row('type' => 'timestamp') }) } + .to raise_error(ConfigurationError, /type "timestamp" is not one of/) + end + + # An empty list would publish a filterable column no operator can reach. + it 'refuses a field declaring no operator, and says where it belongs' do + expect { table(fields: { 'created_at' => field_row('operators' => []) }) } + .to raise_error(ConfigurationError, /belongs in the refused table/) + end + + # YAML reads an unquoted `true` as a boolean and a quoted one as a string, + # which is truthy in Ruby: a typo would publish a sortable column the + # endpoint never sorts. + it 'refuses a sortable flag that is not a boolean' do + expect { table(fields: { 'created_at' => field_row('sortable' => 'true') }) } + .to raise_error(ConfigurationError, /sortable "true" is neither true nor false/) + end + + it 'refuses a provenance that is neither measured nor read off the documentation' do + expect { table(fields: { 'created_at' => field_row('source' => 'guessed') }) } + .to raise_error(ConfigurationError, /source "guessed" is neither measured nor spec/) + end + + it 'refuses a refusal with no provenance of its own' do + expect { table(refused: { 'company_id' => { 'reason' => 'no', 'source' => 'hearsay' } }) } + .to raise_error(ConfigurationError, /tickets.company_id/) + end + + # Operators the DSL does spell, none of which the column's *type* can + # carry: `OperatorTable` maps no Forest operator onto `~` for a date, so + # the column would be called filterable and publish nothing, and the + # translator would refuse every request on it. + it 'refuses operators the column type cannot publish' do + expect { table(fields: { 'created_at' => field_row('operators' => ['~']) }) } + .to raise_error(ConfigurationError, /none of ~ is an operator Intercom answers on a date/) + end + + # `Endpoint#field` is consulted before `#refusal`, so the filterable row + # would win and the reason an operator reads would be dropped in silence. + # The file is script-rewritten, which is the shape a bad rewrite takes. + it 'refuses a column declared filterable and refused at once' do + expect do + table(fields: { 'created_at' => field_row }, + refused: { 'created_at' => { 'reason' => 'no', 'source' => 'spec' } }) + end.to raise_error(ConfigurationError, /filterable and refused at once/) + end + + # The block is a refusal from end to end, so a rewrite flipping the flag + # would leave the file saying the opposite of what the package does with + # it -- publish nothing. + it 'refuses an attribute family declared filterable' do + expect { table(attributes: attributes_row('filterable' => true)) } + .to raise_error(ConfigurationError, /tickets.ticket_attributes: filterable true is not something/) + end + + it 'refuses an attribute family with no provenance of its own' do + expect { table(attributes: attributes_row('source' => 'hearsay')) } + .to raise_error(ConfigurationError, /tickets.ticket_attributes: source "hearsay"/) + end + + it 'names the endpoint and the column it choked on' do + expect { table(fields: { 'created_at' => field_row('type' => 'timestamp') }) } + .to raise_error(ConfigurationError, /search_fields\.yml is malformed at tickets\.created_at/) + end + end + + describe 'what it hands to the schema' do + it 'carries the Intercom field a column is filtered through' do + expect(described_class.fetch('conversations').field('closed_at').field).to eq('statistics.last_close_at') + end + + it 'reads a refusal reason as one line, whatever the YAML wrapping' do + reason = table(refused: { 'company_id' => { 'reason' => "one\ntwo\n", 'source' => 'spec' } }) + .refusal('company_id').reason + + expect(reason).to eq('one two') + end + + it 'refuses an endpoint nothing declares, rather than filtering nothing' do + expect { described_class.fetch('companies') } + .to raise_error(ConfigurationError, /Unknown Intercom search endpoint "companies"/) + end + + it 'is measured once the probe has stamped a date on it' do + expect(table(measured_at: '2026-09-01')).to be_measured + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/rate_limiter_spec.rb b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/rate_limiter_spec.rb new file mode 100644 index 000000000..78c234aa6 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/rate_limiter_spec.rb @@ -0,0 +1,150 @@ +module ForestAdminDatasourceIntercom + RSpec.describe RateLimiter do + # Sleeping moves the clock, the way it does outside a spec: a window waited + # out is a window that has refilled by the time the next request asks. + subject(:limiter) { build_limiter } + + let(:time) { [1_000.0] } + let(:slept) { [] } + + def build_limiter(**options) + sleeper = lambda do |seconds| + slept << seconds + time[0] += seconds + end + + described_class.new(now: -> { time[0] }, sleeper: sleeper, **options) + end + + # What Intercom answers with: the limit of the 10-second window, what is + # left of it, and the epoch second it refills at. + def headers(remaining:, reset_in: 4, limit: 1667) + { 'x-ratelimit-limit' => limit.to_s, + 'x-ratelimit-remaining' => remaining.to_s, + 'x-ratelimit-reset' => (time[0] + reset_in).to_i.to_s } + end + + it 'lets the first request through: the budget is what that request discovers' do + limiter.acquire + + expect(slept).to be_empty + end + + it 'lets a request through while the window still has room' do + limiter.observe(headers(remaining: 5)) + limiter.acquire + + expect(slept).to be_empty + end + + it 'waits for the reset once the window is spent' do + limiter.observe(headers(remaining: 0, reset_in: 4)) + limiter.acquire + + expect(slept).to eq([4.0]) + end + + it 'lets requests through again once the reset has passed' do + limiter.observe(headers(remaining: 0, reset_in: -1)) + limiter.acquire + + expect(slept).to be_empty + end + + # Several requests can be in flight before any of them answers, and a + # `remaining` that only moves on a response lets all of them through on the + # same stale figure. + it 'counts its own requests down rather than trusting the last response' do + limiter.observe(headers(remaining: 2, reset_in: 3)) + 3.times { limiter.acquire } + + expect(slept).to eq([3.0]) + end + + it 'ignores a response from a window older than the one it knows' do + limiter.observe(headers(remaining: 0, reset_in: 5)) + limiter.observe(headers(remaining: 100, reset_in: -10)) + limiter.acquire + + expect(slept).to eq([5.0]) + end + + it 'adopts a new window whole, generous remaining included' do + limiter.observe(headers(remaining: 0, reset_in: 2)) + limiter.observe(headers(remaining: 50, reset_in: 12)) + limiter.acquire + + expect(slept).to be_empty + end + + # A response that left Intercom before the requests now in flight were made + # must not hand their budget back. + it 'keeps the smaller remaining inside one window' do + limiter.observe(headers(remaining: 2, reset_in: 3)) + limiter.acquire + limiter.observe(headers(remaining: 99, reset_in: 3)) + 2.times { limiter.acquire } + + expect(slept).to eq([3.0]) + end + + it 'reads the headers whatever their case' do + limiter.observe('X-RateLimit-Remaining' => '0', 'X-RateLimit-Reset' => (time[0] + 6).to_i.to_s) + limiter.acquire + + expect(slept).to eq([6.0]) + end + + it 'ignores a response carrying no rate-limit headers at all' do + limiter.observe('content-type' => 'application/json') + limiter.acquire + + expect(slept).to be_empty + end + + it 'ignores a remaining that is not a number' do + limiter.observe('x-ratelimit-remaining' => 'many', 'x-ratelimit-reset' => (time[0] + 3).to_i.to_s) + limiter.acquire + + expect(slept).to be_empty + end + + it 'sleeps a reset out when it fits within the bound it was given' do + capped = build_limiter(max_wait: 2.0) + capped.observe(headers(remaining: 0, reset_in: 1)) + capped.acquire + + expect(slept).to eq([1.0]) + end + + describe 'a reset further out than one window' do + # Intercom's reset is a timestamp from its clock. One that far out is the + # two clocks disagreeing, not a window emptying, so the request goes + # through and the log says why -- waiting an hour would read as a hang. + before do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + limiter.observe(headers(remaining: 0, reset_in: 3_600)) + end + + it 'lets the request through instead of waiting it out' do + limiter.acquire + + expect(slept).to be_empty + end + + it 'says so once, not once per request' do + 3.times { limiter.acquire } + + expect(ForestAdminDatasourceIntercom.logger) + .to have_received(:warn).once.with(/rate-limit window is spent.*limit 1667/m) + end + end + + it 'sleeps for real when handed no sleeper' do + real = described_class.new(now: -> { 999.96 }) + real.observe('x-ratelimit-remaining' => '0', 'x-ratelimit-reset' => '1000') + + expect { real.acquire }.not_to raise_error + end + end +end diff --git a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/retry_policy_spec.rb b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/retry_policy_spec.rb new file mode 100644 index 000000000..060565df4 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/retry_policy_spec.rb @@ -0,0 +1,49 @@ +module ForestAdminDatasourceIntercom + RSpec.describe RetryPolicy do + describe '#to_faraday_options' do + subject(:options) { described_class.new.to_faraday_options } + + it 'retries the statuses worth another attempt' do + expect(options[:retry_statuses]).to eq([429, 500, 502, 503, 504]) + end + + # A 502 on the way back from a POST Intercom did perform would be replayed + # into a second reply on the conversation. + it 'only replays the verbs that change nothing' do + expect(options[:methods]).to eq(%i[get head options]) + end + + it 'replays a 429 whatever the verb, Intercom having rejected it unprocessed' do + expect(options[:retry_if].call({ status: 429 }, nil)).to be(true) + end + + it 'leaves any other status to the methods list' do + expect(options[:retry_if].call({ status: 502 }, nil)).to be(false) + end + + # faraday-retry abandons outright when Retry-After exceeds max_interval, + # so the cap has to cover Intercom's whole 10-second window. + it 'waits out a full rate-limit window' do + expect(options[:max_interval]).to be > RateLimiter::WINDOW + end + + it 'absorbs a dropped connection, which faraday-retry does not by default' do + expect(options[:exceptions]).to include(Faraday::ConnectionFailed) + end + end + + describe '.boot' do + subject(:options) { described_class.boot.to_faraday_options } + + it 'retries once: a boot read is never revisited, and never worth a long wait' do + expect(options[:max]).to eq(1) + end + + # Below a rate-limit window on purpose: past the cap faraday-retry gives + # up at once, which is what keeps a 429 from holding the Rails boot. + it 'gives up rather than waiting a 429 out' do + expect(options[:max_interval]).to be < RateLimiter::WINDOW + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/schema/data_attributes_introspector_spec.rb b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/schema/data_attributes_introspector_spec.rb new file mode 100644 index 000000000..6896ccf33 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/schema/data_attributes_introspector_spec.rb @@ -0,0 +1,136 @@ +module ForestAdminDatasourceIntercom + module Schema + RSpec.describe DataAttributesIntrospector do + subject(:introspector) { described_class.new(Client.new(configuration), model: 'contact') } + + let(:configuration) { Configuration.new(access_token: 's3cr3t', rate_limiter: nil) } + let(:base) { configuration.url } + + def json(payload, status = 200) + { status: status, body: payload.to_json, headers: { 'Content-Type' => 'application/json' } } + end + + def attribute(name, overrides = {}) + { 'type' => 'data_attribute', 'model' => 'contact', 'name' => name, 'label' => name, + 'data_type' => 'string', 'custom' => true, 'archived' => false, + 'api_writable' => true }.merge(overrides) + end + + def stub_attributes(*attributes, model: 'contact') + stub_request(:get, "#{base}/data_attributes").with(query: { 'model' => model }) + .to_return(json('type' => 'list', 'data' => attributes)) + end + + it 'reads the attributes of the model it was built for' do + stub_attributes(attribute('paid_subscriber'), model: 'company') + + expect(described_class.new(Client.new(configuration), model: 'company').attributes.map(&:name)) + .to eq(['paid_subscriber']) + end + + # The standard attributes are columns the collection declares by hand, + # with the filters the search table measured. Publishing them again under + # their `custom_attributes` name would show one fact twice, the + # unfilterable copy winning nothing. + it 'leaves out the attributes Intercom defines itself' do + stub_attributes(attribute('paid_subscriber'), attribute('email', 'custom' => false)) + + expect(introspector.attributes.map(&:name)).to eq(['paid_subscriber']) + end + + it 'leaves out an archived attribute, which the workspace stopped offering' do + stub_attributes(attribute('paid_subscriber'), attribute('old_plan', 'archived' => true)) + + expect(introspector.attributes.map(&:name)).to eq(['paid_subscriber']) + end + + it 'leaves out an attribute with no name to be a column of' do + stub_attributes(attribute(''), 'not a hash') + + expect(introspector.attributes).to be_empty + end + + # Read and carried although every column of this lot is published + # read-only: it is what lot 4b needs to tell an attribute it may write + # from one Intercom fills in itself, and reading it again then would be a + # second boot-time round trip. + it 'carries api_writable for the lot that writes' do + stub_attributes(attribute('paid_subscriber'), attribute('lifetime_value', 'api_writable' => false)) + + expect(introspector.attributes.map { |a| [a.name, a.api_writable] }) + .to eq([['paid_subscriber', true], ['lifetime_value', false]]) + end + + it 'maps each Intercom data type onto what Forest renders' do + stub_attributes(attribute('a', 'data_type' => 'integer'), attribute('b', 'data_type' => 'float'), + attribute('c', 'data_type' => 'boolean'), attribute('d', 'data_type' => 'date')) + + expect(introspector.attributes.map(&:column_type)).to eq(%w[Number Number Boolean Date]) + end + + # Showing the value Intercom sent beats hiding a column because its type + # is new. + it 'reads an unknown data type as a string rather than dropping the column' do + stub_attributes(attribute('paid_subscriber', 'data_type' => 'quantum')) + + expect(introspector.attributes.first.column_type).to eq('String') + end + + # Forest lists the fields of a request in a comma-separated query + # parameter and names a field through a relation with a colon: either one + # in a column name breaks the projection before the page is read. + it 'takes the commas and colons out of a column name, keeping the name the payload uses' do + stub_attributes(attribute('Plan, tier: current')) + + expect(introspector.attributes.map { |a| [a.name, a.column_name] }) + .to eq([['Plan, tier: current', 'Plan tier current']]) + end + + it 'unescapes a name Intercom handed back escaped' do + stub_attributes(attribute('Ce que j'ai vérifié')) + + expect(introspector.attributes.first.column_name).to eq("Ce que j'ai vérifié") + end + + # Two attributes landing on one column would share an entry, and the + # second's values would be read under the first's name -- wrong values + # rather than missing ones. + it 'leaves out an attribute colliding with one already kept, and says which' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + stub_attributes(attribute('Plan, tier'), attribute('Plan: tier')) + + expect(introspector.attributes.map(&:name)).to eq(['Plan, tier']) + expect(ForestAdminDatasourceIntercom.logger) + .to have_received(:warn).with(/"Plan: tier" is left out.*"Plan tier"/) + end + + it 'keeps an attribute whose name is nothing but separators out of the schema' do + stub_attributes(attribute(',,')) + + expect(introspector.attributes).to be_empty + end + + # A token without the permission costs the columns, never the boot. + it 'answers no attribute when Intercom refuses the read, and says so' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + stub_request(:get, "#{base}/data_attributes").with(query: { 'model' => 'contact' }) + .to_return(json( + { 'type' => 'error.list', + 'errors' => [{ 'code' => 'forbidden' }] }, 403 + )) + + expect(introspector.attributes).to eq([]) + expect(ForestAdminDatasourceIntercom.logger) + .to have_received(:warn).with(/could not read the contact attributes \(HTTP 403\)/) + end + + it 'reads Intercom once, however many times it is asked' do + stub_attributes(attribute('paid_subscriber')) + + 2.times { introspector.attributes } + + expect(WebMock).to have_requested(:get, /data_attributes/).once + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/schema/ticket_attributes_introspector_spec.rb b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/schema/ticket_attributes_introspector_spec.rb new file mode 100644 index 000000000..a6119eaea --- /dev/null +++ b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/schema/ticket_attributes_introspector_spec.rb @@ -0,0 +1,164 @@ +module ForestAdminDatasourceIntercom + module Schema + RSpec.describe TicketAttributesIntrospector do + subject(:introspector) { described_class.new(Client.new(configuration)) } + + let(:configuration) { Configuration.new(access_token: 's3cr3t', rate_limiter: nil) } + let(:base) { configuration.url } + + def json(payload, status = 200) + { status: status, body: payload.to_json, headers: { 'Content-Type' => 'application/json' } } + end + + def attribute(name, id, data_type: 'string', archived: false) + { 'id' => id, 'name' => name, 'data_type' => data_type, 'archived' => archived } + end + + def ticket_type(id, name, *attributes) + { 'type' => 'ticket_type', 'id' => id, 'name' => name, + 'ticket_type_attributes' => { 'type' => 'list', 'data' => attributes } } + end + + def stub_types(*types) + stub_request(:get, "#{base}/ticket_types").to_return(json('type' => 'list', 'data' => types)) + end + + it 'reads one entry per attribute name' do + stub_types(ticket_type('1', 'Bug', attribute('Severity', '9001')), + ticket_type('2', 'Task', attribute('Due', '9002', data_type: 'datetime'))) + + expect(introspector.attributes.map(&:name)).to contain_exactly('Severity', 'Due') + end + + # Measured: two ticket types share the names `_default_title_` and + # `_default_description_` while carrying different attribute ids. A union + # column has no single id to be filtered by, which is why these ship + # unfilterable -- and why the ids are kept, since that is what a filter + # per ticket type will need. + it 'keeps the id each ticket type gives the same attribute name' do + stub_types(ticket_type('1', 'Bug', attribute('_default_title_', '14162161')), + ticket_type('2', 'Task', attribute('_default_title_', '14162165'))) + + expect(introspector.attributes.map(&:ids_by_ticket_type)) + .to eq([{ '1' => '14162161', '2' => '14162165' }]) + end + + # Forest lists the fields of a request in a comma-separated query + # parameter: a comma in a column name splits the projection into fields no + # collection has, and the agent rejects the page with a 400 before reading + # anything. Measured on a real workspace, several attributes carry one. + it 'takes the commas out of a column name, keeping the name the payload uses' do + stub_types(ticket_type('1', 'Bug', attribute("ID de l'objet (immo, facture, user)", '9001'))) + + expect(introspector.attributes.first) + .to have_attributes(name: "ID de l'objet (immo, facture, user)", + column_name: "ID de l'objet (immo facture user)") + end + + # A colon is how Forest names a field through a relation. + it 'takes a colon out too' do + stub_types(ticket_type('1', 'Bug', attribute('Scope: mobile', '9001'))) + + expect(introspector.attributes.first.column_name).to eq('Scope mobile') + end + + # Intercom hands the names back HTML-escaped, which is an artefact of where + # they were typed rather than part of the name. + it 'unescapes what Intercom escaped' do + stub_types(ticket_type('1', 'Bug', attribute('Ce que j'ai vérifié & validé', '9001'))) + + expect(introspector.attributes.first.column_name).to eq("Ce que j'ai vérifié & validé") + end + + it 'leaves a name that needs nothing alone' do + stub_types(ticket_type('1', 'Bug', attribute('Severity', '9001'))) + + expect(introspector.attributes.first).to have_attributes(name: 'Severity', column_name: 'Severity') + end + + # Two different attributes landing on one column would otherwise share an + # entry, and the second's values would be read under the first's name -- + # wrong values rather than missing ones. + it 'leaves out a second attribute that reads as an existing column' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + stub_types(ticket_type('1', 'Bug', attribute('Scope, mobile', '9001'), attribute('Scope mobile', '9002'))) + + expect(introspector.attributes.map(&:name)).to eq(['Scope, mobile']) + expect(ForestAdminDatasourceIntercom.logger).to have_received(:warn).with(/is left out/) + end + + it 'leaves out an attribute whose name is nothing but separators' do + stub_types(ticket_type('1', 'Bug', attribute(' , : ', '9001'))) + + expect(introspector.attributes).to be_empty + end + + it 'maps the Intercom data types onto what Forest renders' do + stub_types(ticket_type('1', 'Bug', attribute('n', '1', data_type: 'integer'), + attribute('d', '2', data_type: 'decimal'), + attribute('b', '3', data_type: 'boolean'), + attribute('t', '4', data_type: 'datetime'), + attribute('l', '5', data_type: 'list'), + attribute('f', '6', data_type: 'files'))) + + expect(introspector.attributes.map(&:column_type)).to eq(%w[Number Number Boolean Date String Json]) + end + + # Showing the value Intercom sent beats hiding a column because its type + # is one this datasource has not met yet. + it 'reads an unknown data type as a string rather than dropping the column' do + stub_types(ticket_type('1', 'Bug', attribute('x', '1', data_type: 'quantum'))) + + expect(introspector.attributes.map(&:column_type)).to eq(%w[String]) + end + + it 'leaves out an archived attribute, which is not offered any more' do + stub_types(ticket_type('1', 'Bug', attribute('Gone', '1', archived: true), attribute('Here', '2'))) + + expect(introspector.attributes.map(&:name)).to eq(%w[Here]) + end + + it 'leaves out an attribute with no name to be a column of' do + stub_types(ticket_type('1', 'Bug', attribute('', '1'))) + + expect(introspector.attributes).to be_empty + end + + it 'reads a ticket type declaring no attribute as declaring none' do + stub_types({ 'id' => '1', 'name' => 'Bug' }) + + expect(introspector.attributes).to be_empty + end + + # It runs while Rails is starting, so it waits far less than a request + # that already has a page on screen. + it 'reads through the boot connection' do + slow = Configuration.new(access_token: 's3cr3t', rate_limiter: nil, boot_timeout: 2, timeout: 30) + client = Client.new(slow) + stub_types + + described_class.new(client).attributes + + expect(client.send(:boot_connection).options.timeout).to eq(2) + end + + it 'reads once and remembers, a schema being built once' do + stub_types(ticket_type('1', 'Bug', attribute('Severity', '9001'))) + + 2.times { introspector.attributes } + + expect(WebMock).to have_requested(:get, "#{base}/ticket_types").once + end + + # A token without the ticket-types permission costs the attribute columns, + # never the boot of the agent. + it 'degrades to no attribute when the read is refused' do + allow(ForestAdminDatasourceIntercom.logger).to receive(:warn) + stub_request(:get, "#{base}/ticket_types").to_return(json({ 'errors' => [{ 'code' => 'forbidden' }] }, 403)) + + expect(introspector.attributes).to eq([]) + expect(ForestAdminDatasourceIntercom.logger).to have_received(:warn).with(/boots without its attribute/) + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/throttle_spec.rb b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/throttle_spec.rb new file mode 100644 index 000000000..a2209a2a7 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom/throttle_spec.rb @@ -0,0 +1,40 @@ +module ForestAdminDatasourceIntercom + RSpec.describe Throttle do + let(:limiter) { instance_double(RateLimiter, acquire: nil, observe: nil) } + let(:connection) do + Faraday.new(url: 'https://api.intercom.test') do |f| + f.use described_class, limiter: limiter + end + end + + before do + stub_request(:get, 'https://api.intercom.test/me') + .to_return(status: 200, body: '{}', + headers: { 'Content-Type' => 'application/json', 'x-ratelimit-remaining' => '7' }) + end + + it 'asks for room before the request leaves' do + connection.get('me') + + expect(limiter).to have_received(:acquire) + end + + it 'hands the window back what the response says about it' do + connection.get('me') + + expect(limiter).to have_received(:observe).with(hash_including('x-ratelimit-remaining' => '7')) + end + + # The 429 carries the most useful reset of all, so the observation cannot be + # limited to the responses that succeeded. + it 'observes a rejected response too' do + stub_request(:get, 'https://api.intercom.test/me') + .to_return(status: 429, body: '{}', + headers: { 'Content-Type' => 'application/json', 'x-ratelimit-remaining' => '0' }) + + connection.get('me') + + expect(limiter).to have_received(:observe).with(hash_including('x-ratelimit-remaining' => '0')) + end + end +end diff --git a/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom_spec.rb b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom_spec.rb new file mode 100644 index 000000000..f38f4adfe --- /dev/null +++ b/packages/forest_admin_datasource_intercom/spec/forest_admin_datasource_intercom_spec.rb @@ -0,0 +1,49 @@ +RSpec.describe ForestAdminDatasourceIntercom do + describe 'VERSION' do + # The release `sed` in .releaserc.js only matches `VERSION = "x.y.z"`, and a + # format it misses is a version that stays behind with no CI failure to say so. + it 'is a double-quoted semantic version' do + expect(described_class::VERSION).to match(/\A\d+\.\d+\.\d+\z/) + end + end + + describe '.logger' do + around do |example| + previous = described_class.logger + example.run + described_class.logger = previous + end + + it 'defaults to a logger named after the package' do + described_class.logger = nil + + expect(described_class.logger.progname).to eq('forest_admin_datasource_intercom') + end + + it 'takes the logger it is handed' do + logger = Logger.new(File::NULL) + described_class.logger = logger + + expect(described_class.logger).to be(logger) + end + end + + describe 'errors' do + it 'reports a filter Intercom cannot express as a validation error' do + expect(described_class::UnsupportedOperatorError.new('nope')) + .to be_a(ForestAdminDatasourceToolkit::Exceptions::ValidationError) + end + + it 'carries the status and parsed body of a failed call' do + error = described_class::APIError.new('boom', status: 429, body: { 'type' => 'error.list' }) + + expect(error).to have_attributes(message: 'boom', status: 429, body: { 'type' => 'error.list' }) + end + + it 'leaves the status and body unset when the call failed before answering' do + error = described_class::APIError.new('timeout') + + expect(error).to have_attributes(status: nil, body: nil) + end + end +end diff --git a/packages/forest_admin_datasource_intercom/spec/probe_search_fields_spec.rb b/packages/forest_admin_datasource_intercom/spec/probe_search_fields_spec.rb new file mode 100644 index 000000000..b7d79b175 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/spec/probe_search_fields_spec.rb @@ -0,0 +1,192 @@ +require 'tmpdir' + +load File.expand_path('../exe/forest_admin_intercom_probe', __dir__) + +module ForestAdminDatasourceIntercom + RSpec.describe ProbeSearchFields do + let(:base) { Configuration::REGION_HOSTS[:us] } + let(:endpoint) { Query::SearchFields.fetch('tickets') } + let(:client) { Client.new(Configuration.new(access_token: 's3cr3t', rate_limiter: nil)) } + + # The gem installs this as `forest_admin_intercom_probe`, and RubyGems runs + # it through a stub that `load`s it -- so `$PROGRAM_NAME` is the stub and + # not this file. A guard comparing the two paths would ship a command that + # exits without doing anything. + describe '.invoked_as_command?' do + it 'runs behind the stub RubyGems installs' do + expect(described_class.invoked_as_command?('/usr/local/bin/forest_admin_intercom_probe')).to be(true) + end + + it 'runs when the file is the command itself' do + expect(described_class.invoked_as_command?(File.expand_path('../exe/forest_admin_intercom_probe', + __dir__))).to be(true) + end + + it 'stays out of the way of whatever else loads it' do + expect(described_class.invoked_as_command?('/usr/local/bin/rspec')).to be(false) + end + end + + def json(payload, status = 200) + { status: status, body: payload.to_json, headers: { 'Content-Type' => 'application/json' } } + end + + # Every probe is a search asking for one record; what a stub answers is + # therefore an empty page or one of Intercom's refusal codes. + def stub_search(code: nil, operator: nil, value: nil) + body = if code + { 'type' => 'error.list', 'errors' => [{ 'code' => code, 'message' => 'nope' }] } + else + { 'type' => 'list', 'tickets' => [], 'total_count' => 0 } + end + + stub_request(:post, "#{base}/tickets/search").with { |request| matches?(request, operator, value) } + .to_return(json(body, code ? 400 : 200)) + end + + def matches?(request, operator, value) + query = JSON.parse(request.body)['query'] + + (operator.nil? || query['operator'] == operator) && (value.nil? || query['value'] == value) + end + + describe 'the type guessed for a candidate field' do + # A candidate is a name and nothing else -- discovering what it is is the + # point -- so the value shape sent with it is guessed from that name. + it 'reads a timestamp, a count and a flag off the name' do + expect(described_class.guess_type('statistics.last_close_at')).to eq('date') + expect(described_class.guess_type('count_reopens')).to eq('number') + expect(described_class.guess_type('open')).to eq('boolean') + expect(described_class.guess_type('state')).to eq('string') + end + end + + describe described_class::Probe do + subject(:probe) { described_class.new(client, endpoint) } + + it 'keeps the operators the endpoint answers and drops the ones it refuses' do + stub_search(code: 'data_invalid') + stub_search(operator: '=') + stub_search(operator: '!=') + + result = probe.run('category', 'string') + + expect(result[:operators].select { |_, outcome| outcome[:ok] }.keys).to eq(['=', '!=']) + end + + # A field the endpoint does not filter at all is worth one request, not + # twelve: `invalid_field` ends the row. + it 'stops at the first invalid_field and reports the field unfilterable' do + stub_search(code: 'invalid_field') + + result = probe.run('company_id', 'string') + + expect(result[:unfilterable][:code]).to eq('invalid_field') + expect(a_request(:post, "#{base}/tickets/search")).to have_been_made.once + end + + # A wrong value shape is refused with the same code as an unsupported + # operator, so a single attempt would report a filter Intercom does answer + # as refused. + it 'retries a refused cell with the other value shapes its type takes' do + stub_search(code: 'data_invalid') + stub_search(operator: '>', value: '2026-01-01') + + result = probe.run('created_at', 'date') + + expect(result[:operators]['>'][:ok]).to be(true) + expect(result[:operators]['<'][:ok]).to be(false) + end + + it 'names the failure by its HTTP status when Intercom sends no error code' do + stub_request(:post, "#{base}/tickets/search").to_return(json({ 'nope' => true }, 500)) + + result = probe.run('category', 'string') + + expect(result[:operators]['='][:code]).to eq('http_500') + end + end + + describe described_class::Report do + subject(:report) { described_class.new(endpoint) } + + def run_probe(field, type) + ProbeSearchFields::Probe.new(client, endpoint).run(field, type) + end + + # The only reason to run the probe: an operator the table promises and + # Intercom refuses is a filter the interface offers and the read cannot + # honour. + it 'reports an operator the table promises and Intercom refuses' do + stub_search(code: 'data_invalid') + stub_search(operator: '=') + report.record('category', 'category', run_probe('category', 'string')) + + expect { report.print_diff }.to output(/! the table promises != here, Intercom refuses it/).to_stdout + end + + it 'reports an operator the table does not know about yet' do + stub_search(code: 'data_invalid') + ['=', '!=', '~'].each { |operator| stub_search(operator: operator) } + report.record('category', 'category', run_probe('category', 'string')) + + expect { report.print_diff }.to output(/\+ Intercom also accepts ~/).to_stdout + end + + it 'names the column a field the endpoint refuses is declared on' do + stub_search(code: 'invalid_field') + report.record('category', 'category', run_probe('category', 'string')) + + expect { report.print_diff }.to output(/NOT FILTERABLE \(invalid_field\).*column 'category'/).to_stdout + end + + it 'writes the measurement as evidence, refusal codes included' do + stub_search(code: 'data_invalid') + stub_search(operator: '=') + report.record('category', 'category', run_probe('category', 'string')) + + written = YAML.safe_load(report.to_yaml_document) + + expect(written['fields']['category']['operators']).to eq(['=']) + expect(written['fields']['category']['refused']['!=']).to eq('data_invalid') + end + + it 'writes a field the endpoint refuses as unfilterable' do + stub_search(code: 'invalid_field') + report.record('company_id', nil, run_probe('company_id', 'string')) + + written = YAML.safe_load(report.to_yaml_document) + + expect(written['fields']['company_id']).to eq({ 'filterable' => false, 'code' => 'invalid_field' }) + end + end + + describe described_class::CLI do + # A probe with no token would abort on the first request with an Intercom + # 401, which reads as a workspace problem rather than as a missing option. + it 'refuses to start without a token' do + expect { described_class.call(['--endpoint', 'tickets']) } + .to raise_error(SystemExit).and output(/No token/).to_stderr + end + + it 'refuses an endpoint the table does not declare' do + expect { described_class.call(['--endpoint', 'companies', '--token', 's3cr3t']) } + .to raise_error(ConfigurationError, /Unknown Intercom search endpoint/) + end + + it 'probes every declared endpoint and writes the evidence where it was asked to' do + out = File.join(Dir.tmpdir, 'intercom-probe.yml') + stub_search(code: 'invalid_field') + stub_request(:post, "#{base}/conversations/search") + .to_return(json({ 'type' => 'error.list', 'errors' => [{ 'code' => 'invalid_field' }] }, 400)) + stub_request(:post, "#{base}/contacts/search") + .to_return(json({ 'type' => 'error.list', 'errors' => [{ 'code' => 'invalid_field' }] }, 400)) + + expect { described_class.call(['--token', 's3cr3t', '--out', out]) }.to output(/NOT FILTERABLE/).to_stdout + expect(YAML.safe_load_file(out)['endpoint']).to eq('conversations') + ensure + FileUtils.rm_f(out) + end + end + end +end diff --git a/packages/forest_admin_datasource_intercom/spec/spec_helper.rb b/packages/forest_admin_datasource_intercom/spec/spec_helper.rb new file mode 100644 index 000000000..bdf9ae346 --- /dev/null +++ b/packages/forest_admin_datasource_intercom/spec/spec_helper.rb @@ -0,0 +1,78 @@ +require 'simplecov' +# JSON output is consumed by the qlty CI coverage step; HTML is for local +# inspection. simplecov-html and simplecov_json_formatter are required only +# in Gemfile-test, so guard the require for local Gemfile runs. +begin + require 'simplecov_json_formatter' + require 'simplecov-html' + SimpleCov.formatters = [SimpleCov::Formatter::JSONFormatter, SimpleCov::Formatter::HTMLFormatter] +rescue LoadError + # Local Gemfile run without the CI formatters; default text output is fine. +end + +SimpleCov.start do + add_filter '/spec/' + enable_coverage :branch + minimum_coverage 90 +end + +SimpleCov.coverage_dir 'coverage' + +require 'webmock/rspec' +require 'forest_admin_datasource_customizer' +require 'forest_admin_datasource_intercom' + +# Every payload the specs feed in is hand-written from the Intercom OpenAPI 2.16 +# spec, never captured from a workspace: a conversation body is personal data, +# and a fixture is read by everyone who clones the repo. +WebMock.disable_net_connect!(allow_localhost: true) + +# A datasource checks the API version it was served and introspects the +# ticket-type attributes and the contact and company attributes while it +# registers its collections, so every spec building one issues those four +# reads. The base url is not taken from the datasource on purpose: reading it +# would build the datasource, and boot the very reads this stubs. +module IntercomBootStubs + def stub_me(base: ForestAdminDatasourceIntercom::Configuration::REGION_HOSTS[:us], + version: ForestAdminDatasourceIntercom::Configuration::DEFAULT_API_VERSION) + stub_request(:get, "#{base}/me") + .to_return(status: 200, body: { 'type' => 'admin', 'id' => '1', 'email' => 'ops@example.test' }.to_json, + headers: { 'Content-Type' => 'application/json', 'Intercom-Version' => version }) + end + + def stub_ticket_types(*types, base: ForestAdminDatasourceIntercom::Configuration::REGION_HOSTS[:us]) + stub_request(:get, "#{base}/ticket_types") + .to_return(status: 200, body: { 'type' => 'list', 'data' => types }.to_json, + headers: { 'Content-Type' => 'application/json' }) + end + + def stub_data_attributes(model, *attributes, + base: ForestAdminDatasourceIntercom::Configuration::REGION_HOSTS[:us]) + stub_request(:get, "#{base}/data_attributes").with(query: { 'model' => model }) + .to_return(status: 200, + body: { 'type' => 'list', 'data' => attributes }.to_json, + headers: { 'Content-Type' => 'application/json' }) + end +end + +RSpec.configure do |config| + config.include IntercomBootStubs + config.expect_with :rspec do |c| + c.syntax = :expect + end + config.mock_with :rspec do |m| + m.verify_partial_doubles = true + end + config.disable_monkey_patching! + config.warnings = false + config.order = :random + Kernel.srand config.seed + + config.before do + WebMock.reset! + stub_me + stub_ticket_types + stub_data_attributes('contact') + stub_data_attributes('company') + end +end diff --git a/packages/forest_admin_datasource_toolkit/lib/forest_admin_datasource_toolkit/utils/collection.rb b/packages/forest_admin_datasource_toolkit/lib/forest_admin_datasource_toolkit/utils/collection.rb index 537cdf97c..916d9884c 100644 --- a/packages/forest_admin_datasource_toolkit/lib/forest_admin_datasource_toolkit/utils/collection.rb +++ b/packages/forest_admin_datasource_toolkit/lib/forest_admin_datasource_toolkit/utils/collection.rb @@ -56,10 +56,15 @@ def self.other_inverse?(field, relation_field) field.origin_key == relation_field.foreign_key end + # ValidationError rather than a bare ForestException, for the same reason as + # below: every one of these three names a field the request asked for -- a + # filter, a sort, a scope written against another schema, a condition + # reaching through a relation that is not one this can traverse. That is a + # 400 the caller can read and fix, not a 500 saying the agent broke. def self.get_field_schema(collection, field_name) fields = collection.schema[:fields] unless field_name.include?(':') - raise ForestException, "Column not found #{collection.name}.#{field_name}" unless fields.key?(field_name) + raise ValidationError, "Column not found #{collection.name}.#{field_name}" unless fields.key?(field_name) return fields[field_name] end @@ -67,10 +72,11 @@ def self.get_field_schema(collection, field_name) association_name = field_name.split(':')[0] relation_schema = fields[association_name] - raise ForestException, "Relation not found #{collection.name}.#{association_name}" unless relation_schema + raise ValidationError, "Relation not found #{collection.name}.#{association_name}" unless relation_schema if relation_schema.type != 'ManyToOne' && relation_schema.type != 'OneToOne' - raise ForestException, "Unexpected field type #{relation_schema.type}: #{collection.name}.#{association_name}" + raise ValidationError, + "Unexpected field type #{relation_schema.type}: #{collection.name}.#{association_name}" end get_field_schema( @@ -161,7 +167,14 @@ def self.list_relation(collection, primary_key_values, relation_name, caller, fo projection.nest(prefix: foreign_relation) ) - return records.map { |r| r[foreign_relation] } + # Compacted: a through row whose target the foreign collection no + # longer answers -- a record deleted, outside the caller's reach, or + # dropped by a datasource that reads its targets in bounded pages -- + # carries a nil where a record was expected, and every consumer of a + # related list down to the JSON:API serializer reads a row by key. + # A join whose other side is gone yields no row, it does not yield an + # empty one. + return records.filter_map { |r| r[foreign_relation] } end end diff --git a/packages/forest_admin_datasource_toolkit/spec/lib/forest_admin_datasource_toolkit/utils/collection_spec.rb b/packages/forest_admin_datasource_toolkit/spec/lib/forest_admin_datasource_toolkit/utils/collection_spec.rb index fb7d88535..ce0036c25 100644 --- a/packages/forest_admin_datasource_toolkit/spec/lib/forest_admin_datasource_toolkit/utils/collection_spec.rb +++ b/packages/forest_admin_datasource_toolkit/spec/lib/forest_admin_datasource_toolkit/utils/collection_spec.rb @@ -221,7 +221,7 @@ module Utils expect do described_class.get_field_schema(collection_person, 'foo') - end.to raise_error(ForestException, 'Column not found Person.foo') + end.to raise_error(ValidationError, 'Column not found Person.foo') end it 'get_field_schema should work with simple column' do @@ -233,14 +233,14 @@ module Utils expect do described_class.get_field_schema(collection_person, 'unknown:foo') - end.to raise_error(ForestException, 'Relation not found Person.unknown') + end.to raise_error(ValidationError, 'Relation not found Person.unknown') end it 'get_field_schema should throw with invalid relation type' do expect do described_class.get_field_schema(collection_book, 'myBookPersons:bookId') - end.to raise_error(ForestException, 'Unexpected field type OneToMany: Book.myBookPersons') + end.to raise_error(ValidationError, 'Unexpected field type OneToMany: Book.myBookPersons') end it 'get_field_schema should work with relation column' do @@ -311,6 +311,25 @@ module Utils ForestAdminDatasourceToolkit::Components::Query::Projection.new)).to eq([1]) end + # A through row whose target the foreign collection no longer answers -- + # deleted, outside the caller's reach, or dropped by a datasource that + # reads its targets in bounded pages -- carries a nil where a record was + # expected, and every consumer down to the JSON:API serializer reads a + # row by key. A join whose other side is gone yields no row. + it 'list_relation should drop a through row whose target resolved to nothing' do + book_person_class = Struct.new(:bookId, :personId, :myPerson, :myBook) + stub_const('BookPerson', book_person_class) + allow(collection_book_person).to receive(:list).and_return( + [ + BookPerson.new(1, 1, 1, 1), + BookPerson.new(1, 2, nil, 1) + ] + ) + + expect(described_class.list_relation(collection_book, [1], 'myPersons', caller, ForestAdminDatasourceToolkit::Components::Query::Filter.new, + ForestAdminDatasourceToolkit::Components::Query::Projection.new)).to eq([1]) + end + it 'aggregate_relation should work with one to many relation' do allow(collection_book_person).to receive(:aggregate).and_return(1)