feat(datasource-pylon): Pylon datasource for the Ruby agent (EXT-4)

.github/workflows/build.yml

@@ -35,6 +35,7 @@ jobs:
           - forest_admin_datasource_snowflake
           - forest_admin_datasource_mambu_payments
           - forest_admin_datasource_graphql_hasura
+          - forest_admin_datasource_pylon
 
     steps:
       - name: Checkout
@@ -88,6 +89,7 @@ jobs:
           - forest_admin_datasource_snowflake
           - forest_admin_datasource_mambu_payments
           - forest_admin_datasource_graphql_hasura
+          - forest_admin_datasource_pylon
     services:
       mongodb:
         image: mongo:latest
@@ -166,7 +168,7 @@ jobs:
         with:
           verbose: true
           oidc: true
-          files: ${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_agent/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_datasource_active_record/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_datasource_customizer/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_datasource_toolkit/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_rails/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_datasource_mongoid/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_rpc_agent/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_datasource_rpc/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_datasource_zendesk/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_datasource_snowflake/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_datasource_mambu_payments/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_datasource_graphql_hasura/coverage.json
+          files: ${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_agent/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_datasource_active_record/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_datasource_customizer/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_datasource_toolkit/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_rails/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_datasource_mongoid/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_rpc_agent/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_datasource_rpc/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_datasource_zendesk/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_datasource_snowflake/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_datasource_mambu_payments/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_datasource_graphql_hasura/coverage.json,${{ github.workspace }}/reports/${{ matrix.ruby-version }}-forest_admin_datasource_pylon/coverage.json
 
   deploy:
     name: Release package

.releaserc.js

@@ -31,7 +31,8 @@ module.exports = {
             'sed -i \'s/VERSION = ".*"/VERSION = "${nextRelease.version}"/g\' packages/forest_admin_datasource_zendesk/lib/forest_admin_datasource_zendesk/version.rb; '+
             'sed -i \'s/VERSION = ".*"/VERSION = "${nextRelease.version}"/g\' packages/forest_admin_datasource_snowflake/lib/forest_admin_datasource_snowflake/version.rb; '+
             'sed -i \'s/VERSION = ".*"/VERSION = "${nextRelease.version}"/g\' packages/forest_admin_datasource_mambu_payments/lib/forest_admin_datasource_mambu_payments/version.rb; '+
-            'sed -i \'s/VERSION = ".*"/VERSION = "${nextRelease.version}"/g\' packages/forest_admin_datasource_graphql_hasura/lib/forest_admin_datasource_graphql_hasura/version.rb; ',
+            'sed -i \'s/VERSION = ".*"/VERSION = "${nextRelease.version}"/g\' packages/forest_admin_datasource_graphql_hasura/lib/forest_admin_datasource_graphql_hasura/version.rb; '+
+            'sed -i \'s/VERSION = ".*"/VERSION = "${nextRelease.version}"/g\' packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/version.rb; ',
         successCmd:
             '( cd packages/forest_admin_agent && gem build && gem push forest_admin_agent-*.gem );' +
             '( cd packages/forest_admin_datasource_active_record && gem build && gem push forest_admin_datasource_active_record-*.gem );' +
@@ -45,7 +46,8 @@ module.exports = {
             '( cd packages/forest_admin_datasource_zendesk && gem build && gem push forest_admin_datasource_zendesk-*.gem );' +
             '( cd packages/forest_admin_datasource_snowflake && gem build && gem push forest_admin_datasource_snowflake-*.gem );' +
             '( cd packages/forest_admin_datasource_mambu_payments && gem build && gem push forest_admin_datasource_mambu_payments-*.gem );' +
-            '( cd packages/forest_admin_datasource_graphql_hasura && gem build && gem push forest_admin_datasource_graphql_hasura-*.gem );' ,
+            '( cd packages/forest_admin_datasource_graphql_hasura && gem build && gem push forest_admin_datasource_graphql_hasura-*.gem );' +
+            '( cd packages/forest_admin_datasource_pylon && gem build && gem push forest_admin_datasource_pylon-*.gem );' ,
       },
     ],
     [
@@ -68,6 +70,7 @@ module.exports = {
           'packages/forest_admin_datasource_snowflake/lib/forest_admin_datasource_snowflake/version.rb',
           'packages/forest_admin_datasource_mambu_payments/lib/forest_admin_datasource_mambu_payments/version.rb',
           'packages/forest_admin_datasource_graphql_hasura/lib/forest_admin_datasource_graphql_hasura/version.rb',
+          'packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/version.rb',
           'package.json'
         ],
       },

.rubocop.yml

@@ -42,6 +42,7 @@ Gemspec/RequireMFA:
     - 'packages/forest_admin_datasource_snowflake/forest_admin_datasource_snowflake.gemspec'
     - 'packages/forest_admin_datasource_mambu_payments/forest_admin_datasource_mambu_payments.gemspec'
     - 'packages/forest_admin_datasource_graphql_hasura/forest_admin_datasource_graphql_hasura.gemspec'
+    - 'packages/forest_admin_datasource_pylon/forest_admin_datasource_pylon.gemspec'
 
 # Offense count: 1
 # This cop supports unsafe autocorrection (--autocorrect-all).
@@ -133,6 +134,7 @@ Style/MutableConstant:
     - 'packages/forest_admin_datasource_zendesk/lib/forest_admin_datasource_zendesk/version.rb'
     - 'packages/forest_admin_datasource_snowflake/lib/forest_admin_datasource_snowflake/version.rb'
     - 'packages/forest_admin_datasource_mambu_payments/lib/forest_admin_datasource_mambu_payments/version.rb'
+    - 'packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/version.rb'
 
 # Offense count: 38
 # This cop supports safe autocorrection (--autocorrect).
@@ -217,6 +219,7 @@ Style/StringLiterals:
     - 'packages/forest_admin_datasource_zendesk/lib/forest_admin_datasource_zendesk/version.rb'
     - 'packages/forest_admin_datasource_snowflake/lib/forest_admin_datasource_snowflake/version.rb'
     - 'packages/forest_admin_datasource_mambu_payments/lib/forest_admin_datasource_mambu_payments/version.rb'
+    - 'packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/version.rb'
 
 # Offense count: 1
 # This cop supports safe autocorrection (--autocorrect).
@@ -262,6 +265,8 @@ Metrics/ParameterLists:
     - 'packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/store.rb'
     - 'packages/forest_admin_datasource_graphql_hasura/lib/forest_admin_datasource_graphql_hasura/configuration.rb'
     - 'packages/forest_admin_datasource_zendesk/lib/forest_admin_datasource_zendesk/collections/base_collection.rb'
+    - 'packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/collections/base_collection.rb'
+    - 'packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/configuration.rb'
     - 'packages/forest_admin_datasource_snowflake/lib/forest_admin_datasource_snowflake/datasource.rb'
     - 'packages/forest_admin_agent/lib/forest_admin_agent/routes/query_handler.rb'
     - 'packages/forest_admin_agent/lib/forest_admin_agent/services/smart_action_checker.rb'
@@ -294,6 +299,7 @@ Metrics/ModuleLength:
     - 'packages/forest_admin_datasource_customizer/spec/**/*'
     - 'packages/forest_admin_datasource_zendesk/spec/**/*'
     - 'packages/forest_admin_datasource_mambu_payments/spec/**/*'
+    - 'packages/forest_admin_datasource_pylon/spec/**/*'
     - 'packages/forest_admin_rails/spec/**/*'
     - 'packages/forest_admin_rpc_agent/spec/**/*'
     - 'packages/forest_admin_datasource_mongoid/lib/forest_admin_datasource_mongoid/utils/helpers.rb'
@@ -369,6 +375,7 @@ Metrics/ClassLength:
     - 'packages/forest_admin_datasource_graphql_hasura/lib/forest_admin_datasource_graphql_hasura/query/query_builder.rb'
     - 'packages/forest_admin_datasource_graphql_hasura/lib/forest_admin_datasource_graphql_hasura/query/aggregator.rb'
     - 'packages/forest_admin_datasource_zendesk/lib/forest_admin_datasource_zendesk/collections/base_collection.rb'
+    - 'packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon/collections/base_collection.rb'
     - 'packages/forest_admin_datasource_snowflake/lib/forest_admin_datasource_snowflake/collection.rb'
     - 'packages/forest_admin_datasource_snowflake/lib/forest_admin_datasource_snowflake/datasource.rb'
     - 'packages/forest_admin_datasource_snowflake/lib/forest_admin_datasource_snowflake/utils/query.rb'

packages/forest_admin_datasource_customizer/lib/forest_admin_datasource_customizer/decorators/sort/sort_collection_decorator.rb

@@ -51,15 +51,19 @@ def list(caller, filter = nil, projection = nil)
           projection.apply(records)
         end
 
+        # `@sorts` holds nil as the value of an emulated field, so membership is
+        # read with `key?`, the way `emulated?` reads it. The fields hash and the
+        # ColumnSchema objects in it belong to the collection below, so both are
+        # copied before the flag is written.
         def refine_schema(child_schema)
-          child_schema[:fields].each do |name, schema|
-            if schema.type == 'Column'
-              schema.is_sortable = true if @sorts[name].nil?
-              child_schema[:fields][name] = schema
-            end
+          schema = child_schema.dup
+          schema[:fields] = child_schema[:fields].to_h do |name, field|
+            next [name, field] unless field.type == 'Column' && @sorts.key?(name)
+
+            [name, field.dup.tap { |column| column.is_sortable = true }]
           end
 
-          child_schema
+          schema
         end
 
         def rewrite_plain_sort_clause(clause)

packages/forest_admin_datasource_customizer/spec/lib/forest_admin_datasource_customizer/decorators/sort/sort_collection_decorator_spec.rb

@@ -105,6 +105,25 @@ module Sort
           expect { @decorated_book.replace_field_sorting('author_id', nil) }.to raise_error(ForestException, 'A new sorting method should be provided to replace field sorting')
         end
 
+        it 'leaves the sortability of the fields it cannot order as the datasource declared it' do
+          expect(@decorated_book.schema[:fields]['title'].is_sortable).to be false
+          expect(@decorated_book.schema[:fields]['author_id'].is_sortable).to be false
+          expect(@decorated_book.schema[:fields]['id'].is_sortable).to be false
+        end
+
+        it 'marks a field sortable once its sorting is replaced' do
+          @decorated_book.replace_field_sorting('author_id', [{ field: 'id', ascending: true }])
+
+          expect(@decorated_book.schema[:fields]['author_id'].is_sortable).to be true
+        end
+
+        it 'leaves the schema of the collection below untouched' do
+          @decorated_book.emulate_field_sorting('title')
+
+          expect(@decorated_book.schema[:fields]['title'].is_sortable).to be true
+          expect(@collection_book.schema[:fields]['title'].is_sortable).to be false
+        end
+
         context 'when emulating sort on book.title (no relations)' do
           before do
             @decorated_book.emulate_field_sorting('title')

packages/forest_admin_datasource_pylon/.gitignore

@@ -0,0 +1,8 @@
+*.gem
+.bundle/
+Gemfile.lock
+Gemfile-test.lock
+coverage/
+pkg/
+tmp/
+.rspec_status

packages/forest_admin_datasource_pylon/.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper

packages/forest_admin_datasource_pylon/Gemfile

@@ -0,0 +1,16 @@
+source 'https://rubygems.org'
+
+gemspec
+
+gem 'forest_admin_datasource_customizer'
+gem 'forest_admin_datasource_toolkit'
+gem 'rake', '~> 13.0'
+gem 'rubocop', '1.86.1'
+gem 'rubocop-performance', '1.26.1'
+gem 'rubocop-rspec', '3.9.0'
+
+group :development, :test do
+  gem 'rspec', '~> 3.0'
+  gem 'simplecov', '~> 0.22', require: false
+  gem 'webmock', '~> 3.0'
+end

packages/forest_admin_datasource_pylon/Gemfile-test

@@ -0,0 +1,19 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in forest_admin_datasource_pylon.gemspec
+gemspec
+
+gem 'rake', '~> 13.0'
+gem 'rubocop', '1.86.1'
+gem 'rubocop-performance', '1.26.1'
+gem 'rubocop-rspec', '3.9.0'
+
+group :development, :test do
+  gem 'forest_admin_datasource_customizer', path: '../forest_admin_datasource_customizer'
+  gem 'forest_admin_datasource_toolkit', path: '../forest_admin_datasource_toolkit'
+  gem 'rspec', '~> 3.0'
+  gem 'simplecov', '~> 0.22', require: false
+  gem 'simplecov-html', '~> 0.12.3'
+  gem 'simplecov_json_formatter', '~> 0.1.4'
+  gem 'webmock', '~> 3.0'
+end

packages/forest_admin_datasource_pylon/Rakefile

@@ -0,0 +1,6 @@
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec)
+
+task default: :spec

packages/forest_admin_datasource_pylon/forest_admin_datasource_pylon.gemspec

@@ -0,0 +1,36 @@
+lib = File.expand_path('lib', __dir__)
+$LOAD_PATH.unshift lib unless $LOAD_PATH.include?(lib)
+
+require_relative 'lib/forest_admin_datasource_pylon/version'
+
+Gem::Specification.new do |spec|
+  spec.name        = 'forest_admin_datasource_pylon'
+  spec.version     = ForestAdminDatasourcePylon::VERSION
+  spec.authors     = ['Forest Admin']
+  spec.email       = ['contact@forestadmin.com']
+  spec.homepage    = 'https://www.forestadmin.com'
+  spec.summary     = 'Pylon datasource for Forest Admin Ruby agent.'
+  spec.description = 'Surface Pylon issues, accounts, contacts, users and teams as Forest Admin collections.'
+  spec.license     = 'GPL-3.0'
+  spec.required_ruby_version = '>= 3.0.0'
+
+  spec.metadata['homepage_uri']    = spec.homepage
+  spec.metadata['source_code_uri'] = 'https://github.com/ForestAdmin/agent-ruby'
+  spec.metadata['changelog_uri']   = 'https://github.com/ForestAdmin/agent-ruby/blob/main/CHANGELOG.md'
+  spec.metadata['rubygems_mfa_required'] = 'false'
+
+  spec.files = Dir.chdir(__dir__) do
+    `git ls-files -z`.split("\x0").reject do |f|
+      (File.expand_path(f) == __FILE__) ||
+        f.start_with?(*%w[bin/ test/ spec/ features/ .git .circleci appveyor Gemfile])
+    end
+  end
+  spec.bindir        = 'exe'
+  spec.executables   = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
+  spec.require_paths = ['lib']
+
+  spec.add_dependency 'activesupport', '>= 6.1'
+  spec.add_dependency 'faraday', '~> 2.0'
+  spec.add_dependency 'faraday-retry', '~> 2.0'
+  spec.add_dependency 'zeitwerk', '~> 2.3'
+end

packages/forest_admin_datasource_pylon/lib/forest_admin_datasource_pylon.rb

@@ -0,0 +1,69 @@
+require_relative 'forest_admin_datasource_pylon/version'
+require 'json'
+require 'logger'
+require 'uri'
+require 'zeitwerk'
+require 'faraday'
+require 'faraday/retry'
+require 'forest_admin_datasource_toolkit'
+
+loader = Zeitwerk::Loader.for_gem
+loader.setup
+
+module ForestAdminDatasourcePylon
+  class Error < StandardError; end
+  class ConfigurationError < Error; end
+
+  # A filter Pylon cannot express. It descends from the toolkit's ValidationError
+  # rather than from the package's own Error so the agent answers 400 carrying
+  # the message instead of a 500 "Unexpected error": every one of these names a
+  # condition the operator set and can change, and the message is the only place
+  # they learn which one.
+  class UnsupportedOperatorError < ForestAdminDatasourceToolkit::Exceptions::ValidationError; end
+
+  # The three write errors below descend from ValidationError for that same
+  # reason: each names something the operator did and can undo.
+
+  # A verb Pylon's API has no endpoint for, a field it only accepts in the other
+  # direction, or a write reaching more records than one pass may cover.
+  class UnsupportedWriteError < ForestAdminDatasourceToolkit::Exceptions::ValidationError; end
+
+  # A write Pylon performed on some of its records and then failed on: one
+  # record is one request, so the ones before the failure stay written, and a
+  # retry of the whole selection would write them a second time.
+  class PartialWriteError < ForestAdminDatasourceToolkit::Exceptions::ValidationError; end
+
+  # A write Pylon itself refused, carrying the reason it gave — the likeliest
+  # way a write fails. Only its 4xx travels this way: a 5xx or a dropped
+  # connection is not the operator's to act on and stays the APIError it was.
+  class WriteRejectedError < ForestAdminDatasourceToolkit::Exceptions::ValidationError; end
+
+  # Raised when a Pylon API call fails. Carries the HTTP status and the
+  # (parsed) response body so callers — smart actions in particular — can
+  # surface Pylon's own validation message instead of a generic string.
+  class APIError < Error
+    attr_reader :status, :body
+
+    def initialize(message, status: nil, body: nil)
+      super(message)
+      @status = status
+      @body = body
+    end
+  end
+
+  class << self
+    attr_writer :logger
+
+    def logger
+      @logger ||= default_logger
+    end
+
+    private
+
+    def default_logger
+      return Rails.logger if defined?(Rails) && Rails.respond_to?(:logger) && Rails.logger
+
+      Logger.new($stderr).tap { |l| l.progname = 'forest_admin_datasource_pylon' }
+    end
+  end
+end