refactor: use sepa_file_parser instead of camt_parser

we used a forked `camt_parser` version which was outdated.
To use the `sepa_file_parser` gem we extended the mapping factory in
order to keep the code impact as small as possible.

NOTE: thanks to the mapping the transaction_id is now better found which
caused a few spec to break due to invalid expectations. Those were
removed.

Author: janz93

Gemfile

@@ -6,6 +6,7 @@ ruby "3.3.2"
 
 gem "activesupport"
 gem "camt_parser", git: "https://github.com/railslove/camt_parser.git"
+gem "sepa_file_parser"
 gem "cmxl", git: "https://github.com/railslove/cmxl"
 gem "epics"
 gem "faraday"

Gemfile.lock

@@ -55,6 +55,7 @@ GEM
     database_cleaner-sequel (2.0.2)
       database_cleaner-core (~> 2.0.0)
       sequel
+    date (3.4.1)
     diff-lcs (1.5.1)
     domain_name (0.5.20190701)
       unf (>= 0.0.5, < 1.0.0)
@@ -210,6 +211,10 @@ GEM
     sentry-sidekiq (5.22.1)
       sentry-ruby (~> 5.22.1)
       sidekiq (>= 3.0)
+    sepa_file_parser (0.4.0)
+      bigdecimal
+      nokogiri
+      time
     sepa_king (0.12.0)
       activemodel (>= 3.1)
       iban-tools
@@ -239,6 +244,8 @@ GEM
       rubocop-performance (~> 1.21.0)
     statsd-ruby (1.5.0)
     tilt (2.4.0)
+    time (0.4.1)
+      date
     timecop (0.9.10)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
@@ -287,6 +294,7 @@ DEPENDENCIES
   rubocop
   sentry-ruby
   sentry-sidekiq
+  sepa_file_parser
   sepa_king
   sequel
   sidekiq

Rakefile

@@ -86,7 +86,7 @@ namespace :migration_tasks do
 
       bank_statements = Box::BankStatement.where(account_id: account_id).all
       bank_statements.each do |bank_statement|
-        parser = bank_statement.content.starts_with?(":") ? Cmxl : CamtParser::Format053::Statement
+        parser = bank_statement.content.starts_with?(":") ? Cmxl : SepaFileParser::Camt053::Statement
         begin
           result = parser.parse(bank_statement.content)
           transactions = result.is_a?(Array) ? result.first.transactions : result.transactions

box/business_processes/import_bank_statement.rb

@@ -57,7 +57,7 @@ def self.find_or_create_bank_statement(raw_bank_statement, account)
           bs.account_id = account.id
           bs.sequence = raw_bank_statement.sequence
           bs.year = extract_year_from_bank_statement(raw_bank_statement)
-          bs.remote_account = raw_bank_statement.account_identification.source
+          bs.remote_account = raw_bank_statement.account_identification.iban
           bs.opening_balance = as_big_decimal(raw_bank_statement.opening_or_intermediary_balance) # this will be final or intermediate
           bs.closing_balance = as_big_decimal(raw_bank_statement.closing_or_intermediary_balance) # this will be final or intermediate
           bs.transaction_count = raw_bank_statement.transactions.count
@@ -85,7 +85,7 @@ def self.as_big_decimal(input)
 
       def self.extract_year_from_bank_statement(raw_bank_statement)
         first_transaction = raw_bank_statement.transactions.first
-        first_transaction&.date&.year
+        first_transaction.date&.year
       end
 
       def self.checksum(raw_bank_statement, account)

box/business_processes/import_statements.rb

@@ -1,17 +1,18 @@
 # frozen_string_literal: true
 
-require "cmxl"
-require "camt_parser"
+require "sepa_file_parser"
+
 
 require_relative "../models/account"
 require_relative "../models/bank_statement"
 require_relative "../models/event"
 require_relative "../../lib/checksum_generator"
+require_relative "../../lib/data_mapping/statement_factory"
 
 module Box
   module BusinessProcesses
     class ImportStatements
-      PARSERS = {"mt940" => Cmxl, "camt53" => CamtParser::Format053::Statement}.freeze
+      PARSERS = {"mt940" => Cmxl, "camt53" => SepaFileParser::Camt053::Statement}.freeze
 
       def self.from_bank_statement(bank_statement, upcoming = false)
         bank_transactions = parse_bank_statement(bank_statement)
@@ -28,7 +29,9 @@ def self.from_bank_statement(bank_statement, upcoming = false)
       def self.parse_bank_statement(bank_statement)
         parser = PARSERS.fetch(bank_statement.account.statements_format, Cmxl)
         result = parser.parse(bank_statement.content)
-        result.is_a?(Array) ? result.first.transactions : result.transactions
+        statement_data = result.is_a?(Array) ? result.first : result
+        statement = DataMapping::StatementFactory.new(statement_data, bank_statement.account).call
+        statement.transactions
       end
 
       def self.create_statement(bank_statement, bank_transaction, upcoming = false)
@@ -71,51 +74,47 @@ def self.checksum(transaction, bank_statement)
       end
 
       def self.checksum_attributes(transaction, remote_account)
-        return [remote_account, transaction.transaction_id] if transaction.try(:transaction_id).present?
+        return [remote_account, transaction.transaction_id] if transaction.transaction_id.present?
 
         payload_from_transaction_attributes(transaction, remote_account)
       end
 
       def self.payload_from_transaction_attributes(transaction, remote_account)
-        eref = transaction.respond_to?(:eref) ? transaction.eref : transaction.sepa["EREF"]
-        mref = transaction.respond_to?(:mref) ? transaction.mref : transaction.sepa["MREF"]
-        svwz = transaction.respond_to?(:svwz) ? transaction.svwz : transaction.sepa["SVWZ"]
-
         [
           remote_account,
           transaction.date,
           transaction.amount_in_cents,
           transaction.iban,
           transaction.name,
           transaction.sign,
-          eref,
-          mref,
-          svwz,
+          transaction.eref,
+          transaction.mref,
+          transaction.svwz,
           transaction.information.gsub(/\s/, "")
         ]
       end
 
       def self.statement_attributes_from_bank_transaction(transaction, bank_statement)
         {
-          sha: checksum(transaction, bank_statement),
-          date: transaction.date,
-          entry_date: transaction.entry_date,
           amount: transaction.amount_in_cents,
-          sign: transaction.sign,
-          debit: transaction.debit?,
-          swift_code: transaction.swift_code,
-          reference: transaction.reference,
           bank_reference: transaction.bank_reference,
           bic: transaction.bic,
+          creditor_identifier: transaction.creditor_identifier,
+          date: transaction.date,
+          debit: transaction.debit?,
+          description: transaction.description,
+          entry_date: transaction.entry_date,
+          eref: transaction.eref,
           iban: transaction.iban,
-          name: transaction.name,
           information: transaction.information,
-          description: transaction.description,
-          eref: transaction.respond_to?(:eref) ? transaction.eref : transaction.sepa["EREF"],
-          mref: transaction.respond_to?(:mref) ? transaction.mref : transaction.sepa["MREF"],
-          svwz: transaction.respond_to?(:svwz) ? transaction.svwz : transaction.sepa["SVWZ"],
-          tx_id: transaction.try(:primanota) || transaction.try(:transaction_id),
-          creditor_identifier: transaction.respond_to?(:creditor_identifier) ? transaction.creditor_identifier : transaction.sepa["CRED"]
+          mref: transaction.mref,
+          name: transaction.name,
+          reference: transaction.reference,
+          sha: checksum(transaction, bank_statement),
+          sign: transaction.sign,
+          svwz: transaction.svwz,
+          swift_code: transaction.swift_code,
+          tx_id: transaction.transaction_id,
         }
       end
     end

box/jobs/fetch_statements.rb

@@ -2,7 +2,7 @@
 
 require "sidekiq-scheduler"
 require "active_support/all"
-require "camt_parser"
+require "sepa_file_parser"
 require "cmxl"
 require "epics"
 require "sequel"
@@ -84,7 +84,7 @@ def camt53(client, from, to)
         combined_camt = client.C53(from.to_s(:db), to.to_s(:db))
         return unless combined_camt.any?
 
-        combined_camt.map { |chunk| CamtParser::String.parse(chunk).statements }.flatten
+        combined_camt.map { |chunk| SepaFileParser::String.parse(chunk).statements }.flatten
       end
 
       def mt940(client, from, to)

box/jobs/fetch_upcoming_statements.rb

@@ -2,7 +2,7 @@
 
 require "sidekiq-scheduler"
 require "active_support/all"
-require "camt_parser"
+require "sepa_file_parser"
 require "cmxl"
 require "epics"
 require "sequel"

box/jobs/queue_fetch_statements.rb

@@ -2,7 +2,7 @@
 
 require "sidekiq-scheduler"
 require "active_support/all"
-require "camt_parser"
+require "sepa_file_parser"
 require "cmxl"
 require "epics"
 require "sequel"

box/jobs/queue_fetch_upcoming_statements.rb

@@ -2,7 +2,7 @@
 
 require "sidekiq-scheduler"
 require "active_support/all"
-require "camt_parser"
+require "sepa_file_parser"
 require "cmxl"
 require "epics"
 require "sequel"

lib/data_mapping/camt53/statement.rb

@@ -1,3 +1,5 @@
+require_relative "transaction"
+
 module DataMapping
   module Camt53
     class Statement
@@ -12,27 +14,29 @@ def blank?
       end
 
       def account_identification
-        raw_bank_statement.account_identification
+        raw_bank_statement.account
       end
 
       def closing_or_intermediary_balance
-        raw_bank_statement.closing_or_intermediary_balance
+        raw_bank_statement.closing_balance
       end
 
       def sequence
         raw_bank_statement.electronic_sequence_number
       end
 
       def opening_or_intermediary_balance
-        raw_bank_statement.closing_or_intermediary_balance
+        raw_bank_statement.opening_balance
       end
 
       def source
         raw_bank_statement.source
       end
 
       def transactions
-        raw_bank_statement.transactions
+        raw_bank_statement.entries.map do |entry|
+          Transaction.new(entry)
+        end
       end
     end
   end

lib/data_mapping/camt53/transaction.rb

@@ -0,0 +1,73 @@
+module DataMapping
+  module Camt53
+    class Transaction
+      attr_reader :raw_bank_statement, :first_transaction
+
+      delegate :amount_in_cents,
+        :bank_reference,
+        :credit?,
+        :debit?,
+        :reference,
+        :sign,
+        :transaction_id,
+        to: :raw_bank_statement
+
+      def initialize(raw_bank_statement)
+        @raw_bank_statement = raw_bank_statement
+        @first_transaction = raw_bank_statement.transactions.first
+      end
+
+      def bic
+        first_transaction.bic
+      end
+
+      def creditor_identifier
+        first_transaction.creditor_identifier
+      end
+
+      def date
+        raw_bank_statement.value_date
+      end
+
+      def description
+        raw_bank_statement.additional_information
+      end
+
+      def entry_date
+        raw_bank_statement.booking_date
+      end
+
+      def eref
+        first_transaction.end_to_end_reference
+      end
+
+      def iban
+        first_transaction.iban
+      end
+
+      def information
+        first_transaction.payment_information
+      end
+
+      def mref
+        first_transaction.mandate_reference
+      end
+
+      def name
+        first_transaction.name
+      end
+
+      def svwz
+        first_transaction.remittance_information
+      end
+
+      def swift_code
+        first_transaction.swift_code
+      end
+
+      def transaction_id
+        first_transaction.transaction_id
+      end
+    end
+  end
+end

lib/data_mapping/cmxl/account.rb

@@ -0,0 +1,19 @@
+module DataMapping
+  module Cmxl
+    class Account
+      attr_reader :raw_bank_statement
+
+      delegate :account_number,
+        to: :raw_bank_statement
+
+      def initialize(raw_bank_statement)
+        @raw_bank_statement = raw_bank_statement
+      end
+
+      def iban
+        return raw_bank_statement.iban if raw_bank_statement.iban.present?
+        raw_bank_statement.source
+      end
+    end
+  end
+end

lib/data_mapping/cmxl/statement.rb

@@ -1,14 +1,16 @@
+require_relative "account"
+require_relative "transaction"
+
 module DataMapping
   module Cmxl
     class Statement
       attr_reader :raw_bank_statement
 
-      delegate :account_identification,
+      delegate :account_number,
         :blank?,
         :closing_or_intermediary_balance,
         :opening_or_intermediary_balance,
         :source,
-        :transactions,
         to: :raw_bank_statement
 
       def initialize(raw_bank_statement)
@@ -18,6 +20,16 @@ def initialize(raw_bank_statement)
       def sequence
         raw_bank_statement.legal_sequence_number
       end
+
+      def account_identification
+        Account.new(raw_bank_statement.account_identification)
+      end
+
+      def transactions
+        raw_bank_statement.transactions.map do |entry|
+          Transaction.new(entry)
+        end
+      end
     end
   end
 end