autogenerate bank_reference from data_raw by default

Author: ufundo

Civi/Api4/Action/BankTransactionBatch/ImportBase.php

@@ -45,7 +45,7 @@ abstract class ImportBase extends \Civi\Api4\Generic\AbstractAction {
   /**
    * @var string
    *
-   * The column header corresponding to the transaction reference - leave blank to auto-detect
+   * The column header corresponding to the transaction reference - leave blank to hash raw data
    */
   protected string $referenceColumn = '';
   /**
@@ -131,20 +131,11 @@ public function _run(Result $result) {
       $this->autodetectDateFormat($transactions);
     }
 
-    // parse data/amount/reference columns
+    // extract data/amount/reference columns
     $transactions = array_map(function ($tx) {
-      // parse reference
-      $rawReference = $tx['data_parsed'][$this->referenceColumn];
-      $tx['bank_reference'] = ($rawReference !== '') ? $rawReference : NULL;
-
-      // parse amount to float
-      // @todo could we autodetect thousands separator?
-      $rawAmount = $tx['data_parsed'][$this->amountColumn];
-      $tx['amount'] = ($rawAmount !== '') ? (float) \str_replace($this->thousandsSeparator, '', $rawAmount) : NULL;
-
-      // parse date
-      $tx['booking_date'] = $this->parseDate($this->dateFormat, $tx['data_parsed'][$this->dateColumn]);
-
+      $tx['bank_reference'] = $this->getReference($tx);
+      $tx['amount'] = $this->getAmount($tx);
+      $tx['booking_date'] = $this->getDate($tx);
       return $tx;
     }, $transactions);
 
@@ -216,7 +207,8 @@ abstract protected function process(array $statement, array $transactions, Resul
   protected function validHeader(array $header): bool {
     $dateColumn = $this->dateColumn ? in_array($this->dateColumn, $header) : array_find($header, fn ($col) => \str_contains($col, E::ts('Date')));
     $amountColumn = $this->amountColumn ? in_array($this->amountColumn, $header) : array_find($header, fn ($col) => \str_contains($col, E::ts('Amount')));
-    $referenceColumn = $this->referenceColumn ? in_array($this->referenceColumn, $header) : array_find($header, fn ($col) => \str_contains($col, E::ts('Reference')));
+    // note: we only need the reference column if set - if not autogenerated
+    $referenceColumn = $this->referenceColumn ? in_array($this->referenceColumn, $header) : TRUE;
 
     return $dateColumn && $amountColumn && $referenceColumn;
   }
@@ -235,14 +227,13 @@ protected function autodetectHeader(array $sheet): void {
     // get the header row
     $header = $sheet[$this->headerIndex];
 
-    // use autodetect key columns values if needed
+    // use autodetected key columns values if needed
     $this->dateColumn = $this->dateColumn ?: array_find($header, fn ($col) => \str_contains($col, E::ts('Date')));
     $this->amountColumn = $this->amountColumn ?: array_find($header, fn ($col) => \str_contains($col, E::ts('Amount')));
-    $this->referenceColumn = $this->referenceColumn ?: array_find($header, fn ($col) => \str_contains($col, E::ts('Reference')));
 
     // sense check: header should contain all key columns
     // this could fail if header and xColumns are specified that are inconsistent
-    $missing = array_diff([$this->dateColumn, $this->amountColumn, $this->referenceColumn], $header);
+    $missing = array_diff(array_filter([$this->dateColumn, $this->amountColumn, $this->referenceColumn]), $header);
     if ($missing) {
       $message = E::ts('Unable to find %1 in header row %2', [1 => implode(', ', $missing), 2 => $this->headerIndex]);
       throw new \CRM_Core_Exception($message);
@@ -292,4 +283,23 @@ protected function parseDate($format, $value): ?string {
     return NULL;
   }
 
+  protected function getDate(array $tx): ?string {
+    return $this->parseDate($this->dateFormat, $tx['data_parsed'][$this->dateColumn]);
+  }
+
+  protected function getReference(array $tx): ?string {
+    if ($this->referenceColumn) {
+      $rawReference = $tx['data_parsed'][$this->referenceColumn];
+      return ($rawReference !== '') ? $rawReference : NULL;
+    }
+    return \md5($tx['data_raw']);
+  }
+
+  protected function getAmount(array $tx): ?float {
+    // parse amount to float
+    // @todo could we autodetect thousands separator?
+    $rawAmount = $tx['data_parsed'][$this->amountColumn];
+    return ($rawAmount !== '') ? (float) \str_replace($this->thousandsSeparator, '', $rawAmount) : NULL;
+  }
+
 }

Civi/Api4/Action/BankTransactionBatch/Preview.php

@@ -22,7 +22,7 @@ protected function process(array $statement, array $transactions, Result $result
       'amountColumn' => $this->amountColumn,
     ];
 
-    $result['columnOptions'] = array_keys($transactions[0]['data_parsed']);
+    $result['headerColumns'] = array_keys($transactions[0]['data_parsed']);
   }
 
 }

elements/civi-banking-import.js

@@ -105,11 +105,11 @@
     buildOptionSelector(param, labelText, options) {
       const select = document.createElement('select');
 
-      select.append(...options.map((value) => {
+      select.append(...options.map((o) => {
         const option = document.createElement('option');
-        option.value = value;
-        option.innerText = value;
-        option.selected = (value === this.apiParams[param])
+        option.value = o.value;
+        option.innerText = o.label;
+        option.selected = (o.value === this.apiParams[param])
         return option;
       }));
 
@@ -135,11 +135,27 @@
     }
 
     renderParamInputs() {
+      const columnOptions = [
+        {
+          label: '- auto -',
+          value: '',
+        },
+        ...this.previewResult.headerColumns.map((col) => ({
+          label: col,
+          value: col
+        }))
+      ];
+
+      const dateFormatOptions = ['Y-m-d', 'm/d/Y', 'd/m/Y'].map((format) => ({
+        label: format,
+        value: format
+      }));
+
       this.querySelector('fieldset').replaceChildren(
-        this.buildOptionSelector('dateColumn', ts('Date Column'), this.previewResult.columnOptions),
-        this.buildOptionSelector('dateFormat', ts('Date Format'), ['Y-m-d', 'm/d/Y', 'd/m/Y']),
-        this.buildOptionSelector('amountColumn', ts('Amount Column'), this.previewResult.columnOptions),
-        this.buildOptionSelector('referenceColumn', ts('Reference Column'), this.previewResult.columnOptions),
+        this.buildOptionSelector('dateColumn', ts('Date Column'), columnOptions),
+        this.buildOptionSelector('dateFormat', ts('Date Format'), dateFormatOptions),
+        this.buildOptionSelector('amountColumn', ts('Amount Column'), columnOptions),
+        this.buildOptionSelector('referenceColumn', ts('Reference Column'), columnOptions),
     //    this.buildTextInput('statementTitle', ts('Statement Title'))
       );
     }