mormot.db.sql.pas

/// Database Framework Shared Abstract SQL Types and Classes
// - this unit is a part of the Open Source Synopse mORMot framework 2,
// licensed under a MPL/GPL/LGPL three license - see LICENSE.md
unit mormot.db.sql;

{
  *****************************************************************************

   Shared Types and Definitions for SQL Database Access
    - SQL Fields and Columns Definitions
    - Define Database Engine Specific Behavior
    - General SQL Processing Functions
    - Abstract SQL DB Classes and Interfaces
    - Parent Classes for Thread-Safe and Parametrized Connections

  *****************************************************************************
}

interface

{$I ..\mormot.defines.inc}

uses
  sysutils,
  classes,
  variants,
  mormot.core.base,
  mormot.core.os,
  mormot.core.buffers,
  mormot.core.unicode,
  mormot.core.text,
  mormot.core.perf,
  mormot.core.datetime,
  mormot.core.data,
  mormot.core.variants,
  mormot.core.json,
  mormot.core.search,
  mormot.core.fmt,
  mormot.crypt.secure,
  mormot.core.rtti,
  mormot.core.log,
  mormot.db.core;

{.$define SYNDB_SILENCE}
// if defined, this unit won't log the statement execution


{ ************ SQL Fields and Columns Definitions }

type
  /// an array of RawUtf8, for each existing column type
  // - used e.g. by SqlCreate method
  // - ftUnknown maps int32 field (e.g. boolean), ftNull maps RawUtf8 index # field,
  // ftUtf8 maps RawUtf8 blob field, other types map their default kind
  // - for UTF-8 text, ftUtf8 will define the BLOB field, whereas ftNull will
  // expect to be formated with an expected field length in ColumnAttr
  // - the RowID definition will expect the ORM to create an unique identifier,
  // and will use the ftInt64 type definition for this
  // and send it with the INSERT statement (some databases, like Oracle, do not
  // support standard's IDENTITY attribute) - see http://troels.arvin.dk/db/rdbms
  TSqlDBFieldTypeDefinition = array[TSqlDBFieldType] of RawUtf8;

  /// the diverse type of bound parameters during a statement execution
  // - will be paramIn by default, which is the case 90% of time
  // - could be set to paramOut or paramInOut if must be refereshed after
  // execution (for calling a stored procedure expecting such parameters)
  TSqlDBParamInOutType = (
    paramIn,
    paramOut,
    paramInOut);

  /// used to define a field/column layout in a table schema
  // - for TSqlDBConnectionProperties.SqlCreate to describe the new table
  // - for TSqlDBConnectionProperties.GetFields to retrieve the table layout
  TSqlDBColumnDefine = packed record
    /// the Column name
    ColumnName: RawUtf8;
    /// the Column type, as retrieved from the database provider
    // - returned as plain text by GetFields method, to be used e.g. by
    // TSqlDBConnectionProperties.GetFieldDefinitions method
    // - SqlCreate will check for this value to override the default type
    ColumnTypeNative: RawUtf8;
    /// the Column default width (in chars or bytes) of ftUtf8 or ftBlob
    // - can be set to value <0 for CLOB or BLOB column type, i.e. for
    // a value without any maximal length
    ColumnLength: PtrInt;
    /// the Column data precision
    // - used e.g. for numerical values
    ColumnPrecision: PtrInt;
    /// the Column data scale
    // - used e.g. for numerical values
    // - may be -1 if the metadata SQL statement returned NULL
    ColumnScale: PtrInt;
    /// the Column type, as recognized by our mormot.db.sql classes
    // - should not be ftUnknown nor ftNull
    ColumnType: TSqlDBFieldType;
    /// specify if column is indexed
    ColumnIndexed: boolean;
  end;

  /// used to define the column layout of a table schema
  // - e.g. for TSqlDBConnectionProperties.GetFields
  TSqlDBColumnDefineDynArray = array of TSqlDBColumnDefine;

  /// used to describe extended Index definition of a table schema
  TSqlDBIndexDefine = packed record
    /// name of the index
    IndexName: RawUtf8;
    /// description of the index type
    // - for MS SQL possible values are:
    // $ HEAP | CLUSTERED | NONCLUSTERED | XML |SPATIAL
    //  - for Oracle:
    // $ NORMAL | BITMAP | FUNCTION-BASED NORMAL | FUNCTION-BASED BITMAP | DOMAIN
    // see @http://docs.oracle.com/cd/B19306_01/server.102/b14237/statviews_1069.htm
    TypeDesc: RawUtf8;
    /// Expression for the subset of rows included in the filtered index
    // - only set for MS SQL - not retrieved for other DB types yet
    Filter: RawUtf8;
    /// comma separated list of indexed column names, in order of their definition
    KeyColumns: RawUtf8;
    /// comma separaded list of a nonkey column added to the index by using the CREATE INDEX INCLUDE clause
    // - only set for MS SQL - not retrieved for other DB types yet
    IncludedColumns: RawUtf8;
    /// if Index is unique
    IsUnique: boolean;
    /// if Index is part of a PRIMARY KEY constraint
    // - only set for MS SQL - not retrieved for other DB types yet
    IsPrimaryKey: boolean;
    /// if Index is part of a UNIQUE constraint
    // - only set for MS SQL - not retrieved for other DB types yet
    IsUniqueConstraint: boolean;
  end;

  /// used to describe extended Index definition of a table schema
  // - e.g. for TSqlDBConnectionProperties.GetIndexes
  TSqlDBIndexDefineDynArray = array of TSqlDBIndexDefine;

  /// used to define a parameter/column layout in a stored procedure schema
  // - for TSqlDBConnectionProperties.GetProcedureParameters to retrieve the stored procedure parameters
  // - can be extended according to https://msdn.microsoft.com/en-us/library/ms711701(v=vs.85).aspx
  TSqlDBProcColumnDefine = packed record
    /// the Column name
    ColumnName: RawUtf8;
    /// the Column type, as retrieved from the database provider
    // - used e.g. by TSqlDBConnectionProperties.GetProcedureParameters method
    ColumnTypeNative: RawUtf8;
    /// the Column default width (in chars or bytes) of ftUtf8 or ftBlob
    // - can be set to value <0 for CLOB or BLOB column type, i.e. for
    // a value without any maximal length
    ColumnLength: PtrInt;
    /// the Column data precision
    // - used e.g. for numerical values
    ColumnPrecision: PtrInt;
    /// the Column data scale
    // - used e.g. for numerical values
    // - may be -1 if the metadata SQL statement returned NULL
    ColumnScale: PtrInt;
    /// the Column type, as recognized by our mormot.db.sql classes
    // - should not be ftUnknown nor ftNull
    ColumnType: TSqlDBFieldType;
    /// defines the procedure column as a parameter or a result set column
    ColumnParamType: TSqlDBParamInOutType;
  end;

  /// used to define the parameter/column layout of a stored procedure schema
  // - e.g. for TSqlDBConnectionProperties.GetProcedureParameters
  TSqlDBProcColumnDefineDynArray = array of TSqlDBProcColumnDefine;

  /// possible column retrieval patterns
  // - used by TSqlDBColumnProperty.ColumnValueState
  TSqlDBStatementGetCol = (
    colNone,
    colNull,
    colWrongType,
    colDataFilled,
    colDataTruncated);

  /// used to define a field/column layout
  // - for TSqlDBConnectionProperties.SqlCreate to describe the table
  // - for T*Statement.Execute/Column*() methods to map the IRowSet content
  TSqlDBColumnProperty = packed record
    /// the Column name
    ColumnName: RawUtf8;
    /// a general purpose integer value
    // - for SqlCreate: default width (in WideChars or Bytes) of ftUtf8 or ftBlob;
    // if set to 0, a CLOB or BLOB column type will be created - note that
    // UTF-8 encoding is expected when calculating the maximum column byte size
    // for the CREATE TABLE statement (e.g. for Oracle 1333=4000/3 is used)
    // - for TOleDBStatement: the offset of this column in the IRowSet data,
    // starting with a DBSTATUSENUM, the data, then its length (for inlined
    // oftUtf8 and oftBlob only)
    // - for TSqlDBOracleStatement: contains an offset to this column values
    // inside fRowBuffer[] internal buffer
    // - for TSqlDBDatasetStatement: maps TField pointer value
    // - for TSqlDBPostgresStatement: contains the column type OID
    ColumnAttr: PtrUInt;
    /// the Column type, used for storage
    // - for SqlCreate: should not be ftUnknown nor ftNull
    // - for TOleDBStatement: should not be ftUnknown
    // - for mormot.db.sql.oracle: never ftUnknown, may be ftNull (for SQLT_RSET)
    ColumnType: TSqlDBFieldType;
    /// set if the Column must exists (i.e. should not be null)
    ColumnNonNullable: boolean;
    /// set if the Column shall have unique value (add the corresponding constraint)
    ColumnUnique: boolean;
    /// set if the Column data is inlined within the main rows buffer
    // - for TOleDBStatement: set if column was NOT defined as DBTYPE_BYREF
    // which is the most common case, when column data < 4 KB
    // - for TSqlDBOracleStatement: FALSE if column is an array of
    // POCILobLocator (SQLT_CLOB/SQLT_BLOB) or POCIStmt (SQLT_RSET)
    // - for TSqlDBOdbcStatement: FALSE if bigger than 255 WideChar (ftUtf8) or
    // 255 bytes (ftBlob)
    ColumnValueInlined: boolean;
    /// expected column data size
    // - for TSqlDBOracleStatement/TOleDBStatement/TODBCStatement: used to store
    // one column size (in bytes)
    ColumnValueDBSize: cardinal;
    /// optional character set encoding for ftUtf8 columns
    // - for SQLT_STR/SQLT_CLOB (mormot.db.sql.oracle): equals to the OCI char set
    ColumnValueDBCharSet: integer;
    /// internal DB column data type
    // - for TSqlDBOracleStatement: used to store the DefineByPos() TypeCode,
    // can be SQLT_STR/SQLT_CLOB, SQLT_FLT, SQLT_INT, SQLT_DAT, SQLT_BLOB,
    // SQLT_BIN and SQLT_RSET
    // - for TSqlDBOdbcStatement: used to store the DataType as returned
    // by ODBC.DescribeColW() - use private ODBC_TYPE_TO[ColumnType] to
    // retrieve the marshalled type used during column retrieval
    // - for TSqlDBOleDBStatement: used to store the DBTYPE value
    // - for TSqlDBFirebirdStatement: used to store XSQLVAR.sqltype
    // - for TSqlDBDatasetStatement: indicates the TField class type, i.e.
    // 0=TField, 1=TLargeIntField, 2=TWideStringField
    ColumnValueDBType: smallint;
    /// driver-specific encoding information
    // - for mormot.db.sql.oracle: used to store the ftUtf8 column encoding, i.e.
    // for SQLT_CLOB, equals either to SQLCS_NCHAR or SQLCS_IMPLICIT
    ColumnValueDBForm: byte;
    /// may contain the current status of the column value
    // - for mormot.db.sql.odbc: state of the latest SqlGetData() call
    ColumnDataState: TSqlDBStatementGetCol;
    /// may contain the current column size for not FIXEDLENGTH_SQLDBFIELDTYPE
    // - for mormot.db.sql.odbc: size (in bytes) in corresponding fColData[]
    // - TSqlDBProxyStatement: the actual maximum column size
    ColumnDataSize: integer;
  end;

  PSqlDBColumnProperty = ^TSqlDBColumnProperty;

  /// used to define a table/field column layout
  TSqlDBColumnPropertyDynArray = array of TSqlDBColumnProperty;

  /// used to define how a column to be created
  TSqlDBColumnCreate = record
    /// the data type
    // - here, ftUnknown is used for Int32 values, ftInt64 for Int64 values,
    // as expected by TSqlDBFieldTypeDefinition
    DBType: TSqlDBFieldType;
    /// the column name
    Name: RawUtf8;
    /// the width, e.g. for VARCHAR() types
    Width: cardinal;
    /// if the column should be unique
    Unique: boolean;
    /// if the column should be non null
    NonNullable: boolean;
    /// if the column is the ID primary key
    PrimaryKey: boolean;
  end;

  /// used to define how a table is to be created
  TSqlDBColumnCreateDynArray = array of TSqlDBColumnCreate;

  /// identify a CRUD mode of a statement
  // - in addition to CRUD states, cPostgreBulkArray would identify if the ORM
  // should generate unnested/any bound array statements - currently only
  // supported by mormot.db.sql.postgres for bulk insert/update/delete
  TSqlDBStatementCRUD = (
    cCreate,
    cRead,
    cUpdate,
    cDelete,
    cPostgreBulkArray);

  /// identify the CRUD modes of a statement
  // - used e.g. for batch send abilities of a DB engine
  TSqlDBStatementCRUDs = set of TSqlDBStatementCRUD;


const
  /// a magic constant used e.g. by TSqlDBStatement.FetchAllToBinary
  FETCHALLTOBINARY_MAGIC = 1;




{ ************ Define Database Engine Specific Behavior }

type
  /// where the LIMIT clause should be inserted for a given SQL syntax
  // - used by TSqlDBDefinitionLimitClause and SqlLimitClause() method
  TSqlDBDefinitionLimitPosition = (
    posNone,
    posWhere,
    posSelect,
    posAfter,
    posOuter);

  /// defines the LIMIT clause to be inserted for a given SQL syntax
  // - used by TSqlDBDefinitionLimitClause and SqlLimitClause() method
  TSqlDBDefinitionLimitClause = record
    Position: TSqlDBDefinitionLimitPosition;
    InsertFmt: PUtf8Char;
  end;

const
  /// the known column data types corresponding to our TSqlDBFieldType types
  // - will be used e.g. for TSqlDBConnectionProperties.SqlFieldCreate()
  // - see TSqlDBFieldTypeDefinition documentation to find out the mapping:
  // ftUnknown will be used for 32-bit integers, and ftNull for UTF-8 text
  DB_FIELDS: array[TSqlDBDefinition] of TSqlDBFieldTypeDefinition = (

  // dUnknown
    (' INT',             // ftUnknown = int32
     ' NVARCHAR(%)',     // ftNull    = UTF-8
     ' BIGINT',          // ftInt64
     ' DOUBLE',          // ftDouble
     ' NUMERIC(19,4)',   // ftCurrency
     ' TIMESTAMP',       // ftDate
     ' CLOB',            // ftUtf8
     ' BLOB'),           // ftBlob

  // dDefault
    (' INT',              // ftUnknown = int32
     ' NVARCHAR(%)',      // ftNull    = UTF-8
     ' BIGINT',           // ftInt64
     ' DOUBLE',           // ftDouble
     ' NUMERIC(19,4)',    // ftCurrency
     ' TIMESTAMP',        // ftDate
     ' CLOB',             // ftUtf8
     ' BLOB'),            // ftBlob

  // dOracle
    (' NUMBER(22,0)',      // ftUnknown = int32
     ' NVARCHAR2(%)',      // ftNull    = UTF-8
     ' NUMBER(22,0)',      // ftInt64
     ' BINARY_DOUBLE',     // ftDouble
     ' NUMBER(19,4)',      // ftCurrency
     ' DATE',              // ftDate
     ' NCLOB',             // ftUtf8
     ' BLOB'),             // ftBlob
    // NCLOB (National Character Large Object) is an Oracle data type that can hold
    // up to 4 GB of character data. It's similar to a CLOB, but characters are
    // stored in a NLS or multibyte national character set (like NVARCHAR2)

  // dMSSQL
    (' int',                // ftUnknown = int32
     ' nvarchar(%)',        // ftNull    = UTF-8
     ' bigint',             // ftInt64
     ' float',              // ftDouble
     ' money',              // ftCurrency
     ' datetime',           // ftDate
     ' nvarchar(max)',      // ftUtf8
     ' varbinary(max)'),    // ftBlob

  // dJet
    (' Long',                // ftUnknown = int32
     ' VarChar(%)',          // ftNull    = UTF-8
     ' Decimal(19,0)',       // ftInt64
     ' Double',              // ftDouble
     ' Currency',            // ftCurrency
     ' DateTime',            // ftDate
     ' LongText',            // ftUtf8
     ' LongBinary'),         // ftBlob

  // dMySQL
    (' int',                             // ftUnknown = int32
     ' varchar(%) character set utf8',   // ftNull    = UTF-8
     ' bigint',                          // ftInt64
     ' double',                          // ftDouble
     ' decimal(19,4)',                   // ftCurrency
     ' datetime',                        // ftDate
     ' mediumtext character set utf8',   // ftUtf8
     ' mediumblob'),                     // ftBlob

  // dSQLite
    (' INTEGER',              // ftUnknown = int32
     ' TEXT',                 // ftNull    = UTF-8
     ' INTEGER',              // ftInt64
     ' FLOAT',                // ftDouble
     ' FLOAT',                // ftCurrency
     ' TEXT',                 // ftDate
     ' TEXT',                 // ftUtf8
     ' BLOB'),                // ftBlob

  // dFirebird
    (' INTEGER',                                // ftUnknown = int32
     ' VARCHAR(%) CHARACTER SET UTF8',          // ftNull    = UTF-8
     ' BIGINT',                                 // ftInt64
     ' FLOAT',                                  // ftDouble
     ' DECIMAL(18,4)',                          // ftCurrency
     ' TIMESTAMP',                              // ftDate
     ' BLOB SUB_TYPE 1 SEGMENT SIZE 2000 ' +    // ftUtf8
        'CHARACTER SET UTF8',
     ' BLOB SUB_TYPE 0 SEGMENT SIZE 2000'),     // ftBlob
   // about BLOB: http://www.ibphoenix.com/resources/documents/general/doc_54

  // dNexusDB
    (' INTEGER',               // ftUnknown = int32
     ' NVARCHAR(%)',           // ftNull    = UTF-8
     ' LARGEINT',              // ftInt64
     ' REAL',                  // ftDouble
     ' MONEY',                 // ftCurrency
     ' DATETIME',              // ftDate
     ' NCLOB',                 // ftUtf8
     ' BLOB'),                 // ftBlob
    // VARCHAR(%) CODEPAGE 65001 just did not work well with Delphi<2009

  // dPostgreSQL
    (' INTEGER',                // ftUnknown = int32
     ' TEXT',                   // ftNull    = UTF-8
     ' BIGINT',                 // ftInt64
     ' DOUBLE PRECISION',       // ftDouble
     ' NUMERIC(19,4)',          // ftCurrency
     ' TIMESTAMP',              // ftDate
     ' TEXT',                   // ftUtf8
     ' BYTEA'),                 // ftBlob
    // like SQLite3, we will create TEXT column instead of VARCHAR(%), as stated
    // by http://www.postgresql.org/docs/current/static/datatype-character.html

  // dDB2 (for CCSID Unicode tables)
    (' int',                     // ftUnknown = int32
     ' varchar(%)',              // ftNull    = UTF-8
     ' bigint',                  // ftInt64
     ' real',                    // ftDouble
     ' decimal(19,4)',           // ftCurrency
     ' timestamp',               // ftDate
     ' clob',                    // ftUtf8
     ' blob'),                   // ftBlob
    // note: bigint needs 9.1 and up

  // dInformix
    (' int',                            // ftUnknown = int32
     ' lvarchar(%)',                    // ftNull    = UTF-8
     ' bigint',                         // ftInt64
     ' smallfloat',                     // ftDouble
     ' decimal(19,4)',                  // ftCurrency
     ' datetime year to fraction(3)',   // ftDate
     ' clob',                           // ftUtf8
     ' blob'),                          // ftBlob

  // dMariaDB
    (' int',                             // ftUnknown = int32
     ' varchar(%) character set utf8',   // ftNull    = UTF-8
     ' bigint',                          // ftInt64
     ' double',                          // ftDouble
     ' decimal(19,4)',                   // ftCurrency
     ' datetime',                        // ftDate
     ' mediumtext character set utf8',   // ftUtf8
     ' mediumblob'));                    // ftBlob

  /// the known column data types corresponding to our TSqlDBFieldType types
  // - will be used e.g. for TSqlDBConnectionProperties.SqlFieldCreate()
  // - SQLite3 doesn't expect any field length, neither PostgreSQL, so set to 0
  DB_FIELDSMAX: array[TSqlDBDefinition] of cardinal = (
    1000,   // dUnknown
    1000,   // dDefault
    1333,   // dOracle        =4000/3 since WideChar is up to 3 bytes in UTF-8
    4000,   // dMSSQL
    255,    // dJet
    4000,   // dMySQL
    0,      // dSQLite
    32760,  // dFirebird
    32767,  // dNexusDB
    0,      // dPostgreSQL
    32700,  // dDB2
    32700,  // dInformix
    4000);  // dMariaDB

  /// the maximum number of bound parameters to a SQL statement
  // - will be used e.g. for Batch process multi-insert
  // - those values were done empirically, assuring total count is < 656,
  // which is the maximum within :AA..:ZZ range, excuding 20 reserved keywords
  // - see http://stackoverflow.com/a/6582902 for theoritical high limits
  DB_PARAMSMAX: array[TSqlDBDefinition] of cardinal = (
    0,              // dUnknown
    0,              // dDefault
    500,            // dOracle       empirical value (from ODBC)
    500,            // dMSSQL        theoritical=2100
    0,              // dJet
    500,            // dMySQL        theoritical=60000
    MAX_SQLPARAMS,  // dSQLite       theoritical=999 - see mormot.orm.sqlite3
    0,              // dFirebird
    100,            // dNexusDB      empirical limit (above is slower)
    500,            // dPostgreSQL   theoritical=34000
    500,            // dDB2          empirical value (from ODBC)
    0,              // dInformix
    500);           // dMariaDB
  /// the known SQL statement to retrieve the server date and time
  // - contains '' for the engines with local time
  DB_SERVERTIME: array[TSqlDBDefinition] of RawUtf8 = (
    '',                                                // dUnknown
    '',                                                // dDefault
    'select sysdate from dual',                        // dOracle
    'select GETDATE()',                                // dMSSQL
    '',                                                // dJet
    'SELECT NOW()',                                    // dMySQL
    '',                                                // dSQLite
    'select current_timestamp from rdb$database',      // dFirebird
    'SELECT CURRENT_TIMESTAMP',                        // dNexusDB
    'SELECT LOCALTIMESTAMP',                           // dPostgreSQL
    'select current timestamp from sysibm.sysdummy1',  // dDB2
    'select CURRENT YEAR TO FRACTION(3) ' +            // dInformix
       'from SYSTABLES where tabid = 1',
    'select NOW()');                                   // dMariaDB

const
  /// the known SQL syntax to limit the number of returned rows in a SELECT
  // - Position indicates if should be included within the WHERE clause,
  // at the beginning of the SQL statement, or at the end of the SQL statement
  // - InsertFmt will replace '%' with the maximum number of lines to be retrieved
  // - used by TSqlDBConnectionProperties.AdaptSqlLimitForEngineList()
  DB_SQLLIMITCLAUSE: array[TSqlDBDefinition] of TSqlDBDefinitionLimitClause = (
    ( // dUnknown
    Position: posNone;
    InsertFmt: nil
  ),
    ( // dDefault
    Position: posNone;
    InsertFmt: nil
  ),
    ( // dOracle
    Position: posWhere;
    InsertFmt: 'rownum<=%'
  ),
    ( // dMSSQL
    Position: posSelect;
    InsertFmt: 'top(%) '
  ),
    ( // dJet
    Position: posSelect;
    InsertFmt: 'top % '
  ),
    ( // dMySQL
    Position: posAfter;
    InsertFmt: ' limit %'
  ),
    ( // dSQLite
    Position: posAfter;
    InsertFmt: ' limit %'
  ),
    ( // dFireBird
    Position: posSelect;
    InsertFmt: 'first % '
  ),
    ( // dNexusDB
    Position: posSelect;
    InsertFmt: 'top % '
  ),
    ( // dPostgreSQL
    Position: posAfter;
    InsertFmt: ' limit %'
  ),
    ( // dDB2
    Position: posAfter;
    InsertFmt: ' fetch first % rows only'
  ),
    ( // dInformix
    Position: posAfter;
    InsertFmt: ' first % '
  ),
    ( // dMariaDB
    Position: posAfter;
    InsertFmt: ' limit %'
  ));

  /// the known database engines handling CREATE INDEX IF NOT EXISTS statement
  DB_HANDLECREATEINDEXIFNOTEXISTS = [dSQLite, dPostgreSQL, dMariaDB];

  /// the known database engines handling CREATE INDEX on BLOB columns
  // - SQLite3 does not have any issue about indexing any column
  // - PostgreSQL is able to index TEXT columns, which are some kind of CLOB
  DB_HANDLEINDEXONBLOBS = [dSQLite, dPostgreSQL];

  /// where the DESC clause shall be used for a CREATE INDEX statement
  // - only identified syntax exception is for FireBird
  DB_SQLDESENDINGINDEXPOS: array[TSqlDBDefinition] of
    (posWithColumn, posGlobalBefore) =
    (posWithColumn,   // dUnknown
     posWithColumn,   // dDefault
     posWithColumn,   // dOracle
     posWithColumn,   // dMSSQL
     posWithColumn,   // dJet
     posWithColumn,   // dMySQL
     posWithColumn,   // dSQLite
     posGlobalBefore, // dFirebird
     posWithColumn,   // dNexusDB
     posWithColumn,   // dPostgreSQL
     posWithColumn,   // dDB2
     posWithColumn,   // dInformix
     posWithColumn);  // dMariaDB

  /// the SQL text corresponding to the identified WHERE operators for a SELECT
  DB_SQLOPERATOR: array[opEqualTo..opLike] of RawUtf8 = (
    '=',            // opEqualTo
    '<>',           // opNotEqualTo
    '<',            // opLessThan
    '<=',           // opLessThanOrEqualTo
    '>',            // opGreaterThan
    '>=',           // opGreaterThanOrEqualTo
    ' in ',         // opIn
    ' is null',     // opIsNull
    ' is not null', // opIsNotNull
    ' like ');      // opLike


/// retrieve the text of a given Database SQL dialect enumeration
// - see also TSqlDBConnectionProperties.GetDbmsName() method
function ToText(Dbms: TSqlDBDefinition): PShortString; overload;


{ ************ General SQL Processing Functions }

/// function helper logging some column truncation information text
// - is called by low-level mormot.db.sql.* providers when the driver notifies
// that a column content has been truncated when retrieved
procedure LogTruncatedColumn(Sender: TObject; const Col: TSqlDBColumnProperty);

/// retrieve a table name without any left schema
// - e.g. TrimLeftSchema('SCHEMA.TABLENAME')='TABLENAME'
function TrimLeftSchema(const TableName: RawUtf8): RawUtf8;

/// replace all '?' in the SQL statement with named parameters like :AA :AB..
// - returns the number of ? parameters found within aSql
// - won't generate any SQL keyword parameters (e.g. :AS :OF :BY), to be
// compliant with Oracle OCI expectations - allow up to 656 parameters
// - any ending ';' character is deleted, unless aStripSemicolon is unset
function ReplaceParamsByNames(const aSql: RawUtf8; var aNewSql: RawUtf8;
  aStripSemicolon: boolean = true): integer;

/// replace all '?' in the SQL statement with indexed parameters like $1 $2 ...
// - returns the number of ? parameters found within aSql
// - as used e.g. by PostgreSQL & Oracle (:1 :2) library
// - if AllowSemicolon is false (by default), reject any statement with ;
// (Postgres do not allow ; inside prepared statement); it should be
// true for Oracle
function ReplaceParamsByNumbers(const aSql: RawUtf8; var aNewSql: RawUtf8;
  IndexChar: AnsiChar = '$'; AllowSemicolon: boolean = false): integer;

/// create a JSON array from an array of UTF-8 SQL bound values
// - as generated during array binding, i.e. with quoted strings
// 'one','t"wo' -> '{"one","t\"wo"}'   and  1,2,3 -> '{1,2,3}'
// - as used e.g. by PostgreSQL library (note that its syntax as {} not []
// unless you change the Open/Close optional parameters)
function BoundArrayToJsonArray(const Values: TRawUtf8DynArray;
  Open: AnsiChar = '{'; Close: AnsiChar = '}'): RawUtf8; overload;

/// create a JSON array from an array of UTF-8 SQL bound values
procedure BoundArrayToJsonArray(const Values: TRawUtf8DynArray;
  out Result: RawUtf8; Open: AnsiChar = '{'; Close: AnsiChar = '}'); overload;

/// create an array of UTF-8 SQL bound values from a JSON array
// - as generated during array binding, i.e. with quoted strings
// '["one","t\"wo"]' -> 'one','t"wo'  and  '[1,2,3]' -> 1,2,3
// - as used e.g. by BindArrayJson outside of PostgreSQL library
// - warning: input JSON buffer will be parsed in-place, so will be modified
// - here syntax is regular [] so not an exact reverse to BoundArrayToJsonArray
function JsonArrayToBoundArray(Json: PUtf8Char; ParamType: TSqlDBFieldType;
  TimeSeparator: AnsiChar; DateMS: boolean; out Values: TRawUtf8DynArray): boolean;


{ ************ Abstract SQL DB Classes and Interfaces }

var
  /// the TSynLog class used for logging for all our mormot.db.sql related units
  // - since not all exceptions are handled specificaly by this unit, you
  // may better use a common TSynLog class for the whole application or module
  SynDBLog: TSynLogClass = TSynLog;

type
  /// a custom variant type used to have direct access to a result row content
  // - use ISqlDBRows.RowData method to retrieve such a Variant
  TSqlDBRowVariantType = class(TSynInvokeableVariantType)
  public
    /// overriden method for actual getter by name implementation
    function IntGet(var Dest: TVarData; const Instance: TVarData;
      Name: PAnsiChar; NameLen: PtrInt; NoException: boolean): boolean; override;
  end;

  {$M+}
  TSqlDBStatement = class;
  TSqlDBConnection = class;
  TSqlDBConnectionProperties = class;
  {$M-}

  /// generic Exception type, as used by mormot.db.sql and mormot.db.sql.* units
  ESqlDBException = class(ECoreDBException)
  protected
    fStatement: TSqlDBStatement;
  public
    /// constructor which will use FormatUtf8() instead of Format()
    // - if the first Args[0] is a TSqlDBStatement class instance, the current
    // SQL statement will be part of the exception message
    // - will also call SetDbError() with the resulting message text
    constructor CreateUtf8(const Format: RawUtf8; const Args: array of const); override;
  published
    /// associated TSqlDBStatement instance, if supplied as first parameter
    property Statement: TSqlDBStatement
      read fStatement;
  end;

  /// generic interface to access a SQL query result rows
  // - not all TSqlDBStatement methods are available, but only those to retrieve
  // data from a statement result: the purpose of this interface is to make
  // easy access to result rows, not provide all available features - therefore
  // you only have access to the Step() and Column*() methods
  ISqlDBRows = interface
    ['{11291095-9C15-4984-9118-974F1926DB9F}']
    /// after a prepared statement has been prepared returning a ISqlDBRows
    // interface, this method must be called one or more times to evaluate it
    // - you shall call this method before calling any Column*() methods
    // - return TRUE on success, with data ready to be retrieved by Column*()
    // - return FALSE if no more row is available (e.g. if the SQL statement
    // is not a SELECT but an UPDATE or INSERT command)
    // - access the first or next row of data from the SQL Statement result:
    // if SeekFirst is TRUE, will put the cursor on the first row of results,
    // otherwise, it will fetch one row of data, to be called within a loop
    // - should raise an Exception on any error
    // - typical use may be:
    // ! var Customer: Variant;
    // ! begin
    // !   with Props.Execute(
    // !       'select * from Sales.Customer where AccountNumber like ?',
    // !       ['AW000001%'], @Customer) do
    // !   begin
    // !     while Step do //  loop through all matching data rows
    // !       assert(Copy(Customer.AccountNumber, 1, 8)='AW000001');
    // !     ReleaseRows;
    // !   end;
    // ! end;
    function Step(SeekFirst: boolean = false): boolean;
    /// retrieve one row of the result set as a JSON object
    // - see also FetchAllAsJson to retrieve all the raws as a JSON result
    // - returns '' if the step was not possible (i.e. reached end of results)
    function StepAsJson(SeekFirst: boolean = false): RawUtf8;
    /// retrieve one row of the resultset as a JSON object into a TResultsWriter
    // - see also FetchAllToJson to retrieve all the raws into a JSON result
    // - returns false if the step was not possible (i.e. reached end of results)
    function StepToJson(W: TJsonWriter; SeekFirst: boolean = false): boolean;
    /// release cursor memory and resources once Step loop is finished
    // - this method call is optional, but is better be used if the ISqlDBRows
    // statement from taken from cache, and returned a lot of content which
    // may still be in client (and server) memory
    // - will also free all temporary memory used for optional logging
    procedure ReleaseRows;

    /// the column/field count of the current Row
    function ColumnCount: integer;
    /// the Column name of the current Row
    // - Columns numeration (i.e. Col value) starts with 0
    // - it's up to the implementation to ensure than all column names are unique
    function ColumnName(Col: integer): RawUtf8;
    /// returns the Column index of a given Column name
    // - Columns numeration (i.e. Col value) starts with 0
    // - returns -1 if the Column name is not found (via case insensitive search)
    function ColumnIndex(const aColumnName: RawUtf8): integer;
    /// the Column type of the current Row
    // - FieldSize can be set to store the size in chars of a ftUtf8 column
    // (0 means BLOB kind of TEXT column)
    // - warning: FieldSize^ should be a 32-bit "integer" variable, not a PtrInt
    function ColumnType(Col: integer; FieldSize: PInteger = nil): TSqlDBFieldType;
    /// returns TRUE if the column contains NULL, first Col is 0
    function ColumnNull(Col: integer): boolean; overload;
    /// return a Column integer value of the current Row, first Col is 0
    function ColumnInt(Col: integer): Int64; overload;
    /// return a Column floating point value of the current Row, first Col is 0
    function ColumnDouble(Col: integer): double; overload;
    /// return a Column floating point value of the current Row, first Col is 0
    function ColumnDateTime(Col: integer): TDateTime; overload;
    /// return a column date and time value of the current Row, first Col is 0
    function ColumnTimestamp(Col: integer): TTimeLog; overload;
    /// return a Column currency value of the current Row, first Col is 0
    function ColumnCurrency(Col: integer): currency; overload;
    /// return a Column UTF-8 encoded text value of the current Row, first Col is 0
    function ColumnUtf8(Col: integer): RawUtf8; overload;
    /// return a Column UTF-8 text buffer of the current Row, first Col is 0
    // - low-level function: may return nil if not supported by the provider
    // - returned pointer is likely to last only until next Step or Reset call
    function ColumnPUtf8(Col: integer): PUtf8Char;
    /// return a Column text value as RTL string of the current Row, first Col is 0
    function ColumnString(Col: integer): string; overload;
    /// return a Column as a blob value of the current Row, first Col is 0
    function ColumnBlob(Col: integer): RawByteString; overload;
    /// return a Column as a blob value of the current Row, first Col is 0
    function ColumnBlobBytes(Col: integer): TBytes; overload;
    /// read a blob Column into the Stream parameter
    procedure ColumnBlobToStream(Col: integer; Stream: TStream); overload;
    /// write a blob Column into the Stream parameter
    // - expected to be used with 'SELECT .. FOR UPDATE' locking statements
    procedure ColumnBlobFromStream(Col: integer; Stream: TStream); overload;
    /// return a Column as a TSqlVar value, first Col is 0
    // - the specified Temp variable will be used for temporary storage of
    // ftUtf8/ftBlob values
    procedure ColumnToSqlVar(Col: integer; var Value: TSqlVar; var Temp: RawByteString);
    /// append a Column as a JSON value, first Col is 0
    procedure ColumnToJson(Col: integer; W: TJsonWriter);
    /// return a Column as a variant
    // - a ftUtf8 TEXT content will return a varUString on Delphi/FPC or an AnsiString
    // varString on Delphi 7-2007 (Connection.Properties.VariantStringAsWideString=true
    // could return a varOleStr) unless ForceUtf8=true would return a RawUtf8 varString
    // - a ftBlob BLOB content will be mapped into a TBlobData AnsiString variant
    function ColumnVariant(Col: integer; ForceUtf8: boolean = false): Variant; overload;
    /// return a Column as a variant, first Col is 0
    // - this default implementation will call Column*() method above
    // - a ftUtf8 TEXT content will return a varUString on Delphi/FPC or an AnsiString
    // varString on Delphi 7-2007 (Connection.Properties.VariantStringAsWideString=true
    // could return a varOleStr) unless ForceUtf8=true would return a RawUtf8 varString
    // - a ftBlob BLOB content will be mapped into a TBlobData AnsiString variant
    function ColumnToVariant(Col: integer; var Value: Variant;
      ForceUtf8: boolean = false): TSqlDBFieldType;
    /// return a special CURSOR Column content as a mormot.db.sql result set
    // - Cursors are not handled internally by mORMot, but some databases (e.g.
    // Oracle) usually use such structures to get data from stored procedures
    // - such columns are mapped as ftNull internally - so this method is the only
    // one giving access to the data rows
    // - see also BoundCursor() if you want to access a CURSOR out parameter
    function ColumnCursor(Col: integer): ISqlDBRows; overload;

    /// returns TRUE if the column contains NULL, from a supplied column name
    function ColumnNull(const ColName: RawUtf8): boolean; overload;
    /// return a Column integer value of the current Row, from a supplied column name
    function ColumnInt(const ColName: RawUtf8): Int64; overload;
    /// return a Column floating point value of the current Row, from a supplied column name
    function ColumnDouble(const ColName: RawUtf8): double; overload;
    /// return a Column floating point value of the current Row, from a supplied column name
    function ColumnDateTime(const ColName: RawUtf8): TDateTime; overload;
    /// return a column date and time value of the current Row, from a supplied column name
    function ColumnTimestamp(const ColName: RawUtf8): TTimeLog; overload;
    /// return a Column currency value of the current Row, from a supplied column name
    function ColumnCurrency(const ColName: RawUtf8): currency; overload;
    /// return a Column UTF-8 encoded text value of the current Row, from a supplied column name
    function ColumnUtf8(const ColName: RawUtf8): RawUtf8; overload;
    /// return a Column text value as RTL string of the current Row, from a supplied column name
    function ColumnString(const ColName: RawUtf8): string; overload;
    /// return a Column as a blob value of the current Row, from a supplied column name
    function ColumnBlob(const ColName: RawUtf8): RawByteString; overload;
    /// return a Column as a blob value of the current Row, from a supplied column name
    function ColumnBlobBytes(const ColName: RawUtf8): TBytes; overload;
    /// read a blob Column into the Stream parameter
    procedure ColumnBlobToStream(const ColName: RawUtf8; Stream: TStream); overload;
    /// write a blob Column into the Stream parameter
    procedure ColumnBlobFromStream(const ColName: RawUtf8; Stream: TStream); overload;
    /// return a Column as a variant, from a supplied column name
    // - a ftUtf8 TEXT content will return a varUString on Delphi/FPC or an AnsiString
    // varString on Delphi 7-2007 (Connection.Properties.VariantStringAsWideString=true
    // could return a varOleStr) unless ForceUtf8=true would return a RawUtf8 varString
    function ColumnVariant(const ColName: RawUtf8; ForceUtf8: boolean = false): Variant; overload;
    /// return a Column as a variant, from a supplied column name
    // - since a property getter can't be an overloaded method, we define one
    // for the Column[] property
    function GetColumnVariant(const ColName: RawUtf8): Variant;
    /// return a special CURSOR Column content as a mormot.db.sql result set
    // - Cursors are not handled internally by mORMot, but some databases (e.g.
    // Oracle) usually use such structures to get data from strored procedures
    // - such columns are mapped as ftNull internally - so this method is the only
    // one giving access to the data rows
    function ColumnCursor(const ColName: RawUtf8): ISqlDBRows; overload;
    /// return a Column as a variant
    // - this default property can be used to write simple code like this:
    // ! procedure WriteFamily(const aName: RawUtf8);
    // ! var I: ISqlDBRows;
    // ! begin
    // !   I := MyConnProps.Execute('select * from table where name=?',[aName]);
    // !   while I.Step do
    // !     writeln(I['FirstName'],' ',DateToStr(I['BirthDate']));
    // !   I.ReleaseRows;
    // ! end;
    // - of course, using a variant and a column name will be a bit slower than
    // direct access via the Column*() dedicated methods, but resulting code
    // is fast in practice
    // - a ftUtf8 TEXT content will return a varUString on Delphi/FPC or an AnsiString
    // varString on Delphi 7-2007 (Connection.Properties.VariantStringAsWideString=true
    // could return a varOleStr) - use ColumnVariant/ColumnToVariant with ForceUtf8=true
    // if you want to return a RawUtf8 varString on all platforms
    property Column[const ColName: RawUtf8]: Variant
      read GetColumnVariant; default;
    /// create a TSqlDBRowVariantType able to access any field content via late binding
    // - i.e. you can use Data.Name to access the 'Name' column of the current row
    // - this Variant will point to the corresponding TSqlDBStatement instance,
    // so it's not necessary to retrieve its value for each row; but once the
    // associated ISqlDBRows instance is released, you won't be able to access
    // its data - use RowDocVariant/FetchAllToDocVariantArray instead
    // - typical use is:
    // ! var Row: Variant;
    // ! (...)
    // !  with MyConnProps.Execute('select * from table where name=?',[aName]) do
    // !  begin
    // !    Row := RowData;
    // !    while Step do
    // !      writeln(Row.FirstName,Row.BirthDate);
    // !    ReleaseRows;
    // !  end;
    function RowData: Variant;
    /// create a TDocVariant custom variant containing all columns values
    // - will create a "fast" TDocVariant object instance with all fields
    // - see also FetchAllToDocVariantArray() method
    procedure RowDocVariant(out aDocument: variant;
      aOptions: TDocVariantOptions = JSON_FAST);
    /// return the associated statement instance
    function Instance: TSqlDBStatement;
    // return all rows content as a JSON string
    // - JSON data is retrieved with UTF-8 encoding
    // - if Expanded is true, JSON output is a standard array of objects, for
    // direct use with any Ajax or .NET client:
    // & [{"f1":"1v1","f2":1v2},{"f2":"2v1","f2":2v2}...]
    // - if Expanded is false, JSON data is serialized in non-expanded format:
    // & {"fieldCount":2,"values":["f1","f2","1v1",1v2,"2v1",2v2...],"rowCount":20}
    // resulting in lower space use and faster process - it could be parsed by
    // TOrmTableJson or TDocVariantData.InitArrayFromResults
    // - BLOB field value is saved as Base64, in the '"\uFFF0base64encodedbinary"'
    // format and contains true BLOB data
    // - if ReturnedRowCount points to an integer variable, it will be filled with
    // the number of row data returned (excluding field names)
    // - similar to corresponding TSqlRequest.Execute method in the
    // mormot.db.raw.sqlite3 unit
    function FetchAllAsJson(Expanded: boolean; ReturnedRowCount: PPtrInt = nil): RawUtf8;
    // append all rows content as a JSON stream
    // - JSON data is added to the supplied TStream, with UTF-8 encoding
    // - if Expanded is true, JSON output is a standard array of objects, for
    // direct use with any Ajax or .NET client:
    // & [{"f1":"1v1","f2":1v2},{"f2":"2v1","f2":2v2}...]
    // - if Expanded is false, JSON data is serialized in non-expanded format:
    // & {"fieldCount":2,"values":["f1","f2","1v1",1v2,"2v1",2v2...],"rowCount":20}
    // resulting in lower space use and faster process - it could be parsed by
    // TOrmTableJson or TDocVariantData.InitArrayFromResults
    // - BLOB field value is saved as Base64, in the '"\uFFF0base64encodedbinary"'
    // format and contains true BLOB data
    // - similar to corresponding TSqlRequest.Execute method in the
    // mormot.db.raw.sqlite3 unit
    // - returns the number of row data returned (excluding field names)
    function FetchAllToJson(Json: TStream; Expanded: boolean): PtrInt;
    /// append all rows content as binary stream
    // - will save the column types and name, then every data row in optimized
    // binary format (faster and smaller than JSON)
    // - you can specify a LIMIT for the data extent (default 0 meaning all data)
    // - generates the format expected by TSqlDBProxyStatement
    function FetchAllToBinary(Dest: TStream; MaxRowCount: cardinal = 0;
      DataRowPosition: PCardinalDynArray = nil): cardinal;
    /// append all rows content as a new TDocVariant dvArray
    // - save all content directly from the result buffers, with no temporary JSON
    // - you can specify a LIMIT for the data extent (default 0 meaning all data)
    function FetchAllToDocVariantArray(MaxRowCount: cardinal = 0): variant;
  end;

  /// generic interface to bind to prepared SQL query
  // - inherits from ISqlDBRows, so gives access to the result columns data
  // - not all TSqlDBStatement methods are available, but only those to bind
  // parameters and retrieve data after execution
  // - reference counting mechanism of this interface will feature statement
  // cache (if available) for NewThreadSafeStatementPrepared() or PrepareInlined()
  ISqlDBStatement = interface(ISqlDBRows)
    ['{EC27B81C-BD57-47D4-9711-ACFA27B583D7}']
    // some raw properties getter/setter
    function GetForceBlobAsNull: boolean;
    procedure SetForceBlobAsNull(value: boolean);
    function GetForceDateWithMS: boolean;
    procedure SetForceDateWithMS(value: boolean);
    /// bind a NULL value to a parameter
    // - the leftmost SQL parameter has an index of 1
    // - some providers (e.g. OleDB during MULTI INSERT statements) expect the
    // proper column type to be set in BoundType, even for NULL values
    procedure BindNull(Param: integer; IO: TSqlDBParamInOutType = paramIn;
      BoundType: TSqlDBFieldType = ftNull);
    /// bind an integer value to a parameter
    // - the leftmost SQL parameter has an index of 1
    procedure Bind(Param: integer; Value: Int64;
      IO: TSqlDBParamInOutType = paramIn); overload;
    /// bind a double value to a parameter
    // - the leftmost SQL parameter has an index of 1
    procedure Bind(Param: integer; Value: double;
      IO: TSqlDBParamInOutType = paramIn); overload;
    /// bind a TDateTime value to a parameter
    // - the leftmost SQL parameter has an index of 1
    procedure BindDateTime(Param: integer; Value: TDateTime;
      IO: TSqlDBParamInOutType = paramIn); overload;
    /// bind a currency value to a parameter
    // - the leftmost SQL parameter has an index of 1
    procedure BindCurrency(Param: integer; Value: currency;
      IO: TSqlDBParamInOutType = paramIn); overload;
    /// bind a UTF-8 encoded string to a parameter
    // - the leftmost SQL parameter has an index of 1
    procedure BindTextU(Param: integer; const Value: RawUtf8;
      IO: TSqlDBParamInOutType = paramIn); overload;
    /// bind a UTF-8 encoded buffer text (#0 ended) to a parameter
    // - the leftmost SQL parameter has an index of 1
    procedure BindTextP(Param: integer; Value: PUtf8Char;
      IO: TSqlDBParamInOutType = paramIn); overload;
    /// bind a UTF-8 encoded string to a parameter
    // - the leftmost SQL parameter has an index of 1
    procedure BindTextS(Param: integer; const Value: string;
      IO: TSqlDBParamInOutType = paramIn); overload;
    /// bind a UTF-8 encoded string to a parameter
    // - the leftmost SQL parameter has an index of 1
    procedure BindTextW(Param: integer; const Value: WideString;
      IO: TSqlDBParamInOutType = paramIn); overload;
    /// bind a Blob buffer to a parameter
    // - the leftmost SQL parameter has an index of 1
    procedure BindBlob(Param: integer; Data: pointer; Size: integer;
      IO: TSqlDBParamInOutType = paramIn); overload;
    /// bind a Blob buffer to a parameter
    // - the leftmost SQL parameter has an index of 1
    procedure BindBlob(Param: integer; const Data: RawByteString;
      IO: TSqlDBParamInOutType = paramIn); overload;
    /// bind a Variant value to a parameter
    // - the leftmost SQL parameter has an index of 1
    // - will call all virtual Bind*() methods from the Data type
    // - if DataIsBlob is TRUE, will call BindBlob(RawByteString(Data)) instead
    // of BindTextW(WideString(Variant)) - used e.g. by TQuery.AsBlob/AsBytes
    procedure BindVariant(Param: integer; const Data: Variant;
      DataIsBlob: boolean; IO: TSqlDBParamInOutType = paramIn);
    /// bind one TSqlVar value
    // - the leftmost SQL parameter has an index of 1
    procedure Bind(Param: integer; const Data: TSqlVar;
      IO: TSqlDBParamInOutType = paramIn); overload;
    /// bind one RawUtf8 encoded value
    // - the leftmost SQL parameter has an index of 1
    // - the value should match the BindArray() format, i.e. be stored as in SQL
    // (i.e. number, 'quoted string', 'YYYY-MM-DD hh:mm:ss', null)
    procedure Bind(Param: integer; ParamType: TSqlDBFieldType; const Value: RawUtf8;
      ValueAlreadyUnquoted: boolean; IO: TSqlDBParamInOutType = paramIn); overload;
    /// bind an array of const values
    // - parameters marked as ? should be specified as method parameter in Params[]
    // - BLOB parameters can be bound with this method, when set after encoding
    // via BinToBase64WithMagic() call
    // - TDateTime parameters can be bound with this method, when encoded via
    // a DateToSql() or DateTimeToSql() call
    procedure Bind(const Params: array of const;
      IO: TSqlDBParamInOutType = paramIn); overload;
    /// bind an array of fields from an existing SQL statement
    // - can be used e.g. after ColumnsToSqlInsert() method call for fast data
    // conversion between tables
    procedure BindFromRows(const Fields: TSqlDBFieldTypeDynArray; Rows: TSqlDBStatement);
    /// bind a special CURSOR parameter to be returned as a mormot.db.sql result set
    // - Cursors are not handled internally by mORMot, but some databases (e.g.
    // Oracle) usually use such structures to get data from strored procedures
    // - such parameters are mapped as ftUnknown
    // - use BoundCursor() method to retrieve the corresponding ISqlDBRows after
    // execution of the statement
    procedure BindCursor(Param: integer);
    /// return a special CURSOR parameter content as a mormot.db.sql result set
    // - this method is not about a column, but a parameter defined with
    // BindCursor() before method execution
    // - Cursors are not handled internally by mORMot, but some databases (e.g.
    // Oracle) usually use such structures to get data from strored procedures
    // - this method allow direct access to the data rows after execution
    function BoundCursor(Param: integer): ISqlDBRows;

    /// bind an array of values to a parameter
    // - the leftmost SQL parameter has an index of 1
    // - values are stored as in SQL (i.e. number, 'quoted string',
    // 'YYYY-MM-DD hh:mm:ss', null)
    // - this default implementation will raise an exception if the engine
    // does not support array binding
    procedure BindArray(Param: integer; ParamType: TSqlDBFieldType;
      const Values: TRawUtf8DynArray; ValuesCount: integer); overload;
    /// bind an array of integer values to a parameter
    // - the leftmost SQL parameter has an index of 1
    // - this default implementation will raise an exception if the engine
    // does not support array binding
    procedure BindArray(Param: integer; const Values: array of Int64); overload;
    /// bind an array of double values to a parameter
    // - the leftmost SQL parameter has an index of 1
    // - this default implementation will raise an exception if the engine
    // does not support array binding
    procedure BindArray(Param: integer; const Values: array of double); overload;
    /// bind an array of TDateTime values to a parameter
    // - the leftmost SQL parameter has an index of 1
    // - values are stored as in SQL (i.e. 'YYYY-MM-DD hh:mm:ss')
    // - this default implementation will raise an exception if the engine
    // does not support array binding
    procedure BindArrayDateTime(Param: integer; const Values: array of TDateTime);
    /// bind an array of currency values to a parameter
    // - the leftmost SQL parameter has an index of 1
    // - this default implementation will raise an exception if the engine
    // does not support array binding
    procedure BindArrayCurrency(Param: integer; const Values: array of currency);
    /// bind an array of RawUtf8 values to a parameter
    // - the leftmost SQL parameter has an index of 1
    // - values are stored as in SQL (i.e. 'quoted string')
    // - this default implementation will raise an exception if the engine
    // does not support array binding
    procedure BindArray(Param: integer; const Values: array of RawUtf8); overload;
    /// bind an array of JSON values to a parameter
    // - the leftmost SQL parameter has an index of 1
    // - values are stored as JSON Arrays (i.e. [1,2,3] or ["a","b","c"])
    // - this default implementation will call JsonArrayToBoundArray()
    // then BindArray(array of RawUtf8)
    // - warning: JsonArray could be parsed in-place so a private copy is needed
    procedure BindArrayJson(Param: integer; ParamType: TSqlDBFieldType;
      var JsonArray: RawUtf8; ValuesCount: integer);

    /// retrieve the parameter content, after SQL execution
    // - the leftmost SQL parameter has an index of 1
    // - to be used e.g. with stored procedures:
    // ! query :=  'BEGIN TEST_PKG.DUMMY(?, ?, ?, ?, ?); END;';
    // ! stmt := Props.NewThreadSafeStatementPrepared(query, false);
    // ! stmt.Bind(1, in1, paramIn);
    // ! stmt.BindTextU(2, in2, paramIn);
    // ! stmt.BindTextU(3, in3, paramIn);
    // ! stmt.BindTextS(4, '', paramOut); //  to be retrieved with out1: string
    // ! stmt.Bind(5, 0, paramOut);       //  to be retrieved with out2: integer
    // ! stmt.ExecutePrepared;
    // ! stmt.ParamToVariant(4, out1, true);
    // ! stmt.ParamToVariant(5, out2, true);
    // - the parameter should have been bound with IO=paramOut or IO=paramInOut
    // if CheckIsOutParameter is TRUE
    function ParamToVariant(Param: integer; var Value: Variant;
      CheckIsOutParameter: boolean = true): TSqlDBFieldType;

    /// execute a prepared SQL statement
    // - parameters marked as ? should have been already bound with Bind*() functions
    // - should raise an Exception on any error
    // - after execution, you can access any returned data via ISqlDBRows methods
    procedure ExecutePrepared;
    // execute a prepared SQL statement and return all rows content as a JSON string
    // - JSON data is retrieved with UTF-8 encoding
    // - if Expanded is true, JSON output is a standard array of objects, for
    // direct use with any Ajax or .NET client:
    // & [{"f1":"1v1","f2":1v2},{"f1":"2v1","f2":2v2}...]
    // - if Expanded is false, JSON data is serialized in non-expanded format:
    // & {"fieldCount":2,"values":["f1","f2","1v1",1v2,"2v1",2v2...],"rowCount":20}
    // resulting in lower space use and faster process - it could be parsed by
    // TOrmTableJson or TDocVariantData.InitArrayFromResults
    // - BLOB field value is saved as Base64, in the '"\uFFF0base64encodedbinary"'
    // format and contains true BLOB data
    procedure ExecutePreparedAndFetchAllAsJson(Expanded: boolean;
      out Json: RawUtf8; ReturnedRowCount: PPtrInt = nil);
    /// if set, any BLOB field won't be retrieved, and forced to be null
    // - this may be used to speed up fetching the results for SQL requests
    // with * statements
    property ForceBlobAsNull: boolean
      read GetForceBlobAsNull write SetForceBlobAsNull;
    /// if set, any ftDate field will contain the milliseconds information
    // when serialized into ISO-8601 text
    // - this setting is private to each statement, since may vary depending
    // on data definition (e.g. ORM TDateTime/TDateTimeMS)
    property ForceDateWithMS: boolean
      read GetForceDateWithMS write SetForceDateWithMS;
    /// gets a number of updates made by latest executed statement
    function UpdateCount: integer;
  end;

  /// possible events notified to TOnSqlDBProcess callback method
  // - event handler is specified by TSqlDBConnectionProperties.OnProcess or
  // TSqlDBConnection.OnProcess properties
  // - speCreated is called from all NewConnection overriden methods
  // - speConnected / speDisconnected will notify TSqlDBConnection.Connect
  // and TSqlDBConnection.Disconnect calls
  // - speNonActive / speActive will be used to notify external DB blocking
  // access, so can be used e.g. to change the mouse cursor shape (this trigger
  // is re-entrant, i.e. it will be executed only once in case of nested calls)
  // - speReconnected will be called if TSqlDBConnection did successfully
  // recover its database connection (on error, TQuery will call
  // speConnectionLost): this event will be called by TSqlDBConnection.Connect
  // after a regular speConnected notification
  // - speConnectionLost will be called by TQuery in case of broken connection,
  // and if Disconnect/Reconnect did not restore it as expected (i.e. speReconnected)
  // - speStartTransaction / speCommit / speRollback will notify the
  // corresponding TSqlDBConnection.StartTransaction, TSqlDBConnection.Commit
  // and TSqlDBConnection.Rollback methods
  TOnSqlDBProcessEvent = (
    speCreated,
    speConnected,
    speDisconnected,
    speNonActive,
    speActive,
    speConnectionLost,
    speReconnected,
    speStartTransaction,
    speCommit,
    speRollback);

  /// event handler called during all external DB process
  // - event handler is specified by TSqlDBConnectionProperties.OnProcess or
  // TSqlDBConnection.OnProperties properties
  TOnSqlDBProcess = procedure(Sender: TSqlDBConnection;
    Event: TOnSqlDBProcessEvent) of object;

  /// event triggered when an expired password is detected
  // - will allow to provide a new password
  TOnPasswordExpired = function (Sender: TSqlDBConnection;
    var APassword: RawUtf8): boolean of object;

  /// event handler called when the low-level driver send some warning information
  // - errors will trigger Exceptions, but sometimes the database driver returns
  // some non critical information, which is logged and may be intercepted using
  // the TSqlDBConnectionProperties.OnStatementInfo property
  // - may be used e.g. to track ORA-28001 or ORA-28002 about account expire
  // - is currently implemented by mormot.db.sql.oracle, mormot.db.sql.odbc
  // and mormot.db.sql.oledb units
  TOnSqlDBInfo = procedure(Sender: TSqlDBStatement; const Msg: RawUtf8) of object;

  /// actions implemented by TSqlDBConnectionProperties.SharedTransaction()
  TSqlDBSharedTransactionAction = (
    transBegin,
    transCommitWithoutException,
    transCommitWithException,
    transRollback);

  /// defines a callback signature able to handle multiple INSERT
  // - may execute e.g. for 2 fields and 3 data rows on a database engine
  // implementing INSERT with multiple VALUES (like MySQL, PostgreSQL, NexusDB,
  // MSSQL or SQLite3), as implemented by
  // TSqlDBConnectionProperties.MultipleValuesInsert() :
  // $ INSERT INTO TableName(FieldNames[0],FieldNames[1]) VALUES
  // $   (FieldValues[0][0],FieldValues[1][0]),
  // $   (FieldValues[0][1],FieldValues[1][1]),
  // $   (FieldValues[0][2],FieldValues[1][2]);
  // - for other kind of DB which do not support multi values INSERT, may
  // execute a dedicated driver command, like MSSQL "bulk insert" or Firebird
  // "execute block"
  TOnBatchInsert = procedure(Props: TSqlDBConnectionProperties;
    const TableName: RawUtf8; const FieldNames: TRawUtf8DynArray;
    const FieldTypes: TSqlDBFieldTypeArray; RowCount: integer;
    const FieldValues: TRawUtf8DynArrayDynArray;
    BatchOptions: TRestBatchOptions) of object;

  /// event handler called to customize TRestStorageExternal table creation
  // - access the DB using Sender.Properties to properly create the index
  // - the default SQL as generated by the ORM is supplied as reference
  // - should return TRUE if the table has been created, FALSE if not
  TOnTableCreate = function(Sender: TSqlDBConnectionProperties;
    const aTableName: RawUtf8; const Columns: TSqlDBColumnCreateDynArray;
    const SQL: RawUtf8): boolean of object;

  /// event handler called to customize TRestStorageExternal column adding
  // - access the DB using Sender.Properties to properly create the index
  // - the default SQL as generated by the ORM is supplied as reference
  // - should return TRUE if the table has been created, FALSE if not
  TOnTableAddColumn = function(Sender: TSqlDBConnectionProperties;
    const aTableName: RawUtf8; const Column: TSqlDBColumnCreate;
    const SQL: RawUtf8): boolean of object;

  /// event handler called to customize TRestStorageExternal index creation
  // - access the DB using Sender.Properties to properly create the index
  // - the default SQL as generated by the ORM is supplied as reference
  // - should return TRUE if the index has been created, FALSE if not
  TOnTableCreateMultiIndex = function(Sender: TSqlDBConnectionProperties;
    const aTableName: RawUtf8; const FieldNames: array of RawUtf8; Unique: boolean;
    IndexName: RawUtf8; const SQL: RawUtf8): boolean of object;

  /// internal in-memory structure used for transactions
  TSqlDBConnectionTransaction = record
    SessionID: cardinal;
    RefCount: integer;
    Connection: TSqlDBConnection;
  end;

  /// specify the class of TSqlDBConnectionProperties
  // - sometimes used to create connection properties instances, from a set
  // of available classes (see e.g. SynDBExplorer or sample 16)
  TSqlDBConnectionPropertiesClass = class of TSqlDBConnectionProperties;

  /// define TSqlDBConnectionProperties internal flags
  TSqlDBConnectionPropertiesFlag = (
    cpfUseCache,
    cpfStoreVoidStringAsNull,
    cpfLogSqlStatementOnException,
    cpfRollbackOnDisconnect,
    cpfReconnectAfterConnectionError,
    cpfConnectionTimeOutBackground,
    cpfEnsureColumnNameUnique,
    cpfFilterTableViewSchemaName,
    cpfNoBlobBindArray,
    cpfIsThreadSafe,
    cpfVariantStringAsWideString,
    cpfDeleteConnectionInOwnThread,
    cpfArrayParamsAsBinary,
    cpfSQliteUseMormotCollations,
    cpfProxyHandleConnection);
  /// store TSqlDBConnectionProperties internal flags
  // - exposed as boolean properties for mORMot 1 and existing code compatibility
  TSqlDBConnectionPropertiesFlags = set of TSqlDBConnectionPropertiesFlag;

  /// abstract class used to set Database-related properties
  // - handle e.g. the Database server location and connection parameters (like
  // UserID and password)
  // - should also provide some Database-specific generic SQL statement creation
  // (e.g. how to create a Table), to be used e.g. by the mORMot layer
  // - this class level will handle a single "main connection" - you may inherit
  // from TSqlDBConnectionThreadSafe to maintain one connection per thread
  TSqlDBConnectionProperties = class
  protected
    fMainConnectionSafe: TLightLock; // topmost to ensure aarch64 alignment
    fSharedTransactionsSafe: TLightLock;
    fConnectionPoolSafe: TLightLock; // for fConnectionPool[TSynLog.ThreadIndex]
    fMainConnection: TSqlDBConnection;
    fDbms: TSqlDBDefinition;
    fBatchSendingAbilities: TSqlDBStatementCRUDs;
    fDateTimeFirstChar: AnsiChar;
    fFlags: TSqlDBConnectionPropertiesFlags;
    fBatchMaxSentAtOnce: integer;
    fLoggedSqlMaxSize: integer;
    fConnectionTimeOutSecs: integer; // maps ConnectionTimeOutMinutes as secs
    fStatementMaxMemory: PtrInt;
    fEngineName: RawUtf8;
    fServerName: RawUtf8;
    fDatabaseName: RawUtf8;
    fPassWord: SpiUtf8;
    fUserID: RawUtf8;
    fForcedSchemaName: RawUtf8;
    fSqlGetServerTimestamp: RawUtf8;
    fOnProcess: TOnSqlDBProcess;
    fOnStatementInfo: TOnSqlDBInfo;
    fOnBatchInsert: TOnBatchInsert;
    fStatementCacheReplicates: integer;
    fSqlCreateFieldMax: cardinal;
    fSqlCreateField: TSqlDBFieldTypeDefinition;
    fSharedTransactions: array of TSqlDBConnectionTransaction;
    fExecuteWhenConnected: TRawUtf8DynArray;
    fForeignKeys: TSynNameValue;
    fOnTableCreate: TOnTableCreate;
    fOnTableAddColumn: TOnTableAddColumn;
    fOnTableCreateMultiIndex: TOnTableCreateMultiIndex;
    procedure SetFlag(const flag: TSqlDBConnectionPropertiesFlag; const value: boolean);
      {$ifdef HASINLINE}inline;{$endif}
    function GetFlag(const flag: TSqlDBConnectionPropertiesFlag): boolean;
      {$ifdef HASINLINE} inline; {$endif}
    procedure SetConnectionTimeOutMinutes(minutes: cardinal);
    function GetConnectionTimeOutMinutes: cardinal;
    function FindDatabaseNameField(Up: PAnsiChar): RawUtf8;
    // this default implementation just returns the fDbms value or dDefault
    // (never returns dUnknwown)
    function GetDbms: TSqlDBDefinition; virtual;
    function GetDbmsName: RawUtf8; virtual;
    function GetForeignKeysData: RawByteString;
    procedure SetForeignKeysData(const Value: RawByteString);
    function FieldsFromList(const aFields: TSqlDBColumnDefineDynArray;
      aExcludeTypes: TSqlDBFieldTypes): RawUtf8;
    function GetMainConnection: TSqlDBConnection; virtual;
    function GetDatabaseNameSafe: RawUtf8; virtual;
    /// any overriden TSqlDBConnectionProperties class should call it in the
    // initialization section of its implementation unit to be recognized
    class procedure RegisterClassNameForDefinition;
    /// will be called at the end of constructor
    // - this default implementation will do nothing
    procedure SetInternalProperties; virtual;
    /// Assign schema name to owner from ForceSchemaName or UserID or Database name
    procedure SetSchemaNameToOwner(out Owner: RawUtf8); virtual;
    /// SQL statement to get all field/column names for a specified Table
    // - used by GetFieldDefinitions public method
    // - should return a SQL "SELECT" statement with the field names as first
    // column, a textual field type as 2nd column, then field length, then
    // numeric precision and scale as 3rd, 4th and 5th columns, and the index
    // count in 6th column
    // - this default implementation just returns nothing
    // - if this method is overridden, the ColumnTypeNativeToDB() method should
    // also be overridden in order to allow conversion from native column
    // type into the corresponding TSqlDBFieldType
    function SqlGetField(const aTableName: RawUtf8): RawUtf8; virtual;
    /// SQL statement to get advanced information about all indexes for a Table
    // - should return a SQL "SELECT" statement with the index names as first
    function SqlGetIndex(const aTableName: RawUtf8): RawUtf8; virtual;
    /// SQL statement to get all parameter for a specified Stored Procedure
    // - used by GetProcedureParameters public method
    // - should return a SQL "SELECT" statement with the parameter names as first
    // column, a textual field type as 2nd column, then parameter length as 3rd, then
    // parameter direction as 4th
    // - this default implementation just returns nothing
    // - if this method is overridden, the ColumnTypeNativeToDB() method should
    // also be overridden in order to allow conversion from native column
    // type into the corresponding TSqlDBFieldType
    function SqlGetParameter(const aProcName: RawUtf8): RawUtf8; virtual;
    /// SQL statement to get all stored procedure names for current connection
    // - used by GetProcedureNames public method
    // - should return a SQL "SELECT" statement with the procedure names as unique column
    // - this default implementation just returns nothing
    // - if this method is overridden, the ColumnTypeNativeToDB() method should
    // also be overridden in order to allow conversion from native column
    // type into the corresponding TSqlDBFieldType
    function SqlGetProcedure: RawUtf8; virtual;
    /// SQL statement to get all table names
    // - used by GetTableNames public method
    // - should return a SQL "SELECT" statement with the table names as
    // first column (any other columns will be ignored)
    // - this default implementation just returns nothing
    function SqlGetTableNames: RawUtf8; virtual;
    /// SQL statement to get all view names
    // - used by GetViewNames public method
    // - should return a SQL "SELECT" statement with the view names as
    // first column (any other columns will be ignored)
    // - this default implementation just returns nothing
    function SqlGetViewNames: RawUtf8; virtual;
    /// should initialize fForeignKeys content with all foreign keys of this
    // database
    // - used by GetForeignKey method
    procedure GetForeignKeys; virtual; abstract;
    /// will use fSqlCreateField[Max] to create the SQL column definition
    // - this default virtual implementation will handle properly all supported
    // database engines, assuming aField.ColumnType as in TSqlDBFieldTypeDefinition
    // - if the field is a primary key, aAddPrimaryKey may be modified to contain
    // some text to be appended at the end of the ALTER/CREATE TABLE statement
    function SqlFieldCreate(const aField: TSqlDBColumnCreate;
      var aAddPrimaryKey: RawUtf8): RawUtf8; virtual;
    /// wrapper around GetIndexes() + set Fields[].ColumnIndexed in consequence
    // - used by some overridden versions of GetFields() method
    procedure GetIndexesAndSetFieldsColumnIndexed(const aTableName: RawUtf8;
      var Fields: TSqlDBColumnDefineDynArray);
    /// check if the exception or its error message is about DB connection error
    // - will be used by TSqlDBConnection.LastErrorWasAboutConnection method
    // - default method will check for the 'conne' sub-string in the message text
    // - should be overridden depending on the error message returned by the DB
    function ExceptionIsAboutConnection(aClass: ExceptClass;
      const aMessage: RawUtf8): boolean; virtual;
    /// generic method able to implement OnBatchInsert() with parameters
    // - for MySQL, PostgreSQL, MSSQL2008, NexusDB or SQLite3, will execute
    // (with parameters) the extended standard syntax:
    // $ INSERT INTO TableName(FieldNames[0],FieldNames[1]) VALUES
    // $   (FieldValues[0][0],FieldValues[1][0]),
    // $   (FieldValues[0][1],FieldValues[1][1]),
    // $   (FieldValues[0][2],FieldValues[1][2]);
    // - for Firebird, will run the corresponding EXECUTE BLOCK() statement
    // with parameters - but Firebird sounds slower than without any parameter
    // (as tested with ZDBC/ZEOS or UniDAC)
    // - for Oracle, will run (with parameters for values):
    // $ INSERT ALL
    // $  INTO TableName(FieldNames[0],FieldNames[1]) VALUES (?,?)
    // $  INTO TableName(FieldNames[0],FieldNames[1]) VALUES (?,?)
    // $  INTO TableName(FieldNames[0],FieldNames[1]) VALUES (?,?)
    // $ SELECT 1 FROM DUAL;
    procedure MultipleValuesInsert(Props: TSqlDBConnectionProperties;
      const TableName: RawUtf8; const FieldNames: TRawUtf8DynArray;
      const FieldTypes: TSqlDBFieldTypeArray; RowCount: integer;
      const FieldValues: TRawUtf8DynArrayDynArray; BatchOptions: TRestBatchOptions);
    /// Firebird-dedicated method able to implement OnBatchInsert()
    // - will run an EXECUTE BLOCK statement without any parameters, but
    // including inlined values - sounds to be faster on ZEOS/ZDBC!
    procedure MultipleValuesInsertFirebird(Props: TSqlDBConnectionProperties;
      const TableName: RawUtf8; const FieldNames: TRawUtf8DynArray;
      const FieldTypes: TSqlDBFieldTypeArray; RowCount: integer;
      const FieldValues: TRawUtf8DynArrayDynArray; BatchOptions: TRestBatchOptions);
  public
    /// initialize the properties
    // - children may optionaly handle the fact that no UserID or Password
    // is supplied here, by displaying a corresponding Dialog box
    constructor Create(
      const aServerName, aDatabaseName, aUserID, aPassWord: RawUtf8); virtual;
    /// release related memory, and close MainConnection
    destructor Destroy; override;
    /// save the properties into a persistent storage object
    // - you can use TSqlDBConnectionPropertiesDescription.CreateFrom()
    // later on to instantiate the proper TSqlDBConnectionProperties class
    // - current Definition.Key value will be used for the password encryption
    procedure DefinitionTo(Definition: TSynConnectionDefinition); virtual;
    /// save the properties into a JSON file
    // - you could use TSqlDBConnectionPropertiesDescription.CreateFromJson()
    // later on to instantiate the proper TSqlDBConnectionProperties class
    // - you can specify a custom Key, if the default is not enough for you
    function DefinitionToJson(Key: cardinal = 0): RawUtf8; virtual;
    /// save the properties into a JSON file
    // - you could use TSqlDBConnectionPropertiesDescription.CreateFromFile()
    // later on to instantiate the proper TSqlDBConnectionProperties class
    // - you can specify a custom Key, if the default is not enough for you
    procedure DefinitionToFile(const aJsonFile: TFileName; Key: cardinal = 0;
      Fmt: TTextWriterJsonFormat = jsonHumanReadable);
    /// create a new TSqlDBConnectionProperties instance from the stored values
    class function CreateFrom(
      aDefinition: TSynConnectionDefinition): TSqlDBConnectionProperties; virtual;
    /// create a new TSqlDBConnectionProperties instance from a JSON content
    // - as previously serialized with TSqlDBConnectionProperties.DefinitionToJson
    // - you can specify a custom Key, if the default is not safe enough for you
    class function CreateFromJson(
      const aJsonDefinition: RawUtf8; aKey: cardinal = 0): TSqlDBConnectionProperties; virtual;
    /// create a new TSqlDBConnectionProperties instance from a JSON file
    // - as previously serialized with TSqlDBConnectionProperties.DefinitionToFile
    // - you can specify a custom Key, if the default is not safe enough for you
    class function CreateFromFile(
      const aJsonFile: TFileName; aKey: cardinal = 0): TSqlDBConnectionProperties;
    /// retrieve the registered class from the aDefinition.Kind string
    class function ClassFrom(
      aDefinition: TSynConnectionDefinition): TSqlDBConnectionPropertiesClass;

    /// create a new connection
    // - call this method if the shared MainConnection is not enough (e.g. for
    // multi-thread access)
    // - the caller is responsible of freeing this instance
    function NewConnection: TSqlDBConnection; virtual;
    /// get a thread-safe connection
    // - this default implementation will return the MainConnection shared
    // instance, so the provider should be thread-safe by itself
    // - TSqlDBConnectionPropertiesThreadSafe in tmThreadPool mode will implement
    // a per-thread connection pool for efficient process with proper re-use
    function ThreadSafeConnection: TSqlDBConnection; virtual;
    /// release all existing connections
    // - can be called e.g. after a DB connection problem, to purge the
    // connection pool, and allow automatic reconnection
    // - warning: no connection shall still be used on the background (e.g. in
    // multi-threaded applications), or some unexpected border effects may occur
    procedure ClearConnectionPool; virtual;
    /// specify a maximum period of inactivity after which any connection will
    // be flushed and recreated, to avoid potential database issues
    // - is 0 by default, and rely on the database to gracefully reconnect
    // - in practice, recreating the connections after a while is safe and
    // won't slow down the process - on the contrary, it may help reducing the
    // consumpted resources, and stabilize long running n-Tier servers
    // - ThreadSafeConnection method will check for the last activity on each
    // TSqlDBConnection, and release any deprecated instances in their own thread
    // - ConnectionTimeOutBackground=true would make a global periodic search
    // and release, but could make the whole process thread-unsafe
    property ConnectionTimeOutMinutes: cardinal
      read GetConnectionTimeOutMinutes write SetConnectionTimeOutMinutes;
    /// force deprecation check of all connections
    // - by default (false), each thread will check for ConnectionTimeOutMinutes
    // idle connections, and recreate a new one for safety
    // - set true so at most that every 32 seconds, NewThreadSafeStatement will
    // check for deprecated connections in its all internal list and relase them
    // - warning: this could lead to issues, especially in multi-threaded
    // applications, and require e.g. amLocked/amBackgroundThread/amMainThread
    // mode for TRestServer.AcquireExecutionMode[execOrmGet/execOrmWrite]
    property ConnectionTimeOutBackground: boolean
      index cpfConnectionTimeOutBackground read GetFlag write SetFlag;
    /// intercept connection errors at statement preparation and try to reconnect
    // - i.e. detect TSqlDBConnection.LastErrorWasAboutConnection in
    // TSqlDBConnection.NewStatementPrepared
    // - warning: no connection shall still be used on the background (e.g. in
    // multi-threaded applications), or some unexpected issues may occur - see
    // AcquireExecutionMode[] recommendations in ConnectionTimeOutMinutes
    property ReconnectAfterConnectionError: boolean
      index cpfReconnectAfterConnectionError read GetFlag write SetFlag;
    /// create a new thread-safe statement
    // - this method just redirects to ThreadSafeConnection.NewStatement
    function NewThreadSafeStatement: TSqlDBStatement;
      {$ifdef HASINLINE} inline; {$endif}
    /// create a new thread-safe statement from an internal cache (if any)
    // - will call ThreadSafeConnection.NewStatementPrepared
    // - this method should return a prepared statement instance on success
    // - on error, returns nil and you can check Connnection.LastErrorMessage /
    // Connection.LastErrorException to retrieve corresponding error information
    // (if RaiseExceptionOnError is left to default FALSE value, otherwise, it will
    // raise an exception)
    function NewThreadSafeStatementPrepared(const aSql: RawUtf8;
      ExpectResults: boolean; RaiseExceptionOnError: boolean = false): ISqlDBStatement; overload;
    /// create a new thread-safe statement from an internal cache (if any)
    // - this method will call the overloaded NewThreadSafeStatementPrepared method
    // - here Args[] array does not refer to bound parameters, but to values
    // to be changed within SqlFormat in place of '%' characters (this method
    // will call FormatUtf8() internally); parameters will be bound directly
    // on the returned TSqlDBStatement instance
    // - this method should return a prepared statement instance on success
    // - on error, returns nil and you can check Connnection.LastErrorMessage /
    // Connection.LastErrorException to retrieve correspnding error information
    // (if RaiseExceptionOnError is left to default FALSE value, otherwise, it will
    // raise an exception)
    function NewThreadSafeStatementPrepared(const SqlFormat: RawUtf8;
      const Args: array of const; ExpectResults: boolean;
      RaiseExceptionOnError: boolean = false): ISqlDBStatement; overload;
    /// create, prepare and bound inlined parameters to a thread-safe statement
    // - this implementation will call the NewThreadSafeStatement virtual method,
    // then bound inlined parameters as :(1234): and return the resulting statement
    // - raise an exception on error
    // - consider using ExecuteInlined() for direct execution
    function PrepareInlined(const aSql: RawUtf8; ExpectResults: boolean): ISqlDBStatement; overload;
    /// create, prepare and bound inlined parameters to a thread-safe statement
    // - overloaded method using FormatUtf8() and inlined parameters
    // - consider using ExecuteInlined() for direct execution
    function PrepareInlined(const SqlFormat: RawUtf8; const Args: array of const;
      ExpectResults: boolean): ISqlDBStatement; overload;
    /// execute a SQL query, returning a statement interface instance to retrieve
    // the result rows corresponding to the supplied SELECT statement
    // - will call NewThreadSafeStatement method to retrieve a thread-safe
    // statement instance, then run the corresponding Execute() method
    // - raise an exception on error
    // - returns an ISqlDBRows to access any resulting rows (if ExpectResults is
    // TRUE), and provide basic garbage collection, as such:
    // ! procedure WriteFamily(const aName: RawUtf8);
    // ! var I: ISqlDBRows;
    // ! begin
    // !   I := MyConnProps.Execute('select * from table where name=?',[aName]);
    // !   while I.Step do
    // !     writeln(I['FirstName'],' ',DateToStr(I['BirthDate']));
    // !   I.ReleaseRows;
    // ! end;
    // - if RowsVariant is set, you can use it to row column access via late
    // binding, as such:
    // ! procedure WriteFamily(const aName: RawUtf8);
    // ! var R: Variant;
    // ! begin
    // !   with MyConnProps.Execute('select * from table where name=?',[aName],@R) do
    // !   begin
    // !     while Step do
    // !       writeln(R.FirstName,' ',DateToStr(R.BirthDate));
    // !     ReleaseRows;
    // !   end;
    // ! end;
    // - you can any BLOB field to be returned as null with the ForceBlobAsNull
    // optional parameter
    function Execute(const aSql: RawUtf8; const Params: array of const;
      RowsVariant: PVariant = nil; ForceBlobAsNull: boolean = false): ISqlDBRows;
    /// execute a SQL query, without returning any rows
    // - can be used to launch INSERT, DELETE or UPDATE statement, e.g.
    // - will call NewThreadSafeStatement method to retrieve a thread-safe
    // statement instance, then run the corresponding Execute() method
    // - return the number of modified rows, i.e. the ISqlDBStatement.UpdateCount
    // value (or 0 if the DB driver does not supply this value)
    function ExecuteNoResult(const aSql: RawUtf8; const Params: array of const): integer;
    /// create, prepare, bound inlined parameters and execute a thread-safe statement
    // - this implementation will call the NewThreadSafeStatement virtual method,
    // then bound inlined parameters as :(1234): and call its Execute method
    // - raise an exception on error
    function ExecuteInlined(const aSql: RawUtf8;
      ExpectResults: boolean): ISqlDBRows; overload;
    /// create, prepare, bound inlined parameters and execute a thread-safe statement
    // - overloaded method using FormatUtf8() and inlined parameters
    function ExecuteInlined(const SqlFormat: RawUtf8; const Args: array of const;
      ExpectResults: boolean): ISqlDBRows; overload;
    /// handle a transaction process common to all associated connections
    // - could be used to share a single transaction among several connections,
    // or to run nested transactions even on DB engines which do not allow them
    // - will use a simple reference counting mechanism to allow nested
    // transactions, identified by a session identifier
    // - will fail if the same connection is not used for the whole process,
    // which would induce a potentially incorrect behavior
    // - returns the connection corresponding to the session, nil on error
    function SharedTransaction(SessionID: cardinal;
      action: TSqlDBSharedTransactionAction): TSqlDBConnection; virtual;

    /// convert a textual column data type, as retrieved e.g. from SqlGetField,
    // into our internal primitive types
    // - default implementation will always return ftUtf8
    function ColumnTypeNativeToDB(const aNativeType: RawUtf8;
      aScale: integer): TSqlDBFieldType; virtual;
    /// returns the SQL statement used to create a Table
    // - should return the SQL "CREATE" statement needed to create a table with
    // the specified field/column names and types
    // - if aAddID is TRUE, "ID Int64 PRIMARY KEY" column is added as first,
    // and will expect the ORM to create an unique RowID value sent at INSERT
    // (could use "select max(ID) from table" to retrieve the last value) -
    // note that 'ID' is used instead of 'RowID' since it fails on Oracle e.g.
    // - this default implementation will use internal fSqlCreateField and
    // fSqlCreateFieldMax protected values, which contains by default the
    // ANSI SQL Data Types and maximum 1000 inlined WideChars: inherited classes
    // may change the default fSqlCreateField* content or override this method
    function SqlCreate(const aTableName: RawUtf8;
      const aFields: TSqlDBColumnCreateDynArray; aAddID: boolean): RawUtf8; virtual;
    /// returns the SQL statement used to add a column to a Table
    // - should return the SQL "ALTER TABLE" statement needed to add a column to
    // an existing table
    // - this default implementation will use internal fSqlCreateField and
    // fSqlCreateFieldMax protected values, which contains by default the
    // ANSI SQL Data Types and maximum 1000 inlined WideChars: inherited classes
    // may change the default fSqlCreateField* content or override this method
    function SqlAddColumn(const aTableName: RawUtf8;
      const aField: TSqlDBColumnCreate): RawUtf8; virtual;
    /// returns the SQL statement used to add an index to a Table
    // - should return the SQL "CREATE INDEX" statement needed to add an index
    // to the specified column names of an existing table
    // - index will expect UNIQUE values in the specified columns, if Unique
    // parameter is set to true
    // - this default implementation will return the standard SQL statement, i.e.
    // 'CREATE [UNIQUE] INDEX index_name ON table_name (column_name[s])'
    function SqlAddIndex(const aTableName: RawUtf8;
      const aFieldNames: array of RawUtf8; aUnique: boolean;
      aDescending: boolean = false; const aIndexName: RawUtf8 = ''): RawUtf8; virtual;
    /// used to compute a SELECT statement for the given fields
    // - should return the SQL "SELECT ... FROM ..." statement to retrieve
    // the specified column names of an existing table
    // - by default, all columns specified in aFields[] will be available:
    // it will return "SELECT * FROM TableName"
    // - but if you specify a value in aExcludeTypes, it will compute the
    // matching column names to ignore those kind of content (e.g. [stBlob] to
    // save time and space)
    function SqlSelectAll(const aTableName: RawUtf8;
      const aFields: TSqlDBColumnDefineDynArray;
      aExcludeTypes: TSqlDBFieldTypes): RawUtf8; virtual;
    /// SQL statement to create the corresponding database
    // - this default implementation will only handle dFirebird by now
    function SqlCreateDatabase(const aDatabaseName: RawUtf8;
      aDefaultPageSize: integer = 0): RawUtf8; virtual;
    /// convert an ISO-8601 encoded time and date into a date appropriate to
    // be pasted in the SQL request
    // - this default implementation will return the quoted ISO-8601 value, i.e.
    // 'YYYY-MM-DDTHH:MM:SS' (as expected by Microsoft SQL server e.g.)
    // - returns  to_date('....','YYYY-MM-DD HH24:MI:SS')  for Oracle
    function SqlIso8601ToDate(const Iso8601: RawUtf8): RawUtf8; virtual;
    /// convert a TDateTime into a ISO-8601 encoded time and date, as expected
    // by the database provider
    // - e.g. SQLite3, DB2 and PostgreSQL will use non-standard ' ' instead of 'T'
    function SqlDateToIso8601Quoted(DateTime: TDateTime): RawUtf8; virtual;
    /// split a table name to its OWNER.TABLE full name (if applying)
    // - will use ForcedSchemaName property (if applying), or the OWNER. already
    // available within the supplied table name
    procedure SqlSplitTableName(const aTableName: RawUtf8;
      out Owner, Table: RawUtf8); virtual;
    /// split a procedure name to its OWNER.PACKAGE.PROCEDURE full name (if applying)
    // - will use ForcedSchemaName property (if applying), or the OWNER. already
    // available within the supplied table name
    procedure SqlSplitProcedureName(const aProcName: RawUtf8;
      out Owner, package, ProcName: RawUtf8); virtual;
    /// return the fully qualified SQL table name
    // - will use ForcedSchemaName property (if applying), or return aTableName
    // - you can override this method to force the expected format
    function SqlFullTableName(const aTableName: RawUtf8): RawUtf8; virtual;
    /// return a SQL table name with quotes if necessary
    // - can be used e.g. with SELECT statements
    // - you can override this method to force the expected format
    function SqlTableName(const aTableName: RawUtf8): RawUtf8; virtual;

    /// retrieve the column/field layout of a specified table
    // - this default implementation will use protected SqlGetField virtual
    // method to retrieve the field names and properties
    // - used e.g. by GetFieldDefinitions
    // - will call ColumnTypeNativeToDB protected virtual method to guess the
    // each mORMot TSqlDBFieldType
    procedure GetFields(const aTableName: RawUtf8;
      out Fields: TSqlDBColumnDefineDynArray); virtual;
    /// retrieve the advanced indexed information of a specified Table
    //  - this default implementation will use protected SqlGetIndex virtual
    // method to retrieve the index names and properties
    // - currently only MS SQL and Oracle are supported
    procedure GetIndexes(const aTableName: RawUtf8;
      out Indexes: TSqlDBIndexDefineDynArray); virtual;
    /// get all field/column definition for a specified Table as text
    // - call the GetFields method and retrieve the column field name and
    // type as 'Name [Type Length Precision Scale]'
    // - if WithForeignKeys is set, will add external foreign keys as '% tablename'
    procedure GetFieldDefinitions(const aTableName: RawUtf8;
      out Fields: TRawUtf8DynArray; WithForeignKeys: boolean);
    /// get one field/column definition as text
    // - return column type as 'Name [Type Length Precision Scale]'
    class function GetFieldDefinition(
      const Column: TSqlDBColumnDefine): RawUtf8;
    /// get one field/column definition as text, targeting a TOrm
    // published property
    // - return e.g. property type information as:
    // ! 'Name: RawUtf8 read fName write fName index 20;';
    class function GetFieldORMDefinition(const Column: TSqlDBColumnDefine): RawUtf8;
    /// check if the supplied text word is not a keyword for a given database engine
    class function IsSqlKeyword(aDB: TSqlDBDefinition; aWord: RawUtf8): boolean;
      overload; virtual;
    /// check if the supplied text word is not a keyword for the current database engine
    // - just a wrapper around the overloaded class function
    function IsSqlKeyword(aWord: RawUtf8): boolean; overload;
    /// retrieve a list of stored procedure names from current connection
    procedure GetProcedureNames(out Procedures: TRawUtf8DynArray); virtual;
    /// retrieve procedure input/output parameter information
    // - aProcName: stored procedure name to retrieve parameter infomation.
    // - Parameters: parameter list info (name, datatype, direction, default)
    procedure GetProcedureParameters(const aProcName: RawUtf8;
      out Parameters: TSqlDBProcColumnDefineDynArray); virtual;
    /// get all table names
    // - this default implementation will use protected SqlGetTableNames virtual
    // method to retrieve the table names
    procedure GetTableNames(out Tables: TRawUtf8DynArray); virtual;
    /// wrapper to call GetTableNames() then check if TableName is part of it
    function TableExists(const TableName: RawUtf8): boolean;
    /// get all view names
    // - this default implementation will use protected SqlGetViewNames virtual
    // method to retrieve the view names
    procedure GetViewNames(out Views: TRawUtf8DynArray); virtual;
    /// retrieve a foreign key for a specified table and column
    // - first time it is called, it will retrieve all foreign keys from the
    // remote database using virtual protected GetForeignKeys method into
    // the protected fForeignKeys list: this may be slow, depending on the
    // database access (more than 10 seconds waiting is possible)
    // - any further call will use this internal list, so response will be
    // immediate
    // - the whole foreign key list is shared by all connections
    function GetForeignKey(const aTableName, aColumnName: RawUtf8): RawUtf8;

    /// returns the information to adapt the LIMIT # clause in the SQL SELECT
    // statement to a syntax matching the underlying DBMS
    // - e.g. TRestStorageExternal.AdaptSqlForEngineList() calls this
    // to let TRestServer.Uri by-pass virtual table mechanism
    function SqlLimitClause(
      AStmt: TSelectStatement): TSqlDBDefinitionLimitClause; virtual;
    /// determine if the SQL statement can be cached
    // - used by TSqlDBConnection.NewStatementPrepared() for handling its cache
    // of prepared statements (here we don't speak about caching results)
    // - this virtual method redirects to IsCacheableDML() by default
    function IsCacheable(P: PUtf8Char): boolean; virtual;
    /// check if a primary key has already an index
    // - can specify if it is ascending only, which is not the case for Firebird
    function IsPrimaryKeyIndexed(var AscendingOnly: boolean): boolean; virtual;
    /// would replace any matching Password value content from supplied text
    function SanitizeFromPassword(const S: RawUtf8): RawUtf8;
    /// return the database engine name, as computed from the class name
    // - 'TSqlDBConnectionProperties' will be trimmed left side of the class name
    class function EngineName: RawUtf8;

    /// return a shared connection, corresponding to the given database
    // - the returned instance is shared among calls, so not thread-safe
    // - for multi-thread access, call the ThreadSafeConnection method instead
    // or NewThreadSafeStatement for direct retrieval of a new statement
    // - is mainly used internally by TSqlDBConnectionProperties for reading the
    // database metadata
    property MainConnection: TSqlDBConnection
      read GetMainConnection;
    /// the associated User Password, as specified at creation
    // - not published, for security reasons (may be serialized otherwise)
    property PassWord: SpiUtf8
      read fPassWord;
    /// the associated database name, as specified at creation
    // - not published, for security reasons (may be serialized otherwise)
    // - DatabaseNameSafe will be published, and delete any matching
    // PasswordValue in DatabaseName
    property DatabaseName: RawUtf8
      read fDatabaseName;
    /// can be used to store the fForeignKeys[] data in an external BLOB
    // - since GetForeignKeys can be (somewhat) slow, could save a lot of time
    property ForeignKeysData: RawByteString
      read GetForeignKeysData write SetForeignKeysData;
    /// this event handler will be called during all process
    // - can be used e.g. to change the desktop cursor, or be notified
    // on connection/disconnection/reconnection
    // - you can override this property directly in the TSqlDBConnection
    property OnProcess: TOnSqlDBProcess
      read fOnProcess write fOnProcess;
    /// this event handler will be called when statements trigger some low-level
    // information
    property OnStatementInfo: TOnSqlDBInfo
      read fOnStatementInfo write fOnStatementInfo;
    /// you can define a callback method able to handle multiple INSERT
    // - may execute e.g. INSERT with multiple VALUES (like MySQL, MSSQL, NexusDB,
    // PostgreSQL or SQLite3), as defined by MultipleValuesInsert() callback
    property OnBatchInsert: TOnBatchInsert
      read fOnBatchInsert write fOnBatchInsert;

    /// allow to customize the creation of a table by our ORM
    // - may be used e.g. to generate a partitioned table on PostgreSQL
    property OnTableCreate: TOnTableCreate
      read fOnTableCreate write fOnTableCreate;
    /// allow to customize the creation of a table by our ORM
    // - may be used e.g. to generate a partitioned table on PostgreSQL
    property OnTableAddColumn: TOnTableAddColumn
      read fOnTableAddColumn write fOnTableAddColumn;
    /// allow to customize the creation of an index by our ORM
    property OnTableCreateMultiIndex: TOnTableCreateMultiIndex
      read fOnTableCreateMultiIndex write fOnTableCreateMultiIndex;
  published { to be logged as JSON - no UserID nor Password for security :) }
    /// return the database engine name, as computed from the class name
    // - 'TSqlDBConnectionProperties' will be trimmed left side of the class name
    property Engine: RawUtf8
      read fEngineName;
    /// the associated server name, as specified at creation
    property ServerName: RawUtf8
      read fServerName;
    /// the associated database name, safely trimmed from the password
    // - would replace any matching Password value content from DatabaseName
    // by '***' for security reasons, e.g. before serialization
    property DatabaseNameSafe: RawUtf8
      read GetDatabaseNameSafe;
    /// the associated User Identifier, as specified at creation
    property UserID: RawUtf8
      read fUserID;
    /// the remote DBMS type, as stated by the inheriting class itself, or
    // retrieved at connecton time (e.g. for ODBC)
    // - you may force a DB engine to bypass the value returned at runtime
    // (use with caution, but may be useful with unsupported drivers)
    property Dbms: TSqlDBDefinition
      read GetDbms write fDbms;
    /// the remote DBMS type name, retrieved as text from the DBMS property
    property DbmsEngineName: RawUtf8
      read GetDbmsName;
    /// the abilities of the database for batch sending
    // - e.g. Oracle will handle array DML binds, or MS SQL bulk insert
    property BatchSendingAbilities: TSqlDBStatementCRUDs
      read fBatchSendingAbilities;
    /// the maximum number of rows to be transmitted at once for batch sending
    // - e.g. Oracle handles array DML operation with iters <= 32767 at best
    // - if OnBatchInsert points to MultipleValuesInsert(), this value is
    // ignored, and the maximum number of parameters is guessed per DBMS type
    property BatchMaxSentAtOnce: integer
      read fBatchMaxSentAtOnce write fBatchMaxSentAtOnce;
    /// the maximum size, in bytes, of logged SQL statements
    // - setting 0 will log statement and parameters with no size limit
    // - setting -1 will log statement without any parameter value (just ?)
    // - setting any value >0 will log statement and parameters up to the
    // number of bytes (default set to 2048 to log up to 2KB per statement)
    property LoggedSqlMaxSize: integer
      read fLoggedSqlMaxSize write fLoggedSqlMaxSize;
    /// allow to log the SQL statement when any low-level ESqlDBException is raised
    property LogSqlStatementOnException: boolean
      index cpfLogSqlStatementOnException read GetFlag write SetFlag;
    /// an optional Schema name to be used for SqlGetField() instead of UserID
    // - by default, UserID will be used as schema name, if none is specified
    // (i.e. if table name is not set as SCHEMA.TABLE)
    // - depending on the DBMS identified, the class may also set automatically
    // the default 'dbo' for MS SQL or 'public' for PostgreSQL
    // - you can set a custom schema to be used instead
    property ForcedSchemaName: RawUtf8
      read fForcedSchemaName write fForcedSchemaName;
    /// if GetTableNames/GetViewNames should only return the table names
    // starting with 'ForcedSchemaName.' prefix
    property FilterTableViewSchemaName: boolean
      index cpfFilterTableViewSchemaName read GetFlag write SetFlag;
    /// TRUE if an internal cache of SQL statement should be used
    // - cache will be accessed for NewStatementPrepared() method only, by
    // returning ISqlDBStatement interface instances
    // - default value is TRUE for faster process (e.g. TTestSQLite3ExternalDB
    // regression tests will be two times faster with statement caching)
    // - will cache only statements containing ? parameters or a SELECT with no
    // WHERE clause within
    property UseCache: boolean
      index cpfUseCache read GetFlag write SetFlag;
    /// maximum bytes allowed for FetchAllToJSON/FetchAllToBinary methods
    // - if a result set exceeds this limit, an ESQLDBException is raised
    // - default is 512 shl 20, i.e. 512MB which is very high
    // - avoid unexpected OutOfMemory errors when incorrect statement is run
    property StatementMaxMemory: PtrInt
      read fStatementMaxMemory write fStatementMaxMemory;
    /// if UseCache is true, how many statement replicates can be generated
    // if the cached ISqlDBStatement is already used
    // - such replication is normally not needed in a per-thread connection,
    // unless ISqlDBStatement are not released as soon as possible
    // - above this limit, no cache will be made, and a dedicated single-time
    // statement will be prepared
    // - default is 0 to cache statements once - but you may try to increase
    // this value if you run identical SQL with long-standing ISqlDBStatement;
    // or you can set -1 if you don't want the warning log to appear
    property StatementCacheReplicates: integer
      read fStatementCacheReplicates write fStatementCacheReplicates;
    /// defines if TSqlDBConnection.Disconnect shall Rollback any pending
    // transaction
    // - some engines executes a COMMIT when the client is disconnected, others
    // do raise an exception: this parameter ensures that any pending transaction
    // is roll-backed before disconnection
    // - is set to TRUE by default
    property RollbackOnDisconnect: boolean
      index cpfRollbackOnDisconnect read GetFlag write SetFlag;
    /// by default, result column names won't be checked for name unicity
    // - it is up to your SQL statement to return genuine columns
    // - ColumName() lookup will use O(1) brute force, so you should rather
    // use per-index Column*() methods
    // - force true to have the mORMot 1 compatible behavior, in which column
    // names will be hashed for unicity (so will be slower at execution),
    // but will allow faster ColumnName() lookup
    property EnsureColumnNameUnique: boolean
      index cpfEnsureColumnNameUnique read GetFlag write SetFlag;
    /// defines if '' string values are to be stored as SQL null
    // - by default, '' will be stored as ''
    // - but some DB engines (e.g. Jet or MS SQL) does not allow by default to
    // store '' values, but expect NULL to be stored instead
    property StoreVoidStringAsNull: boolean
      index cpfStoreVoidStringAsNull read GetFlag write SetFlag;
    /// customize the ISO-8601 text format expected by the database provider
    // - is 'T' by default, as expected by the ISO-8601 standard
    // - will be changed e.g. for PostgreSQL, which expects ' ' instead
    // - as used by SqlDateToIso8601Quoted() and BindArray()
    property DateTimeFirstChar: AnsiChar
      read fDateTimeFirstChar write fDateTimeFirstChar;
    /// defines if the engine does not support BindArray(ftBlob)
    // - only set for TSqlDBPostgresConnectionProperties by now
    property NoBlobBindArray: boolean
      index cpfNoBlobBindArray read GetFlag write SetFlag;
    {$ifndef UNICODE}
    /// set to true to force all variant conversion to WideString instead of
    // the default faster AnsiString, for pre-Unicode version of Delphi
    // - by default, the conversion to Variant will create an AnsiString
    // varString: for pre-Unicode Delphi, avoiding WideString/OleStr content
    // will speed up the process a lot, if you are sure that the current
    // charset matches the expected one (which is very likely)
    // - set this property to TRUE so that the conversion to Variant will
    // create a WideString kind of variant, to avoid any character data loss:
    // the access to the property will be slower, but you won't have any
    // potential data loss
    // - starting with Delphi 2009, the TEXT content will be stored as an
    // UnicodeString in the variant, so this property is not necessary
    // - this Variant conversion is mostly used for the TQuery wrapper, or for
    // the ISqlDBRows.Column[] property or ISqlDBRows.ColumnVariant() method;
    // this won't affect other Column*() methods, or JSON production
    // - if you want a RawUtf8 varString (as used e.g. within TDocVariantData),
    // use ISqlDBStatement.ColumnVariant/ColumnToVariant() with ForceUtf8=true
    property VariantStringAsWideString: boolean
      index cpfVariantStringAsWideString read GetFlag write SetFlag;
    {$endif UNICODE}
    /// SQL statements what will be executed for each new connection, as typical
    // usage scenarios examples:
    // - Oracle: force case-insensitive like
    // $  ['ALTER SESSION SET NLS_COMP=LINGUISTIC', 'ALTER SESSION SET NLS_SORT=BINARY_CI']
    //  - Postgres: disable notices and warnings
    // $  ['SET client_min_messages to ERROR']
    // - SQLite3: turn foreign keys ON
    // $  ['PRAGMA foreign_keys = ON']
    property ExecuteWhenConnected: TRawUtf8DynArray
      read fExecuteWhenConnected write fExecuteWhenConnected;
  end;

  /// abstract connection created from TSqlDBConnectionProperties
  // - more than one TSqlDBConnection instance can be run for the same
  // TSqlDBConnectionProperties
  TSqlDBConnection = class(TSynPersistent)
  protected
    fProperties: TSqlDBConnectionProperties;
    fErrorException: ExceptClass;
    fErrorMessage: RawUtf8;
    fServerTimestampOffset: TDateTime;
    fCacheSafe: TOSLightLock; // protect fCache - warning: not reentrant!
    fCache: TRawUtf8List;     // statements cache
    fCacheLast: RawUtf8;
    fCacheLastIndex: integer;
    fTotalConnectionCount: integer;
    fInternalProcessActive: integer;
    fTransactionCount: integer;
    fLastAccessTickSec: integer; // from GetTickSec - for IsOutdated()
    fRollbackOnDisconnect: boolean;
    fOnProcess: TOnSqlDBProcess;
    fOwned: TObjectDynArray;
    function IsOutdated(secs: integer): boolean;
      {$ifdef HASINLINE} inline; {$endif}
    function GetInTransaction: boolean; virtual;
    function GetServerTimestamp: TTimeLog;
    function GetServerDateTime: TDateTime; virtual;
    function GetLastErrorWasAboutConnection: boolean;
    /// raise an exception if IsConnected returns false
    procedure CheckConnection;
    /// call OnProcess() call back event, if needed
    procedure InternalProcess(Event: TOnSqlDBProcessEvent);
  public
    /// connect to a specified database engine
    constructor Create(aProperties: TSqlDBConnectionProperties); reintroduce; virtual;
    /// release memory and connection
    destructor Destroy; override;
    /// retrieve one object owned by a thread-safe connection instance
    // - may be used e.g. to reuse some internal processing objects
    // - returns nil if SetThreadOwned() was not previously called
    function GetThreadOwned(aClass: TClass): pointer;
      {$ifdef HASINLINE} inline; {$endif}
    /// register one object owned by a thread-safe connection instance
    // - returns the supplied aObject instance as a fluid-interface mechanism
    // - raise an ESqlDBException if not a TSqlDBConnectionPropertiesThreadSafe
    function SetThreadOwned(aObject: TObject): pointer;

    /// connect to the specified database
    // - should raise an Exception on error
    // - this default implementation will notify OnProgress callback for
    // successful re-connection: it should be called in overridden methods
    // AFTER actual connection process
    procedure Connect; virtual;
    /// stop connection to the specified database
    // - should raise an Exception on error
    // - this default implementation will release all cached statements: so it
    // should be called in overridden methods BEFORE actual disconnection
    procedure Disconnect; virtual;
    /// return TRUE if Connect has been already successfully called
    function IsConnected: boolean; virtual; abstract;
    /// initialize a new SQL query statement for the given connection
    // - the caller should free the instance after use
    function NewStatement: TSqlDBStatement; virtual; abstract;
    /// initialize a new SQL query statement for the given connection
    // - this default implementation will call the NewStatement method, and
    // implement handle statement caching is UseCache=true - in this case,
    // the TSqlDBStatement.Reset method shall have been overridden to allow
    // binding and execution of the very same prepared statement
    // - the same aSql can cache up to 9 statements in this TSqlDBConnection
    // - this method should return a prepared statement instance on success
    // - on error, if RaiseExceptionOnError=false (by default), it returns nil
    // and you can check LastErrorMessage and LastErrorException properties to
    // retrieve corresponding error information
    // - if TSqlDBConnectionProperties.ReconnectAfterConnectionError is set,
    // any connection error will be trapped, unless AllowReconnect is false
    // - on error, if RaiseExceptionOnError=true, an exception is raised
    function NewStatementPrepared(const aSql: RawUtf8; ExpectResults: boolean;
      RaiseExceptionOnError: boolean = false;
      AllowReconnect: boolean = true): ISqlDBStatement; virtual;
    /// begin a Transaction for this connection
    // - this default implementation will check and set TransactionCount
    procedure StartTransaction; virtual;
    /// commit changes of a Transaction for this connection
    // - StartTransaction method must have been called before
    // - this default implementation will check and set TransactionCount
    procedure Commit; virtual;
    /// discard changes of a Transaction for this connection
    // - StartTransaction method must have been called before
    // - this default implementation will check and set TransactionCount
    procedure Rollback; virtual;
    /// allows to change the password of the current connected user
    // - do nothing method by default, returning false
    // - properly overriden e.g. by TSqlDBOracleConnection
    function PasswordChange: boolean; virtual;

    /// direct export of a DB statement rows into a new table of this database
    // - the corresponding table will be created within the current connection,
    // if it does not exist
    // - if the column types are not set, they will be identified from the
    // first row of data
    // - INSERTs will be nested within a transaction if WithinTransaction is TRUE
    // - will raise an Exception in case of error
    function NewTableFromRows(const TableName: RawUtf8; Rows: TSqlDBStatement;
      WithinTransaction: boolean; ColumnForcedTypes: TSqlDBFieldTypeDynArray = nil): integer;
    /// create a new table in this database from an existing table on another DB
    // - could be called instead of NewTableFromRows() if no Rows are available
    function NewTableFrom(const NewTableName, SourceTableName: RawUtf8;
      Source: TSqlDBConnectionProperties): boolean;

    /// the current Date and Time, as retrieved from the server
    // - note that this value is the DB_SERVERTIME[] constant SQL value, so
    // will most likely return a local time, not an UTC time
    // - this property will return the timestamp in TTimeLog / TTimeLogBits /
    // Int64 value
    property ServerTimestamp: TTimeLog
      read GetServerTimestamp;
    /// the current Date and Time, as retrieved from the server
    // - note that this value is the DB_SERVERTIME[] constant SQL value, so
    // will most likely return a local time, not an UTC time
    // - this property will return the value as regular TDateTime
    property ServerDateTime: TDateTime
      read GetServerDateTime;
    /// this event handler will be called during all process
    // - can be used e.g. to change the desktop cursor
    // - please let this callback return as quickly as possible
    // - by default, will follow TSqlDBConnectionProperties.OnProcess property
    property OnProcess: TOnSqlDBProcess
      read fOnProcess write fOnProcess;
  published { to be logged as JSON }
    /// returns TRUE if the connection was set
    property Connected: boolean
      read IsConnected;
    /// number of successful connections for this instance
    // - can be greater than 1 in case of re-connection via Disconnect/Connect
    property TotalConnectionCount: integer
      read fTotalConnectionCount;
    /// number of nested StartTransaction calls
    // - equals 0 if no transaction is active
    property TransactionCount: integer
      read fTransactionCount;
    /// TRUE if StartTransaction has been called
    // - check if TransactionCount>0
    property InTransaction: boolean
      read GetInTransaction;
    /// defines if Disconnect shall Rollback any pending transaction
    // - some engines executes a COMMIT when the client is disconnected, others
    // do raise an exception: this parameter ensures that any pending transaction
    // is roll-backed before disconnection
    // - is set to TRUE by default
    property RollbackOnDisconnect: boolean
      read fRollbackOnDisconnect write fRollbackOnDisconnect;
    /// some error message, e.g. during execution of NewStatementPrepared
    property LastErrorMessage: RawUtf8
      read fErrorMessage write fErrorMessage;
    /// some error exception, e.g. during execution of NewStatementPrepared
    property LastErrorException: ExceptClass
      read fErrorException;
    /// TRUE if last error is a broken connection, e.g. during execution of
    // NewStatementPrepared
    // - i.e. LastErrorException/LastErrorMessage concerns the database connection
    // - will use TSqlDBConnectionProperties.ExceptionIsAboutConnection virtual method
    property LastErrorWasAboutConnection: boolean
      read GetLastErrorWasAboutConnection;
    /// the associated database properties
    property Properties: TSqlDBConnectionProperties
      read fProperties;
  end;

  /// how this statement is cached
  // - scPossible is set by TSqlDBConnection.NewStatementPrepared when
  // TSqlDBConnectionProperties.IsCacheable was true for this SQL statement
  // - scOnClient is set when it is actually cached within the TSqlDBConnection
  // - scOnServer is set when cached on server side (mormot.db.sql.oracle
  // and mormot.db.sql.postgresl only by now)
  TSqlDBStatementCache = set of (
    scPossible,
    scOnClient,
    scOnServer);

  /// define all possible TSqlDBStatement internal flags
  TSqlDBStatementFlag = (
    dsfStripSemicolon,
    dsfExpectResults,
    dsfForceBlobAsNull,
    dsfForceDateWithMS,
    dsfShouldLogSQL);
  /// store TSqlDBStatement internal flags
  // - exposed as boolean properties for mORMot 1 and existing code compatibility
  TSqlDBStatementFlags = set of TSqlDBStatementFlag;

  /// generic abstract class to implement a prepared SQL query
  // - inherited classes should implement the DB-specific connection in its
  // overridden methods, especially Bind*(), Prepare(), ExecutePrepared, Step()
  // and Column*() methods
  TSqlDBStatement = class(TInterfacedObject, ISqlDBRows, ISqlDBStatement)
  protected
    fConnection: TSqlDBConnection;
    fParamCount: integer;
    fColumnCount: integer;
    fTotalRowsRetrieved: integer;
    fCurrentRow: integer;
    fDbms: TSqlDBDefinition;
    fFlags: TSqlDBStatementFlags;
    fCache: TSqlDBStatementCache;
    fSqlLogLevel: TSynLogLevel;
    fSql: RawUtf8;
    fSqlLogLog: TSynLog;
    fSqlWithInlinedParams: RawUtf8;
    fSqlLogTimer: TPrecisionTimer;
    fSqlPrepared: RawUtf8;
    procedure SetFlag(const flag: TSqlDBStatementFlag; const value: boolean);
      {$ifdef HASINLINE}inline;{$endif}
    function GetFlag(const flag: TSqlDBStatementFlag): boolean;
      {$ifdef HASINLINE} inline; {$endif}
    function GetSqlCurrent: RawUtf8;
    function GetSqlWithInlinedParams: RawUtf8;
    procedure ComputeSqlWithInlinedParams;
    function GetForceBlobAsNull: boolean;
    procedure SetForceBlobAsNull(value: boolean);
    function GetForceDateWithMS: boolean;
    procedure SetForceDateWithMS(value: boolean);
    /// raise an exception if Col is out of range according to fColumnCount
    procedure CheckCol(Col: integer);
      {$ifdef HASINLINE}inline;{$endif}
    procedure CheckColInvalid(Col: integer);
    /// will set a Int64/Double/Currency/TDateTime/RawUtf8/TBlobData Dest variable
    // from a given column value
    // - internal conversion will use a temporary Variant and ColumnToVariant method
    // - expects Dest to be of the exact type (e.g. Int64, not integer)
    function ColumnToTypedValue(Col: integer; DestType: TSqlDBFieldType;
      var Dest): TSqlDBFieldType;
    /// append the inlined value of a given parameter, mainly for GetSqlWithInlinedParams
    // - optional MaxCharCount will truncate the text to a given number of chars
    procedure AddParamValueAsText(Param: integer; Dest: TJsonWriter;
      MaxCharCount: integer); virtual;
    /// return a Column as a variant - for ISqlDBRows.Column[] default property
    function GetColumnVariant(const ColName: RawUtf8): Variant;
    /// return the associated statement instance for a ISqlDBRows interface
    function Instance: TSqlDBStatement;
    /// wrappers to compute sllSQL/sllDB/sllResult SQL context with a local timer
    function SqlLogBegin(Level: TSynLogLevel): TSynLog;
      {$ifdef HASINLINE} inline; {$endif}
    function SqlLogEnd(const Fmt: RawUtf8; const Args: array of const): Int64; overload;
    function SqlLogEnd(Msg: PShortString = nil): Int64; overload;
      {$ifdef HASINLINE} inline; {$endif}
    {$ifndef SYNDB_SILENCE}
    function DoSqlLogBegin(Log: TSynLogFamily; Level: TSynLogLevel): TSynLog;
    function DoSqlLogEnd(Msg: PShortString): Int64;
    {$endif SYNDB_SILENCE}
    property fExpectResults: boolean // internal compatibility wrapper
      index dsfExpectResults read GetFlag write SetFlag;
  public
    /// create a statement instance
    constructor Create(aConnection: TSqlDBConnection); virtual;

    /// bind a NULL value to a parameter
    // - the leftmost SQL parameter has an index of 1
    // - some providers (e.g. OleDB during MULTI INSERT statements) expect the
    // proper column type to be set in BoundType, even for NULL values
    procedure BindNull(Param: integer; IO: TSqlDBParamInOutType = paramIn;
      BoundType: TSqlDBFieldType = ftNull); virtual; abstract;
    /// bind an integer value to a parameter
    // - the leftmost SQL parameter has an index of 1
    procedure Bind(Param: integer; Value: Int64;
      IO: TSqlDBParamInOutType = paramIn); overload; virtual; abstract;
    /// bind a double value to a parameter
    // - the leftmost SQL parameter has an index of 1
    procedure Bind(Param: integer; Value: double;
      IO: TSqlDBParamInOutType = paramIn); overload; virtual; abstract;
    /// bind a TDateTime value to a parameter
    // - the leftmost SQL parameter has an index of 1
    procedure BindDateTime(Param: integer; Value: TDateTime;
      IO: TSqlDBParamInOutType = paramIn); overload; virtual; abstract;
    /// bind a currency value to a parameter
    // - the leftmost SQL parameter has an index of 1
    procedure BindCurrency(Param: integer; Value: currency;
      IO: TSqlDBParamInOutType = paramIn); overload; virtual; abstract;
    /// bind a UTF-8 encoded string to a parameter
    // - the leftmost SQL parameter has an index of 1
    procedure BindTextU(Param: integer; const Value: RawUtf8;
      IO: TSqlDBParamInOutType = paramIn); overload; virtual; abstract;
    /// bind a UTF-8 encoded buffer text (#0 ended) to a parameter
    // - the leftmost SQL parameter has an index of 1
    procedure BindTextP(Param: integer; Value: PUtf8Char;
      IO: TSqlDBParamInOutType = paramIn); overload; virtual; abstract;
    /// bind a UTF-8 encoded string to a parameter
    // - the leftmost SQL parameter has an index of 1
    procedure BindTextS(Param: integer; const Value: string;
      IO: TSqlDBParamInOutType = paramIn); overload; virtual; abstract;
    /// bind a UTF-8 encoded string to a parameter
    // - the leftmost SQL parameter has an index of 1
    procedure BindTextW(Param: integer; const Value: WideString;
      IO: TSqlDBParamInOutType = paramIn); overload; virtual; abstract;
    /// bind a Blob buffer to a parameter
    // - the leftmost SQL parameter has an index of 1
    procedure BindBlob(Param: integer; Data: pointer; Size: integer;
      IO: TSqlDBParamInOutType = paramIn); overload; virtual; abstract;
    /// bind a Blob buffer to a parameter
    // - the leftmost SQL parameter has an index of 1
    procedure BindBlob(Param: integer; const Data: RawByteString;
      IO: TSqlDBParamInOutType = paramIn); overload; virtual; abstract;
    /// bind a Variant value to a parameter
    // - the leftmost SQL parameter has an index of 1
    // - will call all virtual Bind*() methods from the Data type
    // - if DataIsBlob is TRUE, will call BindBlob(RawByteString(Data)) instead
    // of BindTextW(WideString(Variant)) - used e.g. by TQuery.AsBlob/AsBytes
    procedure BindVariant(Param: integer; const Data: Variant;
      DataIsBlob: boolean; IO: TSqlDBParamInOutType = paramIn); virtual;
    /// bind one TSqlVar value
    // - the leftmost SQL parameter has an index of 1
    // - this default implementation will call corresponding Bind*() method
    procedure Bind(Param: integer; const Data: TSqlVar;
      IO: TSqlDBParamInOutType = paramIn); overload; virtual;
    /// bind one RawUtf8 encoded value
    // - the leftmost SQL parameter has an index of 1
    // - the value should match the BindArray() format, i.e. be stored as in SQL
    // (i.e. number, 'quoted string', 'YYYY-MM-DD hh:mm:ss', null) - e.g. as
    // computed by TJsonObjectDecoder.Decode()
    procedure Bind(Param: integer; ParamType: TSqlDBFieldType;
      const Value: RawUtf8; ValueAlreadyUnquoted: boolean;
      IO: TSqlDBParamInOutType = paramIn); overload; virtual;
    /// bind an array of const values
    // - parameters marked as ? should be specified as method parameter in Params[]
    // - BLOB parameters can be bound with this method, when set after encoding
    // via BinToBase64WithMagic() call
    // - TDateTime parameters can be bound with this method, when encoded via
    // a DateToSql() or DateTimeToSql() call
    // - any variant parameter will be bound with BindVariant(i,VVariant^,true,IO)
    // i.e. with DataIsBlob=true
    // - this default implementation will call corresponding Bind*() method
    procedure Bind(const Params: array of const;
      IO: TSqlDBParamInOutType = paramIn); overload; virtual;
    /// bind an array of fields from an existing SQL statement
    // - can be used e.g. after ColumnsToSqlInsert() method call for fast data
    // conversion between tables
    procedure BindFromRows(
      const Fields: TSqlDBFieldTypeDynArray; Rows: TSqlDBStatement);
    /// bind a special CURSOR parameter to be returned as a mormot.db.sql result set
    // - Cursors are not handled internally by mORMot, but some databases (e.g.
    // Oracle) usually use such structures to get data from strored procedures
    // - such parameters are mapped as ftUnknown
    // - use BoundCursor() method to retrieve the corresponding ISqlDBRows after
    // execution of the statement
    // - this default method will raise an exception about unexpected behavior
    procedure BindCursor(Param: integer); virtual;
    /// return a special CURSOR parameter content as a mormot.db.sql result set
    // - this method is not about a column, but a parameter defined with
    // BindCursor() before method execution
    // - Cursors are not handled internally by mORMot, but some databases (e.g.
    // Oracle) usually use such structures to get data from strored procedures
    // - this method allow direct access to the data rows after execution
    // - this default method will raise an exception about unexpected behavior
    function BoundCursor(Param: integer): ISqlDBRows; virtual;

    /// bind an array of values to a parameter
    // - the leftmost SQL parameter has an index of 1
    // - values are stored as in SQL (i.e. number, 'quoted string',
    // 'YYYY-MM-DD hh:mm:ss', null)
    // - this default implementation will raise an exception if the engine
    // does not support array binding
    procedure BindArray(Param: integer; ParamType: TSqlDBFieldType;
      const Values: TRawUtf8DynArray; ValuesCount: integer); overload; virtual;
    /// bind an array of integer values to a parameter
    // - the leftmost SQL parameter has an index of 1
    // - this default implementation will raise an exception if the engine
    // does not support array binding
    procedure BindArray(Param: integer;
      const Values: array of Int64); overload; virtual;
    /// bind an array of double values to a parameter
    // - the leftmost SQL parameter has an index of 1
    // - this default implementation will raise an exception if the engine
    // does not support array binding
    procedure BindArray(Param: integer;
      const Values: array of double); overload; virtual;
    /// bind an array of TDateTime values to a parameter
    // - the leftmost SQL parameter has an index of 1
    // - values are stored as in SQL (i.e. 'YYYY-MM-DD hh:mm:ss')
    // - this default implementation will raise an exception if the engine
    // does not support array binding
    procedure BindArrayDateTime(Param: integer;
      const Values: array of TDateTime); virtual;
    /// bind an array of currency values to a parameter
    // - the leftmost SQL parameter has an index of 1
    // - this default implementation will raise an exception if the engine
    // does not support array binding
    procedure BindArrayCurrency(Param: integer;
      const Values: array of currency); virtual;
    /// bind an array of RawUtf8 values to a parameter
    // - the leftmost SQL parameter has an index of 1
    // - values are stored as in SQL (i.e. 'quoted string')
    // - this default implementation will raise an exception if the engine
    // does not support array binding
    procedure BindArray(Param: integer;
      const Values: array of RawUtf8); overload; virtual;
    /// bind an array of JSON values to a parameter
    // - the leftmost SQL parameter has an index of 1
    // - values are stored as JSON Arrays (i.e. [1,2,3] or ["a","b","c"])
    // - this default implementation will call JsonArrayToBoundArray()
    // then BindArray(array of RawUtf8)
    // - warning: JsonArray could be parsed in-place so a private copy is needed
    procedure BindArrayJson(Param: integer; ParamType: TSqlDBFieldType;
      var JsonArray: RawUtf8; ValuesCount: integer); virtual;

    /// Prepare an UTF-8 encoded SQL statement
    // - parameters marked as ? will be bound later, before ExecutePrepared call
    // - if ExpectResults is TRUE, then Step() and Column*() methods are available
    // to retrieve the data rows
    // - should raise an Exception on any error
    // - this default implementation will just store aSql content and the
    // ExpectResults parameter, and connect to the remote server is was not
    // already connected
    procedure Prepare(const aSql: RawUtf8; ExpectResults: boolean); overload; virtual;
    /// Execute a prepared SQL statement
    // - parameters marked as ? should have been already bound with Bind*() functions
    // - should raise an Exception on any error
    // - this void default implementation will call set fConnection.fLastAccess
    procedure ExecutePrepared; virtual;
    /// release cursor memory and resources once Step loop is finished
    // - this method call is optional, but is better be used if the ISqlDBRows
    // statement from taken from cache, and returned a lot of content which
    // may still be in client (and server) memory
    // - override to free cursor memory when ISqlDBStatement is back in cache
    procedure ReleaseRows; virtual;
    /// Reset the previous prepared statement
    // - some drivers expect an explicit reset before binding parameters and
    // executing the statement another time
    // - this default implementation will just do nothing
    procedure Reset; virtual;
    /// Prepare and Execute an UTF-8 encoded SQL statement
    // - parameters marked as ? should have been already bound with Bind*()
    //  functions above
    // - if ExpectResults is TRUE, then Step() and Column*() methods are available
    //  to retrieve the data rows
    // - should raise an Exception on any error
    // - this method will call Prepare then ExecutePrepared methods
    procedure Execute(const aSql: RawUtf8; ExpectResults: boolean); overload;
    /// Prepare and Execute an UTF-8 encoded SQL statement
    // - parameters marked as ? should be specified as method parameter in Params[]
    // - BLOB parameters could not be bound with this method, but need an explicit
    // call to BindBlob() method
    // - if ExpectResults is TRUE, then Step() and Column*() methods are available
    // to retrieve the data rows
    // - should raise an Exception on any error
    // - this method will bind parameters, then call Excecute() virtual method
    procedure Execute(const aSql: RawUtf8; ExpectResults: boolean;
      const Params: array of const); overload;
    /// Prepare and Execute an UTF-8 encoded SQL statement
    // - parameters marked as % will be replaced by Args[] value in the SQL text
    // - parameters marked as ? should be specified as method parameter in Params[]
    // - so could be used as such, mixing both % and ? parameters:
    // ! Statement.Execute('SELECT % FROM % WHERE RowID=?',true,[FieldName,TableName],[ID])
    // - BLOB parameters could not be bound with this method, but need an explicit
    // call to BindBlob() method
    // - if ExpectResults is TRUE, then Step() and Column*() methods are available
    // to retrieve the data rows
    // - should raise an Exception on any error
    // - this method will bind parameters, then call Excecute() virtual method
    procedure Execute(const SqlFormat: RawUtf8; ExpectResults: boolean;
      const Args, Params: array of const); overload;
    /// execute a prepared SQL statement and return all rows content as a JSON string
    // - JSON data is retrieved with UTF-8 encoding
    // - if Expanded is true, JSON data is an array of objects, for direct use
    // with any Ajax or .NET client:
    // & [ {"col1":val11,"col2":"val12"},{"col1":val21,... ]
    // - if Expanded is false, JSON data is serialized (used in TOrmTableJson)
    // & { "FieldCount":1,"Values":["col1","col2",val11,"val12",val21,..] }
    // - BLOB field value is saved as Base64, in the '"\uFFF0base64encodedbinary"'
    // format and contains true BLOB data
    // - this virtual implementation calls ExecutePrepared then FetchAllAsJson()
    procedure ExecutePreparedAndFetchAllAsJson(Expanded: boolean;
      out Json: RawUtf8; ReturnedRowCount: PPtrInt = nil); virtual;
    /// gets a number of updates made by latest executed statement
    // - default implementation returns 0
    function UpdateCount: integer; virtual;
    /// retrieve the parameter content, after SQL execution
    // - the leftmost SQL parameter has an index of 1
    // - to be used e.g. with stored procedures:
    // ! query :=  'BEGIN TEST_PKG.DUMMY(?, ?, ?, ?, ?); END;';
    // ! stmt := Props.NewThreadSafeStatementPrepared(query, false);
    // ! stmt.Bind(1, in1, paramIn);
    // ! stmt.BindTextU(2, in2, paramIn);
    // ! stmt.BindTextU(3, in3, paramIn);
    // ! stmt.BindTextS(4, '', paramOut); //  to be retrieved with out1: string
    // ! stmt.Bind(5, 0, paramOut);       //  to be retrieved with out2: integer
    // ! stmt.ExecutePrepared;
    // ! stmt.ParamToVariant(4, out1, true);
    // ! stmt.ParamToVariant(5, out2, true);
    // - the parameter should have been bound with IO=paramOut or IO=paramInOut
    // if CheckIsOutParameter is TRUE
    // - this implementation just check that Param is correct: overridden method
    // should fill Value content
    function ParamToVariant(Param: integer; var Value: Variant;
      CheckIsOutParameter: boolean = true): TSqlDBFieldType; virtual;

    /// After a statement has been prepared via Prepare() + ExecutePrepared() or
    // Execute(), this method must be called one or more times to evaluate it
    // - you shall call this method before calling any Column*() methods
    // - return TRUE on success, with data ready to be retrieved by Column*()
    // - return FALSE if no more row is available (e.g. if the SQL statement
    // is not a SELECT but an UPDATE or INSERT command)
    // - access the first or next row of data from the SQL Statement result:
    // if SeekFirst is TRUE, will put the cursor on the first row of results,
    // otherwise, it will fetch one row of data, to be called within a loop
    // - should raise an Exception on any error
    // - typical use may be (see also e.g. the mormot.orm.sql unit):
    // ! var Query: ISqlDBStatement;
    // ! begin
    // !   Query := Props.NewThreadSafeStatementPrepared(
    // !    'select AccountNumber from Sales.Customer where AccountNumber like ?',
    // !    ['AW000001%'], true);
    // !   if Query <> nil then
    // !   begin
    // !     assert(SameTextU(Query.ColumnName(0), 'AccountNumber'));
    // !     while Query.Step do //  loop through all matching data rows
    // !       assert(Copy(Query.ColumnUtf8(0), 1, 8) = 'AW000001');
    // !     Query.ReleaseRows;
    // !   end;
    // ! end;
    function Step(SeekFirst: boolean = false): boolean; virtual; abstract;
    /// retrieve one row of the result set as a JSON object
    // - see also FetchAllAsJson to retrieve all the raws as a JSON result
    function StepAsJson(SeekFirst: boolean = false): RawUtf8; virtual;
    /// retrieve one row of the resultset as a JSON object into a TResultsWriter
    // - see also FetchAllToJson to retrieve all the raws into a JSON result
    function StepToJson(W: TJsonWriter; SeekFirst: boolean = false): boolean; virtual;
    /// the column/field count of the current Row
    function ColumnCount: integer;
    /// the Column name of the current Row
    // - Columns numeration (i.e. Col value) starts with 0
    // - it's up to the implementation to ensure than all column names are unique
    function ColumnName(Col: integer): RawUtf8; virtual; abstract;
    /// returns the Column index of a given Column name
    // - Columns numeration (i.e. Col value) starts with 0
    // - returns -1 if the Column name is not found (via case insensitive search)
    function ColumnIndex(const aColumnName: RawUtf8): integer; virtual; abstract;
    /// the Column type of the current Row
    // - FieldSize can be set to store the size in chars of a ftUtf8 column
    // (0 means BLOB kind of TEXT column)
    // - warning: FieldSize^ should be a 32-bit "integer" variable, not a PtrInt
    function ColumnType(Col: integer;
      FieldSize: PInteger = nil): TSqlDBFieldType; virtual; abstract;
    /// returns TRUE if the column contains NULL, first Col is 0
    function ColumnNull(Col: integer): boolean; overload; virtual; abstract;
    /// return a Column integer value of the current Row, first Col is 0
    function ColumnInt(Col: integer): Int64; overload; virtual; abstract;
    /// return a Column floating point value of the current Row, first Col is 0
    function ColumnDouble(Col: integer): double; overload; virtual; abstract;
    /// return a Column date and time value of the current Row, first Col is 0
    function ColumnDateTime(Col: integer): TDateTime; overload; virtual; abstract;
    /// return a column date and time value of the current Row, first Col is 0
    // - call ColumnDateTime or ColumnUtf8 to convert into TTimeLogBits/Int64 time
    // stamp from a TDateTime or text
    function ColumnTimestamp(Col: integer): TTimeLog; overload;
    /// return a Column currency value of the current Row, first Col is 0
    function ColumnCurrency(Col: integer): currency; overload; virtual; abstract;
    /// return a Column UTF-8 encoded text value of the current Row, first Col is 0
    function ColumnUtf8(Col: integer): RawUtf8; overload; virtual; abstract;
    /// return a Column UTF-8 text buffer of the current Row, first Col is 0
    // - default implementation returning nil, i.e. not supported by this provider
    // - returned pointer is likely to last only until next Step or Reset call
    function ColumnPUtf8(Col: integer): PUtf8Char; virtual;
    /// return a Column text value as RTL string of the current Row, first Col is 0
    // - this default implementation will call ColumnUtf8
    function ColumnString(Col: integer): string; overload; virtual;
    /// return a Column as a blob value of the current Row, first Col is 0
    function ColumnBlob(Col: integer): RawByteString; overload; virtual; abstract;
    /// return a Column as a blob value of the current Row, first Col is 0
    // - this function will return the BLOB content as a TBytes
    // - this default virtual method will call ColumnBlob()
    function ColumnBlobBytes(Col: integer): TBytes; overload; virtual;
    /// read a blob Column into the Stream parameter
    // - default implementation will just call ColumnBlob(), whereas some
    // providers (like mormot.db.sql.oracle) may implement direct support
    procedure ColumnBlobToStream(Col: integer; Stream: TStream); overload; virtual;
    /// write a blob Column into the Stream parameter
    // - expected to be used with 'SELECT .. FOR UPDATE' locking statements
    // - default implementation will through an exception, since it is highly
    // provider-specific; mormot.db.sql.oracle e.g. implements it properly
    procedure ColumnBlobFromStream(Col: integer; Stream: TStream); overload; virtual;
    /// return a Column as a variant, first Col is 0
    // - this default implementation will call ColumnToVariant() method
    // - a ftUtf8 TEXT content will return a varUString on Delphi/FPC or an AnsiString
    // varString on Delphi 7-2007 (Connection.Properties.VariantStringAsWideString=true
    // could return a varOleStr) unless ForceUtf8=true would return a RawUtf8 varString
    // - a ftBlob BLOB content will be mapped into a TBlobData AnsiString variant
    function ColumnVariant(Col: integer; ForceUtf8: boolean = false): Variant; overload;
    /// return a Column as a variant, first Col is 0
    // - this default implementation will call efficient ColumnToSqlVar() method
    // - a ftUtf8 TEXT content will return a varUString on Delphi/FPC or an AnsiString
    // varString on Delphi 7-2007 (Connection.Properties.VariantStringAsWideString=true
    // could return a varOleStr) unless ForceUtf8=true would return a RawUtf8 varString
    // - a ftBlob BLOB content will be mapped into a TBlobData AnsiString variant
    function ColumnToVariant(Col: integer; var Value: Variant;
      ForceUtf8: boolean = false): TSqlDBFieldType; virtual;
    /// return a Column as a TSqlVar value, first Col is 0
    // - the specified Temp variable will be used for temporary storage of
    // ftUtf8/ftBlob values
    procedure ColumnToSqlVar(Col: integer; var Value: TSqlVar;
      var Temp: RawByteString); virtual;
    /// return a special CURSOR Column content as a mormot.db.sql result set
    // - Cursors are not handled internally by mORMot, but some databases (e.g.
    // Oracle) usually use such structures to get data from strored procedures
    // - such columns are mapped as ftNull internally - so this method is the only
    // one giving access to the data rows
    // - this default method will raise an exception about unexpected behavior
    function ColumnCursor(Col: integer): ISqlDBRows; overload; virtual;
    /// returns TRUE if the column contains NULL, from a supplied column name
    function ColumnNull(const ColName: RawUtf8): boolean; overload;
    /// return a Column integer value of the current Row, from a supplied column name
    function ColumnInt(const ColName: RawUtf8): Int64; overload;
    /// return a Column floating point value of the current Row, from a supplied column name
    function ColumnDouble(const ColName: RawUtf8): double; overload;
    /// return a Column date and time value of the current Row, from a supplied column name
    function ColumnDateTime(const ColName: RawUtf8): TDateTime; overload;
    /// return a column date and time value of the current Row, from a supplied column name
    // - call ColumnDateTime or ColumnUtf8 to convert into TTimeLogBits/Int64 time
    // stamp from a TDateTime or text
    function ColumnTimestamp(const ColName: RawUtf8): TTimeLog; overload;
    /// return a Column currency value of the current Row, from a supplied column name
    function ColumnCurrency(const ColName: RawUtf8): currency; overload;
    /// return a Column UTF-8 encoded text value of the current Row, from a supplied column name
    function ColumnUtf8(const ColName: RawUtf8): RawUtf8; overload;
    /// return a Column text value as RTL string of the current Row, from a supplied column name
    function ColumnString(const ColName: RawUtf8): string; overload;
    /// return a Column as a blob value of the current Row, from a supplied column name
    function ColumnBlob(const ColName: RawUtf8): RawByteString; overload;
    /// return a Column as a blob value of the current Row, from a supplied column name
    function ColumnBlobBytes(const ColName: RawUtf8): TBytes; overload;
    /// read a blob Column into the Stream parameter
    procedure ColumnBlobToStream(const ColName: RawUtf8; Stream: TStream); overload;
    /// write a blob Column into the Stream parameter
    // - expected to be used with 'SELECT .. FOR UPDATE' locking statements
    procedure ColumnBlobFromStream(const ColName: RawUtf8; Stream: TStream); overload;
    /// return a Column as a variant, from a supplied column name
    function ColumnVariant(const ColName: RawUtf8; ForceUtf8: boolean = false): Variant; overload;
    /// create a TSqlDBRowVariantType able to access any field content via late binding
    // - i.e. you can use Data.Name to access the 'Name' column of the current row
    // - this Variant will point to the corresponding TSqlDBStatement instance,
    // so it's not necessary to retrieve its value for each row
    // - typical use is:
    // ! var Row: Variant;
    // ! (...)
    // !  with MyConnProps.Execute('select * from table where name=?',[aName]) do
    // !  begin
    // !    Row := RowDaa;
    // !    while Step do
    // !      writeln(Row.FirstName,Row.BirthDate);
    // !    ReleaseRows;
    // !  end;
    function RowData: Variant; virtual;
    /// create a TDocVariant custom variant containing all columns values
    // - will create a "fast" TDocVariant object instance with all fields
    // - see also FetchAllToDocVariantArray() method
    procedure RowDocVariant(out aDocument: variant;
      aOptions: TDocVariantOptions = JSON_FAST); virtual;
    /// return a special CURSOR Column content as a mormot.db.sql result set
    // - Cursors are not handled internally by mORMot, but some databases (e.g.
    // Oracle) usually use such structures to get data from strored procedures
    // - such columns are mapped as ftNull internally - so this method is the only
    // one giving access to the data rows
    // - this default method will raise an exception about unexpected behavior
    function ColumnCursor(const ColName: RawUtf8): ISqlDBRows; overload;
    /// append a Column as a JSON value, first Col is 0
    // - in practice, this method is always overriden, so never called as such
    procedure ColumnToJson(Col: integer; W: TJsonWriter); virtual;
    /// append all columns values of the current Row to a JSON stream
    // - will use WR.Expand to guess the expected output format
    // - this default implementation will call Column*() methods above, but you
    // should also implement a custom version with no temporary variable
    // - BLOB field value is saved as Base64, in the '"\uFFF0base64encodedbinary"
    // format and contains true BLOB data (unless ForceBlobAsNull property was set)
    procedure ColumnsToJson(WR: TResultsWriter); virtual;
    /// compute the SQL INSERT statement corresponding to this columns row
    // - and populate the Fields[] array with columns information (type and name)
    // - if the current column value is NULL, will return ftNull: it is up to the
    // caller to set the proper field type
    // - the SQL statement is prepared with bound parameters, e.g.
    // $ insert into TableName (Col1,Col2) values (?,N)
    // - used e.g. to convert some data on the fly from one database to another,
    // via the TSqlDBConnection.NewTableFromRows method
    // - returns '' and Fields= nil if there is not enough column information
    // to compute the SQL
    function ColumnsToSqlInsert(const TableName: RawUtf8;
      var Fields: TSqlDBColumnCreateDynArray): RawUtf8; virtual;
    // append all rows content as a JSON stream
    // - JSON data is added to the supplied TStream, with UTF-8 encoding
    // - if Expanded is true, JSON data is an array of objects, for direct use
    // with any Ajax or .NET client:
    // & [ {"col1":val11,"col2":"val12"},{"col1":val21,... ]
    // - if Expanded is false, JSON data is serialized (used in TOrmTableJson)
    // & { "FieldCount":1,"Values":["col1","col2",val11,"val12",val21,..] }
    // - BLOB field value is saved as Base64, in the '"\uFFF0base64encodedbinary"'
    // format and contains true BLOB data
    // - similar to corresponding TSqlRequest.Execute method in the
    // mormot.db.raw.sqlite3 unit
    // - returns the number of row data returned (excluding field names)
    // - warning: TRestStorageExternal.EngineRetrieve in mormot.orm.sql unit
    // expects the Expanded=true format to return '[{...}]'#10
    function FetchAllToJson(Json: TStream; Expanded: boolean): PtrInt;
    // Append all rows content as a CSV stream
    // - CSV data is added to the supplied TStream, with UTF-8 encoding
    // - if Tab=TRUE, will use TAB instead of ',' between columns
    // - you can customize the ',' separator - use e.g. the global ListSeparator
    // variable (from SysUtils) to reflect the current system definition (some
    // country use ',' as decimal separator, for instance our "douce France")
    // - AddBOM will add a UTF-8 Byte Order Mark at the beginning of the content
    // - BLOB fields will be appended as "blob" with no data
    // - returns the number of row data returned
    function FetchAllToCsvValues(Dest: TStream; Tab: boolean;
      CommaSep: AnsiChar = ','; AddBOM: boolean = true): PtrInt;
    // return all rows content as a JSON string
    // - JSON data is retrieved with UTF-8 encoding
    // - if Expanded is true, JSON data is an array of objects, for direct use
    // with any Ajax or .NET client:
    // & [ {"col1":val11,"col2":"val12"},{"col1":val21,... ]
    // - if Expanded is false, JSON data is serialized (used in TOrmTableJson)
    // & { "FieldCount":1,"Values":["col1","col2",val11,"val12",val21,..] }
    // - BLOB field value is saved as Base64, in the '"\uFFF0base64encodedbinary"'
    // format and contains true BLOB data
    // - if ReturnedRowCount points to an integer variable, it will be filled with
    // the number of row data returned (excluding field names)
    // - similar to corresponding TSqlRequest.Execute method in the
    // mormot.db.raw.sqlite3 unit
    function FetchAllAsJson(Expanded: boolean; ReturnedRowCount: PPtrInt = nil): RawUtf8;
    /// append all rows content as binary stream
    // - will save the column types and name, then every data row in optimized
    // binary format (faster and smaller than JSON)
    // - you can specify a LIMIT for the data extent (default 0 meaning all data)
    // - generates the format expected by TSqlDBProxyStatement
    function FetchAllToBinary(Dest: TStream; MaxRowCount: cardinal = 0;
      DataRowPosition: PCardinalDynArray = nil): cardinal; virtual;
    /// append current row content as binary stream
    // - will save one data row in optimized binary format (if not in Null)
    // - virtual method called by FetchAllToBinary()
    // - follows the format expected by TSqlDBProxyStatement
    procedure ColumnsToBinary(W: TBufferWriter; Null: pointer;
      const ColTypes: TSqlDBFieldTypeDynArray); virtual;
    /// append all rows content as a new TDocVariant dvArray
    // - save all content directly from the result buffers, with no temporary JSON
    // - you can specify a LIMIT for the data extent (default 0 meaning all data)
    function FetchAllToDocVariantArray(MaxRowCount: cardinal = 0): variant; virtual;
    /// low-level access to the Timer used for last DB operation
    property SqlLogTimer: TPrecisionTimer
      read fSqlLogTimer;
    /// after a call to Prepare(), contains the query text to be passed to the DB
    // - depending on the DB, parameters placeholders are replaced by ?, :1, $1 etc
    // - this SQL is ready to be used in any DB tool, e.g. to check the real
    // execution plan/timing
    property SqlPrepared: RawUtf8
      read fSqlPrepared;
    /// the prepared SQL statement, in its current state
    // - if statement is prepared, then equals SqlPrepared, otherwise, contains
    // the raw SQL property content
    // - used internally by the implementation units, e.g. for errors logging
    property SqlCurrent: RawUtf8
      read GetSqlCurrent;
  published
    /// the prepared SQL statement, as supplied to Prepare() method
    property Sql: RawUtf8
      read fSql;
    /// the prepared SQL statement, with all '?' changed into the supplied
    // parameter values
    // - such statement query plan usually differ from a real execution plan for
    // prepared statements with parameters - see SqlPrepared property instead
    property SqlWithInlinedParams: RawUtf8
      read GetSqlWithInlinedParams;
    /// how this statement has been cached, after a call to Prepare()
    property Cache: TSqlDBStatementCache
      read fCache;
    /// the current row after Execute/Step call, corresponding to Column*() methods
    // - contains 0 before initial Step call, or a number >=1 during data retrieval
    property CurrentRow: integer
      read fCurrentRow;
    /// the total number of data rows retrieved by this instance
    // - is not reset when there is no more row of available data (Step returns
    // false), or when Step() is called with SeekFirst=true
    property TotalRowsRetrieved: integer
      read fTotalRowsRetrieved;
    /// the associated database connection
    // - warning: some constructors may left nil here, e.g. after
    // TSqlDBSQLite3Statement.CreateFrom
    property Connection: TSqlDBConnection
      read fConnection;
    /// strip last semicolon in query
    // - expectation may vary, depending on the SQL statement and the engine
    // - default is true
    property StripSemicolon: boolean
      index dsfStripSemicolon read GetFlag write SetFlag;
  end;


function ToText(c: TSqlDBStatementCache): ShortString; overload;


{ ************ Parent Classes for Thread-Safe and Parametrized Connections }

type
  /// abstract connection created from TSqlDBConnectionProperties
  // - we will rely on efficient TSynLog.ThreadIndex round-robin slot numbers
  // to properly implement thread-safety and connection reuse with no TThreadID
  // - e.g. OleDB, ODBC and Oracle connections will inherit from this class
  TSqlDBConnectionThreadSafe = class(TSqlDBConnection)
  public
    /// finalize this connection, and remove it from any internal pool
    destructor Destroy; override;
  end;

  /// threading modes set to TSqlDBConnectionPropertiesThreadSafe.ThreadingMode
  // - default mode is to use a Thread Pool, i.e. one connection per thread
  // - or you can force to use the main connection - to be used for embedded
  // databases (SQLite3/FireBird), when multiple connections may break
  // stability, consume too much resources and/or decrease performance
  TSqlDBConnectionPropertiesThreadSafeThreadingMode = (
    tmThreadPool,
    tmMainConnection);

  /// connection properties with an internal Thread-Safe connection pool
  // - will use TSynLog.ThreadIndex to maintain the internal list of active
  // connections, with proper deprecation and TSynLog.NotifyThreadEnded support
  // - initial implementation used an explicit and confusing EndCurrentThread
  // which is not needed any more since is part of TSynLog.NotifyThreadEnded,
  // and allows efficient reuse between short-living threads
  TSqlDBConnectionPropertiesThreadSafe = class(TSqlDBConnectionProperties)
  protected
    fConnectionPool: array of TSqlDBConnectionThreadSafe;
    fConnectionPoolMin, fConnectionPoolMax, fConnectionPoolCount: integer;
    fConnectionPoolDeprecatedTix32: integer;
    fThreadingMode: TSqlDBConnectionPropertiesThreadSafeThreadingMode;
    procedure DeleteDeprecated(secs: integer);
    procedure RemoveFromPool(conn: TSqlDBConnectionThreadSafe);
    /// overridden method to properly handle multi-thread
    function GetMainConnection: TSqlDBConnection; override;
  public
    /// initialize the properties
    // - this overridden method will initialize the internal per-thread connection pool
    constructor Create(const aServerName, aDatabaseName, aUserID, aPassWord: RawUtf8); override;
    /// release related memory, and all per-thread connections
    destructor Destroy; override;
    /// get a thread-safe connection
    // - tmThreadPool will use its internal fConnectionPool[TSynLog.ThreadIndex]
    // but requires TSynLog.NotifyThreadEnded to be eventually called (which is
    // the case when used with our mormot.rest.* and mormot.threads.* units)
    function ThreadSafeConnection: TSqlDBConnection; override;
    /// release all existing connections
    // - this overridden implementation will release all per-thread
    // TSqlDBConnection internal connection pool
    // - warning: no connection shall still be used on the background (e.g. in
    // multi-threaded applications), or some unexpected border effects may occur
    procedure ClearConnectionPool; override;
    /// use this property if you want to disable the tmThreadPool default mode
    // - could be set e.g. in database embedded mode (SQLite3/FireBird), when
    // multiple connections may break stability and decrease performance
    property ThreadingMode: TSqlDBConnectionPropertiesThreadSafeThreadingMode
      read fThreadingMode write fThreadingMode;
    /// by default, deprecated connections after ConnectionTimeOutMinutes will
    // be released as soon as detected, from any thread
    // - set this property to true to force the connection to be released
    // only when accessed from ThreadSafeConnection(), i.e. their own thread:
    // some non-threadsafe database providers may require to free the connection
    // in the very same thread which created it - see also EndCurrentThread
    property DeleteConnectionInOwnThread: boolean
      index cpfDeleteConnectionInOwnThread read GetFlag write SetFlag;
    /// how many TSqlDBConnectionThreadSafe are currently in tmThreadPool mode
    property ConnectionPoolCount: integer
      read fConnectionPoolCount;
  end;

  /// a structure used to store a standard binding parameter
  // - you can use your own internal representation of parameters
  // (TOleDBStatement use its own TOleDBStatementParam type), but
  // this type can be used to implement a generic parameter
  // - used e.g. by TSqlDBStatementWithParams as a dynamic array
  // (and its inherited TSqlDBOracleStatement)
  // - warning: don't change this structure, since it will be serialized as
  // SaveRecord() binary for TSqlDBProxyConnectionCommandExecute
  TSqlDBParam = packed record
    /// storage used for TEXT (ftUtf8) and BLOB (ftBlob) values
    // - ftBlob are stored as RawByteString
    // - ftUtf8 are stored as RawUtf8
    // - sometimes, may be ftInt64 or ftCurrency provided as SQLT_AVC text,
    // or ftDate value converted to SQLT_TIMESTAMP
    VData: RawByteString;
    /// storage used for bound array values
    // - number of items in array is stored in VInt64
    // - values are stored as in SQL (i.e. number, 'quoted string',
    // 'YYYY-MM-DD hh:mm:ss', null)
    VArray: TRawUtf8DynArray;
    /// storage used for ftInt64, ftDouble, ftDate and ftCurrency value
    VInt64: Int64;
    /// the column/parameter Value type
    VType: TSqlDBFieldType;
    /// define if parameter can be retrieved after a stored procedure execution
    VInOut: TSqlDBParamInOutType;
    /// used e.g. by TSqlDBOracleStatement or TSqlDBPostgresStatement
    VDBType: word;
  end;

  PSqlDBParam = ^TSqlDBParam;

  /// dynamic array used to store standard binding parameters
  // - used e.g. by TSqlDBStatementWithParams (and its
  // inherited TSqlDBOracleStatement)
  TSqlDBParamDynArray = array of TSqlDBParam;

  /// generic abstract class handling prepared statements with binding
  // - will provide protected fields and methods for handling standard
  // TSqlDBParam parameters
  TSqlDBStatementWithParams = class(TSqlDBStatement)
  protected
    fParams: TSqlDBParamDynArray;
    fParam: TDynArray;
    fParamsArrayCount: integer;
    function CheckParam(Param: integer; NewType: TSqlDBFieldType;
      IO: TSqlDBParamInOutType): PSqlDBParam; overload;
    function CheckParam(Param: integer; NewType: TSqlDBFieldType;
      IO: TSqlDBParamInOutType; ArrayCount: integer): PSqlDBParam; overload;
    /// append the inlined value of a given parameter
    // - faster overridden method
    procedure AddParamValueAsText(Param: integer; Dest: TJsonWriter;
      MaxCharCount: integer); override;
  public
    /// create a statement instance
    // - this overridden version will initialize the internal fParam* fields
    constructor Create(aConnection: TSqlDBConnection); override;
    /// bind a NULL value to a parameter
    // - the leftmost SQL parameter has an index of 1
    // - raise an Exception on any error
    // - some providers (only OleDB during MULTI INSERT statements, so never used
    // in this class) expect the  proper column type to be set in BoundType
    procedure BindNull(Param: integer; IO: TSqlDBParamInOutType = paramIn;
      BoundType: TSqlDBFieldType = ftNull); override;
    /// bind an integer value to a parameter
    // - the leftmost SQL parameter has an index of 1
    // - raise an Exception on any error
    procedure Bind(Param: integer; Value: Int64;
      IO: TSqlDBParamInOutType = paramIn); overload; override;
    /// bind a double value to a parameter
    // - the leftmost SQL parameter has an index of 1
    // - raise an Exception on any error
    procedure Bind(Param: integer; Value: double;
      IO: TSqlDBParamInOutType = paramIn); overload; override;
    /// bind a TDateTime value to a parameter
    // - the leftmost SQL parameter has an index of 1
    // - raise an Exception on any error
    procedure BindDateTime(Param: integer; Value: TDateTime;
      IO: TSqlDBParamInOutType = paramIn); overload; override;
    /// bind a currency value to a parameter
    // - the leftmost SQL parameter has an index of 1
    // - raise an Exception on any error
    procedure BindCurrency(Param: integer; Value: currency;
      IO: TSqlDBParamInOutType = paramIn); overload; override;
    /// bind a UTF-8 encoded string to a parameter
    // - the leftmost SQL parameter has an index of 1
    // - raise an Exception on any error
    procedure BindTextU(Param: integer; const Value: RawUtf8;
      IO: TSqlDBParamInOutType = paramIn); overload; override;
    /// bind a UTF-8 encoded buffer text (#0 ended) to a parameter
    // - the leftmost SQL parameter has an index of 1
    procedure BindTextP(Param: integer; Value: PUtf8Char;
      IO: TSqlDBParamInOutType = paramIn); overload; override;
    /// bind a RTL string to a parameter
    // - the leftmost SQL parameter has an index of 1
    // - raise an Exception on any error
    procedure BindTextS(Param: integer; const Value: string;
      IO: TSqlDBParamInOutType = paramIn); overload; override;
    /// bind an OLE WideString to a parameter
    // - the leftmost SQL parameter has an index of 1
    // - raise an Exception on any error }
    procedure BindTextW(Param: integer; const Value: WideString;
      IO: TSqlDBParamInOutType = paramIn); overload; override;
    /// bind a Blob buffer to a parameter
    // - the leftmost SQL parameter has an index of 1
    // - raise an Exception on any error
    procedure BindBlob(Param: integer; Data: pointer; Size: integer;
      IO: TSqlDBParamInOutType = paramIn); overload; override;
    /// bind a Blob buffer to a parameter
    // - the leftmost SQL parameter has an index of 1
    // - raise an Exception on any error M
    procedure BindBlob(Param: integer; const Data: RawByteString;
      IO: TSqlDBParamInOutType = paramIn); overload; override;

    /// bind an array of values to a parameter using OCI bind array feature
    // - the leftmost SQL parameter has an index of 1
    // - values are stored as in SQL (i.e. number, 'quoted string',
    // 'YYYY-MM-DD hh:mm:ss', null)
    // - values are stored as in SQL (i.e. 'YYYY-MM-DD hh:mm:ss')
    procedure BindArray(Param: integer; ParamType: TSqlDBFieldType;
      const Values: TRawUtf8DynArray; ValuesCount: integer); overload; override;
    /// bind an array of integer values to a parameter
    // - the leftmost SQL parameter has an index of 1
    // - this default implementation will call BindArray() after conversion into
    // RawUtf8 items, stored in TSqlDBParam.VArray
    procedure BindArray(Param: integer;
      const Values: array of Int64); overload; override;
    /// bind an array of double values to a parameter
    // - the leftmost SQL parameter has an index of 1
    // - this default implementation will raise an exception if the engine
    // does not support array binding
    // - this default implementation will call BindArray() after conversion into
    // RawUtf8 items, stored in TSqlDBParam.VArray
    procedure BindArray(Param: integer;
      const Values: array of double); overload; override;
    /// bind an array of TDateTime values to a parameter
    // - the leftmost SQL parameter has an index of 1
    // - values are stored as in SQL (i.e. 'YYYY-MM-DD hh:mm:ss')
    // - this default implementation will raise an exception if the engine
    // does not support array binding
    // - this default implementation will call BindArray() after conversion into
    // RawUtf8 items, stored in TSqlDBParam.VArray
    procedure BindArrayDateTime(Param: integer;
      const Values: array of TDateTime); override;
    /// bind an array of currency values to a parameter
    // - the leftmost SQL parameter has an index of 1
    // - this default implementation will raise an exception if the engine
    // does not support array binding
    // - this default implementation will call BindArray() after conversion into
    // RawUtf8 items, stored in TSqlDBParam.VArray
    procedure BindArrayCurrency(Param: integer;
      const Values: array of currency); override;
    /// bind an array of RawUtf8 values to a parameter
    // - the leftmost SQL parameter has an index of 1
    // - values are stored as 'quoted string'
    // - this default implementation will raise an exception if the engine
    // does not support array binding
    procedure BindArray(Param: integer;
      const Values: array of RawUtf8); overload; override;
    /// bind an array of JSON values to a parameter
    // - the leftmost SQL parameter has an index of 1
    // - values are stored as JSON Arrays (i.e. [1,2,3] or ["a","b","c"])
    // - this default implementation will call JsonArrayToBoundArray()
    // then BindArray(array of RawUtf8)
    // - warning: JsonArray could be parsed in-place so a private copy is needed
    procedure BindArrayJson(Param: integer; ParamType: TSqlDBFieldType;
      var JsonArray: RawUtf8; ValuesCount: integer); override;

    /// start parameter array binding per-row process
    // - BindArray*() methods expect the data to be supplied "verticaly": this
    // method allow-per row binding
    // - call this method, then BindArrayRow() with the corresponding values for
    // one statement row, then Execute to send the query
    procedure BindArrayRowPrepare(const aParamTypes: array of TSqlDBFieldType;
      aExpectedMinimalRowCount: integer = 0);
    /// bind a set of parameters for further array binding
    // - supplied parameters shall follow the BindArrayRowPrepare() supplied
    // types (i.e. RawUtf8, integer/Int64, double);  you can also bind directly
    // a TDateTime value if the corresponding binding has been defined as ftDate
    // by BindArrayRowPrepare()
    procedure BindArrayRow(const aValues: array of const);
    /// bind an array of fields from an existing SQL statement for array binding
    // - supplied Rows columns shall follow the BindArrayRowPrepare() supplied
    // types (i.e. RawUtf8, integer/Int64, double, date)
    // - can be used e.g. after ColumnsToSqlInsert() method call for fast data
    // conversion between tables
    procedure BindFromRows(Rows: TSqlDBStatement); virtual;

    /// retrieve the parameter content, after SQL execution
    // - the leftmost SQL parameter has an index of 1
    // - to be used e.g. with stored procedures
    // - this overridden function will retrieve the value stored in the protected
    // fParams[] array: the ExecutePrepared method should have updated its
    // content as exepcted
    function ParamToVariant(Param: integer; var Value: Variant;
      CheckIsOutParameter: boolean = true): TSqlDBFieldType; override;

    /// Reset the previous prepared statement
    // - this overridden implementation will just do reset the internal fParams[]
    procedure Reset; override;
    /// Release used memory
    // - this overridden implementation will free the fParams[] members (e.g.
    // VData) but not the parameters themselves
    procedure ReleaseRows; override;
  end;

  /// generic abstract class handling prepared statements with binding
  // and column description
  // - will provide protected fields and methods for handling both TSqlDBParam
  // parameters and standard TSqlDBColumnProperty column description
  TSqlDBStatementWithParamsAndColumns = class(TSqlDBStatementWithParams)
  protected
    fColumns: TSqlDBColumnPropertyDynArray;
    fColumn: TDynArrayHashed;
    procedure ClearColumns;
    function AddColumn(const ColName: RawUtf8): PSqlDBColumnProperty;
  public
    /// create a statement instance
    // - this overridden version will initialize the internal fColumn* fields
    constructor Create(aConnection: TSqlDBConnection); override;
    /// retrieve one row of the resultset as a JSON object into a TResultsWriter
    function StepToJson(W: TJsonWriter; SeekFirst: boolean = false): boolean; override;
    /// retrieve a column name of the current Row
    // - Columns numeration (i.e. Col value) starts with 0
    // - it's up to the implementation to ensure than all column names are unique
    function ColumnName(Col: integer): RawUtf8; override;
    /// returns the Column index of a given Column name
    // - Columns numeration (i.e. Col value) starts with 0
    // - returns -1 if the Column name is not found (via case insensitive search)
    function ColumnIndex(const aColumnName: RawUtf8): integer; override;
    /// the Column type of the current Row
    // - ftCurrency type should be handled specifically, for faster process and
    // avoid any rounding issue, since currency is a standard OleDB type
    // - FieldSize can be set to store the size in chars of a ftUtf8 column
    // (0 means BLOB kind of TEXT column) - this implementation will store
    // fColumns[Col].ColumnValueDBSize if ColumnValueInlined=true
    // - warning: FieldSize^ should be a 32-bit "integer" variable, not a PtrInt
    function ColumnType(Col: integer;
      FieldSize: PInteger = nil): TSqlDBFieldType; override;
    /// direct access to the columns description
    // - gives more details than the default ColumnType() function
    property Columns: TSqlDBColumnPropertyDynArray
      read fColumns;
  end;



implementation


{ ************ General SQL Processing Functions }

procedure LogTruncatedColumn(Sender: TObject; const Col: TSqlDBColumnProperty);
begin
  SynDBLog.Add.Log(sllDB, 'Truncated column %', [Col.ColumnName], Sender);
end;

function TrimLeftSchema(const TableName: RawUtf8): RawUtf8;
var
  i, j: integer;
begin
  j := 1;
  repeat
    i := PosEx('.', TableName, j);
    if i = 0 then
      break;
    j := i + 1;
  until false;
  if j = 1 then
    result := TableName
  else
    result := copy(TableName, j, maxInt);
end;

function ReplaceParamsByNames(const aSql: RawUtf8; var aNewSql: RawUtf8;
  aStripSemicolon: boolean): integer;
var
  i, j, B, L: PtrInt;
  P: PAnsiChar;
  c: array[0..3] of AnsiChar;
  tmp: RawUtf8;
begin
  result := 0;
  L := Length(aSql);
  if aStripSemicolon then
    while (L > 0) and
          (aSql[L] in [#1..' ', ';']) do
      if (aSql[L] = ';') and
         (L > 5) and
         IdemPChar(@aSql[L - 3], 'END') then
        break
      else // allows 'END;' at the end of a statement
        dec(L);    // trim ' ' or ';' right (last ';' could be found incorrect)
  if PosExChar('?', aSql) > 0 then
  begin
    aNewSql := '';
    // change ? into :AA :BA ..
    c := ':AA';
    i := 0;
    P := pointer(aSql);
    if P <> nil then
      repeat
        B := i;
        while (i < L) and
              (P[i] <> '?') do
        begin
          if P[i] = '''' then
          begin
            repeat // ignore chars inside ' quotes
              inc(i);
            until (i = L) or
                  ((P[i] = '''') and
                   (P[i + 1] <> ''''));
            if i = L then
              break;
          end;
          inc(i);
        end;
        FastSetString(tmp, P + B, i - B);
        aNewSql := aNewSql + tmp;
        if i = L then
          break;
        // store :AA :BA ..
        j := length(aNewSql);
        SetLength(aNewSql, j + 3);
        PCardinal(PtrInt(aNewSql) + j)^ := PCardinal(@c)^;
        repeat
          if c[1] = 'Z' then
          begin
            if c[2] = 'Z' then
              ESqlDBException.RaiseU(
                'Only up to 656 parameters are possible in :AA to :ZZ range');
            c[1] := 'A';
            inc(c[2]);
          end
          else
            inc(c[1]);
        until not IsSqlReservedByTwo(@c[1]);
        inc(result);
        inc(i); // jump '?'
      until i = L;
  end
  else
    aNewSql := copy(aSql, 1, L); // trim right ';' if any
end;

function ReplaceParamsByNumbers(const aSql: RawUtf8; var aNewSql: RawUtf8;
  IndexChar: AnsiChar; AllowSemicolon: boolean): integer;
var
  ndx, L: PtrInt;
  s, d: PUtf8Char;
  c: AnsiChar;
begin
  aNewSql := aSql;
  result := 0;
  ndx := 0;
  L := Length(aSql);
  s := pointer(aSql);
  if (s = nil) or
     (PosExChar('?', aSql) = 0) then
    exit;
  // calculate ? parameters count, check for ;
  while s^ <> #0 do
  begin
    c := s^;
    if c = '?' then
    begin
      inc(ndx);
      if ndx > 9 then  // ? will be replaced by $n $nn $nnn
        if ndx > 99 then
          if ndx > 999 then
            exit
          else
            inc(L, 3)
        else
          inc(L, 2)
        else
          inc(L);
    end
    else if c = '''' then
    begin
      repeat
        inc(s);
        c := s^;
        if c = #0 then
          exit; // quote without proper ending -> reject
        if c = '''' then
          if s[1] = c then
            inc(s) // ignore double quotes between single quotes
          else
            break;
      until false;
    end
    else if (c = ';') and
                not AllowSemicolon then
      exit; // complex expression can not be prepared
    inc(s);
  end;
  if ndx = 0 then // no ? parameter
    exit;
  result := ndx;
  // parse SQL and replace ? into $n $nn $nnn
  d := FastSetString(aNewSql, L);
  s := pointer(aSql);
  ndx := 0;
  repeat
    c := s^;
    if c = '?' then
    begin
      d^ := IndexChar; // e.g. '$'
      inc(d);
      inc(ndx);
      d := Append999ToBuffer(d, ndx);
    end
    else if c = '''' then
    begin
      repeat // ignore double quotes between single quotes
        d^ := c;
        inc(d);
        inc(s);
        c := s^;
        if c = '''' then
          if s[1] = c then
          begin
            d^ := c;
            inc(d);
            inc(s) // ignore double quotes between single quotes
          end
          else
            break;
      until false;
      d^ := c; // store last '''
      inc(d);
    end
    else
    begin
      d^ := c;
      inc(d);
    end;
    inc(s);
  until s^ = #0;
  //assert(d - pointer(aNewSql) = length(aNewSql)); // until stabilized
end;

function BoundArrayToJsonArray(const Values: TRawUtf8DynArray;
  Open, Close: AnsiChar): RawUtf8;
begin
  BoundArrayToJsonArray(Values, result, Open, Close);
end;

procedure BoundArrayToJsonArray(const Values: TRawUtf8DynArray;
  out Result: RawUtf8; Open, Close: AnsiChar);
//  'one', 't"wo' -> '{"one","t\"wo"}'  and  1,2,3 -> '{1,2,3}'
var
  V: ^RawUtf8;
  s, d: PUtf8Char;
  L, vl, n: PtrInt;
  c: AnsiChar;
label
  _dq;
begin
  // fist compute the resulting length
  n := length(Values);
  if n = 0 then
    exit;
  L := 1; // trailing '{'
  inc(L, n); // ',' after each element - and ending '}'
  v := pointer(Values);
  repeat
    vl := length(v^);
    if vl <> 0 then
    begin
      inc(L, vl);
      s := pointer(v^);
      if s^ = '''' then
      begin
        // quoted ftUtf8
        dec(vl, 2);
        if vl > 0 then
          repeat
            inc(s);
            c := s^;
            if c = '''' then
            begin
              if s[1] = '''' then
                dec(L); // double ' into single '
            end
            else if (c = '"') or
                    (c = '\') then
              inc(L); // escape \ before "
            dec(vl);
          until vl = 0;
      end;
    end;
    inc(v);
    dec(n);
  until n = 0;
  // generate the output JSON
  d := FastSetString(Result, L);
  d^ := Open;
  inc(d);
  v := pointer(Values);
  n := length(Values);
  repeat
    vl := length(v^);
    if vl <> 0 then
    begin
      s := pointer(v^);
      if s^ = '''' then
      begin
        // quoted ftUtf8
        d^ := '"';
        inc(d);
        dec(vl, 2);
        if vl > 0 then
          repeat
            inc(s);
            c := s^;
            if c = '''' then
            begin
              if s[1] = '''' then
                goto _dq; // double ' into single '
            end
            else if (c = '"') or
                    (c = '\') then
            begin
              d^ := '\'; // escape \ before "
              inc(d);
            end;
            d^ := c;
            inc(d);
_dq:        dec(vl);
          until vl = 0;
        d^ := '"';
        inc(d);
      end
      else
      begin
        // regular content (e.g. numbers)
        MoveFast(s^, d^, vl);
        inc(d, vl);
      end;
    end;
    d^ := ',';
    inc(d);
    inc(v);
    dec(n);
  until n = 0;
  d[-1] := Close; // replace last ',' by '}'
  //assert(d - pointer(result) = length(result)); // until stabilized
end;

function JsonArrayToBoundArray(Json: PUtf8Char; ParamType: TSqlDBFieldType;
  TimeSeparator: AnsiChar; DateMS: boolean; out Values: TRawUtf8DynArray): boolean;
var
  i, n: PtrInt;
  p: TJsonParserContext;
begin
  result := false;
  if Json = nil then
    exit;
  p.InitParser(Json);
  if not p.ParseArray then
    exit;
  n := JsonArrayCount(p.Json); // prefetch input and compute dest length
  if n <= 0 then
    exit;
  SetLength(Values, n);
  for i := 0 to n - 1 do
  begin
    if not p.ParseNextAny then
      exit; // input parsing error
    case ParamType of
      ftUtf8: // SQL single-quoted string
        QuotedStr(p.Value, p.ValueLen, '''', Values[i]);
      ftDate: // normalize
        DateTimeToIso8601Var(Iso8601ToDateTimePUtf8Char(p.Value, p.ValueLen),
          {expanded=}true, DateMS, TimeSeparator, '''', Values[i]);
    else
      if (p.Value = '') and
         not p.WasString then
        Values[i] := 'null'
      else
        FastSetString(Values[i], p.Value, p.ValueLen);
    end;
  end;
  result := true;
end;


{ ************ Define Database Engine Specific Behavior }

function ToText(Dbms: TSqlDBDefinition): PShortString;
begin
  result := GetEnumName(TypeInfo(TSqlDBDefinition), ord(Dbms));
end;


{ ************ Abstract SQL DB Classes and Interfaces }

{ ESqlDBException }

constructor ESqlDBException.CreateUtf8(const Format: RawUtf8;
  const Args: array of const);
begin
  FormatUtf8(Format, Args, fMessageUtf8);
  {$ifndef SYNDB_SILENCE}
  if high(Args) >= 0 then
  begin
    fStatement := VarRecAs(@Args[0], TSqlDBStatement);
    if (fStatement <> nil) and
       fStatement.Connection.Properties.LogSqlStatementOnException then
      try
        Append(fMessageUtf8, [' - ', fStatement.GetSqlWithInlinedParams]);
      except // if parameter access failed -> append with ?
        Append(fMessageUtf8, [' - ', fStatement.Sql]);
      end;
  end;
  {$endif SYNDB_SILENCE}
  CreateAfterSetMessageUtf8;
end;


{ TSqlDBRowVariantType }

function TSqlDBRowVariantType.IntGet(var Dest: TVarData;
  const Instance: TVarData; Name: PAnsiChar; NameLen: PtrInt;
  NoException: boolean): boolean;
var
  smt: TSqlDBStatement;
  col: RawUtf8;
  ndx: integer;
begin
  smt := TSqlDBStatement(Instance.VPointer);
  if smt = nil then
    ESqlDBException.RaiseUtf8('Invalid % call', [self]);
  FastSetString(col, Name, NameLen);
  ndx := smt.ColumnIndex(col);
  result := ndx >= 0;
  if ndx >= 0 then
    smt.ColumnToVariant(ndx, Variant(Dest)); // text may be set as varSynUnicode
end;


{ TSqlDBConnectionProperties }

constructor TSqlDBConnectionProperties.Create(const aServerName, aDatabaseName,
  aUserID, aPassWord: RawUtf8);
var
  db: TSqlDBDefinition;
begin
  fServerName := aServerName;
  fDatabaseName := aDatabaseName;
  fUserID := aUserID;
  fPassWord := aPassWord;
  fEngineName := EngineName;
  fFlags := fFlags + [cpfRollbackOnDisconnect, cpfUseCache];
  fLoggedSqlMaxSize := 2048; // log up to 2KB of inlined SQL by default
  fStatementMaxMemory := 512 shl 20; // fetch to JSON/Binary up to 512MB
  SetInternalProperties; // virtual method used to override default parameters
  db := GetDbms;
  if db in [dSQLite, dDB2, dPostgreSQL] then // for SqlDateToIso8601Quoted()
    fDateTimeFirstChar := ' '
  else
    fDateTimeFirstChar := 'T';
  if fForcedSchemaName = '' then
    case db of // should make every one life's easier
      dMSSQL:
        fForcedSchemaName := 'dbo';
      dPostgreSql:
        fForcedSchemaName := 'public';
    end;
  if fSqlCreateField[ftUnknown] = '' then
    fSqlCreateField := DB_FIELDS[db];
  if fSqlCreateFieldMax = 0 then
    fSqlCreateFieldMax := DB_FIELDSMAX[db];
  if fSqlGetServerTimestamp = '' then
    fSqlGetServerTimestamp := DB_SERVERTIME[db];
  case db of
    dMSSQL,
    dJet:
      StoreVoidStringAsNull := true;
  end;
  if byte(fBatchSendingAbilities) = 0 then // if not already handled by driver
    case db of
      dSQlite,
      dMySQL,
      dMariaDB,
      dPostgreSQL,
      dNexusDB,
      dMSSQL,
      dDB2, // INSERT with multi VALUES
      //dFirebird,  EXECUTE BLOCK with params is slower (at least for embedded)
      dOracle:
        begin
          // Oracle expects weird INSERT ALL INTO ... statement
          fBatchSendingAbilities := [cCreate];
          fOnBatchInsert := MultipleValuesInsert;
          fBatchMaxSentAtOnce := 4096; // MultipleValuesInsert will do chunking
        end;
      dFirebird:
        begin
          // will run EXECUTE BLOCK without parameters
          fBatchSendingAbilities := [cCreate];
          fOnBatchInsert := MultipleValuesInsertFirebird;
          fBatchMaxSentAtOnce := 4096; // MultipleValuesInsert will do chunking
        end;
    end;
end;

destructor TSqlDBConnectionProperties.Destroy;
begin
  fMainConnection.Free;
  FillZero(fPassword);
  inherited;
end;

function TSqlDBConnectionProperties.Execute(const aSql: RawUtf8;
  const Params: array of const; RowsVariant: PVariant; ForceBlobAsNull: boolean): ISqlDBRows;
var
  stmt: ISqlDBStatement;
begin
  stmt := NewThreadSafeStatementPrepared(aSql, {expectres=}true, true);
  stmt.ForceBlobAsNull := ForceBlobAsNull;
  stmt.Bind(Params);
  stmt.ExecutePrepared;
  result := stmt;
  if RowsVariant <> nil then
    if result = nil then
      SetVariantNull(RowsVariant^)
    else
      RowsVariant^ := result.RowData;
end;

function TSqlDBConnectionProperties.ExecuteNoResult(const aSql: RawUtf8;
  const Params: array of const): integer;
var
  stmt: ISqlDBStatement;
begin
  stmt := NewThreadSafeStatementPrepared(aSql, {expectres=}false, true);
  stmt.Bind(Params);
  stmt.ExecutePrepared;
  try
    result := stmt.UpdateCount;
  except // may occur e.g. for Firebird's CREATE DATABASE
    result := 0;
  end;
end;

function TSqlDBConnectionProperties.PrepareInlined(const aSql: RawUtf8;
  ExpectResults: boolean): ISqlDBStatement;
var
  stmt: ISqlDBStatement;
  i: PtrInt;
  decoder: TExtractInlineParameters;
begin
  result := nil; // returns nil interface on error
  if self = nil then
    exit;
  // convert inlined :(1234): parameters into Values[] for Bind*() calls
  decoder.Parse(aSql);
  stmt := NewThreadSafeStatementPrepared(decoder.GenericSql, ExpectResults, true);
  if stmt = nil then
    exit;
  // bind the inlined parameters
  for i := 0 to decoder.Count - 1 do
    case decoder.Types[i] of
      sptNull:
        stmt.BindNull(i + 1);
      sptInteger:
        stmt.Bind(i + 1, GetInt64(pointer(decoder.Values[i])));
      sptFloat:
        stmt.Bind(i + 1, GetExtended(pointer(decoder.Values[i])));
      sptText:
        stmt.BindTextU(i + 1, decoder.Values[i]);
      sptBlob:
        if decoder.Values[i] = '' then
          stmt.BindNull(i + 1)
        else
          stmt.BindBlob(i + 1, pointer(decoder.Values[i]), length(decoder.Values[i]));
      sptDateTime:
        stmt.BindDateTime(i + 1, Iso8601ToDateTime(decoder.Values[i]));
    end;
  result := stmt;
end;

function TSqlDBConnectionProperties.PrepareInlined(const SqlFormat: RawUtf8;
  const Args: array of const; ExpectResults: boolean): ISqlDBStatement;
var
  sql: RawUtf8;
begin
  FormatUtf8(SqlFormat, Args, sql);
  result := PrepareInlined(sql, ExpectResults);
end;

function TSqlDBConnectionProperties.ExecuteInlined(const aSql: RawUtf8;
  ExpectResults: boolean): ISqlDBRows;
var
  stmt: ISqlDBStatement;
begin
  result := nil; // returns nil interface on error
  if self = nil then
    exit;
  stmt := PrepareInlined(aSql, ExpectResults);
  if stmt = nil then
    exit; // e.g. invalid aSql
  stmt.ExecutePrepared;
  result := stmt;
end;

function TSqlDBConnectionProperties.ExecuteInlined(const SqlFormat: RawUtf8;
  const Args: array of const; ExpectResults: boolean): ISqlDBRows;
var
  sql: RawUtf8;
begin
  FormatUtf8(SqlFormat, Args, sql);
  result := ExecuteInlined(sql, ExpectResults);
end;

procedure TSqlDBConnectionProperties.SetFlag(
  const flag: TSqlDBConnectionPropertiesFlag; const value: boolean);
begin
  if value then
    include(fFlags, flag)
  else
    exclude(fFlags, flag);
end;

function TSqlDBConnectionProperties.GetFlag(
  const flag: TSqlDBConnectionPropertiesFlag): boolean;
begin
  result := flag in fFlags;
end;

procedure TSqlDBConnectionProperties.SetConnectionTimeOutMinutes(minutes: cardinal);
begin
  fConnectionTimeOutSecs := minutes * SecsPerMin;
end;

function TSqlDBConnectionProperties.GetConnectionTimeOutMinutes: cardinal;
begin
  result := fConnectionTimeOutSecs div SecsPerMin;
end;

function TSqlDBConnectionProperties.FindDatabaseNameField(Up: PAnsiChar): RawUtf8;
begin
  result := FindIniNameValueU(StringReplaceAll(fDatabaseName, ';', #10), Up);
end;

function TSqlDBConnectionProperties.GetMainConnection: TSqlDBConnection;
var
  secs: integer;
begin
  secs := 0;
  if fConnectionTimeOutSecs <> 0 then
    secs := GetTickSec;
  fMainConnectionSafe.Lock;
  result := fMainConnection;
  if (result = nil) or
     ((secs <> 0) and // recreate after ConnectionTimeOutMinutes idle
      result.IsOutdated(secs)) then
    try
      FreeAndNilSafe(fMainConnection);
      fMainConnection := NewConnection; // fast enough method
      result := fMainConnection;
    except
      result := nil;
    end;
  if (result <> nil) and
     (secs <> 0) then
    result.fLastAccessTickSec := secs;
  fMainConnectionSafe.UnLock;
end;

function TSqlDBConnectionProperties.{%H-}NewConnection: TSqlDBConnection;
begin
  raise ESqlDBException.CreateUtf8('Unexpected %.NewConnection', [self]);
end;

function TSqlDBConnectionProperties.ThreadSafeConnection: TSqlDBConnection;
begin
  result := MainConnection; // provider should be thread-safe
end;

procedure TSqlDBConnectionProperties.ClearConnectionPool;
begin
  fMainConnectionSafe.Lock;
  FreeAndNilSafe(fMainConnection); // contains its own try..finally
  fMainConnectionSafe.UnLock;
end;

function TSqlDBConnectionProperties.NewThreadSafeStatement: TSqlDBStatement;
begin
  result := ThreadSafeConnection.NewStatement;
end;

function TSqlDBConnectionProperties.NewThreadSafeStatementPrepared(
  const aSql: RawUtf8; ExpectResults, RaiseExceptionOnError: boolean): ISqlDBStatement;
begin
  result := ThreadSafeConnection.NewStatementPrepared(
    aSql, ExpectResults, RaiseExceptionOnError);
end;

function TSqlDBConnectionProperties.NewThreadSafeStatementPrepared(
  const SqlFormat: RawUtf8; const Args: array of const; ExpectResults,
  RaiseExceptionOnError: boolean): ISqlDBStatement;
var
  sql: RawUtf8;
begin
  FormatUtf8(SqlFormat, Args, sql);
  result := NewThreadSafeStatementPrepared(sql, ExpectResults, RaiseExceptionOnError);
end;

function TSqlDBConnectionProperties.SharedTransaction(SessionID: cardinal;
  action: TSqlDBSharedTransactionAction): TSqlDBConnection;
var
  i, n: PtrInt;
  found: boolean;
  t: ^TSqlDBConnectionTransaction;
begin
  try
    result := ThreadSafeConnection;
    // thread-safe found transactions support
    fSharedTransactionsSafe.Lock;
    try
      n := Length(fSharedTransactions);
      t := pointer(fSharedTransactions);
      found := false;
      for i := 0 to n - 1 do
        if t^.SessionID = SessionID then
        begin
          if result <> t^.Connection then
            ESqlDBException.RaiseUtf8(
              '%.SharedTransaction(sessionID=%) with mixed connections: % and %',
              [self, SessionID, result, t^.Connection]);
          if action = transBegin then
          begin
            // found StartTransaction
            inc(t^.RefCount);
            exit;
          end
          else
          begin
            // (found) commit/rollback
            dec(t^.RefCount);
            if t^.RefCount <> 0 then
              exit; // nested transaction
            dec(n);
            if n = 0 then
              fSharedTransactions := nil
            else
              DynArrayFakeDelete(fSharedTransactions, i, n, SizeOf(t^));
            found := true;
          end;
          break;
        end
        else
          inc(t);
      if not found then
        if action = transBegin then
        begin
          t := pointer(fSharedTransactions);
          for i := 1 to n do
            if t^.Connection = result then
              ESqlDBException.RaiseUtf8(
                'Dup %.SharedTransaction(sessionID=%,transBegin) sessionID=%',
                [self, SessionID, t^.SessionID])
            else
              inc(t);
          SetLength(fSharedTransactions, n + 1);
          t := @fSharedTransactions[n];
          t^.SessionID := SessionID;
          t^.RefCount := 1;
          t^.Connection := result;
        end
        else
          ESqlDBException.RaiseUtf8('Unexpected %.SharedTransaction(%,%)',
            [self, SessionID, ord(action)]);
    finally
      fSharedTransactionsSafe.Unlock;
    end;
    // perform the actual transaction SQL operation outside the lock
    case action of
      transBegin:
        begin
          if not result.Connected then
            result.Connect;
          result.StartTransaction;
        end;
      transCommitWithException,
      transCommitWithoutException:
        result.Commit;
      transRollback:
        result.Rollback;
    end;
  except
    on Exception do
    begin
      result := nil; // result.StartTransaction/Commit/Rollback failed
      if action = transCommitWithException then
        raise;
    end;
  end;
end;

procedure TSqlDBConnectionProperties.SetInternalProperties;
begin
  // nothing to do yet
end;

procedure TSqlDBConnectionProperties.SetSchemaNameToOwner(out Owner: RawUtf8);
begin
  if fForcedSchemaName = '' then
    case fDbms of
      dMySql,
      dMariaDB:
        Owner := DatabaseName;
      dInformix:
        Owner := '';
    else
      Owner := UserID;
    end
  else
    Owner := fForcedSchemaName;
end;

function TSqlDBConnectionProperties.IsCacheable(P: PUtf8Char): boolean;
begin
  result := (cpfUseCache in fFlags) and
            IsCacheableDML(P);
end;

function TSqlDBConnectionProperties.IsPrimaryKeyIndexed(
  var AscendingOnly: boolean): boolean;
begin
  // our ORM expects an index to quickly run "select max(RowID)"
  result := false;
  case GetDbms of
    dSQLite,
    dPostgreSQL,
    dMSSQL,
    dMySQL,
    dMariaDB,
    dOracle,
    dNexusDB:
      // most DB create an implicit index on their primary key,
      // which is fine for max(RowID)
      result := true;
    dFirebird:
      // Firebird only creates an ASC index by default on its primary key
      // so max(RowID) is slow - see http://www.firebirdfaq.org/faq205
      // -> need to create a separated DESC index
      AscendingOnly := true;
      // note: TSqlDBIbxConnectionProperties overrides this method
      // and support a CreateDescendingPK property to create a DESC index only
  end;
end;

class function TSqlDBConnectionProperties.GetFieldDefinition(
  const Column: TSqlDBColumnDefine): RawUtf8;
begin
  FormatUtf8('% [%', [Column.ColumnName, Column.ColumnTypeNative], result);
  if (Column.ColumnLength <> 0) or
     (Column.ColumnPrecision <> 0) or
     (Column.ColumnScale <> 0) then
    result := FormatUtf8('% % % %]',
      [result, Column.ColumnLength, Column.ColumnPrecision, Column.ColumnScale])
  else
    AppendShortToUtf8(']', result);
  if Column.ColumnIndexed then
    AppendShortToUtf8(' *', result);
end;

class function TSqlDBConnectionProperties.GetFieldORMDefinition(
  const Column: TSqlDBColumnDefine): RawUtf8;
begin
  // 'Name: RawUtf8 index 20 read fName write fName;';
  FormatUtf8('property %: %',
    [Column.ColumnName, SqlDBFIELDTYPE_TO_DELPHITYPE[Column.ColumnType]], result);
  if (Column.ColumnType = ftUtf8) and
     (Column.ColumnLength > 0) then
    result := FormatUtf8('% index %', [result, Column.ColumnLength]);
  result := FormatUtf8('% read f% write f%;',
    [result, Column.ColumnName, Column.ColumnName]);
end;

var
  DB_KEYWORDS: array[TSqlDBDefinition] of TRawUtf8DynArray;

const
  // reused for MySQL and MariaDB
  MYSQL_KEYWORDS_CSV = 
    'accessible,analyze,asensitive,auto_increment,before,bigint,binary,blob,call,change,' +
    'condition,database,databases,day_hour,day_microsecond,day_minute,day_second,' +
    'delayed,deterministic,distinctrow,div,dual,each,elseif,enclosed,enum,escaped,exit,' +
    'explain,float4,float8,force,fulltext,general,high_priority,hour_microsecond,' +
    'hour_minute,hour_second,if,ignore,ignore_server_ids,infile,inout,int1,int2,int3,int4,' +
    'int8,iterate,keys,kill,leave,limit,linear,linear,lines,load,localtime,localtimestamp,' +
    'lock,long,longblob,longtext,loop,low_priority,master_heartbeat_period,' +
    'master_ssl_verify_server_cert,master_ssl_verify_server_cert,maxvalue,' +
    'mediumblob,mediumint,mediumtext,middleint,minute_microsecond,minute_second,mod,' +
    'modifies,no_write_to_binlog,optimize,optionally,out,outfile,purge,range,range,' +
    'read_only,read_only,read_write,read_write,reads,regexp,release,rename,repeat,replace,' +
    'require,resignal signal,return,rlike,schemas,second_microsecond,sensitive,' +
    'separator,show,slow,spatial,specific,sql_big_result,sql_calc_found_rows,' +
    'sql_small_result,sqlexception,ssl,starting,straight_join,terminated,text,tinyblob,' +
    'tinyint,tinytext,trigger,undo,unlock,unsigned,use,utc_date,utc_time,utc_timestamp,' +
    'varbinary,varcharacter,while,x509,xor,year_month,zerofillaccessible';

  /// CSV of the known reserved keywords per database engine, in alphabetic order
  DB_KEYWORDS_CSV: array[TSqlDBDefinition] of PUtf8Char = (
    // dUnknown
    '',
    // dDefault = ODBC / SQL-92 keywords (always checked first)
    'absolute,action,ada,add,all,allocate,alter,and,any,are,as,asc,assertion,at,authorization,' +
    'avg,begin,between,bit,bit_length,both,by,cascade,cascaded,case,cast,catalog,char,' +
    'char_length,character,character_length,check,close,coalesce,collate,collation,' +
    'column,commit,connect,connection,constraint,constraints,continue,convert,' +
    'corresponding,count,create,cross,current,current_date,current_time,' +
    'current_timestamp,current_user,cursor,date,day,deallocate,dec,decimal,declare,' +
    'default,deferrable,deferred,delete,desc,describe,descriptor,diagnostics,disconnect,' +
    'distinct,domain,double,drop,else,end,end-exec,escape,except,exception,exec,execute,' +
    'exists,external,extract,false,fetch,first,float,for,foreign,fortran,found,from,full,get,' +
    'global,go,goto,grant,group,having,hour,identity,immediate,in,include,index,indicator,' +
    'initially,inner,input,insensitive,insert,int,integer,intersect,interval,into,is,' +
    'isolation,join,key,language,last,leading,left,level,like,local,lower,match,max,min,minute,' +
    'module,month,n,names,national,natural,nchar,next,no,none,not,null,nullif,numeric,' +
    'octet_length,of,on,only,open,option,or,order,outer,output,overlaps,pad,partial,pascal,' +
    'position,precision,prepare,preserve,primary,prior,privileges,procedure,public,read,' +
    'real,references,relative,restrict,revoke,right,rollback,rows,schema,scroll,second,' +
    'section,select,session,session_user,set,size,smallint,some,space,sql,sqlca,sqlcode,' +
    'sqlerror,sqlstate,sqlwarning,substring,sum,system_user,table,temporary,then,time,' +
    'timestamp,timezone_hour,timezone_minute,to,trailing,transaction,translate,' +
    'translation,trim,true,union,unique,unknown,update,upper,usage,user,using,value,values,' +
    'varchar,varying,view,when,whenever,where,with,work,write,year,zone',
    // dOracle specific keywords (in addition to dDefault)
    'access,audit,cluster,comment,compress,exclusive,file,identified,increment,initial,' +
    'lock,long,maxextents,minus,mode,noaudit,nocompress,nowait,number,offline,online,' +
    'pctfree',
    // dMSSQL specific keywords (in addition to dDefault)
    'admin,after,aggregate,alias,array,asensitive,asymmetric,atomic,backup,before,binary,' +
    'blob,boolean,breadth,break,browse,bulk,call,called,cardinality,checkpoint,class,clob,' +
    'clustered,collect,completion,compute,condition,constructor,contains,containstable,' +
    'corr,covar_pop,covar_samp,cube,cume_dist,current_catalog,' +
    'current_default_transform_group,current_path,current_role,current_schema,' +
    'current_transform_group_for_type,cycle,data,database,dbcc,deny,depth,deref,destroy,' +
    'destructor,deterministic,dictionary,disk,distributed,dump,dynamic,each,element,' +
    'equals,errlvl,every,exit,file,fillfactor,filter,free,freetext,freetexttable,' +
    'fulltexttable,function,fusion,general,grouping,hold,holdlock,host,identity_insert,' +
    'identitycol,if,ignore,initialize,inout,intersection,iterate,kill,large,lateral,less,' +
    'like_regex,limit,lineno,ln,load,localtime,localtimestamp,locator,map,member,merge,' +
    'method,mod,modifies,modify,multiset,nclob,new,nocheck,nonclustered,normalize,object,' +
    'occurrences_regex,off,offsets,old,opendatasource,openquery,openrowset,openxml,' +
    'operation,ordinality,out,over,overlay,parameter,parameters,partition,path,percent,' +
    'percent_rank,percentile_cont,percentile_disc,pivot,plan,position_regex,postfix,' +
    'prefix,preorder,print,proc,raiserror,range,reads,readtext,reconfigure,recursive,ref,' +
    'referencing,regr_avgx,regr_avgy,regr_count,regr_intercept,regr_r2,regr_slope,' +
    'regr_sxx,regr_sxy,regr_syy,release,replication,restore,result,return,returns,revert,' +
    'role,rollup,routine,row,rowcount,rowguidcol,rule,save,savepoint,scope,search,' +
    'securityaudit,semantickeyphrasetable,semanticsimilaritydetailstable,' +
    'semanticsimilaritytable,sensitive,sequence,sets,setuser,shutdown,similar,specific,' +
    'specifictype,sqlexception,start,state,statement,static,statistics,stddev_pop,' +
    'stddev_samp,structure,submultiset,substring_regex,symmetric,system,tablesample,' +
    'terminate,textsize,than,top,tran,translate_regex,treat,trigger,truncate,try_convert,' +
    'tsequal,uescape,under,unnest,unpivot,updatetext,use,var_pop,var_samp,variable,waitfor,' +
    'while,width_bucket,window,within,within group,without,writetext,xmlagg,' +
    'xmlattributes,xmlbinary,xmlcast,xmlcomment,xmlconcat,xmldocument,xmlelement,' +
    'xmlexists,xmlforest,xmliterate,xmlnamespaces,xmlparse,xmlpi,xmlquery,xmlserialize,' +
    'xmltable,xmltext,xmlvalidate',
    // dJet specific keywords (in addition to dDefault)
    'longtext,memo,money,note,number,oleobject,owneraccess,parameters,percent,pivot,short,' +
    'single,singlefloat,stdev,stdevp,string,tableid,text,top,transform,unsignedbyte,var,' +
    'varbinary,varp,yesno',
    // dMySQL specific keywords (in addition to dDefault)
     MYSQL_KEYWORDS_CSV,
    // dSQLite keywords will use IsSqliteReserved() from mormot.db.core
    '',
    // dFirebird specific keywords (in addition to dDefault)
    'active,after,ascending,base_name,before,blob,cache,check_point_length,computed,' +
    'conditional,containing,cstring,currency,database,debug,descending,deterministic,do,' +
    'entry_point,exit,file,filter,function,gdscode,gen_id,generator,' +
    'group_commit_wait_time,if,inactive,input_type,log_buffer_size,logfile,manual,' +
    'maximum_segment,merge,message,module_name,num_log_buffers,output_type,over,' +
    'overflow,page,page_size,pages,parameter,parent,password,plan,post_event,protected,' +
    'raw_partitions,rdb$db_key,record_version,reserv,reserving,retain,return,' +
    'returning_values,returns,segment,shadow,shared,singular,snapshot,sort,stability,' +
    'start,starting,starts,statistics,sub_type,suspend,trigger,type,variable,wait,while',
    // dNexusDB specific keywords (in addition to dDefault)
    'abs,achar,assert,astring,autoinc,blob,block,blocksize,bool,boolean,byte,bytearray,' +
    'ceiling,chr,datetime,dword,empty,exp,floor,grow,growsize,ignore,image,initial,' +
    'initialsize,kana,largeint,locale,log,money,nullstring,nvarchar,percent,power,rand,' +
    'round,shortint,sort,string,symbols,text,tinyint,top,type,use,width,word',
  // dPostgreSQL specific keywords (in addition to dDefault)
    'abort,access,admin,after,aggregate,also,always,analyse,analyze,array,assignment,' +
    'asymmetric,backward,before,bigint,binary,boolean,cache,called,chain,characteristics,' +
    'checkpoint,class,cluster,comment,committed,concurrently,configuration,content,' +
    'conversion,copy,cost,createdb,createrole,createuser,csv,ctid,current_role,cycle,database,' +
    'defaults,definer,delimiter,delimiters,dictionary,disable,discard,do,document,each,' +
    'enable,encoding,encrypted,enum,excluding,exclusive,explain,family,force,forward,' +
    'freeze,function,granted,greatest,handler,header,hold,if,ilike,immutable,implicit,' +
    'including,increment,indexes,inherit,inherits,inout,instead,invoker,isnull,' +
    'lancompiler,large,least,limit,listen,load,localtime,localtimestamp,location,lock,' +
    'login,mapping,maxvalue,minvalue,mode,move,new,nocreatedb,nocreaterole,nocreateuser,' +
    'noinherit,nologin,nosuperuser,nothing,notify,notnull,nowait,nulls,object,off,offset,' +
    'oids,old,operator,out,overlay,owned,owner,parser,password,placing,plans,prepared,' +
    'procedural,quote,reassign,recheck,reindex,release,rename,repeatable,replace,replica,' +
    'reset,restart,returning,returns,role,row,rule,savepoint,search,security,sequence,' +
    'serializable,setof,share,show,similar,simple,stable,standalone,start,statement,' +
    'statistics,stdin,stdout,storage,strict,strip,superuser,symmetric,sysid,system,' +
    'tablespace,temp,template,text,treat,trigger,truncate,trusted,type,uncommitted,' +
    'unencrypted,unlisten,until,vacuum,valid,validator,verbose,version,volatile,' +
    'whitespace,without,xml,xmlattributes,xmlconcat,xmlelement,xmlforest,xmlparse,xmlpi,' +
    'xmlroot,xmlserialize,yes',
    // dDB2 specific keywords (in addition to dDefault)
    'activate,document,dssize,dynamic,each,editproc,elseif,enable,encoding,encryption,' +
    'ending,erase,every,excluding,exclusive,exit,explain,fenced,fieldproc,file,final,free,' +
    'function,general,generated,graphic,handler,hash,hashed_value,hint,hold,hours,if,' +
    'including,inclusive,increment,inf,infinity,inherit,inout,integrity,isobid,iterate,jar,' +
    'java,keep,label,lateral,lc_ctype,leave,linktype,localdate,locale,localtime,' +
    'localtimestamp,locator,locators,lock,lockmax,locksize,long,loop,maintained,' +
    'materialized,maxvalue,microsecond,microseconds,minutes,minvalue,mode,modifies,' +
    'months,nan,new,new_table,nextval,nocache,nocycle,nodename,nodenumber,nomaxvalue,' +
    'nominvalue,noorder,normalized,nulls,numparts,obid,old,old_table,optimization,' +
    'optimize,out,over,overriding,package,padded,pagesize,parameter,part,partition,' +
    'partitioned,partitioning,partitions,password,path,piecesize,plan,prevval,priqty,' +
    'program,psid,query,queryno,range,rank,reads,recovery,referencing,refresh,release,' +
    'rename,repeat,reset,resignal,restart,result,result_set_locator,return,returns,role,' +
    'round_ceilingadd,round_downafter,round_flooralias,round_half_downall,' +
    'round_half_evenallocate,round_half_upallow,round_upalter,routineand,' +
    'row_numberas,rowany,rownumberasensitive,rowsassociate,rowsetasutime,rrnat,' +
    'runattributes,savepointaudit,schemaauthorization,scratchpadaux,scrollauxiliary,' +
    'searchbefore,secondbegin,secondsbetween,secqtybinary,securitybufferpool,selectby,' +
    'sensitivecache,sequencecall,session_usercapture,sessioncalled,setcardinality,' +
    'signalcascaded,simplecase,snancast,someccsid,sourcechar,specificcharacter,' +
    'sqlcheck,sqlidclone,stackedclose,standardcluster,startcollection,startingcollid,' +
    'statementcolumn,staticcomment,statmentcommit,stayconcat,stogroupcondition,' +
    'storesconnect,styleconnection,substringconstraint,summarycontains,' +
    'synonymcontinue,sysfuncount,sysibmcount_big,sysproccreate,system_usercurrent,' +
    'systemcross,tablecurrent_date,tablespacecurrent_lc_ctype,thencurrent_path,' +
    'timecurrent_schema,timestampcurrent_server,tocurrent_time,' +
    'transactioncurrent_timestamp,triggercurrent_timezone,trimcurrent_user,' +
    'truncatecursor,typecycle,undodata,uniondatabase,uniquedatapartitionname,' +
    'untildatapartitionnum,updatedate,usageday,userdays,usingdb2general,' +
    'validprocdb2genrl,valuedb2sql,valuesdbinfo,variabledbpartitionname,' +
    'variantdbpartitionnum,vcatdeallocate,versiondeclare,viewdefault,' +
    'volatiledefaults,volumesdefinition,whendelete,wheneverdense_rank,wheredenserank,' +
    'whiledescribe,withdescriptor,withoutdeter',
  // dInformix specific keywords (in addition to dDefault)
    '',
  // dMariaDB specific keywords (in addition to dDefault)
    MYSQL_KEYWORDS_CSV
    );

class function TSqlDBConnectionProperties.IsSqlKeyword(aDB: TSqlDBDefinition;
  aWord: RawUtf8): boolean;
var
  db: TSqlDBDefinition;
begin
  // use SQLite3 function using https://sqlite.org/lang_keywords.html
  if aDB = dSQLite then
  begin
    result := IsSqliteReserved(aWord);
    exit;
  end;
  // prepare the keywords arrays from the per-DB CSV references
  if DB_KEYWORDS[dDefault] = nil then
    for db := low(DB_KEYWORDS) to high(DB_KEYWORDS) do
      CsvToRawUtf8DynArray(DB_KEYWORDS_CSV[db], DB_KEYWORDS[db]);
  // search using fast binary lookup in the alphabetic ordered arrays
  aWord := TrimU(LowerCase(aWord));
  if (FastFindPUtf8CharSorted(pointer(DB_KEYWORDS[dDefault]),
       high(DB_KEYWORDS[dDefault]), pointer(aWord)) < 0) then
    if aDB <= dDefault then
      result := false
    else
      result := FastFindPUtf8CharSorted(pointer(DB_KEYWORDS[aDB]),
        high(DB_KEYWORDS[aDB]), pointer(aWord)) >= 0
  else
    result := true;
end;

function TSqlDBConnectionProperties.IsSqlKeyword(aWord: RawUtf8): boolean;
begin
  result := IsSqlKeyword(GetDbms, aWord);
end;

procedure TSqlDBConnectionProperties.GetFieldDefinitions(
  const aTableName: RawUtf8; out Fields: TRawUtf8DynArray; WithForeignKeys: boolean);
var
  f: TSqlDBColumnDefineDynArray;
  ref: RawUtf8;
  i: integer;
begin
  GetFields(aTableName, f);
  SetLength(Fields, length(f));
  for i := 0 to high(f) do
  begin
    Fields[i] := GetFieldDefinition(f[i]);
    if WithForeignKeys then
    begin
      ref := GetForeignKey(aTableName, f[i].ColumnName);
      if ref <> '' then
        Append(Fields[i], ' % ', ref);
    end;
  end;
end;

procedure TSqlDBConnectionProperties.GetFields(const aTableName: RawUtf8;
  out Fields: TSqlDBColumnDefineDynArray);
var
  sql: RawUtf8;
  n, i: integer;
  f: TSqlDBColumnDefine;
  fa: TDynArray;
begin
  fa.Init(TypeInfo(TSqlDBColumnDefineDynArray), Fields, @n);
  fa.Compare := SortDynArrayAnsiStringI; // fa.Find() case insensitive
  FillCharFast(f, SizeOf(f), 0);
  if fDbms = dSQLite then
  begin
    // SQLite3 has a specific PRAGMA metadata query
    try
      with Execute('PRAGMA table_info(`' + aTableName + '`)', []) do
        while Step do
        begin
          // cid=0,name=1,type=2,notnull=3,dflt_value=4,pk=5
          f.ColumnName := ColumnUtf8(1);
          f.ColumnTypeNative := ColumnUtf8(2);
          f.ColumnType := ColumnTypeNativeToDB(f.ColumnTypeNative, 0);
          f.ColumnIndexed := (ColumnInt(5) = 1); // by definition for SQLite3
          fa.Add(f);
        end;
    except
      on Exception do
        n := 0; // external SQLite3 providers (e.g. UniDAC) are buggy
    end;
    try
      with Execute('PRAGMA index_list(`' + aTableName + '`)', []) do
        while Step do        // seq=0,name=1,unique=2
          with Execute('PRAGMA index_info(' + ColumnUtf8(1) + ')', []) do
            while Step do
            begin
              f.ColumnName := ColumnUtf8(2); // seqno=0,cid=1,name=2
              i := fa.Find(f);
              if i >= 0 then
                Fields[i].ColumnIndexed := true;
            end;
    except
      on Exception do
        ; // ignore any exception if no index is defined
    end;
  end
  else
  begin
    sql := SqlGetField(aTableName);
    if sql = '' then
      exit;
    with Execute(sql, []) do
    begin
      while Step do
      begin
        f.ColumnName := TrimU(ColumnUtf8(0));
        f.ColumnTypeNative := TrimU(ColumnUtf8(1));
        f.ColumnLength := ColumnInt(2);
        f.ColumnPrecision := ColumnInt(3);
        if ColumnNull(4) then // e.g. for plain NUMERIC in Oracle
          f.ColumnScale := -1
        else
          f.ColumnScale := ColumnInt(4);
        f.ColumnType := ColumnTypeNativeToDB(f.ColumnTypeNative, f.ColumnScale);
        if ColumnInt(5) > 0 then
          f.ColumnIndexed := true;
        fa.Add(f);
      end;
      ReleaseRows;
    end;
  end;
  SetLength(Fields, n);
end;

procedure TSqlDBConnectionProperties.GetIndexes(const aTableName: RawUtf8;
  out Indexes: TSqlDBIndexDefineDynArray);
var
  sql: RawUtf8;
  n: integer;
  f: TSqlDBIndexDefine;
  fa: TDynArray;
begin
  sql := SqlGetIndex(aTableName);
  if sql = '' then
    exit;
  fa.Init(TypeInfo(TSqlDBIndexDefineDynArray), Indexes, @n);
  with Execute(sql, []) do
  begin
    while Step do
    begin
      f.IndexName := TrimU(ColumnUtf8(0));
      f.IsUnique := ColumnInt(1) > 0;
      f.TypeDesc := TrimU(ColumnUtf8(2));
      f.IsPrimaryKey := ColumnInt(3) > 0;
      f.IsUniqueConstraint := ColumnInt(4) > 0;
      f.Filter := TrimU(ColumnUtf8(5));
      f.KeyColumns := TrimU(ColumnUtf8(6));
      f.IncludedColumns := TrimU(ColumnUtf8(7));
      fa.Add(f);
    end;
    ReleaseRows;
  end;
  SetLength(Indexes, n);
end;

procedure TSqlDBConnectionProperties.GetProcedureNames(out Procedures: TRawUtf8DynArray);
var
  sql: RawUtf8;
  count: integer;
begin
  sql := SqlGetProcedure;
  if sql <> '' then
  try
    with Execute(sql, []) do
    begin
      count := 0;
      while Step do
        AddSortedRawUtf8(Procedures, count, TrimU(ColumnUtf8(0)));
      SetLength(Procedures, count);
      ReleaseRows;
    end;
  except
    on Exception do
      SetLength(Procedures, 0); // if the supplied sql query is wrong, just ignore
  end;
end;

procedure TSqlDBConnectionProperties.GetProcedureParameters(
  const aProcName: RawUtf8; out Parameters: TSqlDBProcColumnDefineDynArray);
var
  sql: RawUtf8;
  n: integer;
  f: TSqlDBProcColumnDefine;
  fa: TDynArray;
begin
  fa.Init(TypeInfo(TSqlDBColumnDefineDynArray), Parameters, @n);
  fa.Compare := SortDynArrayAnsiStringI; // fa.Find() case insensitive
  FillcharFast(f, SizeOf(f), 0);
  sql := SqlGetParameter(aProcName);
  if sql = '' then
    exit;
  with Execute(sql, []) do
  begin
    while Step do
    begin
      f.ColumnName := TrimU(ColumnUtf8(0));
      f.ColumnTypeNative := TrimU(ColumnUtf8(1));
      f.ColumnLength := ColumnInt(2);
      f.ColumnPrecision := ColumnInt(3);
      if ColumnNull(4) then // e.g. for plain NUMERIC in Oracle
        f.ColumnScale := -1
      else
        f.ColumnScale := ColumnInt(4);
      f.ColumnType := ColumnTypeNativeToDB(f.ColumnTypeNative, f.ColumnScale);
      case FindCsvIndex('IN,OUT,INOUT', ColumnUtf8(5), ',', false) of
        0:
          f.ColumnParamType := paramIn;
        2:
          f.ColumnParamType := paramInOut;
      else // any other is assumed as out
        f.ColumnParamType := paramOut;
      end;
      fa.Add(f);
    end;
    ReleaseRows;
  end;
  SetLength(Parameters, n);
end;

procedure TSqlDBConnectionProperties.GetTableNames(out Tables: TRawUtf8DynArray);
var
  sql, table, checkschema: RawUtf8;
  count: integer;
begin
  sql := SqlGetTableNames;
  if sql <> '' then
  try
    if FilterTableViewSchemaName and
       (fForcedSchemaName <> '') then
      checkschema := UpperCase(fForcedSchemaName) + '.';
    with Execute(sql, []) do
    begin
      count := 0;
      while Step do
      begin
        table := TrimU(ColumnUtf8(0));
        if (checkschema = '') or
           IdemPChar(pointer(table), pointer(checkschema)) then
          AddSortedRawUtf8(Tables, count, table);
      end;
      SetLength(Tables, count);
      ReleaseRows;
    end;
  except
    on Exception do
      SetLength(Tables, 0); // if the supplied sql query is wrong, just ignore
  end;
end;

function TSqlDBConnectionProperties.TableExists(const TableName: RawUtf8): boolean;
var
  tables: TRawUtf8DynArray;
begin
  GetTableNames(tables);
  result := FindRawUtf8(tables, TableName, {casesensitive=}false) >= 0;
end;

procedure TSqlDBConnectionProperties.GetViewNames(out Views: TRawUtf8DynArray);
var
  sql, table, checkschema: RawUtf8;
  count: integer;
begin
  sql := SqlGetViewNames;
  if sql <> '' then
  try
    if FilterTableViewSchemaName and
       (fForcedSchemaName <> '') then
      checkschema := UpperCase(fForcedSchemaName) + '.';
    with Execute(sql, []) do
    begin
      count := 0;
      while Step do
      begin
        table := TrimU(ColumnUtf8(0));
        if (checkschema = '') or
           IdemPChar(pointer(table), pointer(checkschema)) then
          AddSortedRawUtf8(Views, count, table);
      end;
      SetLength(Views, count);
      ReleaseRows;
    end;
  except
    on Exception do
      SetLength(Views, 0); // if the supplied sql query is wrong, just ignore
  end;
end;

procedure TSqlDBConnectionProperties.SqlSplitTableName(
  const aTableName: RawUtf8; out Owner, Table: RawUtf8);
begin
  case fDbms of
    dSQLite:
      Table := aTableName;
  else
    begin
      Split(aTableName, '.', Owner, Table);
      if Table = '' then
      begin
        Table := Owner;
        if fForcedSchemaName = '' then
          case fDbms of
            dMySql,
            dMariaDB:
              Owner := DatabaseName;
          else
            Owner := UserID;
          end
        else
          Owner := fForcedSchemaName;
      end;
    end;
  end;
end;

procedure TSqlDBConnectionProperties.SqlSplitProcedureName(
  const aProcName: RawUtf8; out Owner, package, ProcName: RawUtf8);
var
  dotcount, i: PtrInt;
begin
  dotcount := 0;
  for i := 1 to length(aProcName) do
    if aProcName[i] = '.' then
      inc(dotcount);
  if dotcount = 0 then
  begin
    ProcName := aProcName;
    SetSchemaNameToOwner(Owner);
    exit;
  end;
  case fDbms of
    dSQLite:
      ProcName := aProcName;
    dOracle,
    dFirebird:
      begin
        // Firebird 3 has packages
        if dotcount = 2 then
        begin
          // OWNER.PACKAGE.PROCNAME
          Split(aProcName, '.', Owner, package);
          Split(package, '.', package, ProcName);
        end
        else
        begin
          // PACKAGE.PROCNAME
          Split(aProcName, '.', package, ProcName);
          Owner := UserID;
        end;
      end
  else
    begin
      // OWNER.PROCNAME
      Split(aProcName, '.', Owner, ProcName);
      if ProcName = '' then
      begin
        ProcName := Owner;
        SetSchemaNameToOwner(Owner);
      end
      else if fDbms = dMSSQL then
        // discard ;1 when MSSQL stored procedure name is ProcName;1
        Split(ProcName, ';', ProcName);
    end;
  end;
end;

function TSqlDBConnectionProperties.SqlFullTableName(
  const aTableName: RawUtf8): RawUtf8;
begin
  if (aTableName <> '') and
     (fForcedSchemaName <> '') and
     (PosExChar('.', aTableName) = 0) then
    Join([fForcedSchemaName, '.', aTableName], result)
  else
    result := aTableName;
end;

function TSqlDBConnectionProperties.SqlGetField(
  const aTableName: RawUtf8): RawUtf8;
var
  owner, table, fmt: RawUtf8;
begin
  result := '';
  case GetDbms of
    dOracle:
      fmt :=
        'select c.column_name, c.data_type, c.data_length, c.data_precision, c.data_scale, ' +
        ' (select count(*) from sys.all_indexes a, sys.all_ind_columns b' +
        '  where a.table_owner=c.owner and a.table_name=c.table_name and b.column_name=c.column_name' +
        '  and a.owner=b.index_owner and a.index_name=b.index_name and' +
        '  a.table_owner=b.table_owner and a.table_name=b.table_name) index_count' +
        ' from sys.all_tab_columns c' +
        ' where c.owner like ''%'' and c.table_name like ''%'';';
    dMSSQL,
    dMySQL,
    dMariaDB,
    dPostgreSQL:
      fmt :=
        'select COLUMN_NAME, DATA_TYPE, CHARACTER_MAXIMUM_LENGTH, NUMERIC_PRECISION,' +
        ' NUMERIC_SCALE, 0 INDEX_COUNT' + // INDEX_COUNT=0 here (done via OleDB)
        ' from INFORMATION_SCHEMA.COLUMNS' +
        ' where UPPER(TABLE_SCHEMA) = ''%'' and UPPER(TABLE_NAME) = ''%''';
    dFirebird:
      begin
        Join([// see http://edn.embarcadero.com/article/25259
          'select a.rdb$field_name, b.rdb$field_type || coalesce(b.rdb$field_sub_type, '''') as rdb$field_type,' +
          ' b.rdb$field_length, b.rdb$field_length, abs(b.rdb$field_scale) as rdb$field_scale,' +
          ' (select count(*) from rdb$indices i, rdb$index_segments s' +
          ' where i.rdb$index_name=s.rdb$index_name and i.rdb$index_name not like ''RDB$%''' +
          ' and i.rdb$relation_name=a.rdb$relation_name) as index_count ' +
          'from rdb$relation_fields a left join rdb$fields b on a.rdb$field_source=b.rdb$field_name' +
          ' left join rdb$relations c on a.rdb$relation_name=c.rdb$relation_name ' +
          'where a.rdb$relation_name=''', UpperCase(aTableName), ''''], result);
        exit;
      end;
    dNexusDB:
      begin
        Join([
          'select FIELD_NAME, FIELD_TYPE_SQL, FIELD_LENGTH, FIELD_UNITS,' +
          ' FIELD_DECIMALS, FIELD_INDEX from #fields where TABLE_NAME = ''',
          aTableName, ''''], result);
        exit;
      end;
  else
    exit; // others (e.g. dDB2) will retrieve info from (ODBC) driver
  end;
  SqlSplitTableName(aTableName, owner, table);
  FormatUtf8(fmt, [UpperCase(owner), UpperCase(table)], result);
end;

function TSqlDBConnectionProperties.SqlGetIndex(const aTableName: RawUtf8): RawUtf8;
var
  owner, table: RawUtf8;
  fmt: RawUtf8;
begin
  result := '';
  case GetDbms of
    dOracle:
      fmt :=
        'select  index_name, decode(max(uniqueness),''NONUNIQUE'',0,1) as is_unique, ' +
        ' max(index_type) as type_desc, 0 as is_primary_key, 0 as unique_constraint, ' +
        ' cast(null as varchar(100)) as index_filter, ' +
        ' ltrim(max(sys_connect_by_path(column_name, '', '')), '', '') as key_columns, ' +
        ' cast(null as varchar(100)) as included_columns from ' +
        '( select c.index_name as index_name, i.index_type, i.uniqueness, c.column_name, ' +
        '   row_number() over (partition by c.table_name, c.index_name order by c.column_position) as rn ' +
        '  from user_ind_columns c inner join user_indexes i on c.table_name = i.table_name and c.index_name = i.index_name ' +
        '  where c.table_name = ''%'' ' +
        ') start with rn = 1 connect by prior rn = rn - 1 and prior index_name = index_name ' +
        'group by index_name order by index_name';
    dMSSQL:
      fmt :=
        'select i.name as index_name, i.is_unique, i.type_desc, is_primary_key, is_unique_constraint, ' +
        '  i.filter_definition as index_filter, key_columns, included_columns as included_columns, ' +
        '  t.name as table_name from ' +
        ' sys.tables t inner join sys.indexes i on i.object_id = t.object_id ' +
        ' cross apply(select STUFF(' +
        '   (select '',''+c.name from sys.index_columns ic ' +
        '   inner join sys.columns c on c.object_id = t.object_id and c.column_id = ic.column_id ' +
        '   where i.index_id = ic.index_id and i.object_id = ic.object_id and ic.is_included_column = 0 ' +
        '   order by ic.key_ordinal for xml path('''') ' +
        '   ),1,1,'''') as key_columns) AS c cross apply(select STUFF( ' +
        '   (select '','' + c.name  from sys.index_columns ic ' +
        '   inner join	sys.columns c on	c.object_id = t.object_id	and c.column_id = ic.column_id ' +
        '   where i.index_id = ic.index_id and i.object_id = ic.object_id and ic.is_included_column = 1 ' +
        '   order by ic.key_ordinal for xml path('''') ' +
        '   ),1,1,'''') as included_columns) AS ic ' +
        'where t.type = ''U'' and t.name like ''%''';
    dFirebird:
      fmt :=
        'select i.rdb$index_name, i.rdb$unique_flag, i.rdb$index_type, case rc.rdb$constraint_type ' +
        ' when ''PRIMARY KEY'' then 1 else 0 end as is_primary_key, 0 as unique_constraint, ' +
        ' null as index_filter, (select list(trim(rdb$field_name), '', '') from ' +
        '  (select * from rdb$index_segments where rdb$index_name = i.rdb$index_name ' +
        '  order by rdb$field_position)) as key_columns, null as included_columns ' +
        'from rdb$indices i ' +
        'left outer join rdb$relation_constraints rc on rc.rdb$index_name = i.rdb$index_name and ' +
        '  rc.rdb$constraint_type=''PRIMARY KEY'' ' +
        'where exists(select * from rdb$index_segments where rdb$index_name = i.rdb$index_name) and ' +
        ' i.rdb$relation_name = ''%''';
  else
    exit; // others (e.g. dMySQL or dDB2) will retrieve info from (ODBC) driver
  end;
  Split(aTableName, '.', owner, table);
  if table = '' then
  begin
    table := owner;
    owner := UserID;
  end;
  FormatUtf8(fmt, [UpperCase(table)], result);
end;

function TSqlDBConnectionProperties.SqlGetParameter(const aProcName: RawUtf8): RawUtf8;
var
  owner, package, proc: RawUtf8;
  fmt: RawUtf8;
begin
  result := '';
  SqlSplitProcedureName(aProcName, owner, package, proc);
  case GetDbms of
    dOracle:
      Join([
        'select a.argument_name, a.data_type, a.char_length, a.data_precision, a.data_scale, a.in_out ' +
        'from   sys.all_arguments a where  a.owner like ''%''' +
        '  and  a.package_name like ''', UpperCase(package), '''' +
        '  and  a.object_name like ''%'' order by position'], fmt);
    dMSSQL,
    dMySQL,
    dMariaDB,
    dPostgreSQL:
      fmt :=
        'select PARAMETER_NAME, DATA_TYPE, CHARACTER_MAXIMUM_LENGTH, NUMERIC_PRECISION, NUMERIC_SCALE, PARAMETER_MODE ' +
        'from INFORMATION_SCHEMA.PARAMETERS ' +
        'where UPPER(SPECIFIC_SCHEMA) = ''%'' and UPPER(SPECIFIC_NAME) = ''%''' +
        ' order by ORDINAL_POSITION';
    dFirebird:
      begin
        if package = '' then
          Join([
            'select a.rdb$parameter_name, b.rdb$field_type || coalesce(b.rdb$field_sub_type, '''') as rdb$field_type,' +
            ' b.rdb$field_length, b.rdb$field_precision, b.rdb$field_scale,' +
            ' case a.rdb$parameter_type when 0 then ''IN'' else ''OUT'' end ' +
            'from rdb$procedure_parameters a, rdb$fields b ' +
            'where b.rdb$field_name = a.rdb$field_source and a.rdb$procedure_name = ''',
            UpperCase(proc), ''' order by a.rdb$parameter_number'], result)
        else
          Join([
            'select a.rdb$parameter_name, b.rdb$field_type || coalesce(b.rdb$field_sub_type, '''') as rdb$field_type,' +
            ' b.rdb$field_length, b.rdb$field_precision, b.rdb$field_scale,' +
            ' case a.rdb$parameter_type when 0 then ''IN'' else ''OUT'' end ' +
            'from rdb$procedure_parameters a, rdb$fields b ' +
            'where b.rdb$field_name = a.rdb$field_source and a.rdb$package_name = ''',
            UpperCase(package), '''  and a.rdb$procedure_name = ''',
            UpperCase(proc), ''' order by a.rdb$parameter_number'], result);
        exit;
      end;
    dNexusDB:
      begin
        // NOT TESTED !!!
        Join([
          'select PROCEDURE_ARGUMENT_NAME, PROCEDURE_ARGUMENT_TYPE, PROCEDURE_ARGUMENT_UNITS,' +
          ' PROCEDURE_ARGUMENT_UNITS, PROCEDURE_ARGUMENT_DECIMALS, PROCEDURE_ARGUMENT_KIND,' +
          ' from #procedure_arguments where PROCEDURE_NAME = ''', aProcName,
          ''' order by PROCEDURE_ARGUMENT_INDEX'], result);
        exit;
      end;
  else
    exit; // others (e.g. dDB2) will retrieve info from (ODBC) driver
  end;
  FormatUtf8(fmt, [UpperCase(owner), UpperCase(proc)], result);
end;

function TSqlDBConnectionProperties.SqlGetProcedure: RawUtf8;
var
  fmt, owner: RawUtf8;
begin
  result := '';
  case GetDbms of
    dOracle:
      fmt := 'select case P.OBJECT_TYPE' +
        '  when ''PACKAGE'' then P.OBJECT_NAME || ''.'' || P.PROCEDURE_NAME' +
        '  else P.OBJECT_NAME end NAME_ROUTINE from SYS.ALL_PROCEDURES P ' +
        'where P.owner = ''%'' and P.SUBPROGRAM_ID > 0 order by NAME_ROUTINE';
    dMSSQL,
    dMySQL,
    dMariaDB,
    dPostgreSQL:
      fmt := 'select R.SPECIFIC_NAME NAME_ROUTINE ' +
        'from INFORMATION_SCHEMA.ROUTINES R ' +
        'where UPPER(R.SPECIFIC_SCHEMA) = ''%'' order by NAME_ROUTINE';
    dFirebird:
      fmt := 'select P.RDB$PROCEDURE_NAME NAME_ROUTINE ' +
        'from RDB$PROCEDURES P where P.RDB$OWNER_NAME = ''%'' ' +
        'order by NAME_ROUTINE';
    dNexusDB:
      begin
        // NOT TESTED !!!
        result := 'select P.PROCEDURE_NAME NAME_ROUTINE ' +
          'from #PROCEDURES P order by NAME_ROUTINE';
        exit;
      end;
  else
    exit; // others (e.g. dDB2) will retrieve info from (ODBC) driver
  end;
  SetSchemaNameToOwner(owner);
  FormatUtf8(fmt, [UpperCase(owner)], result);
end;

function TSqlDBConnectionProperties.SqlGetTableNames: RawUtf8;
begin
  case GetDbms of
    dOracle:
      result := 'select owner||''.''||table_name name ' +
        'from sys.all_tables order by owner, table_name';
    dMSSQL:
      result := 'select (TABLE_SCHEMA + ''.'' + TABLE_NAME) as name ' +
        'from INFORMATION_SCHEMA.TABLES where TABLE_TYPE=''BASE TABLE'' order by name';
    dMySQL,
    dMariaDB:
      result := 'select CONCAT(TABLE_SCHEMA,''.'',TABLE_NAME) as name ' +
        'from INFORMATION_SCHEMA.TABLES where TABLE_TYPE=''BASE TABLE'' order by name';
    dPostgreSQL:
      result := 'select (TABLE_SCHEMA||''.''||TABLE_NAME) as name ' +
        'from INFORMATION_SCHEMA.TABLES where TABLE_TYPE=''BASE TABLE'' order by name';
    dSQLite:
      result := 'select name from sqlite_master where type=''table'' ' +
        'and name not like ''sqlite_%''';
    dFirebird:
      result := 'select rdb$relation_name from rdb$relations ' +
        'where rdb$view_blr is null and (rdb$system_flag is null or rdb$system_flag=0)';
    dNexusDB:
      result := 'select table_name name from #tables order by table_name';
  else
    result := ''; // others (e.g. dDB2) will retrieve info from (ODBC) driver
  end;
end;

function TSqlDBConnectionProperties.SqlGetViewNames: RawUtf8;
begin
  case GetDbms of
    dOracle:
      result := 'select owner||''.''||view_name name ' +
        'from sys.all_views order by owner, view_name';
    dMSSQL:
      result := 'select (TABLE_SCHEMA + ''.'' + TABLE_NAME) as name ' +
        'from INFORMATION_SCHEMA.VIEWS order by name';
    dMySQL,
    dMariaDB:
      result := 'select CONCAT(TABLE_SCHEMA,''.'',TABLE_NAME) as name ' +
        'from INFORMATION_SCHEMA.VIEWS order by name';
    dPostgreSQL:
      result := 'select (TABLE_SCHEMA||''.''||TABLE_NAME) as name ' +
        'from INFORMATION_SCHEMA.VIEWS order by name';
    dSQLite:
      result := 'select name from sqlite_master where type=''view'' ' +
        'and name not like ''sqlite_%''';
    dFirebird:
      result := 'select rdb$relation_name from rdb$relations ' +
        'where rdb$view_blr is not null and (rdb$system_flag is null or rdb$system_flag=0)';
    dNexusDB:
      result := 'select view_name name from #views order by view_name'; // NOT TESTED !!!
  else
    result := ''; // others (e.g. dDB2) will retrieve info from (ODBC) driver
  end;
end;

function TSqlDBConnectionProperties.SqlCreateDatabase(
  const aDatabaseName: RawUtf8; aDefaultPageSize: integer): RawUtf8;
begin
  case GetDbms of
    dFirebird:
      begin
        if (aDefaultPageSize <> 8192) or
           (aDefaultPageSize <> 16384) then
          aDefaultPageSize := 4096;
        FormatUtf8('create database ''%'' user ''sysdba'' password ''masterkey''' +
          ' page_size % default character set utf8;',
          [aDatabaseName, aDefaultPageSize], result);
      end;
  else
    result := '';
  end;
end;

const
  COL_DECIMAL = 18; // change it if you update PCHARS[] below before 'DECIMAL'
  COL_NUMERIC = COL_DECIMAL + 1;
  COL_NAMES =       // in efficient IdemPCharSep() format
    'TEXT COLLATE ISO8601|' + // should be before plain 'TEXT'
    'TEXT|CHAR|NCHAR|VARCHAR|NVARCHAR|CLOB|NCLOB|DBCLOB|BIT|INT|BIGINT|DOUBLE|' +
    'NUMBER|FLOAT|REAL|DECFLOAT|CURR|DECIMAL|' +  // see COL_DECIMAL above
    'NUMERIC|' +  // see COL_NUMERIC above
    'BLOB SUB_TYPE 1|BLOB|DATE|SMALLDATE|TIME|TINYINT|BOOL|SMALLINT|MEDIUMINT|' +
    'SERIAL|YEAR|TINYTEXT|MEDIUMTEXT|LONGTEXT|NTEXT|XML|ENUM|SET|' +
    'UNIQUEIDENTIFIER|MONEY|SMALLMONEY|NUM|VARRAW|RAW|LONG RAW|LONG VARRAW|' +
    'TINYBLOB|MEDIUMBLOB|BYTEA|VARBIN|IMAGE|LONGBLOB|BINARY|VARBINARY|GRAPHIC|' +
    'VARGRAPHIC|NULL|';
  COL_TYPES: array[-1 .. 56] of TSqlDBFieldType = (
    ftUnknown,
    ftDate,      // 'TEXT COLLATE ISO8601'
    ftUtf8,      // 'TEXT'
    ftUtf8,      // 'CHAR'
    ftUtf8,      // 'NCHAR'
    ftUtf8,      // 'VARCHAR'
    ftUtf8,      // 'NVARCHAR'
    ftUtf8,      // 'CLOB'
    ftUtf8,      // 'NCLOB'
    ftUtf8,      // 'DBCLOB'
    ftInt64,     // 'BIT'
    ftInt64,     // 'INT'
    ftInt64,     // 'BIGINT'
    ftDouble,    // 'DOUBLE'
    ftDouble,    // 'NUMBER'
    ftDouble,    // 'FLOAT'
    ftDouble,    // 'REAL'
    ftDouble,    // 'DECFLOAT'
    ftCurrency,  // 'CURR'
    ftCurrency,  // 'DECIMAL'
    ftCurrency,  // 'NUMERIC'
    ftUtf8,      // 'BLOB SUB_TYPE 1'
    ftBlob,      // 'BLOB'
    ftDate,      // 'DATE'
    ftDate,      // 'SMALLDATE'
    ftDate,      // 'TIME'
    ftInt64,     // 'TINYINT'
    ftInt64,     // 'BOOL'
    ftInt64,     // 'SMALLINT'
    ftInt64,     // 'MEDIUMINT'
    ftInt64,     // 'SERIAL'
    ftInt64,     // 'YEAR'
    ftUtf8,      // 'TINYTEXT'
    ftUtf8,      // 'MEDIUMTEXT'
    ftUtf8,      // 'LONGTEXT'
    ftUtf8,      // 'NTEXT'
    ftUtf8,      // 'XML'
    ftUtf8,      // 'ENUM'
    ftUtf8,      // 'SET'
    ftUtf8,      // 'UNIQUEIDENTIFIER'
    ftCurrency,  // 'MONEY'
    ftCurrency,  // 'SMALLMONEY'
    ftCurrency,  // 'NUM'
    ftBlob,      // 'VARRAW'
    ftBlob,      // 'RAW'
    ftBlob,      // 'LONG RAW'
    ftBlob,      // 'LONG VARRAW'
    ftBlob,      // 'TINYBLOB'
    ftBlob,      // 'MEDIUMBLOB'
    ftBlob,      // 'BYTEA'
    ftBlob,      // 'VARBIN'
    ftBlob,      // 'IMAGE'
    ftBlob,      // 'LONGBLOB'
    ftBlob,      // 'BINARY'
    ftBlob,      // 'VARBINARY'
    ftBlob,      // 'GRAPHIC'
    ftBlob,      // 'VARGRAPHIC'
    ftNull);     // 'NULL'

function TSqlDBConnectionProperties.ColumnTypeNativeToDB(
  const aNativeType: RawUtf8; aScale: integer): TSqlDBFieldType;

  function ColumnTypeNativeDefault: TSqlDBFieldType;
  var
    ndx: PtrInt;
  begin
    //assert(StrComp(COL_NAMES[COL_DECIMAL],'DECIMAL')=0);
    ndx := IdemPCharSep(pointer(aNativeType), COL_NAMES);
    if (aScale = 0) and
       ((ndx = COL_DECIMAL) or
        (ndx = COL_NUMERIC)) then
      result := ftInt64
    else
      result := COL_TYPES[ndx]; // Types[-1]=ftUnknown
  end;

  function ColumnTypeNativeToDBOracle: TSqlDBFieldType;
  begin
    if PosEx('CHAR', aNativeType) > 0 then
      result := ftUtf8
    else if PropNameEquals(aNativeType, 'NUMBER') then
      case aScale of
        0:
          result := ftInt64;
        1..4:
          result := ftCurrency;
      else
        result := ftDouble;
      end
    else if (PosEx('RAW', aNativeType) > 0) or
            PropNameEquals(aNativeType, 'BLOB') or
            PropNameEquals(aNativeType, 'BFILE') then
      result := ftBlob
    else if IdemPChar(pointer(aNativeType), 'BINARY_') or
            PropNameEquals(aNativeType, 'FLOAT') then
      result := ftDouble
    else if PropNameEquals(aNativeType, 'DATE') or
            IdemPChar(pointer(aNativeType), 'TIMESTAMP') then
      result := ftDate
    else    // all other types will be converted to text
      result := ftUtf8;
  end;

  function ColumnTypeNativeToDBFirebird: TSqlDBFieldType;
  var
    i, err: integer;
  begin
    i := GetInteger(pointer(aNativeType), err);
    if err <> 0 then
      result := ColumnTypeNativeDefault
    else
      case i of // see blr_* definitions
        10, 11, 27:
          result := ftDouble;
        12, 13, 35, 120:
          result := ftDate;
        7, 8, 9, 16, 23, 70, 80, 160:
          result := ftInt64;
        161..169:
          case abs(aScale) of
            0:
              result := ftInt64;
            1..4:
              result := ftCurrency;
          else
            result := ftDouble;
          end;
        2610:
          result := ftBlob;
      else
        result := ftUtf8;
      end;
  end;

begin
  case GetDbms of
    dOracle:
      result := ColumnTypeNativeToDBOracle;
    dFireBird:
      result := ColumnTypeNativeToDBFirebird;
  else
    result := ColumnTypeNativeDefault;
  end;
end;

function TSqlDBConnectionProperties.GetForeignKey(
  const aTableName, aColumnName: RawUtf8): RawUtf8;
begin
  if not fForeignKeys.Initialized then
  begin
    fForeignKeys.Init(false);
    GetForeignKeys;
  end;
  result := fForeignKeys.Value(Join([aTableName, '.', aColumnName]));
end;

function TSqlDBConnectionProperties.GetForeignKeysData: RawByteString;
begin
  if not fForeignKeys.Initialized then
  begin
    fForeignKeys.Init(false);
    GetForeignKeys;
  end;
  result := fForeignKeys.BlobData;
end;

procedure TSqlDBConnectionProperties.SetForeignKeysData(const Value: RawByteString);
begin
  if not fForeignKeys.Initialized then
    fForeignKeys.Init(false);
  fForeignKeys.BlobData := Value;
end;

function TSqlDBConnectionProperties.SqlIso8601ToDate(const Iso8601: RawUtf8): RawUtf8;
begin
  result := Iso8601;
  if (length(result) > 10) and
     (result[11] = 'T') then
    result[11] := ' '; // 'T' -> ' '
  case GetDbms of
    dSQLite:
      ; // use date without 'T'
    dOracle:
      result := Join(['to_date(''', result, ''',''YYYY-MM-DD HH24:MI:SS'')']);
    dNexusDB:
      Join(['DATE ', Iso8601], result);
    dDB2:
      result := Join(['TIMESTAMP ''', result, '''']);
    dPostgreSQL:
      result := Join(['''', result, '''']);
  else
    Join(['''', Iso8601, ''''], result); // default to use ISO-8601 format with 'T'
  end;
end;

function TSqlDBConnectionProperties.SqlDateToIso8601Quoted(DateTime: TDateTime): RawUtf8;
begin
  DateTimeToIso8601Var(DateTime, true, false, DateTimeFirstChar, '''', result);
end;

function TSqlDBConnectionProperties.SqlCreate(const aTableName: RawUtf8;
  const aFields: TSqlDBColumnCreateDynArray; aAddID: boolean): RawUtf8;
var
  i: PtrInt;
  f: RawUtf8;
  col: TSqlDBColumnCreate;
  addprimarykey: RawUtf8;
begin
  // use 'ID' instead of 'RowID' here since some DB (e.g. Oracle) use it
  result := '';
  if high(aFields) < 0 then
    exit; // nothing to create
  if aAddID then
  begin
    col.DBType := ftInt64;
    col.Name := ID_TXT;
    col.Unique := true;
    col.NonNullable := true;
    col.PrimaryKey := true;
    Join([SqlFieldCreate(col, addprimarykey), ','], result);
  end;
  for i := 0 to high(aFields) do
  begin
    f := SqlFieldCreate(aFields[i], addprimarykey);
    if i <> high(aFields) then
      AppendShortToUtf8(',', f);
    Append(result, f);
  end;
  if addprimarykey <> '' then
    Append(result, [', PRIMARY KEY(', addprimarykey, ')']);
  result := Join(['CREATE TABLE ', aTableName, ' (', result, ')']);
  case GetDbms of
    dDB2:
      AppendShortToUtf8(' CCSID Unicode', result);
  end;
end;

function TSqlDBConnectionProperties.SqlFieldCreate(
  const aField: TSqlDBColumnCreate; var aAddPrimaryKey: RawUtf8): RawUtf8;
begin
  if (aField.DBType = ftUtf8) and
     (cardinal(aField.Width - 1) < fSqlCreateFieldMax) then
    FormatUtf8(fSqlCreateField[ftNull], [aField.Width], result)
  else
    result := fSqlCreateField[aField.DBType];
  if aField.NonNullable or aField.Unique or aField.PrimaryKey then
    AppendShortToUtf8(' NOT NULL', result);
  if aField.Unique and
     not aField.PrimaryKey then
    AppendShortToUtf8(' UNIQUE', result); // see http://www.w3schools.com/sql/sql_unique.asp
  if aField.PrimaryKey then
    case GetDbms of
      dSQLite,
      dMSSQL,
      dOracle,
      dJet,
      dPostgreSQL,
      dFirebird,
      dNexusDB,
      dInformix:
        AppendShortToUtf8(' PRIMARY KEY', result);
      dDB2,
      dMySQL,
      dMariaDB:
        aAddPrimaryKey := aField.Name;
    end;
  result := Join([aField.Name, result]);
end;

function TSqlDBConnectionProperties.SqlAddColumn(const aTableName: RawUtf8;
  const aField: TSqlDBColumnCreate): RawUtf8;
var
  addprimarykey: RawUtf8;
begin
  FormatUtf8('ALTER TABLE % ADD %',
    [aTableName, SqlFieldCreate(aField, addprimarykey)], result);
end;

function TSqlDBConnectionProperties.SqlAddIndex(const aTableName: RawUtf8;
  const aFieldNames: array of RawUtf8; aUnique, aDescending: boolean;
  const aIndexName: RawUtf8): RawUtf8;
const
  CREATNDXIFNE: array[boolean] of RawUtf8 = (
    '', 'IF NOT EXISTS ');
var
  indexname, fieldscsv, coldesc, owner, table: RawUtf8;
begin
  result := '';
  if (self = nil) or
     (aTableName = '') or
     (high(aFieldNames) < 0) then
    exit;
  if aUnique then
    result := 'UNIQUE ';
  if aIndexName = '' then
  begin
    SqlSplitTableName(aTableName, owner, table);
    if (owner <> '') and
       not (fDbms in [dMSSQL, dPostgreSQL, dMySQL, dMariaDB, dFirebird, dDB2, dInformix]) then
      // some DB engines do not expect any schema in the index name
      Join([owner, '.'], indexname);
    fieldscsv := JoinCsv('', aFieldNames);
    if length(fieldscsv) + length(table) > 27 then
      // sounds like if some DB limit the identifier length to 32 chars
      Append(indexname,
        ['INDEX', crc32cUtf8ToHex(table), crc32cUtf8ToHex(fieldscsv)])
    else
      Append(indexname, ['NDX', table, fieldscsv]);
  end
  else
    indexname := aIndexName;
  if aDescending then
    case DB_SQLDESENDINGINDEXPOS[GetDbms] of
      posGlobalBefore:
        AppendShortToUtf8('DESC ', result);
      posWithColumn:
        coldesc := JoinCsv(' DESC,', aFieldNames) + ' DESC';
    end;
  if {%H-}coldesc = '' then
    coldesc := JoinCsv(',', aFieldNames);
  result := FormatUtf8('CREATE %INDEX %% ON %(%)', [result,
    CREATNDXIFNE[GetDbms in DB_HANDLECREATEINDEXIFNOTEXISTS],
    indexname, aTableName, coldesc]);
end;

function TSqlDBConnectionProperties.SqlTableName(const aTableName: RawUtf8): RawUtf8;
var
  beginquote, endquote: RawUtf8;
  needquote: boolean;
begin
  beginquote := '"';
  endquote := '"';
  needquote := PosExChar(' ', aTableName) > 0;
  case fDbms of
    dPostgresql:
      if PosExChar('.', aTableName) = 0 then
        needquote := true; // quote if not schema.identifier format
    dMySQL,
    dMariaDB:
      begin
        beginquote := '`';  // backtick/grave accent
        endquote := '`';
      end;
    dJet:
      begin  // note: dMSSQL may SET IDENTIFIER ON to use doublequotes
        beginquote := '[';
        endquote := ']';
      end;
    dSQLite:
      begin
        if PosExChar('.', aTableName) > 0 then
          needquote := true;
        beginquote := '`';  // backtick/grave accent
        endquote := '`';
      end;
  end;
  if needquote and
     (PosEx(beginquote, aTableName) = 0) then
    Join([beginquote, aTableName, endquote], result)
  else
    result := aTableName;
end;

procedure TSqlDBConnectionProperties.GetIndexesAndSetFieldsColumnIndexed(
  const aTableName: RawUtf8; var Fields: TSqlDBColumnDefineDynArray);
var
  i, j: PtrInt;
  colname: RawUtf8;
  indexes: TSqlDBIndexDefineDynArray;
begin
  if Fields = nil then
    exit;
  GetIndexes(aTableName, indexes);
  for i := 0 to high(indexes) do
  begin
    colname := TrimU(GetFirstCsvItem(indexes[i].KeyColumns));
    if colname <> '' then
      for j := 0 to high(Fields) do
        if PropNameEquals(Fields[j].ColumnName, colname) then
        begin
          Fields[j].ColumnIndexed := true;
          break;
        end;
  end;
end;

function TSqlDBConnectionProperties.ExceptionIsAboutConnection(
  aClass: ExceptClass; const aMessage: RawUtf8): boolean;

  function PosErrorNumber(const aMessage: RawUtf8; aSepChar: AnsiChar): PUtf8Char;
  begin
    // search aSepChar followed by a number
    result := pointer(aMessage);
    repeat
      result := PosChar(result, aSepChar);
      if result = nil then
        exit;
      inc(result);
    until result^ in ['0'..'9'];
  end;

begin
  // see more complete list in feature request [f024266c0839]
  case fDbms of
    dOracle:
      result := IdemPCharSep(PosErrorNumber(aMessage, '-'),
        '00028|01012|01017|01033|01089|02396|03113|03114|03135|12152|12154|' +
        '12157|12514|12520|12537|12545|12560|12571|') >= 0;
    dInformix:
      // error codes based on {IBM INFORMIX ODBC DRIVER} on wrong data connection
      result := IdemPCharSep(PosErrorNumber(aMessage, '-'),
        '329|761|902|908|930|931|951|11017|23101|23104|25567|25582|27002|') >= 0;
    dMSSQL:
      // error codes based on {SQL Server Native Client 11.0} tested with wrong
      // data connection using general error codes because MS SQL SERVER has
      // multiple error codes in the error message
      result := IdemPCharSep(PosErrorNumber(aMessage, '['),
        '08001|08S01|08007|28000|42000|') >= 0;
    dMySQL,
    dMariaDB:
      result := (PosEx('Lost connection to', aMessage) > 0) or
                (PosEx('server has gone away', aMessage) > 0);
  else
    result := PosI(' CONNE', aMessage) > 0;
  end;
end;

procedure TSqlDBConnectionProperties.MultipleValuesInsert(
  Props: TSqlDBConnectionProperties; const TableName: RawUtf8;
  const FieldNames: TRawUtf8DynArray; const FieldTypes: TSqlDBFieldTypeArray;
  RowCount: integer; const FieldValues: TRawUtf8DynArrayDynArray;
  BatchOptions: TRestBatchOptions);
var
  sql: RawUtf8;
  sqlcached: boolean;
  prevrowcount: integer;
  maxf: PtrInt;

  procedure ComputeSql(rowcount, offset: integer);
  var
    f, r, p, len: PtrInt;
    W: TTextWriter;
    tmp: TTextWriterStackBuffer; // 8KB work buffer on stack
  begin
    if (fDbms <> dFireBird) and
       (rowcount = prevrowcount) then
      exit;
    prevrowcount := rowcount;
    W := TTextWriter.CreateOwnedStream(tmp);
    try
      case Props.fDbms of
        dFirebird:
          begin
            W.AddShort('execute block('#10);
            p := 0;
            for r := offset to offset + rowcount - 1 do
            begin
              for f := 0 to maxf do
              begin
                W.AddDirect('i');
                inc(p);
                W.AddU(p);
                if FieldValues[f, r] = 'null' then
                  W.AddShorter(' CHAR(1)')
                else
                  case FieldTypes[f] of
                    ftNull:
                      W.AddShorter(' CHAR(1)');
                    ftUtf8:
                      begin
                        len := length(FieldValues[f, r]) - 2;
                        if len < 1 then
                          len := 1; // unquoted UTF-8 text length
                        W.AddShort(' VARCHAR(');
                        W.AddU(len);
                        W.AddShort(') CHARACTER SET UTF8');
                      end;
                  else
                    W.AddString(DB_FIELDS[dFirebird, FieldTypes[f]]);
                  end;
                W.AddDirect('=', '?', ',');
              end;
              W.CancelLastComma;
              W.AddDirect(#10, ',');
            end;
            W.CancelLastComma;
            W.AddShort(') as begin'#10);
            p := 0;
            for r := 1 to rowcount do
            begin
              EncodeInsertPrefix(W, BatchOptions, dFirebird);
              W.AddString(TableName);
              W.AddDirect(' ', '(');
              for f := 0 to maxf do
              begin
                W.AddString(FieldNames[f]);
                W.AddComma;
              end;
              W.CancelLastComma;
              W.AddShort(') values (');
              for f := 0 to maxf do
              begin
                inc(p);
                W.AddDirect(':', 'i');
                W.AddU(p);
                W.AddComma;
              end;
              W.CancelLastComma;
              W.AddDirect(')', ';', #10);
            end;
            W.AddDirect('e', 'n', 'd');
            if W.TextLength > 32700 then
              ESqlDBException.RaiseUtf8(
                '%.MultipleValuesInsert: Firebird Execute Block length=%',
                [self, W.TextLength]);
            sqlcached := false; // ftUtf8 values will have varying field length
          end;
        dOracle:
          begin
            // INSERT ALL INTO ... VALUES ... SELECT 1 FROM DUAL
            W.AddShort('insert all'#10); // see http://stackoverflow.com/a/93724
            for r := 1 to rowcount do
            begin
              W.AddShorter('into ');
              W.AddString(TableName);
              W.AddDirect(' ', '(');
              for f := 0 to maxf do
              begin
                W.AddString(FieldNames[f]);
                W.AddComma;
              end;
              W.CancelLastComma;
              W.AddShort(') values (');
              for f := 0 to maxf do
                W.Add('?', ',');
              W.CancelLastComma;
              W.AddDirect(')', #10);
            end;
            W.AddShort('select 1 from dual');
            sqlcached := true;
          end;
      else
        begin
          //  e.g. NexusDB/SQLite3/MySQL/PostgreSQL/MSSQL2008/DB2/INFORMIX
          // INSERT .. VALUES (..),(..),(..),..
          EncodeInsertPrefix(W, BatchOptions, Props.fDbms);
          W.AddString(TableName);
          W.AddDirect(' ', '(');
          for f := 0 to maxf do
          begin
            W.AddString(FieldNames[f]);
            W.AddComma;
          end;
          W.CancelLastComma;
          W.AddShort(') values ');
          for r := 1 to rowcount do
          begin
            W.AddDirect('(');
            for f := 0 to maxf do
              W.Add('?', ',');
            W.CancelLastComma;
            W.AddDirect(')', ',');
          end;
          W.CancelLastComma;
          sqlcached := true;
        end;
      end;
      W.SetText(sql);
    finally
      W.Free;
    end;
  end;

var
  batchRowCount, paramCountLimit, currentRow, sqllen, p: integer;
  f, i: PtrInt;
  stmt: TSqlDBStatement;
  query: ISqlDBStatement;
begin
  maxf := length(FieldNames);     // e.g. 2 fields
  if (Props = nil) or
     (FieldNames = nil) or
     (TableName = '') or
     (length(FieldValues) <> maxf) then
    ESqlDBException.RaiseUtf8('Invalid %.MultipleValuesInsert(%) call',
      [self, TableName]);
  batchRowCount := 0;
  paramCountLimit := DB_PARAMSMAX[Props.fDbms];
  case Props.fDbms of
    dFirebird:
      begin
        // compute from max sql statement size of 32KB
        sqllen := maxf * 48; // worse case (with BLOB param)
        for f := 0 to maxf - 1 do
          inc(sqllen, Length(FieldNames[f]));
        batchRowCount := 32000 div sqllen;
        if batchRowCount > RowCount then
          batchRowCount := RowCount;
      end;
  end;
  if paramCountLimit <> 0 then
    if RowCount * maxf > paramCountLimit then
      batchRowCount := paramCountLimit div maxf
    else
      batchRowCount := RowCount;
  if batchRowCount = 0 then
    ESqlDBException.RaiseUtf8('%.MultipleValuesInsert(%) with # params = %>%',
      [self, TableName, RowCount * maxf, paramCountLimit]);
  dec(maxf);
  prevrowcount := 0;
  sqlcached := false;
  currentRow := 0;
  repeat
    if RowCount - currentRow > batchRowCount then
      // max number of params -> try cache
      ComputeSql(batchRowCount, currentRow)
    else
    begin
      ComputeSql(RowCount - currentRow, currentRow);
      sqlcached := false; // truncate number of parameters should not be unique
    end;
    if sqlcached then
      query := Props.NewThreadSafeStatementPrepared(sql, false)
    else
    begin
      stmt := Props.NewThreadSafeStatement;
      try
        stmt.Prepare(sql, false);
        query := stmt; // stmt will be released by query := nil below
      except
        on Exception do
          stmt.Free; // avoid memory leak in case of invalid sql statement
        // exception leaves query=nil to raise exception
      end;
    end;
    if query = nil then
      ESqlDBException.RaiseUtf8(
        '%.MultipleValuesInsert: query=nil for [%]', [self, sql]);
    try
      p := 1;
      for i := 1 to prevrowcount do
      begin
        for f := 0 to maxf do
        begin
          query.Bind(p, FieldTypes[f], FieldValues[f, currentRow], false);
          inc(p);
        end;
        inc(currentRow);
      end;
      query.ExecutePrepared;
    finally
      query := nil; // will release the uncached local stmt, if applying
    end;
  until currentRow = RowCount;
end;

procedure TSqlDBConnectionProperties.MultipleValuesInsertFirebird(
  Props: TSqlDBConnectionProperties; const TableName: RawUtf8;
  const FieldNames: TRawUtf8DynArray; const FieldTypes: TSqlDBFieldTypeArray;
  RowCount: integer; const FieldValues: TRawUtf8DynArrayDynArray;
  BatchOptions: TRestBatchOptions);
var
  W: TTextWriter;
  maxf, sqllenwitoutvalues, sqllen, r, f, i: PtrInt;
  v: RawUtf8;
begin
  maxf := length(FieldNames);     // e.g. 2 fields
  if (Props = nil) or
     (FieldNames = nil) or
     (TableName = '') or
     (length(FieldValues) <> maxf) or
     (Props.fDbms <> dFirebird) then
    ESqlDBException.RaiseUtf8(
      'Invalid %.MultipleValuesInsertFirebird(%,%)', [self, Props, TableName]);
  sqllenwitoutvalues := 3 * maxf + 24;
  dec(maxf);
  for f := 0 to maxf do
    case FieldTypes[f] of
      ftBlob:
        begin
          // not possible to inline BLOBs -> fallback to regular
          MultipleValuesInsert(Props, TableName, FieldNames, FieldTypes,
            RowCount, FieldValues, BatchOptions);
          exit;
        end;
      ftDate:
        inc(sqllenwitoutvalues, Length(FieldNames[f]) + 20); // 'timestamp '
    else
      inc(sqllenwitoutvalues, Length(FieldNames[f]));
    end;
  W := TTextWriter.CreateOwnedStream(49152);
  try
    r := 0;
    repeat
      W.AddShort('execute block as begin'#10);
      sqllen := sqllenwitoutvalues;
      repeat
        for f := 0 to maxf do
          inc(sqllen, length(FieldValues[f, r]));
        if sqllen + PtrInt(W.TextLength) > 30000 then
          break;
        EncodeInsertPrefix(W, BatchOptions, dFirebird);
        W.AddString(TableName);
        W.AddDirect(' ', '(');
        for f := 0 to maxf do
        begin
          W.AddString(FieldNames[f]);
          W.AddComma;
        end;
        W.CancelLastComma;
        W.AddShort(') values (');
        for f := 0 to maxf do
        begin
          v := FieldValues[f, r]; // includes single quotes (#39)
          if (v = '') or
             (v = 'null') then
            W.AddNull
          else if FieldTypes[f] = ftDate then
            if v = #39#39 then
              W.AddNull
            else
            begin
              W.AddShort('timestamp ');
              if length(v) > 12 then
              begin
                // not 'CCYY-MM-DD' -> fix needed?
                if v[12] = 'T' then // handle 'CCYY-MM-DDTHH:MM:SS' common case
                  v[12] := ' '
                else
                begin
                  i := PosExChar('T', v);
                  if i > 0 then
                    v[i] := ' ';
                end; // see https://firebirdsql.org/en/firebird-date-literals
              end;
              W.AddString(v)
            end
          else
            W.AddString(v);
          W.AddComma;
        end;
        W.CancelLastComma;
        W.AddDirect(')', ';', #10);
        inc(r);
      until r = RowCount;
      W.AddDirect('e', 'n', 'd');
      with Props.NewThreadSafeStatement do
        try
          Execute(W.Text, false);
        finally
          Free;
        end;
      if r = RowCount then
        break;
      W.CancelAll;
    until false;
  finally
    W.Free;
  end;
end;

function TSqlDBConnectionProperties.FieldsFromList(
  const aFields: TSqlDBColumnDefineDynArray;
  aExcludeTypes: TSqlDBFieldTypes): RawUtf8;
var
  i, n: PtrInt;
begin
  result := '';
  if byte(aExcludeTypes) <> 0 then
  begin
    n := length(aFields);
    for i := 0 to n - 1 do
      with aFields[i] do
        if not (ColumnType in aExcludeTypes) then
        begin
          dec(n);
          if result = '' then
            result := ColumnName
          else
            Append(result, ',', ColumnName);
        end;
    if n = 0 then
      result := '*';
  end
  else
    result := '*';
end;

function TSqlDBConnectionProperties.SqlSelectAll(const aTableName: RawUtf8;
  const aFields: TSqlDBColumnDefineDynArray;
  aExcludeTypes: TSqlDBFieldTypes): RawUtf8;
begin
  if (self = nil) or
     (aTableName = '') then
    result := ''
  else
    Join(['select ', FieldsFromList(aFields, aExcludeTypes),
          ' from ', SqlTableName(aTableName)], result);
end;

{$ifdef ISDELPHI20062007}
  {$warnings off} // avoid paranoid Delphi 2007 warning
{$endif ISDELPHI20062007}

class function TSqlDBConnectionProperties.EngineName: RawUtf8;
var
  L: PtrInt;
begin
  result := '';
  if self = nil then
    exit;
  ClassToText(self, result);
  if IdemPChar(pointer(result), 'TSQLDB') then
    Delete(result, 1, 6)
  else if result[1] = 'T' then
    Delete(result, 1, 1);
  L := length(result);
  if (L > 20) and
     IdemPropName('ConnectionProperties', @result[L - 19], 20) then
    SetLength(result, L - 20);
  if (L > 5) and
     IdemPropName('OLEDB', pointer(result), 5) then
    Delete(result, 1, 5)
  else if (L > 4) and
       IdemPropName('ODBC', pointer(result), 4) then
      Delete(result, 1, 4);
end;

{$ifdef ISDELPHI20062007}
  {$warnings on}
{$endif ISDELPHI20062007}

function TSqlDBConnectionProperties.GetDbms: TSqlDBDefinition;
begin
  if fDbms = dUnknown then
    result := dDefault
  else
    result := fDbms;
end;

function TSqlDBConnectionProperties.GetDbmsName: RawUtf8;
var
  ps: PShortString;
begin
  ps := ToText(GetDbms);
  FastSetString(result, @ps^[2], ord(ps^[0]) - 1);
end;

function TSqlDBConnectionProperties.SanitizeFromPassword(const S: RawUtf8): RawUtf8;
begin
  result := StringReplaceAll(S, fPassWord, '***');
end;

function TSqlDBConnectionProperties.GetDatabaseNameSafe: RawUtf8;
begin
  result := SanitizeFromPassword(fDatabaseName);
end;

function TSqlDBConnectionProperties.SqlLimitClause(AStmt: TSelectStatement):
  TSqlDBDefinitionLimitClause;
begin
  result := DB_SQLLIMITCLAUSE[GetDbms];
end;

var
  GlobalDefinitions: array of TSqlDBConnectionPropertiesClass;

class procedure TSqlDBConnectionProperties.RegisterClassNameForDefinition;
begin
  ObjArrayAddOnce(GlobalDefinitions, TObject(self)); // TClass stored as TObject
end;

procedure TSqlDBConnectionProperties.DefinitionTo(
  Definition: TSynConnectionDefinition);
begin
  if Definition = nil then
    exit;
  Definition.Kind := ClassName;
  Definition.ServerName := ServerName;
  Definition.DatabaseName := DatabaseName;
  Definition.User := UserID;
  Definition.PassWordPlain := PassWord;
end;

function TSqlDBConnectionProperties.DefinitionToJson(Key: cardinal): RawUtf8;
var
  def: TSynConnectionDefinition;
begin
  def := TSynConnectionDefinition.Create;
  try
    def.Key := Key;
    DefinitionTo(def);
    result := def.SaveToJson;
  finally
    def.Free;
  end;
end;

procedure TSqlDBConnectionProperties.DefinitionToFile(
  const aJsonFile: TFileName; Key: cardinal; Fmt: TTextWriterJsonFormat);
begin
  FileFromString(JsonReformat(DefinitionToJson(Key), Fmt), aJsonFile);
end;

class function TSqlDBConnectionProperties.ClassFrom(
  aDefinition: TSynConnectionDefinition): TSqlDBConnectionPropertiesClass;
var
  ndx: PtrInt;
begin
  for ndx := 0 to length(GlobalDefinitions) - 1 do
    if GlobalDefinitions[ndx].ClassNameIs(aDefinition.Kind) then
    begin
      result := GlobalDefinitions[ndx];
      exit;
    end;
  result := nil;
end;

class function TSqlDBConnectionProperties.CreateFrom(
  aDefinition: TSynConnectionDefinition): TSqlDBConnectionProperties;
var
  c: TSqlDBConnectionPropertiesClass;
  pwd: SpiUtf8;
begin
  c := ClassFrom(aDefinition);
  if c = nil then
    ESqlDBException.RaiseUtf8('%.CreateFrom: unknown % class - please ' +
      'add a reference to its implementation unit', [self, aDefinition.Kind]);
  aDefinition.GetPasswordSafe(pwd);
  result := c.Create(aDefinition.ServerName, aDefinition.DatabaseName,
    aDefinition.User, pwd);
  FillZero(pwd); // anti-forensic
end;

class function TSqlDBConnectionProperties.CreateFromJson(
  const aJsonDefinition: RawUtf8; aKey: cardinal): TSqlDBConnectionProperties;
var
  def: TSynConnectionDefinition;
begin
  def := TSynConnectionDefinition.CreateFromJson(aJsonDefinition, aKey);
  try
    result := CreateFrom(def);
  finally
    def.Free;
  end;
end;

class function TSqlDBConnectionProperties.CreateFromFile(
  const aJsonFile: TFileName; aKey: cardinal): TSqlDBConnectionProperties;
begin
  result := CreateFromJson(JsonNormalizeFromFile(aJsonFile), aKey);
end;


{ TSqlDBStatement }

procedure TSqlDBStatement.SetFlag(
  const flag: TSqlDBStatementFlag; const value: boolean);
begin
  if value then
    include(fFlags, flag)
  else
    exclude(fFlags, flag);
end;

function TSqlDBStatement.GetFlag(const flag: TSqlDBStatementFlag): boolean;
begin
  result := flag in fFlags;
end;


procedure TSqlDBStatement.Bind(Param: integer; const Data: TSqlVar;
  IO: TSqlDBParamInOutType);
begin
  with Data do
    case VType of
      ftNull:
        BindNull(Param, IO);
      ftInt64:
        Bind(Param, VInt64, IO);
      ftDate:
        BindDateTime(Param, VDateTime, IO);
      ftDouble:
        Bind(Param, VDouble, IO);
      ftCurrency:
        BindCurrency(Param, VCurrency, IO);
      ftUtf8:
        BindTextP(Param, VText, IO);
      ftBlob:
        BindBlob(Param, VBlob, VBlobLen, IO);
    else
      ESqlDBException.RaiseUtf8('%.Bind(Param=%,VType=%)',
        [self, Param, ord(VType)]);
    end;
end;

procedure TSqlDBStatement.Bind(Param: integer; ParamType: TSqlDBFieldType;
  const Value: RawUtf8; ValueAlreadyUnquoted: boolean; IO: TSqlDBParamInOutType);
var
  tmp: RawUtf8;
begin
  if not ValueAlreadyUnquoted and
     (Value = 'null') then
    // bind null (ftUtf8 should be '"null"')
    BindNull(Param, IO)
  else
    case ParamType of
      ftNull:
        BindNull(Param, IO);
      ftInt64:
        Bind(Param, GetInt64(pointer(Value)), IO);
      ftDouble:
        Bind(Param, GetExtended(pointer(Value)), IO);
      ftCurrency:
        BindCurrency(Param, StrToCurrency(pointer(Value)), IO);
      ftBlob:
        BindBlob(Param, Value, IO); // already decoded
      ftDate:
        begin
          if ValueAlreadyUnquoted then
            tmp := Value
          else
            UnQuoteSqlStringVar(pointer(Value), tmp);
          BindDateTime(Param, Iso8601ToDateTime(tmp), IO);
        end;
      ftUtf8:
        if (fConnection <> nil) and
           fConnection.fProperties.StoreVoidStringAsNull and
           ((Value = '') or // check if '' or '""' should be stored as null
            ((PInteger(Value)^ and $ffffff = $2727) and
             not ValueAlreadyUnquoted)) then
          BindNull(Param, IO, ftUtf8)
        else
        begin
          if ValueAlreadyUnquoted then
            tmp := Value
          else
            UnQuoteSqlStringVar(pointer(Value), tmp);
          BindTextU(Param, tmp, IO);
        end;
    else
      ESqlDBException.RaiseUtf8('Invalid %.Bind(%,TSqlDBFieldType(%),%)',
        [self, Param, ord(ParamType), Value]);
    end;
end;

function VariantIsBlob(const V: variant): boolean;
var
  vd: TVarData absolute V;
begin
  result := (cardinal(vd.VType) = varNull) or
            ((cardinal(vd.VType) = varString) and
             (vd.VString <> nil) and
             (PCardinal(vd.VString)^ and $ffffff = JSON_BASE64_MAGIC_C));
end;

procedure TSqlDBStatement.Bind(const Params: array of const;
  IO: TSqlDBParamInOutType);
var
  arg, c: integer;
  p: PVarRec;
  tmp: RawUtf8;
 begin
  p := @Params[0];
  for arg := 1 to high(Params) + 1 do // bind parameter index starts at 1
  begin
    case p^.VType of
      vtAnsiString:
        if p^.VAnsiString = nil then
          BindTextU(arg, '', IO)
        else
        begin
          c := PInteger(p^.VAnsiString)^ and $00ffffff;
          if c = JSON_BASE64_MAGIC_C then
          begin
            Base64ToBin(p^.VPChar + 3, PStrLen(p^.VPChar - _STRLEN)^ - 3,
              RawByteString(tmp));
            BindBlob(arg, tmp, IO);
          end
          else if c = JSON_SQLDATE_MAGIC_C then
            BindDateTime(arg, Iso8601ToDateTimePUtf8Char(
              PUtf8Char(p^.VAnsiString) + 3, PStrLen(p^.VPChar - _STRLEN)^ - 3))
          else
          begin
            AnyAnsiToUtf8Var(RawByteString(p^.VAnsiString), tmp);
            BindTextU(arg, tmp, IO);
          end;
        end;
      vtBoolean:
        if p^.VBoolean then // normalize
          Bind(arg, 1, IO)
        else
          Bind(arg, 0, IO);
      vtInteger:
        Bind(arg, p^.VInteger, IO);
      {$ifdef FPC} // won't support huge positive QWord values
      vtQWord,
      {$endif FPC}
      vtInt64:
        Bind(arg, p^.VInt64^, IO);
      vtCurrency:
        BindCurrency(arg, p^.VCurrency^, IO);
      vtExtended:
        Bind(arg, p^.VExtended^, IO);
      vtPointer: // see TJsonWriter.AddJsonEscape(TVarRec) or VarRecToVariant()
        if p^.VPointer = nil then
          BindNull(arg, IO)
        else
          Bind(arg, PtrInt(p^.VPointer), IO);
      vtVariant:
        BindVariant(arg, p^.VVariant^, VariantIsBlob(p^.VVariant^), IO);
      {$ifdef UNICODE}
      vtUnicodeString: // optimize Delphi string constants
        BindTextS(arg, string(p^.VUnicodeString), IO);
      {$endif HASVARUSTRING}
    else
      begin
        VarRecToUtf8(p, tmp);
        BindTextU(arg, tmp, IO); // bind e.g. vtPChar/vtWideString as UTF-8
      end;
    end;
    inc(p);
  end;
end;

procedure TSqlDBStatement.BindVariant(Param: integer; const Data: Variant;
  DataIsBlob: boolean; IO: TSqlDBParamInOutType);
var
  I64: Int64Rec;
  vd: TVarData absolute Data;
begin
  case cardinal(vd.VType) of
    varEmpty,
    varNull:
      BindNull(Param, IO);
    varBoolean:
      if vd.VBoolean then
        Bind(Param, 1, IO)
      else
        Bind(Param, 0, IO);
    varByte:
      Bind(Param, vd.VInteger, IO);
    varSmallint:
      Bind(Param, vd.VSmallInt, IO);
    varShortInt:
      Bind(Param, vd.VShortInt, IO);
    varWord:
      Bind(Param, vd.VWord, IO);
    varLongWord:
      begin
        I64.Lo := vd.VLongWord;
        I64.Hi := 0;
        Bind(Param, Int64(I64), IO);
      end;
    varInteger:
      Bind(Param, vd.VInteger, IO);
    varInt64,
    varWord64:
      Bind(Param, vd.VInt64, IO);
    varSingle:
      Bind(Param, vd.VSingle, IO);
    varDouble:
      Bind(Param, vd.VDouble, IO);
    varDate:
      BindDateTime(Param, vd.VDate, IO);
    varCurrency:
      BindCurrency(Param, vd.VCurrency, IO);
    varOleStr:
      // handle special case if was bound explicitly as WideString
      BindTextW(Param, WideString(vd.VAny), IO);
    {$ifdef HASVARUSTRING}
    varUString:
      if DataIsBlob then
        ESqlDBException.RaiseUtf8(
          '%.BindVariant: BLOB should not be UnicodeString', [self])
      else
        BindTextU(Param, UnicodeStringToUtf8(UnicodeString(vd.VAny)), IO);
    {$endif HASVARUSTRING}
    varString:
      if DataIsBlob then
        if (vd.VAny <> nil) and
           (PInteger(vd.VAny)^ and $00ffffff = JSON_BASE64_MAGIC_C) then
          // recognized as Base64 encoded text
          BindBlob(Param, Base64ToBin(PAnsiChar(vd.VAny) + 3,
                                      PStrLen(PAnsiChar(vd.VAny) - _STRLEN)^ - 3))
        else
          // no conversion if was set via TQuery.AsBlob property e.g.
          BindBlob(Param, RawByteString(vd.VAny), IO)
      else
        // direct bind of AnsiString as UTF-8 value
        BindTextU(Param, AnyAnsiToUtf8(RawByteString(vd.VAny)), IO);
  else
    if cardinal(vd.VType) = varVariantByRef then
      BindVariant(Param, PVariant(vd.VPointer)^, DataIsBlob, IO)
    else if cardinal(vd.VType) = varOleStrByRef then
      BindTextW(Param, PWideString(vd.VAny)^, IO)
    else
      // also use TEXT for any non native VType parameter
      BindTextU(Param, VariantToUtf8(Data), IO);
  end;
end;

procedure TSqlDBStatement.BindArray(Param: integer; ParamType: TSqlDBFieldType;
  const Values: TRawUtf8DynArray; ValuesCount: integer);
begin
  if (Param <= 0) or
     (ParamType in [ftUnknown, ftNull]) or
     (ValuesCount <= 0) or
     (length(Values) < ValuesCount) or
     (fConnection = nil) or
     (fConnection.fProperties.BatchSendingAbilities *
      [cCreate, cUpdate, cDelete] = []) then
    ESqlDBException.RaiseUtf8('Invalid call to %.BindArray(Param=%,Type=%)',
      [self, Param, ToText(ParamType)^]);
end;

procedure TSqlDBStatement.BindArray(Param: integer; const Values: array of Int64);
begin
  BindArray(Param, ftInt64, nil, 0); // will raise an exception (Values=nil)
end;

procedure TSqlDBStatement.BindArray(Param: integer; const Values: array of RawUtf8);
begin
  BindArray(Param, ftUtf8, nil, 0); // will raise an exception (Values=nil)
end;

procedure TSqlDBStatement.BindArray(Param: integer; const Values: array of double);
begin
  BindArray(Param, ftDouble, nil, 0); // will raise an exception (Values=nil)
end;

procedure TSqlDBStatement.BindArrayCurrency(Param: integer;
  const Values: array of currency);
begin
  BindArray(Param, ftCurrency, nil, 0); // will raise an exception (Values=nil)
end;

procedure TSqlDBStatement.BindArrayDateTime(Param: integer;
  const Values: array of TDateTime);
begin
  BindArray(Param, ftDate, nil, 0); // will raise an exception (Values=nil)
end;

procedure TSqlDBStatement.BindArrayJson(Param: integer; ParamType: TSqlDBFieldType;
  var JsonArray: RawUtf8; ValuesCount: integer);
begin
  BindArray(Param, ParamType, nil, 0); // will raise an exception (Values=nil)
end;

procedure TSqlDBStatement.CheckCol(Col: integer);
begin
  if (self = nil) or
     (cardinal(Col) >= cardinal(fColumnCount)) then
    CheckColInvalid(Col);
end;

procedure TSqlDBStatement.CheckColInvalid(Col: integer);
begin
  ESqlDBException.RaiseUtf8('Invalid call to %.Column*(Col=%)', [self, Col]);
end;

function TSqlDBStatement.GetForceBlobAsNull: boolean;
begin
  result := GetFlag(dsfForceBlobAsNull);
end;

procedure TSqlDBStatement.SetForceBlobAsNull(value: boolean);
begin
  SetFlag(dsfForceBlobAsNull, value);
end;

function TSqlDBStatement.GetForceDateWithMS: boolean;
begin
  result := GetFlag(dsfForceDateWithMS);
end;

procedure TSqlDBStatement.SetForceDateWithMS(value: boolean);
begin
  SetFlag(dsfForceDateWithMS, value);
end;

constructor TSqlDBStatement.Create(aConnection: TSqlDBConnection);
begin
  inherited Create;
  fConnection := aConnection;
  SetFlag(dsfStripSemicolon, true);
  if aConnection <> nil then
    fDbms := aConnection.fProperties.GetDbms;
end;

function TSqlDBStatement.ColumnCount: integer;
begin
  if self = nil then
    result := 0
  else
    result := fColumnCount;
end;

function TSqlDBStatement.ColumnPUtf8(Col: integer): PUtf8Char;
begin
  result := nil; // not supported by this class yet
end;

function TSqlDBStatement.ColumnBlobBytes(Col: integer): TBytes;
begin
  RawByteStringToBytes(ColumnBlob(Col), result);
end;

procedure TSqlDBStatement.ColumnBlobToStream(Col: integer; Stream: TStream);
var
  tmp: RawByteString;
begin
  tmp := ColumnBlob(Col); // default implementation
  Stream.WriteBuffer(pointer(tmp)^, Length(tmp));
end;

procedure TSqlDBStatement.ColumnBlobFromStream(Col: integer; Stream: TStream);
begin
  ESqlDBException.RaiseUtf8('%.ColumnBlobFromStream not implemented', [self]);
end;

function TSqlDBStatement.ColumnVariant(Col: integer; ForceUtf8: boolean): Variant;
begin
  ColumnToVariant(Col, result, ForceUtf8); // may be SynUnicode
end;

function TSqlDBStatement.ColumnToVariant(Col: integer;
  var Value: Variant; ForceUtf8: boolean): TSqlDBFieldType;
var
  tmp: RawByteString;
  V: TSqlVar;
  d: TSynVarData absolute Value;
begin
  ColumnToSqlVar(Col, V, tmp);
  result := V.VType;
  VarClear(Value);
  d.VType := MAP_FIELDTYPE2VARTYPE[result];
  case result of
    ftNull:
      ; // do nothing
    ftInt64:
      d.VInt64 := V.VInt64;
    ftDouble:
      d.VDouble := V.VDouble;
    ftDate:
      d.VDate := V.VDateTime;
    ftCurrency:
      d.VCurrency := V.VCurrency;
    ftBlob: // as varString
      begin
        d.VAny := nil; // avoid GPF below
        if V.VBlob <> nil then
          if V.VBlob = pointer(tmp) then
            RawByteString(d.VAny) := tmp // increment RefCount
          else
            FastSetRawByteString(RawByteString(d.VAny), V.VBlob, V.VBlobLen);
      end;
    ftUtf8: // VType is varSynUnicode
      begin
        d.VAny := nil; // avoid GPF below
        if V.VText = nil then
          d.VType := varString // avoid obscure "Invalid variant type" in FPC
        else if ForceUtf8 then
        begin
          d.VType := varString;
          if V.VText = pointer(tmp) then
          begin
            FakeCodePage(tmp, CP_UTF8);
            d.VAny := pointer(tmp); // direct assign with no refcount
            pointer(tmp) := nil;
          end
          else
            FastSetString(RawUtf8(d.VAny), V.VText, StrLen(V.VText));
        end
        else
        begin
          if V.VText = pointer(tmp) then
            V.VBlobLen := length(tmp)
          else
            V.VBlobLen := StrLen(V.VText);
          {$ifndef UNICODE}
          if (fConnection <> nil) and
             not fConnection.Properties.VariantStringAsWideString then
          begin
            d.VType := varString;
            if (V.VText = pointer(tmp)) and
               ((Unicode_CodePage = CP_UTF8) or
                IsAnsiCompatible(tmp)) then
              RawByteString(d.VAny) := tmp
            else
              CurrentAnsiConvert.Utf8BufferToAnsi(
                V.VText, V.VBlobLen, RawByteString(d.VAny));
          end
          else
          {$endif UNICODE}
            Utf8ToSynUnicode(V.VText, V.VBlobLen, SynUnicode(d.VAny));
        end;
      end;
  else
    ESqlDBException.RaiseUtf8(
      '%.ColumnToVariant: Invalid ColumnType(%)=%', [self, Col, ord(result)]);
  end;
end;

function TSqlDBStatement.ColumnTimestamp(Col: integer): TTimeLog;
begin
  case ColumnType(Col) of // will call GetCol() to check Col
    ftNull:
      result := 0;
    ftInt64:
      result := ColumnInt(Col);
    ftDate:
      PTimeLogBits(@result)^.From(ColumnDateTime(Col));
  else
    PTimeLogBits(@result)^.From(TrimU(ColumnUtf8(Col)));
  end;
end;

function TSqlDBStatement.ColumnTimestamp(const ColName: RawUtf8): TTimeLog;
begin
  result := ColumnTimestamp(ColumnIndex(ColName));
end;

procedure TSqlDBStatement.ColumnsToJson(WR: TResultsWriter);
var
  col: integer;
begin
  if WR.Expand then
    WR.Add('{');
  for col := 0 to fColumnCount - 1 do
  begin
    if WR.Expand then
      WR.AddString(WR.ColNames[col]); // add '"ColumnName":'
    ColumnToJson(col, WR);
    WR.AddComma;
  end;
  WR.CancelLastComma; // cancel last ','
  if WR.Expand then
    WR.AddDirect('}');
end;

procedure TSqlDBStatement.ColumnToSqlVar(Col: integer; var Value: TSqlVar;
  var Temp: RawByteString);
begin
  Value.Options := [];
  if ColumnNull(Col) then // will call GetCol() to check Col
    Value.VType := ftNull
  else
    Value.VType := ColumnType(Col);
  case Value.VType of
    ftInt64:
      Value.VInt64 := ColumnInt(Col);
    ftDouble:
      Value.VDouble := ColumnDouble(Col);
    ftDate:
      Value.VDateTime := ColumnDateTime(Col);
    ftCurrency:
      Value.VCurrency := ColumnCurrency(Col);
    ftUtf8:
      begin
        Temp := ColumnUtf8(Col);
        Value.VText := pointer(Temp);
      end;
    ftBlob:
      if dsfForceBlobAsNull in fFlags then
      begin
        Value.VBlob := nil;
        Value.VBlobLen := 0;
        Value.VType := ftNull;
      end
      else
      begin
        Temp := ColumnBlob(Col);
        Value.VBlob := pointer(Temp);
        Value.VBlobLen := length(Temp);
      end;
  end;
end;

procedure TSqlDBStatement.ColumnToJson(Col: integer; W: TJsonWriter);
var
  blob: RawByteString;
  p: pointer;
begin // default implementation (never called in practice)
  if ColumnNull(col) then
    W.AddNull
  else
    case ColumnType(col) of
      ftNull:
        W.AddNull;
      ftInt64:
        W.Add(ColumnInt(col));
      ftDouble:
        W.AddDouble(ColumnDouble(col));
      ftCurrency:
        W.AddCurr(ColumnCurrency(col));
      ftDate:
        begin
          W.Add('"');
          W.AddDateTime(ColumnDateTime(col), dsfForceDateWithMS in fFlags);
          W.Add('"');
        end;
      ftUtf8:
        begin
          W.Add('"');
          p := ColumnPUtf8(col);
          if p = nil then
            p := pointer(ColumnUtf8(col));
          W.AddJsonEscape(p);
          W.Add('"');
        end;
      ftBlob:
        if dsfForceBlobAsNull in fFlags then
          W.AddNull
        else
        begin
          blob := ColumnBlob(col);
          W.WrBase64(pointer(blob), length(blob), {withMagic=}true);
        end;
    else
      ESqlDBException.RaiseUtf8('%.ColumnToJson: invalid ColumnType(%)=%',
        [self, col, ord(ColumnType(col))]);
    end;
end;

function TSqlDBStatement.ColumnToTypedValue(Col: integer;
  DestType: TSqlDBFieldType; var Dest): TSqlDBFieldType;
var
  Temp: Variant; // rely on a temporary variant value for the conversion
  wasString: boolean;
begin
  result := ColumnToVariant(Col, Temp, {forceUtf8=}true);
  case DestType of
    ftInt64:
      Int64(Dest) := Temp;
    ftDouble:
      Double(Dest) := Temp;
    ftCurrency:
      currency(Dest) := Temp;
    ftDate:
      TDateTime(Dest) := Temp;
    ftUtf8:
      VariantToUtf8(Temp, RawUtf8(Dest), wasString);
    ftBlob:
      VariantToRawByteString(Temp, RawByteString(Dest));
  else
    ESqlDBException.RaiseUtf8('%.ColumnToTypedValue: Invalid Type [%]',
      [self, ToText(result)^]);
  end;
end;

function TSqlDBStatement.ParamToVariant(Param: integer; var Value: Variant;
  CheckIsOutParameter: boolean): TSqlDBFieldType;
begin
  dec(Param); // start at #1
  if (self = nil) or
     (cardinal(Param) >= cardinal(fParamCount)) then
    ESqlDBException.RaiseUtf8('%.ParamToVariant(%)', [self, Param]);
  // overridden method should fill Value with proper data
  result := ftUnknown;
end;

function TSqlDBStatement.StepAsJson(SeekFirst: boolean): RawUtf8;
var
  w: TJsonWriter;
  tmp: TTextWriterStackBuffer; // 8KB work buffer on stack
begin
  w := TJsonWriter.CreateOwnedStream(tmp);
  try
    StepToJson(w, SeekFirst);
    w.SetText(result);
  finally
    w.Free;
  end;
end;

function TSqlDBStatement.StepToJson(W: TJsonWriter; SeekFirst: boolean): boolean;
var
  col: integer;
begin
  result := Step(SeekFirst);
  if not result then
    exit;
  W.Add('{');
  for col := 0 to fColumnCount - 1 do
  begin
    W.AddFieldName(ColumnName(col)); // add '"ColumnName":'
    ColumnToJson(col, W);
    W.AddComma;
  end;
  W.ReplaceLastComma('}');
end;

procedure TSqlDBStatement.Execute(const aSql: RawUtf8; ExpectResults: boolean);
begin
  Connection.InternalProcess(speActive);
  try
    Prepare(aSql, ExpectResults);
    SetForceBlobAsNull(true);
    ExecutePrepared;
  finally
    Connection.InternalProcess(speNonActive);
  end;
end;

function TSqlDBStatement.FetchAllToJson(Json: TStream; Expanded: boolean): PtrInt;
var
  W: TResultsWriter;
  col: integer;
  maxmem: PtrInt;
  tmp: TTextWriterStackBuffer; // 8KB work buffer on stack
begin
  result := 0;
  W := TResultsWriter.Create(Json, Expanded, false, nil, 0, @tmp);
  try
    Connection.InternalProcess(speActive);
    maxmem := Connection.Properties.StatementMaxMemory;
    // get col names and types
    SetLength(W.ColNames, ColumnCount);
    for col := 0 to ColumnCount - 1 do
      W.ColNames[col] := ColumnName(col);
    W.AddColumns; // write or init field names for appropriate JSON Expand
    if Expanded then
      W.Add('[');
    // write rows data
    {$ifdef SYNDB_SILENCE}
    fSqlLogTimer.Resume; // log fetch duration
    {$endif SYNDB_SILENCE}
    while Step do
    begin
      ColumnsToJson(W);
      W.AddComma;
      inc(result);
      if (maxmem > 0) and
         (W.WrittenBytes > maxmem) then // TextLength is slower
        ESqlDBException.RaiseUtf8('%.FetchAllToJson: overflow %',
          [self, KB(maxmem)]);
    end;
    {$ifdef SYNDB_SILENCE}
    fSqlLogTimer.Pause;
    {$endif SYNDB_SILENCE}
    ReleaseRows;
    if (result = 0) and
       W.Expand then
    begin
      // we want the field names at least, even with no data (RowCount=0)
      W.Expand := false; //  {"FieldCount":2,"Values":["col1","col2"]}
      W.CancelAll;
      for col := 0 to ColumnCount - 1 do
        W.ColNames[col] := ColumnName(col); // previous W.AddColumns did nothing
      W.AddColumns;
    end;
    W.EndJsonObject(0, result);
  finally
    W.Free;
    Connection.InternalProcess(speNonActive);
  end;
end;

function TSqlDBStatement.FetchAllToCsvValues(Dest: TStream; Tab: boolean;
  CommaSep: AnsiChar; AddBOM: boolean): PtrInt;
const
  NULL: array[boolean] of TShort7 = (
    '"null"', 'null');
  BLOB: array[boolean] of TShort7 = (
    '"blob"', 'blob');
var
  F, FMax: integer;
  maxmem: PtrInt;
  W: TJsonWriter;
  tmp: RawByteString;
  V: TSqlVar;
begin
  result := 0;
  if (Dest = nil) or
     (self = nil) or
     (ColumnCount = 0) then
    exit;
  SetForceBlobAsNull(true);
  if Tab then
    CommaSep := #9;
  FMax := ColumnCount - 1;
  maxmem := Connection.Properties.StatementMaxMemory;
  W := TJsonWriter.Create(Dest, 65536);
  try
    // add optional/deprecated/Windows-centric UTF-8 Byte Order Mark
    if AddBOM then
      W.AddDirect(#$ef, #$bb, #$bf);
    // add CSV header
    for F := 0 to FMax do
    begin
      if not Tab then
        W.AddDirect('"');
      W.AddString(ColumnName(F));
      if Tab then
        W.AddDirect(#9)
      else
        W.AddDirect('"', CommaSep);
    end;
    W.CancelLastChar;
    W.AddCR;
    // add CSV rows
    {$ifdef SYNDB_SILENCE}
    fSqlLogTimer.Resume;
    {$endif SYNDB_SILENCE}
    while Step do
    begin
      for F := 0 to FMax do
      begin
        ColumnToSqlVar(F, V, tmp);
        case V.VType of
          ftNull:
            W.AddShorter(NULL[Tab]);
          ftInt64:
            W.Add(V.VInt64);
          ftDouble:
            W.AddDouble(V.VDouble);
          ftCurrency:
            W.AddCurr(V.VCurrency);
          ftDate:
            begin
              if not Tab then
                W.Add('"');
              W.AddDateTime(V.VDateTime, svoDateWithMS in V.Options);
              if not Tab then
                W.AddDirect('"');
            end;
          ftUtf8:
            if not Tab then
            begin
              W.Add('"');
              W.AddJsonEscape(V.VText);
              W.AddDirect('"');
            end
            else
              W.AddNoJsonEscape(V.VText, StrLen(V.VText));
          ftBlob:
            W.AddShorter(BLOB[Tab]);  // ForceBlobAsNull should be true
        else
          ESqlDBException.RaiseUtf8(
            '%.FetchAllToCsvValues: Invalid ColumnType(%) %',
            [self, F, ToText(ColumnType(F))^]);
        end;
        if F = FMax then
          W.AddCR
        else
          W.AddDirect(CommaSep);
      end;
      inc(result);
      if (maxmem > 0) and
         (W.WrittenBytes > maxmem) then // TextLength is slower
        ESqlDBException.RaiseUtf8('%.FetchAllToCsvValues: overflow %',
          [self, KB(maxmem)]);
    end;
    {$ifdef SYNDB_SILENCE}
    fSqlLogTimer.Pause;
    {$endif SYNDB_SILENCE}
    ReleaseRows;
    W.FlushFinal;
  finally
    W.Free;
  end;
end;

function TSqlDBStatement.FetchAllAsJson(
  Expanded: boolean; ReturnedRowCount: PPtrInt): RawUtf8;
var
  stream: TRawByteStringStream;
  RowCount: PtrInt;
begin
  stream := TRawByteStringStream.Create;
  try
    RowCount := FetchAllToJson(stream, Expanded);
    if ReturnedRowCount <> nil then
      ReturnedRowCount^ := RowCount;
    result := stream.DataString;
  finally
    stream.Free;
  end;
end;

procedure TSqlDBStatement.ColumnsToBinary(W: TBufferWriter; Null: pointer;
  const ColTypes: TSqlDBFieldTypeDynArray);
var
  f: PtrInt;
  vdouble: double;
  vcurrency: currency absolute vdouble;
  vdatetime: TDateTime absolute vdouble;
  ft: TSqlDBFieldType;
begin
  for f := 0 to length(ColTypes) - 1 do
    if (Null = nil) or
       not GetBitPtr(Null, f) then
    begin
      ft := ColTypes[f];
      if ft < ftInt64 then
      begin
        // ftUnknown,ftNull
        ft := ColumnType(f); // per-row column type (SQLite3 only)
        W.Write1(ord(ft));
      end;
      case ft of
        ftInt64:
          W.WriteVarInt64(ColumnInt(f));
        ftDouble:
          begin
            vdouble := ColumnDouble(f);
            W.Write8(@vdouble);
          end;
        ftCurrency:
          begin
            vcurrency := ColumnCurrency(f);
            W.Write8(@vcurrency);
          end;
        ftDate:
          begin
            vdatetime := ColumnDateTime(f);
            W.Write8(@vdatetime);
          end;
        ftUtf8:
          W.Write(ColumnUtf8(f));
        ftBlob:
          W.Write(ColumnBlob(f));
      else
        ESqlDBException.RaiseUtf8('%.ColumnsToBinary: Invalid ColumnType(%)=%',
          [self, ColumnName(f), ord(ft)]);
      end;
    end;
end;

function TSqlDBStatement.FetchAllToBinary(Dest: TStream; MaxRowCount: cardinal;
  DataRowPosition: PCardinalDynArray): cardinal;
var
  f, fmax, fieldsize, nullrowlast: integer;
  startpos: Int64;
  maxmem: PtrInt;
  W: TBufferWriter;
  ft: TSqlDBFieldType;
  coltypes: TSqlDBFieldTypeDynArray;
  nullbits: TByteDynArray;
  tmp: TTextWriterStackBuffer; // 8KB work buffer on stack
begin
  result := 0;
  maxmem := Connection.Properties.StatementMaxMemory;
  W := TBufferWriter.Create(Dest, @tmp, SizeOf(tmp));
  try
    W.WriteVarUInt32(FETCHALLTOBINARY_MAGIC);
    fmax := ColumnCount;
    W.WriteVarUInt32(fmax);
    if fmax > 0 then
    begin
      // write column description
      SetLength(coltypes, fmax);
      dec(fmax);
      for f := 0 to fmax do
      begin
        W.Write(ColumnName(f));
        ft := ColumnType(f, @fieldsize);
        if (ft = ftUnknown) and
           (currentRow = 0) and
           Step then
          ft := ColumnType(f, @fieldsize); // e.g. SQLite3 -> fetch and guess
        coltypes[f] := ft;
        W.Write1(ord(ft));
        W.WriteVarUInt32(fieldsize);
      end;
      // initialize null handling
      nullrowlast := 0;
      // save all data rows
      startpos := W.TotalWritten;
      if (currentRow = 1) or
         Step then // Step may already be done (e.g. TQuery.Open)
        repeat
          // save row position in DataRowPosition[] (if any)
          if DataRowPosition <> nil then
          begin
            if Length(DataRowPosition^) <= integer(result) then
              SetLength(DataRowPosition^, NextGrow(result));
            DataRowPosition^[result] := W.TotalWritten - startpos;
          end;
          // first write null columns flags
          if nullrowlast > 0 then
          begin
            FillCharFast(nullbits[0], nullrowlast, 0);
            nullrowlast := 0;
          end;
          for f := 0 to fmax do
            if ColumnNull(f) then
            begin
              if nullbits = nil then
                SetLength(nullbits, (fmax shr 3) + 1);
              SetBitPtr(pointer(nullbits), f);
              nullrowlast := (f shr 3) + 1;
            end;
          if nullrowlast > 0 then
          begin
            W.WriteVarUInt32(nullrowlast);
            W.Write(pointer(nullbits), nullrowlast);
          end
          else
            W.Write1(0); // = W.WriteVarUInt32(0)
          // then write data values
          ColumnsToBinary(W, pointer(nullbits), coltypes);
          inc(result);
          if (MaxRowCount > 0) and
             (result >= MaxRowCount) then
            break;
          if (maxmem > 0) and
             (W.TotalWritten > maxmem) then // Dest.Position is slower
            ESqlDBException.RaiseUtf8('%.FetchAllToBinary: overflow %',
              [self, KB(maxmem)]);
        until not Step;
      ReleaseRows;
    end;
    W.Write(@result, SizeOf(result)); // fixed size at the end for row count
    W.Flush;
  finally
    W.Free;
  end;
end;

function TSqlDBStatement.FetchAllToDocVariantArray(MaxRowCount: cardinal): variant;
var
  r: TDocVariantData absolute result;
  f, fieldcount: PtrInt;
  rowcount: cardinal;
  v: PVariant;
  proto: TDocVariantData; // object prototype with reused VName[]
begin
  VarClear(result);
  r.Init(JSON_FAST_FLOAT, dvArray);
  fieldcount := ColumnCount;
  // initialize the object prototype with the column names
  proto.Init(JSON_FAST_FLOAT, dvObject);
  proto.Capacity := fieldcount;
  dec(fieldcount);
  for f := 0 to fieldcount do
    proto.AddValue(ColumnName(f), null);
  // save all data rows
  rowcount := 0;
  if (currentRow = 1) or
     Step then // Step may already be done (e.g. TQuery.Open)
    repeat
      v := PDocVariantData(r.NewItem)^.InitFrom(proto, {values=}false);
      for f := 0 to fieldcount do
      begin
        ColumnToVariant(f, v^, {forceUtf8=}true);
        inc(v);
      end;
      inc(rowcount);
    until ((MaxRowCount > 0) and
           (rowcount >= MaxRowCount)) or
          not Step;
  ReleaseRows;
end;

procedure TSqlDBStatement.Execute(const aSql: RawUtf8; ExpectResults: boolean;
  const Params: array of const);
begin
  Connection.InternalProcess(speActive);
  try
    Prepare(aSql, ExpectResults);
    Bind(Params);
    ExecutePrepared;
  finally
    Connection.InternalProcess(speNonActive);
  end;
end;

procedure TSqlDBStatement.Execute(const SqlFormat: RawUtf8;
  ExpectResults: boolean; const Args, Params: array of const);
var
  sql: RawUtf8;
begin
  FormatUtf8(SqlFormat, Args, sql);
  Execute(sql, ExpectResults, Params);
end;

function TSqlDBStatement.UpdateCount: integer;
begin
  result := 0;
end;

procedure TSqlDBStatement.ExecutePreparedAndFetchAllAsJson(Expanded: boolean;
  out Json: RawUtf8; ReturnedRowCount: PPtrInt);
begin
  ExecutePrepared;
  Json := FetchAllAsJson(Expanded, ReturnedRowCount);
end;

function TSqlDBStatement.ColumnString(Col: integer): string;
begin
  Utf8ToStringVar(ColumnUtf8(Col), result);
end;

function TSqlDBStatement.ColumnString(const ColName: RawUtf8): string;
begin
  result := ColumnString(ColumnIndex(ColName));
end;

function TSqlDBStatement.ColumnBlob(const ColName: RawUtf8): RawByteString;
begin
  result := ColumnBlob(ColumnIndex(ColName));
end;

function TSqlDBStatement.ColumnBlobBytes(const ColName: RawUtf8): TBytes;
begin
  result := ColumnBlobBytes(ColumnIndex(ColName));
end;

procedure TSqlDBStatement.ColumnBlobToStream(
  const ColName: RawUtf8; Stream: TStream);
begin
  ColumnBlobToStream(ColumnIndex(ColName), Stream);
end;

procedure TSqlDBStatement.ColumnBlobFromStream(
  const ColName: RawUtf8; Stream: TStream);
begin
  ColumnBlobFromStream(ColumnIndex(ColName), Stream);
end;

function TSqlDBStatement.ColumnCurrency(const ColName: RawUtf8): currency;
begin
  result := ColumnCurrency(ColumnIndex(ColName));
end;

function TSqlDBStatement.ColumnDateTime(const ColName: RawUtf8): TDateTime;
begin
  result := ColumnDateTime(ColumnIndex(ColName));
end;

function TSqlDBStatement.ColumnDouble(const ColName: RawUtf8): double;
begin
  result := ColumnDouble(ColumnIndex(ColName));
end;

function TSqlDBStatement.ColumnNull(const ColName: RawUtf8): boolean;
begin
  result := ColumnNull(ColumnIndex(ColName));
end;

function TSqlDBStatement.ColumnInt(const ColName: RawUtf8): Int64;
begin
  result := ColumnInt(ColumnIndex(ColName));
end;

function TSqlDBStatement.ColumnUtf8(const ColName: RawUtf8): RawUtf8;
begin
  result := ColumnUtf8(ColumnIndex(ColName));
end;

function TSqlDBStatement.ColumnVariant(
  const ColName: RawUtf8; ForceUtf8: boolean): Variant;
begin
  ColumnToVariant(ColumnIndex(ColName), result, ForceUtf8);
end;

function TSqlDBStatement.GetColumnVariant(const ColName: RawUtf8): Variant;
begin
  ColumnToVariant(ColumnIndex(ColName), result, {forceUtf8=}false);
end;

function TSqlDBStatement.ColumnCursor(const ColName: RawUtf8): ISqlDBRows;
begin
  result := ColumnCursor(ColumnIndex(ColName));
end;

function TSqlDBStatement.{%H-}ColumnCursor(Col: integer): ISqlDBRows;
begin
  ESqlDBException.RaiseUtf8('% does not support CURSOR columns', [self]);
end;

function TSqlDBStatement.Instance: TSqlDBStatement;
begin
  result := Self;
end;

{$ifdef SYNDB_SILENCE}

function TSqlDBStatement.SqlLogBegin(Level: TSynLogLevel): TSynLog;
begin
  result := nil;
  if Level = sllDB then  // sllDB = prepare
    fSqlLogTimer.Start
  else
    fSqlLogTimer.Resume; // sllSQL or sllResult
end;

function TSqlDBStatement.SqlLogEnd(Msg: PShortString): Int64;
begin
  fSqlLogTimer.Pause;
  result := fSqlLogTimer.LastTimeInMicroSec;
end;

{$else}

function TSqlDBStatement.DoSqlLogBegin(Log: TSynLogFamily; Level: TSynLogLevel): TSynLog;
begin
  result := Log.Add;
  fSqlLogLevel := Level;
  if Level = sllSQL then // sllSQL = executeprepared
    ComputeSqlWithInlinedParams;
  if Level = sllDB then  // sllDB = prepare
    fSqlLogTimer.Start
  else
    fSqlLogTimer.Resume; // sllSQL or sllResult
end;

function TSqlDBStatement.SqlLogBegin(Level: TSynLogLevel): TSynLog;
var
  fam: TSynLogFamily;
begin
  fam := SynDBLog.Family;
  if (fam <> nil) and
     (Level in fam.Level) then
    result := DoSqlLogBegin(fam, Level)
  else
    result := nil; // fSqlLogLog=nil if this Level is disabled
  fSqlLogLog := result;
end;

function TSqlDBStatement.DoSqlLogEnd(Msg: PShortString): Int64;
var
  tmp, elapsed: TShort16;
begin
  fSqlLogTimer.Pause;
  tmp[0] := #0;
  if fSqlLogLevel <> sllDB then // sllSQL or sllResult
  begin
    if Msg = nil then
    begin
      if not (dsfExpectResults in fFlags) then
      begin
        AppendShort(' wr=', tmp);
        AppendShortCardinal(UpdateCount, tmp);
      end;
      Msg := @tmp;
    end;
    MicroSecToString(fSqlLogTimer.StopInMicroSec, elapsed);
    if fSqlLogLevel = sllSQL then
      fSqlLogLog.Log(sllSQL, 'Execute t=%% q=%',
        [elapsed, Msg^, fSqlWithInlinedParams], self)
    else // from TSqlDBPostgresStatement.GetPipelineResult
      fSqlLogLog.Log(fSqlLogLevel, 'Return t=%%',
        [elapsed, Msg^], self);
  end
  else
  begin
    if Msg = nil then
      Msg := @tmp;
    fSqlLogLog.Log(fSqlLogLevel, 'Prepare t=%% q=%',
      [fSqlLogTimer.Stop, Msg^, fSql], self);
  end;
  result := fSqlLogTimer.LastTimeInMicroSec;
  fSqlLogLog := nil;
end;

function TSqlDBStatement.SqlLogEnd(Msg: PShortString): Int64;
begin
  result := 0;
  if fSqlLogLog <> nil then // fSqlLogLog=nil if this level is disabled
    result := DoSqlLogEnd(Msg);
end;

{$endif SYNDB_SILENCE}

function TSqlDBStatement.SqlLogEnd(const Fmt: RawUtf8;
  const Args: array of const): Int64;
var
  tmp: ShortString;
begin
  tmp[0] := #0;
  {$ifndef SYNDB_SILENCE}
  result := 0;
  if fSqlLogLog = nil then
    exit;
  if Fmt <> '' then
    FormatShort(Fmt, Args, tmp);
  {$endif SYNDB_SILENCE}
  result := SqlLogEnd(@tmp);
end;

function TSqlDBStatement.GetSqlCurrent: RawUtf8;
begin
  if fSqlPrepared <> '' then
    result := fSqlPrepared
  else
    result := fSql;
end;

function TSqlDBStatement.GetSqlWithInlinedParams: RawUtf8;
begin
  if fSql = '' then
    result := ''
  else
  begin
    if fSqlWithInlinedParams = '' then
      ComputeSqlWithInlinedParams;
    result := fSqlWithInlinedParams;
  end;
end;

function GotoNextParam(P: PUtf8Char): PUtf8Char;
  {$ifdef HASINLINE}inline;{$endif}
var
  c: AnsiChar;
begin
  repeat
    c := P^;
    if (c = #0) or
       (c = '?') then
      break;
    if (c = '''') and
       (P[1] <> '''') then
    begin
      repeat // ignore ? inside ' quotes
        inc(P);
        c := P^;
      until (c = #0) or
            ((c = '''') and
             (P[1] <> ''''));
      if c = #0 then
        break;
    end;
    inc(P);
  until false;
  result := P;
end;

procedure TSqlDBStatement.ComputeSqlWithInlinedParams;
var
  P, B: PUtf8Char;
  num: integer;
  maxSize, maxAllowed: cardinal;
  W: TJsonWriter; // at least TJsonWriter since W.AddVariant() is needed
  tmp: TTextWriterStackBuffer; // maxsize is typically 2048 so all on stack
begin
  fSqlWithInlinedParams := fSql;
  if fConnection = nil then
    maxSize := 2048 // LoggedSqlMaxSize default seems fair enough
  else
    maxSize := fConnection.fProperties.fLoggedSqlMaxSize;
  if (integer(maxSize) < 0) or
     (PosExChar('?', fSql) = 0) then
    // maxsize=-1 -> log statement without any parameter value (just ?)
    exit;
  P := pointer(fSql);
  num := 1;
  W := nil;
  try
    repeat
      B := P;
      P := GotoNextParam(P);
      if W = nil then
        if P^ = #0 then
          exit // no parameter
        else
          W := TJsonWriter.CreateOwnedStream(tmp);
      W.AddNoJsonEscape(B, P - B);
      if P^ = #0 then
        break;
      inc(P); // jump P^='?'
      if maxSize > 0 then
        maxAllowed := W.TextLength - maxSize
      else
        maxAllowed := maxInt;
      AddParamValueAsText(num, W, maxAllowed);
      inc(num);
    until (P^ = #0) or
          ((maxSize > 0) and
           (W.TextLength >= maxSize));
    W.SetText(fSqlWithInlinedParams);
  finally
    W.Free;
  end;
end;

procedure TSqlDBStatement.AddParamValueAsText(Param: integer; Dest: TJsonWriter;
  MaxCharCount: integer);
var
  v: variant;
  vd: TVarData absolute v;
  ft: TSqlDBFieldType;
begin
  ft := ParamToVariant(Param, v, false);
  case cardinal(vd.VType) of
    varString:
      if ft = ftBlob then
        Dest.AddU(length(RawByteString(vd.VPointer))) // only add binary length
      else
        Dest.AddQuotedStr(
          vd.VString, length(RawByteString(vd.VPointer)), '''', MaxCharCount);
    varOleStr:
      Dest.AddQuotedStrW(
        vd.VString, length(WideString(vd.VPointer)), '''', MaxCharCount);
    {$ifdef HASVARUSTRING}
    varUString:
      Dest.AddQuotedStrW(
        vd.VString, length(UnicodeString(vd.VPointer)), '''', MaxCharCount);
    {$endif HASVARUSTRING}
  else
    if (ft = ftDate) and
       (cardinal(vd.VType) in [varDouble, varDate]) then
      Dest.AddDateTime(vd.VDate)
    else
      Dest.AddVariant(v);
  end;
end;

var
  SqlDBRowVariantType: TCustomVariantType = nil;

function TSqlDBStatement.RowData: Variant;
begin
  if SqlDBRowVariantType = nil then
    SqlDBRowVariantType := SynRegisterCustomVariantType(TSqlDBRowVariantType);
  VarClear(result);
  TSynVarData(result).VType := SqlDBRowVariantType.VarType;
  TSynVarData(result).VAny  := self;
end;

procedure TSqlDBStatement.RowDocVariant(out aDocument: variant;
  aOptions: TDocVariantOptions);
var
  n, F: PtrInt;
  names: TRawUtf8DynArray;
  values: TVariantDynArray;
begin
  n := ColumnCount;
  SetLength(names, n); // faster to assign internal arrays per reference
  SetLength(values, n);
  for F := 0 to n - 1 do
  begin
    names[F] := ColumnName(F);
    ColumnToVariant(F, values[F], {forceutf8=}true);
  end;
  TDocVariantData(aDocument).InitObjectFromVariants(names, values, aOptions);
end;

procedure TSqlDBStatement.Prepare(const aSql: RawUtf8; ExpectResults: boolean);
var
  L: PtrInt;
begin
  Connection.InternalProcess(speActive);
  try
    L := length(aSql);
    if StripSemicolon then
      if (L > 5) and
         (aSql[L] = ';') and // avoid syntax error for some drivers
         not IdemPChar(@aSql[L - 4], ' END') then
        fSql := copy(aSql, 1, L - 1)
      else
        fSql := aSql
    else
      fSql := aSql;
    fExpectResults := ExpectResults;
    if (fConnection <> nil) and
       not fConnection.IsConnected then
      fConnection.Connect;
  finally
    Connection.InternalProcess(speNonActive);
  end;
end;

procedure TSqlDBStatement.ExecutePrepared;
begin
  if (fConnection <> nil) and
     (fConnection.fProperties.fConnectionTimeOutSecs <> 0) then
    fConnection.fLastAccessTickSec := GetTickSec;
  // a do-nothing default method
end;

procedure TSqlDBStatement.Reset;
begin
  fSqlWithInlinedParams := '';
  fSqlLogTimer.Init; // reset timer (for cached statement for example)
  ClearDbError;
end;

procedure TSqlDBStatement.ReleaseRows;
begin
  fSqlWithInlinedParams := '';
end;

function TSqlDBStatement.ColumnsToSqlInsert(const TableName: RawUtf8;
  var Fields: TSqlDBColumnCreateDynArray): RawUtf8;
var
  F: PtrInt;
  size: integer;
begin
  result := '';
  if (self = nil) or
     (TableName = '') then
    exit;
  SetLength(Fields, ColumnCount);
  if Fields = nil then
    exit;
  Join(['insert into ', TableName, ' ('], result);
  for F := 0 to high(Fields) do
  begin
    Fields[F].Name := ColumnName(F);
    Fields[F].DBType := ColumnType(F, @size);
    Fields[F].Width := size;
    case Fields[F].DBType of
      ftNull:
        Fields[F].DBType := ftBlob; // if not identified, assume it is a BLOB
      ftUnknown:
        begin
          Fields := nil;
          result := ''; // not enough information
          exit;
        end;
    end;
    Append(result, Fields[F].Name, ',');
  end;
  result[length(result)] := ')';
  AppendShortToUtf8(' values (', result);
  for F := 0 to high(Fields) do
    AppendShortToUtf8('?,', result); // MUCH faster with a prepared statement
  result[length(result)] := ')';
end;

procedure TSqlDBStatement.BindFromRows(const Fields: TSqlDBFieldTypeDynArray;
  Rows: TSqlDBStatement);
var
  F: integer;
begin
  if (self <> nil) and
     (Fields <> nil) and
     (Rows <> nil) then
    for F := 0 to high(Fields) do
      if Rows.ColumnNull(F) then
        BindNull(F + 1)
      else
        case Fields[F] of
          ftNull:
            BindNull(F + 1);
          ftInt64:
            Bind(F + 1, Rows.ColumnInt(F));
          ftDouble:
            Bind(F + 1, Rows.ColumnDouble(F));
          ftCurrency:
            BindCurrency(F + 1, Rows.ColumnCurrency(F));
          ftDate:
            BindDateTime(F + 1, Rows.ColumnDateTime(F));
          ftUtf8:
            BindTextU(F + 1, Rows.ColumnUtf8(F));
          ftBlob:
            BindBlob(F + 1, Rows.ColumnBlob(F));
        end;
end;

procedure TSqlDBStatement.BindCursor(Param: integer);
begin
  ESqlDBException.RaiseUtf8('% does not support CURSOR parameter', [self]);
end;

function TSqlDBStatement.{%H-}BoundCursor(Param: integer): ISqlDBRows;
begin
  ESqlDBException.RaiseUtf8('% does not support CURSOR parameter', [self]);
end;


function ToText(c: TSqlDBStatementCache): ShortString;
begin
  GetSetNameShort(TypeInfo(TSqlDBStatementCache), c, result, {trim=}true);
end;


{ TSqlDBConnection }

procedure TSqlDBConnection.CheckConnection;
begin
  if self = nil then
    ESqlDBException.RaiseU('TSqlDBConnection(nil).CheckConnection');
  if not Connected then
    ESqlDBException.RaiseUtf8('% on %/% should be connected',
      [self, Properties.ServerName, Properties.DataBaseName]);
end;

procedure TSqlDBConnection.InternalProcess(Event: TOnSqlDBProcessEvent);
begin
  if (self = nil) or
     (not Assigned(OnProcess)) then
    exit;
  case Event of // thread-safe handle of speActive/peNonActive nested calls
    speActive:
      if InterlockedIncrement(fInternalProcessActive) = 1 then
        OnProcess(self, Event);
    speNonActive:
      if InterlockedDecrement(fInternalProcessActive) = 0 then
        OnProcess(self, Event);
  else
    OnProcess(self, Event);
  end;
end;

procedure TSqlDBConnection.Commit;
begin
  CheckConnection;
  if TransactionCount <= 0 then
    ESqlDBException.RaiseUtf8('Invalid %.Commit call', [self]);
  dec(fTransactionCount);
  InternalProcess(speCommit);
end;

constructor TSqlDBConnection.Create(aProperties: TSqlDBConnectionProperties);
begin
  inherited Create;
  fCacheSafe.Init; // mandatory for TOSLightLock
  fProperties := aProperties;
  if aProperties <> nil then
  begin
    fOnProcess := aProperties.OnProcess;
    fRollbackOnDisconnect := aProperties.RollbackOnDisconnect;
  end;
end;

procedure TSqlDBConnection.Connect;
var
  i: PtrInt;
begin
  inc(fTotalConnectionCount);
  InternalProcess(speConnected);
  if fTotalConnectionCount > 1 then
    InternalProcess(speReconnected);
  for i := 0 to length(fProperties.ExecuteWhenConnected) - 1 do
    with NewStatement do
    try
      Execute(fProperties.ExecuteWhenConnected[i], false);
    finally
      Free;
    end;
end;

procedure TSqlDBConnection.Disconnect;
var
  i: PtrInt;
  obj: PPointerArray;
begin
  InternalProcess(speDisconnected);
  if fCache <> nil then
  begin
    InternalProcess(speActive);
    fCacheSafe.Lock; // protect fCache access e.g. for a single SQLite3 DB
    try
      obj := fCache.ObjectPtr;
      if obj <> nil then
        for i := 0 to fCache.Count - 1 do
          TSqlDBStatement(obj[i]).FRefCount := 0; // force clean release
      FreeAndNilSafe(fCache); // release all cached statements
    finally
      fCacheSafe.UnLock;
      InternalProcess(speNonActive);
    end;
  end;
  if InTransaction then
  try
    if RollbackOnDisconnect then
    begin
      fTransactionCount := 1; // flush transaction nesting level
      Rollback;
    end;
  finally
    fTransactionCount := 0; // flush transaction nesting level
  end;
end;

destructor TSqlDBConnection.Destroy;
begin
  try
    Disconnect;
  except
    on E: Exception do
      SynDBLog.Add.Log(sllError, 'Destroy: Disconnect raised %', [E], self);
  end;
  ObjArrayClear(fOwned, {continueonexc=}true);
  inherited;
  fCacheSafe.Done;
end;

function TSqlDBConnection.GetThreadOwned(aClass: TClass): pointer;
begin
  if (self = nil) or
     not (cpfIsThreadSafe in fProperties.fFlags) then
    result := nil
  else
  begin
    result := pointer(fOwned);
    if result <> nil then
      result := FindPrivateSlot(aClass, result); // reuse logic from RTTI
  end;
end;

function TSqlDBConnection.SetThreadOwned(aObject: TObject): pointer;
begin
  if (self = nil) or
     not (cpfIsThreadSafe in fProperties.fFlags) then
    ESqlDBException.RaiseUtf8('Unsupported %.SetThreadOwned', [self]);
  result := aObject;
  if result <> nil then
    PtrArrayAdd(fOwned, aObject);
end;

function TSqlDBConnection.IsOutdated(secs: integer): boolean;
var
  last: integer;
begin // caller ensured self <> nil and secs <> 0 and timeout <> 0
  last := fLastAccessTickSec;
  result := (last < 0) or       // -1 = deprecated by ClearConnectionPool
            ((last <> 0) and    //  0 = new connection
             (secs - last >= fProperties.fConnectionTimeOutSecs)); // too idle
end;

function TSqlDBConnection.GetInTransaction: boolean;
begin
  result := TransactionCount > 0;
end;

function TSqlDBConnection.GetServerTimestamp: TTimeLog;
begin
  PTimeLogBits(@result)^.From(GetServerDateTime);
end;

function TSqlDBConnection.GetServerDateTime: TDateTime;
var
  current: TDateTime;
begin
  current := NowUtc; // so won't conflict with any potential time zone change
  if (fServerTimestampOffset = 0) and
     (fProperties.fSqlGetServerTimestamp <> '') then
  begin
    with fProperties do
      with Execute(fSqlGetServerTimestamp, []) do
      begin
        if Step then
          fServerTimestampOffset := ColumnDateTime(0) - current;
        ReleaseRows;
      end;
    if fServerTimestampOffset = 0 then
      fServerTimestampOffset := 0.000001; // request server only once
  end;
  result := current + fServerTimestampOffset;
end;

function TSqlDBConnection.GetLastErrorWasAboutConnection: boolean;
begin
  result := (self <> nil) and
            (Properties <> nil) and
            (fErrorMessage <> '') and
    Properties.ExceptionIsAboutConnection(fErrorException, fErrorMessage);
end;

function TSqlDBConnection.NewStatementPrepared(const aSql: RawUtf8;
  ExpectResults: boolean; RaiseExceptionOnError: boolean;
  AllowReconnect: boolean): ISqlDBStatement;
var
  stmt: TSqlDBStatement;
  iscacheable, tocache: boolean;
  ndx, altern: integer;
  cachedsql: RawUtf8;

  procedure TryPrepare(doraise: boolean);
  var
    stmt: TSqlDBStatement;
    ndx: integer;
  begin
    stmt := nil;
    try
      InternalProcess(speActive);
      try
        stmt := NewStatement;
        if iscacheable then
          include(stmt.fCache, scPossible);
        stmt.Prepare(aSql, ExpectResults);
        if tocache then
        begin
          fCacheSafe.Lock; // protect fCache access
          try
            if fCache = nil then
              fCache := TRawUtf8List.CreateEx(
                [fObjectsOwned, fNoDuplicate, fCaseSensitive]);
            ndx := fCache.AddObject(cachedsql, stmt);
            if ndx >= 0 then
            begin
              stmt._AddRef; // instance will be owned by fCache.Objects[]
              include(stmt.fCache, scOnClient);
              fCacheLastIndex := ndx;
              fCacheLast := cachedsql;
            end
            else
              SynDBLog.Add.Log(sllWarning, 'NewStatementPrepared: unexpected ' +
                'cache duplicate for %', [stmt.SqlWithInlinedParams], self);
          finally
            fCacheSafe.UnLock;
          end;
        end;
        result := stmt;
      finally
        InternalProcess(speNonActive);
      end;
    except
      on E: Exception do
      begin
        {$ifndef SYNDB_SILENCE}
        if SynDBLog.HasLevel([sllSQL, sllDB, sllException, sllError]) then
          SynDBLog.Add.LogLines(sllSQL, pointer(stmt.SqlWithInlinedParams), self, '--');
        {$endif SYNDB_SILENCE}
        stmt.Free;
        result := nil;
        ExceptionUtf8(E, fErrorMessage);
        fErrorException := PPointer(E)^;
        if doraise then
          raise;
      end;
    end;
  end;

begin
  result := nil;
  fErrorMessage := '';
  fErrorException := nil;
  if length(aSql) < 5 then
    exit;
  // first check if could be retrieved from cache
  cachedsql := aSql;
  stmt := nil;
  fCacheSafe.Lock; // protect fCache (never lock on per-thread connection)
  try
    if fCache <> nil then
      if not fProperties.ReconnectAfterConnectionError or
         not AllowReconnect or
         IsConnected then // don't reuse a statement on a broken connection
      begin
        // fast lookup of the requested SQL in cache
        if (fCacheLast = cachedsql) and
           fCache.EqualValueAt(fCacheLastIndex, cachedsql) then
          ndx := fCacheLastIndex // no need to use the hash lookup
        else
          ndx := fCache.IndexOf(cachedsql); // O(1) hash lookup from fNoDuplicate
        if ndx >= 0 then
        begin
          stmt := fCache.ObjectPtr[ndx];
          if stmt.RefCount = 1 then
          begin
            // this statement is not currently in use and can be returned
            if ndx <> fCacheLastIndex then
            begin
              fCacheLastIndex := ndx;
              fCacheLast := cachedsql;
            end;
            result := stmt; // acquire the statement
          end
        end;
      end;
    // full cache support
    if result = nil then
    begin
      iscacheable := fProperties.IsCacheable(pointer(aSql));
      if iscacheable and
         (stmt <> nil) then // RefCount > 1
      begin
        // main statement in use -> create cached alternatives
        tocache := false; // if all slots are used, won't cache this statement
        if fProperties.StatementCacheReplicates = 0 then
          SynDBLog.Add.Log(sllWarning,
            'NewStatementPrepared: cached statement still in use ' +
            '-> you should release ISqlDBStatement ASAP [%]', [cachedsql], self)
        else
          for altern := 1 to fProperties.StatementCacheReplicates do
          begin
            Make([aSql, #0, altern], cachedsql); // not valid SQL
            ndx := fCache.IndexOf(cachedsql);
            if ndx >= 0 then
            begin
              stmt := fCache.Objects[ndx];
              if stmt.RefCount = 1 then
              begin
                result := stmt;
                break;
              end;
            end
            else
            begin
              tocache := true; // cache the statement in this void slot
              break;
            end;
          end;
      end
      else
        tocache := iscacheable;
    end;
  finally
    fCacheSafe.UnLock;
  end;
  if result <> nil then
  begin
    stmt.Reset; // process outside the lock
    exit;       // return the cached statement
  end;
  // not in cache (or not cacheable) -> prepare now
  if fProperties.ReconnectAfterConnectionError and
     AllowReconnect then
  begin
    TryPrepare({doraise=}false);
    if result <> nil then
      exit; // success
    if LastErrorWasAboutConnection then
    try
      SynDBLog.Add.Log(sllDB, 'NewStatementPrepared: reconnect after %',
        [fErrorException], self);
      Disconnect;
      Connect;
      TryPrepare(RaiseExceptionOnError);
      if result = nil then
      begin
        SynDBLog.Add.Log(sllDB, 'NewStatementPrepared: unable to reconnect', self);
        InternalProcess(speConnectionLost);
      end;
    except
      if RaiseExceptionOnError then
        raise
      else
        result := nil;
    end
    else if RaiseExceptionOnError and
            (fErrorException <> nil) then
      // propagate error not related to connection (e.g. SQL syntax error)
      if fErrorException.InheritsFrom(ESynException) then
        ECoreDBException.RaiseU(fErrorMessage)
      else
        raise fErrorException.Create(Utf8ToString(fErrorMessage));
  end
  else
    // regular preparation, with no connection error interception
    TryPrepare(RaiseExceptionOnError);
end;

procedure TSqlDBConnection.Rollback;
begin
  CheckConnection;
  if TransactionCount <= 0 then
    ESqlDBException.RaiseUtf8('Invalid %.Rollback call', [self]);
  dec(fTransactionCount);
  InternalProcess(speRollback);
end;

function TSqlDBConnection.PasswordChange: boolean;
begin
  result := false;
end;

procedure TSqlDBConnection.StartTransaction;
begin
  CheckConnection;
  inc(fTransactionCount);
  InternalProcess(speStartTransaction);
end;

function TSqlDBConnection.NewTableFromRows(const TableName: RawUtf8;
  Rows: TSqlDBStatement; WithinTransaction: boolean;
  ColumnForcedTypes: TSqlDBFieldTypeDynArray): integer;

  function CreateDestTableAndComputeSql: RawUtf8;
  var
    i, n: PtrInt;
    tableu: RawUtf8;
    fields: TSqlDBColumnCreateDynArray;
  begin
    tableu := Properties.SqlTableName(TableName);
    result := Rows.ColumnsToSqlInsert(tableu, fields);
    n := length(fields);
    if (result = '') or
       (n = 0) then
      exit;
    if Length(ColumnForcedTypes) <> n then
    begin
      SetLength(ColumnForcedTypes, n);
      for i := 0 to n - 1 do
        case fields[i].DBType of
          ftUnknown:
            ColumnForcedTypes[i] := ftInt64;
          ftNull:
            ColumnForcedTypes[i] := ftBlob; // assume NULL is a BLOB
        else
          ColumnForcedTypes[i] := fields[i].DBType;
        end;
    end;
    if not Properties.TableExists(TableName) then
      Properties.ExecuteNoResult(
        Properties.SqlCreate(tableu, fields, false), []);
  end;

var
  sql: RawUtf8;
  stmt: TSqlDBStatement;
begin
  result := 0;
  if (self = nil) or
     (Rows = nil) or
     (Rows.ColumnCount = 0) then
    exit;
  if WithinTransaction then
    StartTransaction; // MUCH faster within a transaction
  try
    stmt := nil;
    InternalProcess(speActive);
    try
      while Rows.Step do
      begin
        // init when first row of data is available
        if stmt = nil then
        begin
          sql := CreateDestTableAndComputeSql;
          stmt := NewStatement;
          stmt.Prepare(sql, false);
        end;
        // write row data
        stmt.BindFromRows(ColumnForcedTypes, Rows);
        stmt.ExecutePrepared;
        stmt.Reset;
        inc(result);
      end;
      if (stmt = nil) and
         not Properties.TableExists(TableName) then
        // no data was retrieved: but generate the void table anyway
        CreateDestTableAndComputeSql;
      if WithinTransaction then
        Commit;
    finally
      Rows.ReleaseRows;
      stmt.Free;
      InternalProcess(speNonActive);
    end;
  except
    on Exception do
    begin
      if WithinTransaction then
        Rollback;
      raise;
    end;
  end;
end;

function TSqlDBConnection.NewTableFrom(
  const NewTableName, SourceTableName: RawUtf8;
  Source: TSqlDBConnectionProperties): boolean;
var
  src: TSqlDBColumnDefineDynArray;
  dst: TSqlDBColumnCreateDynArray;
  f, n: PtrInt;
begin
  result := false;
  if (self = nil) or
     (Source = nil) or
     (NewTableName = '') or
     (SourceTableName = '') or
     Properties.TableExists(NewTableName) then
    exit;
  Source.GetFields(SourceTableName, src);
  if src = nil then
    exit;
  SetLength(dst, length(src));
  n := 0;
  for f := 0 to high(src) do
  with src[f] do
    if not (ColumnType in [ftUnknown, ftNull]) then
    begin
      dst[n].Name := ColumnName;
      dst[n].DBType := ColumnType;
      if ColumnType = ftUtf8 then
        dst[n].Width := ColumnLength;
      inc(n);
    end;
  if n = 0 then
    exit;
  DynArrayFakeLength(dst, n);
  Properties.ExecuteNoResult(Properties.SqlCreate(
    Properties.SqlTableName(NewTableName), dst, false), []);
  result := true;
end;


{ ************ Parent Classes for Thread-Safe and Parametrized Connections }

{ TSqlDBConnectionThreadSafe }

destructor TSqlDBConnectionThreadSafe.Destroy;
begin
  if fProperties <> nil then
    (fProperties as TSqlDBConnectionPropertiesThreadSafe).RemoveFromPool(self);
  inherited Destroy;
end;


{ TSqlDBConnectionPropertiesThreadSafe }

constructor TSqlDBConnectionPropertiesThreadSafe.Create(
  const aServerName, aDatabaseName, aUserID, aPassWord: RawUtf8);
begin
  include(fFlags, cpfIsThreadSafe);
  fConnectionPoolMax := -1;
  inherited Create(aServerName, aDatabaseName, aUserID, aPassWord);
end;

destructor TSqlDBConnectionPropertiesThreadSafe.Destroy;
var
  i: PtrInt;
begin
  for i := fConnectionPoolMin to fConnectionPoolMax do
    if fConnectionPool[i] <> nil then
      FreeAndNilSafe(fConnectionPool[i]);
  inherited Destroy; // do MainConnection.Free
end;

function TSqlDBConnectionPropertiesThreadSafe.GetMainConnection: TSqlDBConnection;
begin
  result := ThreadSafeConnection;
end;

procedure TSqlDBConnectionPropertiesThreadSafe.ClearConnectionPool;
var
  i: PtrInt;
  delete: TObjectDynArray; // outside non-reentrant lock
  deletecount: integer;
  log: ISynLog;
begin
  SynDBLog.EnterLocal(log, self, 'ClearConnectionPool');
  if DeleteConnectionInOwnThread then
  begin
    // mark all connections as deprecated - Destroy will be done later on, in
    // the proper thread (some providers may require to keep the same thread)
    fMainConnectionSafe.Lock;
    if fMainConnection <> nil then
      fMainConnection.fLastAccessTickSec := -1; // force IsOutdated()=true
    fMainConnectionSafe.UnLock;
    fConnectionPoolSafe.Lock;
    try
      for i := fConnectionPoolMin to fConnectionPoolMax do
        if fConnectionPool[i] <> nil then
          fConnectionPool[i].fLastAccessTickSec := -1;
    finally
      fConnectionPoolSafe.UnLock;
    end;
    fConnectionPoolDeprecatedTix32 := 0; // trigger DeleteDeprecated()
  end
  else
  begin
    // we can delete all existing connections now
    inherited ClearConnectionPool; // FreeAndNilSafe(fMainConnection)
    deletecount := 0;
    fConnectionPoolSafe.Lock;
    try
      for i := fConnectionPoolMin to fConnectionPoolMax do
        if fConnectionPool[i] <> nil then
          ObjArrayAddCount(delete, fConnectionPool[i], deletecount);
    finally
      fConnectionPoolSafe.UnLock;
    end;
    if deletecount = 0 then
      exit;
    if Assigned(log) then
      log.Log(sllTrace, 'ClearConnectionPool: frompool=%', [deletecount], self);
    ObjArrayClear(delete, {continueonexc=}true, @deletecount);
  end;
end;

procedure TSqlDBConnectionPropertiesThreadSafe.RemoveFromPool(
  conn: TSqlDBConnectionThreadSafe);
var
  i: PtrInt;
begin // called from TSqlDBConnectionThreadSafe.Destroy
  fConnectionPoolSafe.Lock;
  try
    for i := fConnectionPoolMin to fConnectionPoolMax do
      if fConnectionPool[i] = conn then
      begin
        fConnectionPool[i] := nil;
        dec(fConnectionPoolCount);
        if fConnectionPoolCount = 0 then
        begin
          fConnectionPoolMin := 0; // reset range
          fConnectionPoolMax := -1;
        end;
        break;
      end;
  finally
    fConnectionPoolSafe.UnLock;
  end;
end;

procedure TSqlDBConnectionPropertiesThreadSafe.DeleteDeprecated(secs: integer);
var
  i: PtrInt;
  c: TSqlDBConnectionThreadSafe;
  delete: TObjectDynArray; // outside non-reentrant lock
  deletecount: integer;    // not PtrInt
  log: ISynLog;
begin // called at most every 32 seconds - ensured timeout <> 0 and secs <> 0
  if fConnectionPoolCount = 0 then
    exit;
  // detect outdated connection instances into a local delete[] list
  deletecount := 0;
  fConnectionPoolSafe.Lock;
  try
    for i := fConnectionPoolMin to fConnectionPoolMax do
    begin
      c := fConnectionPool[i];
      if (c = nil) or
         not c.IsOutdated(secs) then
        continue;
      ObjArrayAddCount(delete, c, deletecount);
      fConnectionPool[i] := nil; // instance is owned by delete[] now
    end;
  finally
    fConnectionPoolSafe.UnLock;
  end;
  if deletecount = 0 then
    exit;
  // delete all deprecated connections outside of the lock
  SynDBLog.EnterLocal(log, 'DeleteDeprecated=%', [deletecount], self);
  ObjArrayClear(delete, {continueonexc=}true, @deletecount);
end;

function TSqlDBConnectionPropertiesThreadSafe.ThreadSafeConnection: TSqlDBConnection;
var
  secs: integer;
  ndx: PtrInt;
begin
  case fThreadingMode of
    tmThreadPool:
      begin
        // first delete any deprecated connection(s) - check every 32 seconds
        secs := 0;
        if fConnectionTimeOutSecs <> 0 then
        begin
          secs := GetTickSec;
          if ConnectionTimeOutBackground and // disabled by default
             (not (cpfDeleteConnectionInOwnThread in fFlags)) and
             (fConnectionPoolDeprecatedTix32 <> secs shr 5) then
          begin
            fConnectionPoolDeprecatedTix32 := secs shr 5;
            DeleteDeprecated(secs);
          end;
        end;
        // search for an existing connection
        result := nil;
        ndx := TSynLog.ThreadIndex; // efficient slots thread list with reuse
        fConnectionPoolSafe.Lock;
        if (ndx >= fConnectionPoolMin) and
           (ndx <= fConnectionPoolMax) then
          result := fConnectionPool[ndx]; // O(1) lookup
        fConnectionPoolSafe.UnLock;
        // is this connection deprecated?
        if result <> nil then
          if (result.fLastAccessTickSec < 0) or // from ClearConnectionPool
             ((secs <> 0) and
              result.IsOutdated(secs)) then
            FreeAndNilSafe(result) // release this deprecated idle connection
          else
          begin
            if secs <> 0 then
              result.fLastAccessTickSec := secs; // mark for IsOutDated()
            exit; // valid connection for this thread
          end;
        // we need to (re)create a new connection for this thread
        result := NewConnection; // run outside of the lock (even if fast)
        // store in fConnectionPool[] internal list
        fConnectionPoolSafe.Lock;
        if ndx < fConnectionPoolMin then
          fConnectionPoolMin := ndx;
        if ndx > fConnectionPoolMax then
          fConnectionPoolMax := ndx;
        if ndx >= length(fConnectionPool) then
          SetLength(fConnectionPool, ndx + 32);
        fConnectionPool[ndx] := TSqlDBConnectionThreadSafe(result);
        inc(fConnectionPoolCount);
        fConnectionPoolSafe.UnLock;
      end;
    tmMainConnection:
      result := inherited GetMainConnection; // has its own TLightLock
  else
    result := nil;
  end;
end;


{ TSqlDBStatementWithParams }

function TSqlDBStatementWithParams.CheckParam(Param: integer;
  NewType: TSqlDBFieldType; IO: TSqlDBParamInOutType): PSqlDBParam;
begin
  if self = nil then
    ESqlDBException.RaiseU('TSqlDBStatementWithParams(nil).Bind');
  if Param > fParamCount then
    fParam.Count := Param; // resize fParams[] dynamic array if necessary
  result := @fParams[Param - 1];
  result^.VType := NewType;
  result^.VInOut := IO;
end;

function TSqlDBStatementWithParams.CheckParam(Param: integer;
  NewType: TSqlDBFieldType; IO: TSqlDBParamInOutType;
  ArrayCount: integer): PSqlDBParam;
begin
  result := CheckParam(Param, NewType, IO);
  if (NewType in [ftUnknown, ftNull]) or
     (fConnection = nil) or
     (fConnection.fProperties.BatchSendingAbilities *
       [cCreate, cUpdate, cDelete] = []) then
    ESqlDBException.RaiseUtf8('Invalid call to %.BindArray(Param=%,Type=%)',
      [self, Param, ToText(NewType)^]);
  SetLength(result^.VArray, ArrayCount);
  result^.VInt64 := ArrayCount;
  fParamsArrayCount := ArrayCount;
end;

constructor TSqlDBStatementWithParams.Create(aConnection: TSqlDBConnection);
begin
  inherited Create(aConnection);
  fParam.Init(TypeInfo(TSqlDBParamDynArray), fParams, @fParamCount);
end;

procedure TSqlDBStatementWithParams.Bind(Param: integer; Value: double;
  IO: TSqlDBParamInOutType);
begin
  CheckParam(Param, ftDouble, IO)^.VInt64 := PInt64(@Value)^;
end;

procedure TSqlDBStatementWithParams.Bind(Param: integer; Value: Int64;
  IO: TSqlDBParamInOutType);
begin
  CheckParam(Param, ftInt64, IO)^.VInt64 := Value;
end;

procedure TSqlDBStatementWithParams.BindBlob(Param: integer;
  const Data: RawByteString; IO: TSqlDBParamInOutType);
begin
  CheckParam(Param, ftBlob, IO)^.VData := Data;
end;

procedure TSqlDBStatementWithParams.BindBlob(Param: integer; Data: pointer;
  Size: integer; IO: TSqlDBParamInOutType);
begin
  FastSetRawByteString(CheckParam(Param, ftBlob, IO)^.VData, Data, Size);
end;

procedure TSqlDBStatementWithParams.BindCurrency(Param: integer;
  Value: currency; IO: TSqlDBParamInOutType);
begin
  CheckParam(Param, ftCurrency, IO)^.VInt64 := PInt64(@Value)^;
end;

procedure TSqlDBStatementWithParams.BindDateTime(Param: integer;
  Value: TDateTime; IO: TSqlDBParamInOutType);
begin
  CheckParam(Param, ftDate, IO)^.VInt64 := PInt64(@Value)^;
end;

procedure TSqlDBStatementWithParams.BindNull(Param: integer;
  IO: TSqlDBParamInOutType; BoundType: TSqlDBFieldType);
begin
  CheckParam(Param, ftNull, IO);
end;

procedure TSqlDBStatementWithParams.BindTextS(Param: integer;
  const Value: string; IO: TSqlDBParamInOutType);
begin
  if (Value = '') and
     (fConnection <> nil) and
     fConnection.fProperties.StoreVoidStringAsNull then
    CheckParam(Param, ftNull, IO)
  else
    CheckParam(Param, ftUtf8, IO)^.VData := StringToUtf8(Value);
end;

procedure TSqlDBStatementWithParams.BindTextU(Param: integer;
  const Value: RawUtf8; IO: TSqlDBParamInOutType);
begin
  if (Value = '') and
     (fConnection <> nil) and
     fConnection.fProperties.StoreVoidStringAsNull then
    CheckParam(Param, ftNull, IO)
  else
    CheckParam(Param, ftUtf8, IO)^.VData := Value;
end;

procedure TSqlDBStatementWithParams.BindTextP(Param: integer; Value: PUtf8Char;
  IO: TSqlDBParamInOutType);
begin
  if (Value = nil) and
     (fConnection <> nil) and
     fConnection.fProperties.StoreVoidStringAsNull then
    CheckParam(Param, ftNull, IO)
  else
    FastSetString(RawUtf8(CheckParam(Param, ftUtf8, IO)^.VData),
      Value, StrLen(Value));
end;

procedure TSqlDBStatementWithParams.BindTextW(Param: integer; const Value:
  WideString; IO: TSqlDBParamInOutType);
begin
  if (Value = '') and
     (fConnection <> nil) and
     fConnection.fProperties.StoreVoidStringAsNull then
    CheckParam(Param, ftNull, IO)
  else
    CheckParam(Param, ftUtf8, IO)^.VData := RawUnicodeToUtf8(pointer(Value),
      length(Value));
end;

function TSqlDBStatementWithParams.ParamToVariant(Param: integer;
  var Value: Variant; CheckIsOutParameter: boolean): TSqlDBFieldType;
begin
  inherited ParamToVariant(Param, Value); // raise exception if Param incorrect
  dec(Param); // start at #1
  if CheckIsOutParameter and
     (fParams[Param].VInOut = paramIn) then
    ESqlDBException.RaiseUtf8(
      '%.ParamToVariant expects an [In]Out parameter', [self]);
  // OleDB provider should have already modified the parameter in-place, i.e.
  // in our fParams[] buffer, especialy for TEXT parameters (OleStr/WideString)
  // -> we have nothing to do but return the current value! :)
  with fParams[Param] do
  begin
    result := VType;
    if VArray = nil then
      case VType of
        ftInt64:
          Value := VInt64;
        ftDouble:
          Value := unaligned(PDouble(@VInt64)^);
        ftCurrency:
          Value := PCurrency(@VInt64)^;
        ftDate:
          Value := unaligned(PDateTime(@VInt64)^);
        ftUtf8:
          RawUtf8ToVariant(RawUtf8(VData), Value);
        ftBlob:
          RawByteStringToVariant(VData, Value);
      else
        SetVariantNull(Value)
      end
    else
      SetVariantNull(Value);
  end;
end;

procedure TSqlDBStatementWithParams.AddParamValueAsText(Param: integer;
  Dest: TJsonWriter; MaxCharCount: integer);
begin
  dec(Param);
  if cardinal(Param) >= cardinal(fParamCount) then
    Dest.AddNull
  else
    with fParams[Param] do
      if VArray = nil then
        case VType of
          ftInt64:
            Dest.Add(VInt64);
          ftDouble:
            Dest.AddDouble(unaligned(PDouble(@VInt64)^));
          ftCurrency:
            Dest.AddCurr64(@VInt64);
          ftDate:
            Dest.AddDateTime(PDateTime(@VInt64), ' ', '''');
          ftUtf8:
            Dest.AddQuotedStr(pointer(VData), length(RawUtf8(VData)), '''',
              MaxCharCount);
          ftBlob:
            Dest.AddU(length(VData));
        else
          Dest.AddNull;
        end
      else
      begin
        Dest.Add('[');
        Dest.AddString(VArray[0]); // first item is enough in the logs
        Dest.AddDirect('.', '.', ']');
      end;
end;

procedure TSqlDBStatementWithParams.BindArray(Param: integer;
  const Values: array of double);
var
  i: PtrInt;
begin
  with CheckParam(Param, ftDouble, paramIn, length(Values))^ do
    for i := 0 to high(Values) do
      VArray[i] := DoubleToStr(Values[i]);
end;

procedure TSqlDBStatementWithParams.BindArray(Param: integer;
  const Values: array of Int64);
var
  i: PtrInt;
begin
  with CheckParam(Param, ftInt64, paramIn, length(Values))^ do
    for i := 0 to high(Values) do
      VArray[i] := Int64ToUtf8(Values[i]);
end;

procedure TSqlDBStatementWithParams.BindArray(Param: integer;
  ParamType: TSqlDBFieldType; const Values: TRawUtf8DynArray; ValuesCount: integer);
var
  i: PtrInt;
  p: PSqlDBParam;
  dt: TDateTime;
begin
  inherited; // raise an exception in case of invalid parameter
  p := CheckParam(Param, ParamType, paramIn);
  p^.VInt64 := ValuesCount;
  p^.VArray := Values; // immediate COW reference-counted assignment
  if (ParamType = ftDate) and
     (Connection.Properties.DateTimeFirstChar <> 'T') then
    for i := 0 to ValuesCount - 1 do // fix e.g. for PostgreSQL
      if (p^.VArray[i] <> '') and
         (p^.VArray[i][1] = '''') then
      begin
        // not only replace 'T'->timeseparator, but force expanded format
        dt := Iso8601ToDateTime(p^.VArray[i]);
        DateTimeToIso8601Var(dt, {expanded=}true, dsfForceDateWithMS in fFlags,
          Connection.Properties.DateTimeFirstChar, '''', p^.VArray[i]);
      end;
  fParamsArrayCount := ValuesCount;
end;

procedure TSqlDBStatementWithParams.BindArrayJson(Param: integer;
  ParamType: TSqlDBFieldType; var JsonArray: RawUtf8; ValuesCount: integer);
var
  p: PSqlDBParam;
begin
  // default implementation is to convert JSON values into SQL values
  p := CheckParam(Param, ParamType, paramIn, 0);
  if not JsonArrayToBoundArray(pointer(JsonArray), ParamType,
     Connection.Properties.DateTimeFirstChar, dsfForceDateWithMS in fFlags,
     p^.VArray) or
    (length(p^.VArray) <> ValuesCount) then
    BindArray(Param, ParamType, nil, 0); // raise exception
  p^.VInt64 := ValuesCount;
  fParamsArrayCount := ValuesCount;
end;

procedure TSqlDBStatementWithParams.BindArray(Param: integer;
  const Values: array of RawUtf8);
var
  i: PtrInt;
  StoreVoidStringAsNull: boolean;
begin
  StoreVoidStringAsNull := (fConnection <> nil) and
                           fConnection.Properties.StoreVoidStringAsNull;
  with CheckParam(Param, ftUtf8, paramIn, length(Values))^ do
    for i := 0 to high(Values) do
      if StoreVoidStringAsNull and
         (Values[i] = '') then
        VArray[i] := 'null'
      else
        QuotedStr(Values[i], '''', VArray[i]);
end;

procedure TSqlDBStatementWithParams.BindArrayCurrency(Param: integer;
  const Values: array of currency);
var
  i: PtrInt;
begin
  with CheckParam(Param, ftCurrency, paramIn, length(Values))^ do
    for i := 0 to high(Values) do
      VArray[i] := Curr64ToStr(PInt64(@Values[i])^);
end;

procedure TSqlDBStatementWithParams.BindArrayDateTime(Param: integer;
  const Values: array of TDateTime);
var
  i: PtrInt;
begin
  with CheckParam(Param, ftDate, paramIn, length(Values))^ do
    for i := 0 to high(Values) do
      VArray[i] := Connection.Properties.SqlDateToIso8601Quoted(Values[i]);
end;

procedure TSqlDBStatementWithParams.BindArrayRowPrepare(
  const aParamTypes: array of TSqlDBFieldType; aExpectedMinimalRowCount: integer);
var
  i: PtrInt;
begin
  fParam.Count := 0;
  for i := 0 to high(aParamTypes) do
    CheckParam(i + 1, aParamTypes[i], paramIn, aExpectedMinimalRowCount);
  fParamsArrayCount := 0;
end;

procedure TSqlDBStatementWithParams.BindArrayRow(const aValues: array of const);
var
  i: PtrInt;
  StoreVoidStringAsNull: boolean;
begin
  if length(aValues) <> fParamCount then
    ESqlDBException.RaiseUtf8('Invalid %.BindArrayRow call', [self]);
  StoreVoidStringAsNull := (fConnection <> nil) and
                           fConnection.Properties.StoreVoidStringAsNull;
  for i := 0 to high(aValues) do
    with fParams[i] do
    begin
      if length(VArray) <= fParamsArrayCount then
        SetLength(VArray, NextGrow(fParamsArrayCount));
      VInt64 := fParamsArrayCount;
      if (VType = ftDate) and
         (aValues[i].VType = vtExtended) then
        VArray[fParamsArrayCount] := // direct binding of TDateTime value
          Connection.Properties.SqlDateToIso8601Quoted(aValues[i].VExtended^)
      else
      begin
        VarRecToUtf8(@aValues[i], VArray[fParamsArrayCount]);
        case VType of
          ftUtf8:
            if StoreVoidStringAsNull and
               (VArray[fParamsArrayCount] = '') then
              VArray[fParamsArrayCount] := 'null'
            else
              QuotedStr(VArray[fParamsArrayCount], '''', VArray[fParamsArrayCount]);
          ftDate:
            QuotedStr(VArray[fParamsArrayCount], '''', VArray[fParamsArrayCount]);
        end;
      end;
    end;
  inc(fParamsArrayCount);
end;

procedure TSqlDBStatementWithParams.BindFromRows(Rows: TSqlDBStatement);
var
  F: PtrInt;
  U: RawUtf8;
begin
  if Rows <> nil then
    if Rows.ColumnCount <> fParamCount then
      ESqlDBException.RaiseUtf8('Invalid %.BindFromRows call', [self])
    else
      for F := 0 to fParamCount - 1 do
        with fParams[F] do
        begin
          if length(VArray) <= fParamsArrayCount then
            SetLength(VArray, NextGrow(fParamsArrayCount));
          if Rows.ColumnNull(F) then
            VArray[fParamsArrayCount] := 'null'
          else
            case Rows.ColumnType(F) of
              ftNull:
                VArray[fParamsArrayCount] := 'null';
              ftInt64:
                Int64ToUtf8(Rows.ColumnInt(F), VArray[fParamsArrayCount]);
              ftDouble:
                DoubleToStr(Rows.ColumnDouble(F), VArray[fParamsArrayCount]);
              ftCurrency:
                VArray[fParamsArrayCount] := CurrencyToStr(Rows.ColumnCurrency(F));
              ftDate:
                Join(['''', DateTimeToSql(Rows.ColumnDateTime (F)), ''''],
                  VArray[fParamsArrayCount]);
              ftUtf8:
                begin
                  U := Rows.ColumnUtf8(F);
                  if (U = '') and
                     (fConnection <> nil) and
                     fConnection.Properties.StoreVoidStringAsNull then
                    VArray[fParamsArrayCount] := 'null'
                  else
                    QuotedStr(U, '''', VArray[fParamsArrayCount]);
                end;
              ftBlob:
                VArray[fParamsArrayCount] := Rows.ColumnBlob(F);
            end;
        end;
  inc(fParamsArrayCount);
end;

procedure TSqlDBStatementWithParams.Reset;
begin
  fParamCount := 0; // no need of fParam.Clear
  fParamsArrayCount := 0;
  inherited Reset;
end;

procedure TSqlDBStatementWithParams.ReleaseRows;
var
  i: PtrInt;
  p: PSqlDBParam;
begin
  p := pointer(fParams);
  if p <> nil then
    for i := 1 to fParamCount do
    begin
      if p^.VData <> '' then
        p^.VData := ''; // release bound value, but keep fParams[] reusable
      if p^.VArray <> nil then
        RawUtf8DynArrayClear(p^.VArray);
      inc(p);
    end;
  inherited ReleaseRows;
end;


{ TSqlDBStatementWithParamsAndColumns }

constructor TSqlDBStatementWithParamsAndColumns.Create(
  aConnection: TSqlDBConnection);
begin
  inherited Create(aConnection);
  if (aConnection <> nil) and
     aConnection.Properties.EnsureColumnNameUnique then
    fColumn.InitSpecific(TypeInfo(TSqlDBColumnPropertyDynArray),
      fColumns, ptRawUtf8, @fColumnCount, {caseinsens=}true)
  else
    PDynArray(@fColumn)^.Init(TypeInfo(TSqlDBColumnPropertyDynArray),
      fColumns, @fColumnCount); // no hash index created nor unicity check
end;

function TSqlDBStatementWithParamsAndColumns.StepToJson(W: TJsonWriter;
  SeekFirst: boolean): boolean;
var
  col: integer;
begin
  result := Step(SeekFirst);
  if not result then
    exit;
  W.Add('{');
  for col := 0 to fColumnCount - 1 do
  begin
    W.AddFieldName(fColumns[col].ColumnName); // add '"ColumnName":'
    ColumnToJson(col, W);
    W.AddComma;
  end;
  W.ReplaceLastComma('}');
end;

procedure TSqlDBStatementWithParamsAndColumns.ClearColumns;
begin
  if fColumnCount = 0 then
    exit;
  fColumn.Clear;
  if (fConnection <> nil) and
     fConnection.Properties.EnsureColumnNameUnique then
    fColumn.ForceReHash;
end;

function TSqlDBStatementWithParamsAndColumns.AddColumn(
  const ColName: RawUtf8): PSqlDBColumnProperty;
begin
  if (fConnection <> nil) and
     fConnection.Properties.EnsureColumnNameUnique then
    result := fColumn.AddAndMakeUniqueName(ColName)
  else
  begin
    result := PDynArray(@fColumn)^.NewPtr;
    result^.ColumnName := ColName;
  end;
end;

function TSqlDBStatementWithParamsAndColumns.ColumnIndex(
  const aColumnName: RawUtf8): integer;
var
  c: PSqlDBColumnProperty;
begin
  if (fConnection <> nil) and
     fConnection.Properties.EnsureColumnNameUnique then
    result := fColumn.FindHashed(aColumnName)
  else
  begin
    c := pointer(fColumns);
    for result := 0 to fColumnCount - 1 do
      if IdemPropNameU(aColumnName, c^.ColumnName) then
        exit // brute force O(n) case insensitive check
      else
        inc(c);
    result := -1
  end;
end;

function TSqlDBStatementWithParamsAndColumns.ColumnName(Col: integer): RawUtf8;
begin
  CheckCol(Col);
  result := fColumns[Col].ColumnName;
end;

function TSqlDBStatementWithParamsAndColumns.ColumnType(
  Col: integer; FieldSize: PInteger): TSqlDBFieldType;
begin
  with fColumns[Col] do
  begin
    result := ColumnType;
    if FieldSize <> nil then
      if ColumnValueInlined then
        FieldSize^ := ColumnValueDBSize
      else
        FieldSize^ := 0;
  end;
end;


const
  // used e.g. by TRestStorageExternal.LogFields() to log external columns info
  __TSqlDBColumnDefine = 'ColumnName,ColumnTypeNative:RawUtf8 ' +
    'ColumnLength,ColumnPrecision,ColumnScale:PtrInt ' +
    'ColumnType:TSqlDBFieldType ColumnIndexed:boolean';

initialization
  assert(SizeOf(TSqlDBColumnProperty) = SizeOf(PtrUInt) * 2 + 20);
  Rtti.RegisterType(TypeInfo(TSqlDBFieldType));
  Rtti.RegisterFromText(TypeInfo(TSqlDBColumnDefine), __TSqlDBColumnDefine);

end.