Skip to content

Commit 2da4c33

Browse files
authored
Introduce a new DateAndTime sturct for > millisecond precision support (#178)
Fixes #165 (and #172). So the proposal here is a bit tricky, but here's what's going on in this PR: * Introduce a new DateAndTime struct, which just contains separate `Date` and `Time` fields, where the `time` field can support up to Nanosecond precision * For _prepared_ statement execution, we actually return a DateTime column for > millisecond precision, and there used to be no warning. We now emit a warning that precision is being lost and that `mysql_date_and_time=true` should be passed to avoid such loss * For _directly executed_ queries, we throw an InexactError and now emit the same warning that `mysql_date_and_time=true` should be passed to avoid such errors. We could perhaps match the prepare behavior and return `DateTime`, but I wanted to avoid changing too much all at once * Why introduce a keyword arg instead of just making it the default? Well, that'd be breaking. It's also tricky because we don't necessarily want to force this new `DateAndTime` struct on _all_ DATETIME/TIMESTAMP columns, especially if their precision is within the millisecond range. i.e. it's much more convenient for users to work direclty with `DateTime` objects instead of this new `DateAndTime` struct. So the behavior now is: emit clear warnings in cases where DateTime can't handle the precision, and users can pass `mysql_date_and_time=true` and then handle `DateAndTime` objects accordingly.
1 parent ba8048d commit 2da4c33

File tree

8 files changed

+97
-21
lines changed

8 files changed

+97
-21
lines changed

src/MySQL.jl

+4-3
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ module MySQL
22

33
using Dates, DBInterface, Tables, Parsers, DecFP
44

5-
export DBInterface
5+
export DBInterface, DateAndTime
66

77
# For non-C-api errors that happen in MySQL.jl
88
struct MySQLInterfaceError
@@ -300,11 +300,12 @@ end
300300
Base.close(conn::Connection) = DBInterface.close!(conn)
301301
Base.isopen(conn::Connection) = API.isopen(conn.mysql)
302302

303-
function juliatype(field_type, notnullable, isunsigned, isbinary)
303+
function juliatype(field_type, notnullable, isunsigned, isbinary, date_and_time)
304304
T = API.juliatype(field_type)
305305
T2 = isunsigned && !(T <: AbstractFloat) ? unsigned(T) : T
306306
T3 = !isbinary && T2 == Vector{UInt8} ? String : T2
307-
return notnullable ? T3 : Union{Missing, T3}
307+
T4 = date_and_time && T3 <: DateTime ? DateAndTime : T3
308+
return notnullable ? T4 : Union{Missing, T4}
308309
end
309310

310311
include("execute.jl")

src/api/API.jl

+2
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@ module API
22

33
using Dates, DecFP
44

5+
export DateAndTime
6+
57
if VERSION < v"1.3.0"
68

79
# Load libmariadb from our deps.jl

src/api/apitypes.jl

+23-7
Original file line numberDiff line numberDiff line change
@@ -116,28 +116,44 @@ const MYSQL_TIME_FORMAT = Dates.DateFormat("HH:MM:SS.s")
116116
const MYSQL_DATE_FORMAT = Dates.DateFormat("yyyy-mm-dd")
117117
const MYSQL_DATETIME_FORMAT = Dates.DateFormat("yyyy-mm-dd HH:MM:SS.s")
118118

119+
@noinline dateandtime_warning() = @warn """a datetime value from a column has a microsecond precision > 3,
120+
by default, MySQL.jl attempts to return a DateTime object, which only supports millisecond precision.
121+
To avoid loss in precision or InexactErrors, pass `mysql_date_and_time=true` to `DBInterface.execute(stmt, sql; mysql_date_and_time=true)` or `DBInterface.prepare(stmt, sql; mysql_date_and_time=true)`.
122+
This will result in a column element type of `DateAndTime`, which is a simple struct of separate Date and Time parts, accessed like `dt.date` and `dt.time`.
123+
"""
124+
119125
function Base.convert(::Type{DateTime}, mtime::MYSQL_TIME)
126+
millis, micros = divrem(mtime.second_part, 1000)
120127
if mtime.year == 0 || mtime.month == 0 || mtime.day == 0
121-
DateTime(1970, 1, 1,
122-
mtime.hour, mtime.minute, mtime.second)
128+
dt = DateTime(1970, 1, 1,
129+
mtime.hour, mtime.minute, mtime.second, millis)
123130
else
124-
DateTime(mtime.year, mtime.month, mtime.day,
125-
mtime.hour, mtime.minute, mtime.second)
131+
dt = DateTime(mtime.year, mtime.month, mtime.day,
132+
mtime.hour, mtime.minute, mtime.second, millis)
126133
end
134+
micros > 0 && dateandtime_warning()
135+
return dt
127136
end
128137
Base.convert(::Type{Dates.Time}, mtime::MYSQL_TIME) =
129-
Dates.Time(mtime.hour, mtime.minute, mtime.second)
138+
Dates.Time(mtime.hour, mtime.minute, mtime.second, divrem(mtime.second_part, 1000)...)
130139
Base.convert(::Type{Date}, mtime::MYSQL_TIME) =
131140
Date(mtime.year, mtime.month, mtime.day)
141+
Base.convert(::Type{DateAndTime}, mtime::MYSQL_TIME) =
142+
DateAndTime(Date(mtime.year, mtime.month, mtime.day),
143+
Time(mtime.hour, mtime.minute, mtime.second, divrem(mtime.second_part, 1000)...))
132144

133145
Base.convert(::Type{MYSQL_TIME}, t::Dates.Time) =
134-
MYSQL_TIME(0, 0, 0, Dates.hour(t), Dates.minute(t), Dates.second(t), 0, 0, 0)
146+
MYSQL_TIME(0, 0, 0, Dates.hour(t), Dates.minute(t), Dates.second(t), Dates.millisecond(t) * 1000 + Dates.microsecond(t), 0, 0)
135147
Base.convert(::Type{MYSQL_TIME}, dt::Date) =
136148
MYSQL_TIME(Dates.year(dt), Dates.month(dt), Dates.day(dt), 0, 0, 0, 0, 0, 0)
137149

138150
Base.convert(::Type{MYSQL_TIME}, dtime::DateTime) =
139151
MYSQL_TIME(Dates.year(dtime), Dates.month(dtime), Dates.day(dtime),
140-
Dates.hour(dtime), Dates.minute(dtime), Dates.second(dtime), 0, 0, 0)
152+
Dates.hour(dtime), Dates.minute(dtime), Dates.second(dtime), Dates.millisecond(dtime) * 1000, 0, 0)
153+
154+
Base.convert(::Type{MYSQL_TIME}, dat::DateAndTime) =
155+
MYSQL_TIME(Dates.year(dat), Dates.month(dat), Dates.day(dat),
156+
Dates.hour(dat), Dates.minute(dat), Dates.second(dat), Dates.millisecond(dat) * 1000 + Dates.microsecond(dat), 0, 0)
141157

142158
# this is a helper struct, because MYSQL_BIND needs
143159
# to know where the bound data should live, by using this helper

src/api/consts.jl

+20
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,25 @@ end
4848
Base.show(io::IO, b::Bit) = print(io, "MySQL.API.Bit(\"$(string(b))\")")
4949
Base.unsigned(::Type{Bit}) = Bit
5050

51+
struct DateAndTime <: Dates.AbstractDateTime
52+
date::Date
53+
time::Time
54+
end
55+
56+
Dates.Date(x::DateAndTime) = x.date
57+
Dates.Time(x::DateAndTime) = x.time
58+
Dates.year(x::DateAndTime) = Dates.year(Date(x))
59+
Dates.month(x::DateAndTime) = Dates.month(Date(x))
60+
Dates.day(x::DateAndTime) = Dates.day(Date(x))
61+
Dates.hour(x::DateAndTime) = Dates.hour(Time(x))
62+
Dates.minute(x::DateAndTime) = Dates.minute(Time(x))
63+
Dates.second(x::DateAndTime) = Dates.second(Time(x))
64+
Dates.millisecond(x::DateAndTime) = Dates.millisecond(Time(x))
65+
Dates.microsecond(x::DateAndTime) = Dates.microsecond(Time(x))
66+
67+
import Base.==
68+
==(a::DateAndTime, b::DateAndTime) = ==(a.date, b.date) && ==(a.time, b.time)
69+
5170
mysqltype(::Type{Bit}) = MYSQL_TYPE_BIT
5271
mysqltype(::Union{Type{Cchar}, Type{Cuchar}}) = MYSQL_TYPE_TINY
5372
mysqltype(::Union{Type{Cshort}, Type{Cushort}}) = MYSQL_TYPE_SHORT
@@ -58,6 +77,7 @@ mysqltype(::Type{Dec64}) = MYSQL_TYPE_DECIMAL
5877
mysqltype(::Type{Cdouble}) = MYSQL_TYPE_DOUBLE
5978
mysqltype(::Type{Vector{UInt8}}) = MYSQL_TYPE_BLOB
6079
mysqltype(::Type{DateTime}) = MYSQL_TYPE_TIMESTAMP
80+
mysqltype(::Type{DateAndTime}) = MYSQL_TYPE_DATETIME
6181
mysqltype(::Type{Date}) = MYSQL_TYPE_DATE
6282
mysqltype(::Type{Time}) = MYSQL_TYPE_TIME
6383
mysqltype(::Type{Missing}) = MYSQL_TYPE_NULL

src/execute.jl

+29-7
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ mutable struct TextCursor{buffered} <: DBInterface.Cursor
1010
lookup::Dict{Symbol, Int}
1111
current_rownumber::Int
1212
current_resultsetnumber::Int
13+
mysql_date_and_time::Bool
1314
end
1415

1516
struct TextRow{buffered} <: Tables.AbstractRow
@@ -64,13 +65,34 @@ const DATETIME_OPTIONS = Parsers.Options(dateformat=dateformat"yyyy-mm-dd HH:MM:
6465

6566
function cast(::Type{DateTime}, ptr, len)
6667
buf = unsafe_wrap(Array, ptr, len)
67-
x, code, pos = Parsers.typeparser(DateTime, buf, 1, len, buf[1], Int16(0), DATETIME_OPTIONS)
68-
if code > 0
69-
return x
68+
try
69+
x, code, pos = Parsers.typeparser(DateTime, buf, 1, len, buf[1], Int16(0), DATETIME_OPTIONS)
70+
if code > 0
71+
return x
72+
end
73+
catch e
74+
e isa InexactError && API.dateandtime_warning()
7075
end
7176
casterror(DateTime, ptr, len)
7277
end
7378

79+
const DATEANDTIME_OPTIONS = Parsers.Options(dateformat=dateformat"yyyy-mm-dd HH:MM:SS")
80+
81+
function cast(::Type{DateAndTime}, ptr, len)
82+
buf = unsafe_wrap(Array, ptr, len)
83+
i = findfirst(==(UInt8('.')), buf)
84+
x, code, pos = Parsers.typeparser(DateTime, buf, 1, something(i, len), buf[1], Int16(0), DATETIME_OPTIONS)
85+
if code > 0
86+
dt, tm = Date(x), Time(x)
87+
if i !== nothing
88+
y, code, pos = Parsers.typeparser(Int, buf, i + 1, len, buf[1], Int16(0), Parsers.OPTIONS)
89+
tm += Dates.Microsecond(y)
90+
end
91+
return DateAndTime(dt, tm)
92+
end
93+
casterror(DateAndTime, ptr, len)
94+
end
95+
7496
@noinline wrongrow(i) = throw(ArgumentError("row $i is no longer valid; mysql results are forward-only iterators where each row is only valid when iterated"))
7597

7698
function Tables.getcolumn(r::TextRow, ::Type{T}, i::Int, nm::Symbol) where {T}
@@ -129,7 +151,7 @@ Specifying `mysql_store_result=false` will avoid buffering the full resultset to
129151
the query, which has memory use advantages, though ties up the database server since resultset rows must be
130152
fetched one at a time.
131153
"""
132-
function DBInterface.execute(conn::Connection, sql::AbstractString, params=(); mysql_store_result::Bool=true)
154+
function DBInterface.execute(conn::Connection, sql::AbstractString, params=(); mysql_store_result::Bool=true, mysql_date_and_time::Bool=false)
133155
checkconn(conn)
134156
params != () && error("`DBInterface.execute(conn, sql)` does not support parameter binding; see `?DBInterface.prepare(conn, sql)`")
135157
clear!(conn)
@@ -154,7 +176,7 @@ function DBInterface.execute(conn::Connection, sql::AbstractString, params=(); m
154176
nfields = API.numfields(result)
155177
fields = API.fetchfields(result, nfields)
156178
names = [ccall(:jl_symbol_n, Ref{Symbol}, (Ptr{UInt8}, Csize_t), x.name, x.name_length) for x in fields]
157-
types = [juliatype(x.field_type, API.notnullable(x), API.isunsigned(x), API.isbinary(x)) for x in fields]
179+
types = [juliatype(x.field_type, API.notnullable(x), API.isunsigned(x), API.isbinary(x), mysql_date_and_time) for x in fields]
158180
elseif API.fieldcount(conn.mysql) == 0
159181
rows_affected = API.affectedrows(conn.mysql)
160182
names = Symbol[]
@@ -163,7 +185,7 @@ function DBInterface.execute(conn::Connection, sql::AbstractString, params=(); m
163185
error("error with mysql resultset columns")
164186
end
165187
lookup = Dict(x => i for (i, x) in enumerate(names))
166-
return TextCursor{buffered}(conn, sql, nfields, nrows, Core.bitcast(Int64, rows_affected), result, names, types, lookup, 0, 1)
188+
return TextCursor{buffered}(conn, sql, nfields, nrows, Core.bitcast(Int64, rows_affected), result, names, types, lookup, 0, 1, mysql_date_and_time)
167189
end
168190

169191
struct TextCursors{T}
@@ -186,7 +208,7 @@ function Base.iterate(cursor::TextCursors{buffered}, first=true) where {buffered
186208
cursor.cursor.nfields = API.numfields(cursor.cursor.result)
187209
fields = API.fetchfields(cursor.cursor.result, cursor.cursor.nfields)
188210
cursor.cursor.names = [ccall(:jl_symbol_n, Ref{Symbol}, (Ptr{UInt8}, Csize_t), x.name, x.name_length) for x in fields]
189-
cursor.cursor.types = [juliatype(x.field_type, API.notnullable(x), API.isunsigned(x), API.isbinary(x)) for x in fields]
211+
cursor.cursor.types = [juliatype(x.field_type, API.notnullable(x), API.isunsigned(x), API.isbinary(x), cursor.cursor.mysql_date_and_time) for x in fields]
190212
else
191213
return nothing
192214
end

src/load.jl

+1
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ const SQLTYPES = Dict{Type, String}(
2929
Date => "DATE",
3030
Time => "TIME",
3131
DateTime => "DATETIME",
32+
DateAndTime => "DATETIME(6)",
3233
)
3334

3435
checkdupnames(names) = length(unique(map(x->lowercase(String(x)), names))) == length(names) || error("duplicate case-insensitive column names detected; sqlite doesn't allow duplicate column names and treats them case insensitive")

src/prepare.jl

+5-4
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ Note that `DBInterface.close!(stmt)` should be called once statement executions
4040
freeing resources, it has been noted that too many unclosed statements and resultsets, used in conjunction
4141
with streaming queries (i.e. `mysql_store_result=false`) has led to occasional resultset corruption.
4242
"""
43-
function DBInterface.prepare(conn::Connection, sql::AbstractString)
43+
function DBInterface.prepare(conn::Connection, sql::AbstractString; mysql_date_and_time::Bool=false)
4444
clear!(conn)
4545
stmt = API.stmtinit(conn.mysql)
4646
API.prepare(stmt, sql)
@@ -52,7 +52,7 @@ function DBInterface.prepare(conn::Connection, sql::AbstractString)
5252
if result.ptr != C_NULL
5353
fields = API.fetchfields(result, nfields)
5454
names = [ccall(:jl_symbol_n, Ref{Symbol}, (Ptr{UInt8}, Csize_t), x.name, x.name_length) for x in fields]
55-
types = [juliatype(x.field_type, API.notnullable(x), API.isunsigned(x), API.isbinary(x)) for x in fields]
55+
types = [juliatype(x.field_type, API.notnullable(x), API.isunsigned(x), API.isbinary(x), mysql_date_and_time) for x in fields]
5656
valuehelpers = [API.BindHelper() for i = 1:nfields]
5757
values = [API.MYSQL_BIND(valuehelpers[i].length, valuehelpers[i].is_null) for i = 1:nfields]
5858
foreach(1:nfields) do i
@@ -147,7 +147,7 @@ Specifying `mysql_store_result=false` will avoid buffering the full resultset to
147147
the query, which has memory use advantages, though ties up the database server since resultset rows must be
148148
fetched one at a time.
149149
"""
150-
function DBInterface.execute(stmt::Statement, params=(); mysql_store_result::Bool=true)
150+
function DBInterface.execute(stmt::Statement, params=(); mysql_store_result::Bool=true, mysql_date_and_time::Bool=false)
151151
checkstmt(stmt)
152152
paramcheck(stmt, params)
153153
clear!(stmt.conn)
@@ -179,7 +179,7 @@ function DBInterface.execute(stmt::Statement, params=(); mysql_store_result::Boo
179179
if result.ptr != C_NULL
180180
fields = API.fetchfields(result, nfields)
181181
names = [ccall(:jl_symbol_n, Ref{Symbol}, (Ptr{UInt8}, Csize_t), x.name, x.name_length) for x in fields]
182-
types = [juliatype(x.field_type, API.notnullable(x), API.isunsigned(x), API.isbinary(x)) for x in fields]
182+
types = [juliatype(x.field_type, API.notnullable(x), API.isunsigned(x), API.isbinary(x), mysql_date_and_time) for x in fields]
183183
valuehelpers = [API.BindHelper() for i = 1:nfields]
184184
values = [API.MYSQL_BIND(valuehelpers[i].length, valuehelpers[i].is_null) for i = 1:nfields]
185185
foreach(1:nfields) do i
@@ -250,6 +250,7 @@ ptrhelper(helper, x::API.MYSQL_TIME) = pointer(helper.time)
250250
getvalue(stmt, helper, values, i, ::Type{Time}) = convert(Time, helper.time[1])
251251
getvalue(stmt, helper, values, i, ::Type{Date}) = convert(Date, helper.time[1])
252252
getvalue(stmt, helper, values, i, ::Type{DateTime}) = convert(DateTime, helper.time[1])
253+
getvalue(stmt, helper, values, i, ::Type{DateAndTime}) = convert(DateAndTime, helper.time[1])
253254

254255
inithelper!(helper, x::String) = nothing
255256
ptrhelper(helper, x::String) = C_NULL

test/runtests.jl

+13
Original file line numberDiff line numberDiff line change
@@ -242,6 +242,19 @@ res = DBInterface.execute(resstmt) |> columntable
242242
@test length(res) == 2
243243
@test res[2][1] == DateTime(1970, 1, 1, 3)
244244

245+
# https://github.com/JuliaDatabases/MySQL.jl/issues/165
246+
DBInterface.execute(conn, "DROP TABLE if exists datetime6_field")
247+
DBInterface.execute(conn, "CREATE TABLE datetime6_field (id int(11), t DATETIME(6))")
248+
stmt = DBInterface.prepare(conn, "INSERT INTO datetime6_field (id, t) VALUES (?, ?);")
249+
DBInterface.execute(stmt, [1, DateAndTime(Date(2021, 1, 2), Time(1, 2, 3, 456, 789))])
250+
resstmt = DBInterface.prepare(conn, "select id, t from datetime6_field"; mysql_date_and_time=true)
251+
res = DBInterface.execute(resstmt) |> columntable
252+
@test length(res) == 2
253+
@test res[2][1] == DateAndTime(Date(2021, 1, 2), Time(1, 2, 3, 456, 789))
254+
res = DBInterface.execute(conn, "select id, t from datetime6_field"; mysql_date_and_time=true) |> columntable
255+
@test length(res) == 2
256+
@test res[2][1] == DateAndTime(Date(2021, 1, 2), Time(1, 2, 3, 456, 789))
257+
245258
DBInterface.execute(conn, """
246259
CREATE PROCEDURE get_employee()
247260
BEGIN

0 commit comments

Comments
 (0)