From 1fc8b4445fbdecb98c50c0496a63714a1e9e0f61 Mon Sep 17 00:00:00 2001 From: Jacob Quinn Date: Thu, 11 Jun 2026 13:22:16 -0600 Subject: [PATCH] fix(detection): prefer header delimiters Prefer delimiter candidates that appear in the header before falling back to aggregate consistency across sampled rows. This avoids treating comma decimal markers as the delimiter when a semicolon-delimited file has a clear header. Fixes #1185 --- docs/src/examples.md | 6 +++--- src/detection.jl | 25 +++++++++++++++++++------ test/basics.jl | 6 ++++++ 3 files changed, 28 insertions(+), 9 deletions(-) diff --git a/docs/src/examples.md b/docs/src/examples.md index dc7b32c3..c4ff93f7 100644 --- a/docs/src/examples.md +++ b/docs/src/examples.md @@ -525,9 +525,9 @@ file = CSV.File(IOBuffer(data); dateformat="yyyy/mm/dd") ```julia using CSV -# In many places in the world, floating point number decimals are separated with a comma instead of a period (`3,14` vs. `3.14`) -# . We can correctly parse these numbers by passing in the `decimal=','` keyword argument. Note that we probably need to -# explicitly pass `delim=';'` in this case, since the parser will probably think that it detected `','` as the delimiter. +# In many places in the world, floating point number decimals are separated with a comma instead of a period (`3,14` vs. `3.14`). +# We can correctly parse these numbers by passing in the `decimal=','` keyword argument. If the file has no header or +# delimiter detection is otherwise ambiguous, pass `delim=';'` explicitly so commas are treated only as decimal markers. data = """ col1;col2;col3 1,01;2,02;3,03 diff --git a/src/detection.jl b/src/detection.jl index b2289bc4..a8cdd75f 100644 --- a/src/detection.jl +++ b/src/detection.jl @@ -119,12 +119,25 @@ function detectdelimandguessrows(buf, headerpos, datapos, len, oq, eq, cq, @nosp if delim == UInt8('\n') if nlines > 0 d = UInt8('\n') - for attempted_delim in (UInt8(','), UInt8('\t'), UInt8(' '), UInt8('|'), UInt8(';'), UInt8(':')) - cnt = bvc.counts[Int(attempted_delim) + 1] - # @show Char(attempted_delim), cnt, nlines - if cnt > 0 && cnt % nlines == 0 - d = attempted_delim - break + if headerpos > 0 + # Prefer delimiters represented in the header over incidental data characters. + for attempted_delim in (UInt8(','), UInt8('\t'), UInt8(' '), UInt8('|'), UInt8(';'), UInt8(':')) + hcnt = headerbvc.counts[Int(attempted_delim) + 1] + cnt = bvc.counts[Int(attempted_delim) + 1] + if hcnt > 0 && cnt > 0 && cnt % nlines == 0 + d = attempted_delim + break + end + end + end + if d == UInt8('\n') + for attempted_delim in (UInt8(','), UInt8('\t'), UInt8(' '), UInt8('|'), UInt8(';'), UInt8(':')) + cnt = bvc.counts[Int(attempted_delim) + 1] + # @show Char(attempted_delim), cnt, nlines + if cnt > 0 && cnt % nlines == 0 + d = attempted_delim + break + end end end if d == UInt8('\n') diff --git a/test/basics.jl b/test/basics.jl index 135c2dfc..af52905f 100644 --- a/test/basics.jl +++ b/test/basics.jl @@ -132,6 +132,12 @@ f = CSV.File(IOBuffer("x\ty\n1\t\n2\t\n")) f = CSV.File(IOBuffer("x|y\n1|\n2|\n")) @test (length(f), length(f.names)) == (2, 2) +f = CSV.File(IOBuffer("a;b;c\n1,1;2,2;3,3\n4,4;5,5;6,6\n"); decimal=',') +@test (length(f), length(f.names)) == (2, 3) +@test f.a == [1.1, 4.4] +@test f.b == [2.2, 5.5] +@test f.c == [3.3, 6.6] + # type promotion # int => float f = CSV.File(IOBuffer("x\n1\n3.14"))