Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

add deviation tolerancy #4

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

add deviation tolerancy #4

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions detector/Fixtures/test3.csv
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
Email Address,,first name,last name,Gender,Date of birth,country,city,STREET,POSTAL CODE,INTEREST 1 - Producer,INTEREST 2 - 2-Dutch Recs,INTEREST 3 - SKINK,INTEREST 4 - Musical Madness,INTEREST 5 - Dutch Master Works,INTEREST 6 - 4-Dots,INTEREST 7 - Blue Forest,INTEREST 8 - Brooks,INTEREST 9 - PROMOTER GENERAL,INTEREST 10 - PROMOTER CLEAN
[email protected],First,Last,,,,,,,,,,,,,,Brooks,,
[email protected],First,Last,,,,,,,,,,,,,,Brooks,,
[email protected],First,Last,,,,,,,,,,,,,,Brooks,,
[email protected],First,Last,,,,,,,,,,,,,,Brooks,,
14 changes: 13 additions & 1 deletion detector/detect.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package detector
import (
"bufio"
"io"
// "log"
"math"
"regexp"
)
Expand Down Expand Up @@ -145,11 +146,22 @@ func (d *detector) analyze(ft frequencyTable, sampleLine int) []byte {
}

var candidates []byte
var minDeviation float64
var minDelimiter byte
for delimiter, frequencyOfLine := range ft {
if float64(0.0) == deviation(frequencyOfLine, sampleLine) {
dev := deviation(frequencyOfLine, sampleLine)
if float64(0.0) == dev {
candidates = append(candidates, delimiter)
} else if minDeviation > dev || minDeviation == 0 {
// find minimum deviation available
minDeviation = dev
minDelimiter = delimiter
}
}
// if zero deviation candidates are not found, pick the minimum one
if len(candidates) == 0 && minDeviation > 0 {
candidates = append(candidates, minDelimiter)
}

return candidates
}
Expand Down
13 changes: 13 additions & 0 deletions detector/detect_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,19 @@ func TestDetectDelimiterComma(t *testing.T) {
assert.Equal(t, []string{","}, delimiters)
}

func TestDetectDelimiterComma2(t *testing.T) {
detector := New()
sampleLines := 4
detector.Configure(&sampleLines, nil)
file, err := os.OpenFile("./Fixtures/test3.csv", os.O_RDONLY, os.ModePerm)
assert.NoError(t, err)
defer file.Close()

delimiters := detector.DetectDelimiter(file, '"')

assert.Equal(t, []string{","}, delimiters)
}

func TestDetectDelimiterSemicolon(t *testing.T) {
detector := New()
sampleLines := 4
Expand Down