diff --git a/.claude/codebase/types.md b/.claude/codebase/types.md index 1ac09653..430f8609 100644 --- a/.claude/codebase/types.md +++ b/.claude/codebase/types.md @@ -120,12 +120,13 @@ Col* (e.g., ColID = "id", ColName = "name", ColDeletedAt = "deleted_at") - LLM (provider, model, baseURL, apiKey, timeout, thinking, extraContext) - Chat/Extraction overrides (LLMChatOverride, LLMExtractionOverride) - Documents (MaxFileSize ByteSize, CacheTTL Duration) -- Extraction (MaxPages int, Enabled *bool, TextTimeout, LLMTimeout) +- Extraction (MaxPages int, Enable *bool, LLMTimeout) + - OCR (Enable *bool, ConfidenceThreshold int) - Locale (Currency string) ### Defaults - Provider: "ollama", Model: "qwen3", BaseURL: "http://localhost:11434" -- MaxPages: 20, CacheTTL: 30 days, TextTimeout: 30s, LLMTimeout: 5m +- MaxPages: 20, CacheTTL: 30 days, LLMTimeout: 5m ## LLM Types (internal/llm/) diff --git a/cmd/micasa/main.go b/cmd/micasa/main.go index be6906ff..348a54b1 100644 --- a/cmd/micasa/main.go +++ b/cmd/micasa/main.go @@ -161,7 +161,9 @@ func (cmd *runCmd) Run() error { exCfg := cfg.LLM.ExtractionConfig() extractors := extract.DefaultExtractors( cfg.Extraction.MaxPages, - cfg.Extraction.TextTimeoutDuration(), + 0, // pdftotext uses its own internal default timeout (30s) + cfg.Extraction.IsOCREnabled(), + cfg.Extraction.OCR.ConfidenceThreshold, ) opts.SetExtraction( exCfg.Provider, diff --git a/docs/content/docs/reference/configuration.md b/docs/content/docs/reference/configuration.md index 56533326..29e21546 100644 --- a/docs/content/docs/reference/configuration.md +++ b/docs/content/docs/reference/configuration.md @@ -111,18 +111,19 @@ You can always infer the env var name from the config key. | `MICASA_LLM_MODEL` | `qwen3` | `llm.model` | LLM model name | | `MICASA_LLM_API_KEY` | (empty) | `llm.api_key` | LLM API key for cloud providers | | `MICASA_LLM_EXTRA_CONTEXT` | (empty) | `llm.extra_context` | Custom context appended to LLM system prompts | -| `MICASA_LLM_TIMEOUT` | `5s` | `llm.timeout` | LLM operation timeout | +| `MICASA_LLM_TIMEOUT` | `5m` | `llm.timeout` | Max time for a single LLM response | | `MICASA_LLM_THINKING` | (unset) | `llm.thinking` | Enable model thinking for chat | | `MICASA_DOCUMENTS_MAX_FILE_SIZE` | `50 MiB` | `documents.max_file_size` | Max document import size | | `MICASA_DOCUMENTS_CACHE_TTL` | `30d` | `documents.cache_ttl` | Document cache lifetime | | `MICASA_DOCUMENTS_CACHE_TTL_DAYS` | -- | `documents.cache_ttl_days` | Deprecated; use `MICASA_DOCUMENTS_CACHE_TTL` | | `MICASA_DOCUMENTS_FILE_PICKER_DIR` | (Downloads) | `documents.file_picker_dir` | Starting directory for the file picker | | `MICASA_EXTRACTION_MODEL` | (chat model) | `extraction.model` | LLM model for document extraction | -| `MICASA_EXTRACTION_ENABLED` | `true` | `extraction.enabled` | Enable/disable LLM extraction | +| `MICASA_EXTRACTION_ENABLE` | `true` | `extraction.enable` | Enable/disable LLM extraction | | `MICASA_EXTRACTION_THINKING` | `false` | `extraction.thinking` | Enable model thinking for extraction | -| `MICASA_EXTRACTION_TEXT_TIMEOUT` | `30s` | `extraction.text_timeout` | pdftotext timeout | | `MICASA_EXTRACTION_MAX_PAGES` | `0` | `extraction.max_pages` | Max pages to OCR per document (0 = no limit) | | `MICASA_EXTRACTION_LLM_TIMEOUT` | `5m` | `extraction.llm_timeout` | LLM extraction timeout | +| `MICASA_EXTRACTION_OCR_ENABLE` | `true` | `extraction.ocr.enable` | Enable/disable OCR on documents | +| `MICASA_EXTRACTION_OCR_CONFIDENCE_THRESHOLD` | `0` | `extraction.ocr.confidence_threshold` | Min tesseract confidence (0-100) | | `MICASA_LOCALE_CURRENCY` | (auto-detect) | `locale.currency` | ISO 4217 currency code (e.g. `USD`, `EUR`, `GBP`) | {{% details title="Deprecated env var names" closed="true" %}} @@ -139,8 +140,8 @@ warning. They will be removed in a future release. | `MICASA_CURRENCY` | `MICASA_LOCALE_CURRENCY` | | `MICASA_EXTRACTION_MAX_EXTRACT_PAGES` | `MICASA_EXTRACTION_MAX_PAGES` | | `MICASA_MAX_EXTRACT_PAGES` | `MICASA_EXTRACTION_MAX_PAGES` | -| `MICASA_TEXT_TIMEOUT` | `MICASA_EXTRACTION_TEXT_TIMEOUT` | | `MICASA_MAX_OCR_PAGES` | `MICASA_EXTRACTION_MAX_PAGES` | +| `MICASA_EXTRACTION_ENABLED` | `MICASA_EXTRACTION_ENABLE` | | `MICASA_EXTRACTION_MODEL` | `MICASA_LLM_EXTRACTION_MODEL` | | `MICASA_EXTRACTION_THINKING` | `MICASA_LLM_EXTRACTION_THINKING` | @@ -177,12 +178,12 @@ micasa # uses llama3.3 instead of the default qwen3 ### `MICASA_LLM_TIMEOUT` -Sets the LLM timeout for quick operations (ping, model listing), overriding -the config file value. Uses Go duration syntax: +Sets the maximum time for a single LLM response (including streaming), +overriding the config file value. Uses Go duration syntax: ```sh -export MICASA_LLM_TIMEOUT=15s -micasa # waits up to 15s for LLM server responses +export MICASA_LLM_TIMEOUT=10m +micasa # waits up to 10m for LLM responses ``` ### `MICASA_DOCUMENTS_MAX_FILE_SIZE` @@ -278,10 +279,10 @@ model = "qwen3" # Use this to inject domain-specific details about your house, region, etc. # extra_context = "My house is a 1920s craftsman in Portland, OR." -# Timeout for quick LLM server operations (ping, model listing). -# Go duration syntax: "5s", "10s", "500ms", etc. Default: "5s". -# Increase if your LLM server is slow to respond. -# timeout = "5s" +# Max time for a single LLM response (including streaming). +# Go duration syntax: "5m", "10m", etc. Default: "5m". +# Increase for slow models or complex queries. +# timeout = "5m" # Enable model thinking mode for chat (e.g. qwen3 blocks). # Unset = don't send (server default), true = enable, false = disable. @@ -302,10 +303,6 @@ model = "qwen3" # with small, fast models optimized for structured JSON output. # model = "qwen2.5:7b" -# Timeout for pdftotext. Go duration syntax: "30s", "1m", etc. Default: "30s". -# Increase if you routinely process very large PDFs. -# text_timeout = "30s" - # Maximum pages to OCR for scanned documents. 0 = no limit. Default: 0. # max_pages = 0 @@ -338,7 +335,7 @@ set in `[llm.chat]` and `[llm.extraction]`. | `model` | string | `qwen3` | Model identifier sent in chat requests. Must be available on the server. | | `api_key` | string | (empty) | Authentication credential. Required for cloud providers (Anthropic, OpenAI, etc.). Leave empty for local servers. | | `extra_context` | string | (empty) | Free-form text appended to all LLM system prompts. Useful for telling the model about your house or regional conventions. Currency is handled automatically via `[locale]`. | -| `timeout` | string | `"5s"` | Max wait time for quick LLM operations (ping, model listing). Go duration syntax, e.g. `"10s"`, `"500ms"`. Increase for slow servers. | +| `timeout` | string | `"5m"` | Max time for a single LLM response (including streaming). Go duration syntax, e.g. `"10m"`. Increase for slow models. | | `thinking` | bool | (unset) | Enable model thinking mode (e.g. qwen3 `` blocks). Unset = don't send the option (server default). | ### `[llm.chat]` section @@ -391,11 +388,20 @@ dates, vendor matching) from uploaded documents. | Key | Type | Default | Description | |-----|------|---------|-------------| | `model` | string | (chat model) | **Deprecated.** Use `[llm.extraction] model` instead. Falls back to `llm.model` if empty. | -| `text_timeout` | string | `"30s"` | Max time for `pdftotext` to run. Go duration syntax, e.g. `"1m"`. Increase for very large PDFs. | | `max_pages` | int | `0` | Maximum pages to OCR per scanned document. 0 means no limit. | -| `enabled` | bool | `true` | Set to `false` to disable LLM-powered extraction. When disabled, no structured data is extracted from documents. | +| `enable` | bool | `true` | Set to `false` to disable LLM-powered structured extraction. OCR and pdftotext still run (see `[extraction.ocr]`). | +| `enabled` | bool | -- | **Deprecated.** Use `enable` instead. | | `thinking` | bool | `false` | **Deprecated.** Use `[llm.extraction] thinking` instead. | +### `[extraction.ocr]` section + +OCR sub-pipeline settings. Requires `tesseract` and `pdftocairo`. + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| `enable` | bool | `true` | Set to `false` to disable OCR on documents. When disabled, scanned pages and images produce no text. | +| `confidence_threshold` | int | `0` | Minimum tesseract word confidence (0-100) to keep. Words below this threshold are dropped. 0 means no filtering. | + ### `[locale]` section Locale and currency settings. Controls currency formatting across all money diff --git a/internal/config/config.go b/internal/config/config.go index af9108b4..880ed530 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -264,14 +264,14 @@ type Extraction struct { // documents. 0 means no limit (all pages). Default: 0. MaxPages int `toml:"max_pages"` - // Enabled controls whether LLM-powered extraction runs when a document - // is uploaded. When disabled, no structured data is extracted -- OCR and - // pdftotext are internal pipeline steps, not standalone features. Default: true. - Enabled *bool `toml:"enabled,omitempty"` + // Enable controls whether LLM-powered structured extraction runs when + // a document is uploaded. When disabled, no structured data is extracted + // from documents. OCR and pdftotext still run independently (controlled + // by [extraction.ocr]) to populate the document's stored text. Default: true. + Enable *bool `toml:"enable,omitempty"` - // TextTimeout is the maximum time to wait for pdftotext. Go duration - // string, e.g. "30s", "1m". Default: "30s". - TextTimeout string `toml:"text_timeout"` + // Enabled is the deprecated spelling; migrated to Enable on load. + Enabled *bool `toml:"enabled,omitempty"` // LLMTimeout is the maximum time to wait for the LLM extraction // inference step. Go duration string, e.g. "5m", "90s". Default: "5m". @@ -281,28 +281,39 @@ type Extraction struct { // Supported values: none, low, medium, high, auto. // Empty string = don't send (server default). Default: empty. Thinking string `toml:"thinking,omitempty"` + + // OCR holds settings for the OCR sub-pipeline. + OCR OCR `toml:"ocr" doc:"OCR sub-pipeline. Requires tesseract and pdftocairo."` +} + +// OCR holds settings for the OCR sub-pipeline within extraction. +type OCR struct { + // Enable controls whether OCR runs on uploaded documents. + // When disabled, scanned pages and images produce no text. Default: true. + Enable *bool `toml:"enable,omitempty"` + + // ConfidenceThreshold is the minimum tesseract word confidence (0-100) + // to keep in OCR output. Words below this threshold are dropped. + // 0 means no filtering (all words kept). Default: 0. + ConfidenceThreshold int `toml:"confidence_threshold"` } // IsEnabled returns whether LLM extraction is enabled. Defaults to true // when the field is unset. func (e Extraction) IsEnabled() bool { - if e.Enabled != nil { - return *e.Enabled + if e.Enable != nil { + return *e.Enable } return true } -// TextTimeoutDuration returns the parsed text extraction timeout, falling -// back to DefaultTextTimeout if the value is empty or unparseable. -func (e Extraction) TextTimeoutDuration() time.Duration { - if e.TextTimeout == "" { - return DefaultTextTimeout - } - d, err := time.ParseDuration(e.TextTimeout) - if err != nil { - return DefaultTextTimeout +// IsOCREnabled returns whether OCR is enabled. Defaults to true when +// the field is unset. +func (e Extraction) IsOCREnabled() bool { + if e.OCR.Enable != nil { + return *e.OCR.Enable } - return d + return true } // LLMTimeoutDuration returns the parsed LLM extraction timeout, falling @@ -341,7 +352,6 @@ const ( DefaultLLMExtractionTimeout = DefaultLLMTimeout DefaultCacheTTL = 30 * 24 * time.Hour // 30 days DefaultMaxPages = 0 - DefaultTextTimeout = 30 * time.Second configRelPath = "micasa/config.toml" ) @@ -378,6 +388,15 @@ func LoadFromPath(path string) (Config, error) { return cfg, err } + // Clear deprecated Enabled again: applyEnvOverrides may have + // repopulated it from MICASA_EXTRACTION_ENABLED. + if cfg.Extraction.Enabled != nil { + if cfg.Extraction.Enable == nil { + cfg.Extraction.Enable = cfg.Extraction.Enabled + } + cfg.Extraction.Enabled = nil + } + // Normalize base URLs: strip trailing slash and /v1 suffix -- // providers handle their own path construction. cfg.LLM.BaseURL = normalizeBaseURL(cfg.LLM.BaseURL) @@ -488,22 +507,6 @@ func LoadFromPath(path string) (Config, error) { ) } - if cfg.Extraction.TextTimeout != "" { - d, err := time.ParseDuration(cfg.Extraction.TextTimeout) - if err != nil { - return cfg, fmt.Errorf( - "extraction.text_timeout: invalid duration %q -- use Go syntax like \"30s\" or \"1m\"", - cfg.Extraction.TextTimeout, - ) - } - if d <= 0 { - return cfg, fmt.Errorf( - "extraction.text_timeout must be positive, got %s", - cfg.Extraction.TextTimeout, - ) - } - } - if cfg.Extraction.LLMTimeout != "" { d, err := time.ParseDuration(cfg.Extraction.LLMTimeout) if err != nil { @@ -527,6 +530,13 @@ func LoadFromPath(path string) (Config, error) { ) } + if cfg.Extraction.OCR.ConfidenceThreshold < 0 || cfg.Extraction.OCR.ConfidenceThreshold > 100 { + return cfg, fmt.Errorf( + "extraction.ocr.confidence_threshold must be 0-100, got %d", + cfg.Extraction.OCR.ConfidenceThreshold, + ) + } + checkFilePermissions(&cfg, path) return cfg, nil @@ -905,6 +915,17 @@ func migrateRenamedKeys(cfg *Config, md toml.MetaData, path string) { ) } + // extraction.enabled -> extraction.enable (v1.78) + if md.IsDefined("extraction", "enabled") { + if !md.IsDefined("extraction", "enable") { + cfg.Extraction.Enable = cfg.Extraction.Enabled + } + cfg.Warnings = append(cfg.Warnings, + "extraction.enabled is deprecated -- use extraction.enable instead", + ) + } + cfg.Extraction.Enabled = nil // never propagate the deprecated field + // extraction.model -> llm.extraction.model (v1.59) if md.IsDefined("extraction", "model") && !md.IsDefined("llm", "extraction", "model") { cfg.LLM.Extraction.Model = cfg.Extraction.Model @@ -926,6 +947,9 @@ func migrateRenamedKeys(cfg *Config, md toml.MetaData, path string) { // replacements. Processed newest-first so that the most recent intermediate // name wins when multiple generations of the same variable are set. var envRenames = []struct{ old, canonical string }{ + // v1.78: extraction.enabled -> extraction.enable. + {"MICASA_EXTRACTION_ENABLED", "MICASA_EXTRACTION_ENABLE"}, + // v1.77: env var names now derived from dotted config paths. {"MICASA_CURRENCY", "MICASA_LOCALE_CURRENCY"}, {"MICASA_MAX_DOCUMENT_SIZE", "MICASA_DOCUMENTS_MAX_FILE_SIZE"}, @@ -934,7 +958,6 @@ var envRenames = []struct{ old, canonical string }{ {"MICASA_FILE_PICKER_DIR", "MICASA_DOCUMENTS_FILE_PICKER_DIR"}, {"MICASA_EXTRACTION_MAX_EXTRACT_PAGES", "MICASA_EXTRACTION_MAX_PAGES"}, {"MICASA_MAX_EXTRACT_PAGES", "MICASA_EXTRACTION_MAX_PAGES"}, - {"MICASA_TEXT_TIMEOUT", "MICASA_EXTRACTION_TEXT_TIMEOUT"}, // v1.59 {"MICASA_EXTRACTION_MODEL", "MICASA_LLM_EXTRACTION_MODEL"}, @@ -1102,9 +1125,9 @@ model = "` + DefaultModel + `" # file_picker_dir = "/home/user/Documents" [extraction] -# Timeout for pdftotext. Go duration syntax: "30s", "1m", etc. Default: "30s". -# Increase if you routinely process very large PDFs. -# text_timeout = "30s" +# Set to false to disable LLM-powered structured extraction. OCR and pdftotext +# still run (see [extraction.ocr]) to populate document text for search/display. +# enable = true # Timeout for LLM extraction inference. Go duration syntax: "5m", "90s", etc. # Default: "5m". Increase for slow local models or complex documents. @@ -1113,9 +1136,14 @@ model = "` + DefaultModel + `" # Maximum pages for async extraction of scanned documents. 0 = no limit. Default: 0. # max_pages = 0 -# Set to false to disable LLM-powered extraction even when LLM is configured. -# When disabled, no structured data is extracted from documents. -# enabled = true +# [extraction.ocr] +# Set to false to disable OCR on uploaded documents. When disabled, scanned +# pages and images produce no text. Default: true. +# enable = true + +# Minimum tesseract word confidence (0-100) to keep. Words below this +# threshold are dropped. 0 = no filtering. Default: 0. +# confidence_threshold = 0 [locale] # ISO 4217 currency code. Stored in the database on first run; after that the diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 486f57a0..e8b616d4 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -462,7 +462,7 @@ func TestExtractionFromFile(t *testing.T) { path := writeConfig(t, `[extraction] model = "qwen2.5:7b" max_pages = 10 -enabled = false +enable = false `) cfg, err := LoadFromPath(path) require.NoError(t, err) @@ -485,7 +485,7 @@ func TestExtractionResolvedModel(t *testing.T) { func TestExtractionEnvOverrides(t *testing.T) { t.Setenv("MICASA_EXTRACTION_MODEL", "phi3") t.Setenv("MICASA_EXTRACTION_MAX_PAGES", "5") - t.Setenv("MICASA_EXTRACTION_ENABLED", "false") + t.Setenv("MICASA_EXTRACTION_ENABLE", "false") cfg, err := LoadFromPath(noConfig(t)) require.NoError(t, err) @@ -510,7 +510,7 @@ func TestInvalidEnvVarReturnsError(t *testing.T) { wantMsg string }{ {"MICASA_EXTRACTION_MAX_PAGES", "not-a-number", "expected integer"}, - {"MICASA_EXTRACTION_ENABLED", "maybe", "expected true or false"}, + {"MICASA_EXTRACTION_ENABLE", "maybe", "expected true or false"}, {"MICASA_DOCUMENTS_MAX_FILE_SIZE", "lots", "expected byte size"}, {"MICASA_DOCUMENTS_CACHE_TTL", "forever", "expected duration"}, {"MICASA_DOCUMENTS_CACHE_TTL_DAYS", "many", "expected integer"}, @@ -629,12 +629,15 @@ func TestEnvVars(t *testing.T) { "MICASA_DOCUMENTS_CACHE_TTL_DAYS": "documents.cache_ttl_days", "MICASA_DOCUMENTS_FILE_PICKER_DIR": "documents.file_picker_dir", - "MICASA_EXTRACTION_MODEL": "extraction.model", - "MICASA_EXTRACTION_MAX_PAGES": "extraction.max_pages", - "MICASA_EXTRACTION_ENABLED": "extraction.enabled", - "MICASA_EXTRACTION_TEXT_TIMEOUT": "extraction.text_timeout", - "MICASA_EXTRACTION_LLM_TIMEOUT": "extraction.llm_timeout", - "MICASA_EXTRACTION_THINKING": "extraction.thinking", + "MICASA_EXTRACTION_MODEL": "extraction.model", + "MICASA_EXTRACTION_MAX_PAGES": "extraction.max_pages", + "MICASA_EXTRACTION_ENABLE": "extraction.enable", + "MICASA_EXTRACTION_ENABLED": "extraction.enabled", + "MICASA_EXTRACTION_LLM_TIMEOUT": "extraction.llm_timeout", + "MICASA_EXTRACTION_THINKING": "extraction.thinking", + + "MICASA_EXTRACTION_OCR_ENABLE": "extraction.ocr.enable", + "MICASA_EXTRACTION_OCR_CONFIDENCE_THRESHOLD": "extraction.ocr.confidence_threshold", "MICASA_LOCALE_CURRENCY": "locale.currency", @@ -664,7 +667,7 @@ func TestEnvVarName(t *testing.T) { }{ {"llm.model", "MICASA_LLM_MODEL"}, {"documents.max_file_size", "MICASA_DOCUMENTS_MAX_FILE_SIZE"}, - {"extraction.text_timeout", "MICASA_EXTRACTION_TEXT_TIMEOUT"}, + {"extraction.llm_timeout", "MICASA_EXTRACTION_LLM_TIMEOUT"}, {"locale.currency", "MICASA_LOCALE_CURRENCY"}, {"llm.chat.provider", "MICASA_LLM_CHAT_PROVIDER"}, } @@ -739,6 +742,86 @@ func TestMaxPagesEnvMigrationChain(t *testing.T) { "MICASA_MAX_EXTRACT_PAGES (newer) should take precedence") } +func TestEnabledTOMLMigration(t *testing.T) { + path := writeConfig(t, "[extraction]\nenabled = false\n") + cfg, err := LoadFromPath(path) + require.NoError(t, err) + assert.False(t, cfg.Extraction.IsEnabled()) + require.Len(t, cfg.Warnings, 1) + assert.Contains(t, cfg.Warnings[0], "extraction.enabled") + assert.Contains(t, cfg.Warnings[0], "extraction.enable") +} + +func TestEnabledTOMLMigrationIgnoredWhenNewKeySet(t *testing.T) { + path := writeConfig(t, "[extraction]\nenabled = true\nenable = false\n") + cfg, err := LoadFromPath(path) + require.NoError(t, err) + assert.False(t, cfg.Extraction.IsEnabled(), "new key takes precedence") + require.Len(t, cfg.Warnings, 1) + assert.Contains(t, cfg.Warnings[0], "extraction.enabled") +} + +func TestEnabledEnvMigration(t *testing.T) { + t.Setenv("MICASA_EXTRACTION_ENABLED", "false") + cfg, err := LoadFromPath(noConfig(t)) + require.NoError(t, err) + assert.False(t, cfg.Extraction.IsEnabled()) + assert.Nil(t, cfg.Extraction.Enabled, "deprecated Enabled must be cleared after env migration") + require.Len(t, cfg.Warnings, 1) + assert.Contains(t, cfg.Warnings[0], "MICASA_EXTRACTION_ENABLED") + assert.Contains(t, cfg.Warnings[0], "MICASA_EXTRACTION_ENABLE") +} + +func TestEnabledEnvIgnoredWhenNewEnvSet(t *testing.T) { + t.Setenv("MICASA_EXTRACTION_ENABLED", "true") + t.Setenv("MICASA_EXTRACTION_ENABLE", "false") + cfg, err := LoadFromPath(noConfig(t)) + require.NoError(t, err) + assert.False(t, cfg.Extraction.IsEnabled()) + assert.Empty(t, cfg.Warnings) +} + +// --- OCR config --- + +func TestOCRDefaults(t *testing.T) { + cfg, err := LoadFromPath(noConfig(t)) + require.NoError(t, err) + assert.True(t, cfg.Extraction.IsOCREnabled()) + assert.Equal(t, 0, cfg.Extraction.OCR.ConfidenceThreshold) +} + +func TestOCRFromFile(t *testing.T) { + path := writeConfig(t, "[extraction.ocr]\nenable = false\nconfidence_threshold = 70\n") + cfg, err := LoadFromPath(path) + require.NoError(t, err) + assert.False(t, cfg.Extraction.IsOCREnabled()) + assert.Equal(t, 70, cfg.Extraction.OCR.ConfidenceThreshold) +} + +func TestOCREnvOverrides(t *testing.T) { + t.Setenv("MICASA_EXTRACTION_OCR_ENABLE", "false") + t.Setenv("MICASA_EXTRACTION_OCR_CONFIDENCE_THRESHOLD", "80") + cfg, err := LoadFromPath(noConfig(t)) + require.NoError(t, err) + assert.False(t, cfg.Extraction.IsOCREnabled()) + assert.Equal(t, 80, cfg.Extraction.OCR.ConfidenceThreshold) +} + +func TestOCRConfidenceThresholdValidation(t *testing.T) { + t.Run("rejects negative", func(t *testing.T) { + path := writeConfig(t, "[extraction.ocr]\nconfidence_threshold = -1\n") + _, err := LoadFromPath(path) + require.Error(t, err) + assert.Contains(t, err.Error(), "confidence_threshold must be 0-100") + }) + t.Run("rejects over 100", func(t *testing.T) { + path := writeConfig(t, "[extraction.ocr]\nconfidence_threshold = 101\n") + _, err := LoadFromPath(path) + require.Error(t, err) + assert.Contains(t, err.Error(), "confidence_threshold must be 0-100") + }) +} + // --- Per-pipeline LLM config --- func TestChatConfigInheritsBase(t *testing.T) { @@ -1075,7 +1158,6 @@ func TestDeprecatedEnvVarRenames(t *testing.T) { {"MICASA_MAX_DOCUMENT_SIZE", "MICASA_DOCUMENTS_MAX_FILE_SIZE", "100 MiB"}, {"MICASA_CACHE_TTL", "MICASA_DOCUMENTS_CACHE_TTL", "7d"}, {"MICASA_FILE_PICKER_DIR", "MICASA_DOCUMENTS_FILE_PICKER_DIR", "/tmp"}, - {"MICASA_TEXT_TIMEOUT", "MICASA_EXTRACTION_TEXT_TIMEOUT", "1m"}, } for _, tt := range tests { t.Run(tt.old, func(t *testing.T) { diff --git a/internal/config/show.go b/internal/config/show.go index 57933dc6..2042a5d8 100644 --- a/internal/config/show.go +++ b/internal/config/show.go @@ -28,6 +28,7 @@ var hiddenPaths = map[string]bool{ // "DEPRECATED: use " warning. var deprecatedPaths = map[string]string{ "documents.cache_ttl_days": "documents.cache_ttl", + "extraction.enabled": "extraction.enable", "extraction.model": "llm.extraction.model", "extraction.thinking": "llm.extraction.thinking", } @@ -74,12 +75,13 @@ func (c Config) forDisplay() Config { d.Documents.CacheTTL = &Duration{dur} } // CacheTTLDays preserved when user-set so the dump warns about it. - if d.Extraction.Enabled == nil { + if d.Extraction.Enable == nil { t := true - d.Extraction.Enabled = &t + d.Extraction.Enable = &t } - if d.Extraction.TextTimeout == "" { - d.Extraction.TextTimeout = DefaultTextTimeout.String() + if d.Extraction.OCR.Enable == nil { + t := true + d.Extraction.OCR.Enable = &t } if d.Locale.Currency == "" { d.Locale.Currency = detectCurrencyCode() @@ -335,16 +337,21 @@ func writeAligned(w io.Writer, blk sectionBlock) error { return nil } -// FormatDuration formats a duration in a human-friendly way, using day -// notation for whole-day multiples. +// FormatDuration formats a duration in a human-friendly way, using +// clean notation for whole-unit multiples (days, hours, minutes). func FormatDuration(d time.Duration) string { - if d == 0 { + switch { + case d == 0: return "0s" - } - if d%(24*time.Hour) == 0 { + case d%(24*time.Hour) == 0: return fmt.Sprintf("%dd", d/(24*time.Hour)) + case d%time.Hour == 0: + return fmt.Sprintf("%dh", d/time.Hour) + case d%time.Minute == 0: + return fmt.Sprintf("%dm", d/time.Minute) + default: + return d.String() } - return d.String() } // formatTOMLValue formats a reflected value as a TOML value string. diff --git a/internal/config/show_test.go b/internal/config/show_test.go index 2a84940b..57bf27ac 100644 --- a/internal/config/show_test.go +++ b/internal/config/show_test.go @@ -38,8 +38,7 @@ func TestShowConfigDefaults(t *testing.T) { assert.Contains(t, out, `max_file_size = "50 MiB"`) assert.Contains(t, out, `cache_ttl = "30d"`) assert.Contains(t, out, "max_pages = 0") - assert.Contains(t, out, "enabled = true") - assert.Contains(t, out, `text_timeout = "30s"`) + assert.Contains(t, out, "enable = true") assert.NotContains(t, out, "cache_ttl_days") assert.NotContains(t, out, "[llm.chat]") @@ -63,7 +62,7 @@ cache_ttl = "7d" [extraction] max_pages = 10 -enabled = false +enable = false `) cfg, err := LoadFromPath(path) require.NoError(t, err) @@ -95,8 +94,7 @@ cache_ttl = "7d" [extraction] max_pages = 10 -enabled = false -text_timeout = "1m" +enable = false `) orig, err := LoadFromPath(path) require.NoError(t, err) @@ -123,7 +121,6 @@ text_timeout = "1m" parsed.Documents.CacheTTLDuration()) assert.Equal(t, orig.Extraction.MaxPages, parsed.Extraction.MaxPages) assert.Equal(t, orig.Extraction.IsEnabled(), parsed.Extraction.IsEnabled()) - assert.Equal(t, orig.Extraction.TextTimeout, parsed.Extraction.TextTimeout) // API keys are hidden -- the parsed config must NOT have them. assert.Empty(t, parsed.LLM.APIKey) @@ -352,9 +349,12 @@ func TestFormatDuration(t *testing.T) { }{ {0, "0s"}, {5 * time.Second, "5s"}, + {5 * time.Minute, "5m"}, + {90 * time.Minute, "90m"}, + {2 * time.Hour, "2h"}, {30 * 24 * time.Hour, "30d"}, {7 * 24 * time.Hour, "7d"}, - {90 * time.Minute, "1h30m0s"}, + {90*time.Minute + 30*time.Second, "1h30m30s"}, {500 * time.Millisecond, "500ms"}, } for _, tt := range tests { diff --git a/internal/extract/extractor.go b/internal/extract/extractor.go index bf491f0b..ac51ee0e 100644 --- a/internal/extract/extractor.go +++ b/internal/extract/extractor.go @@ -31,13 +31,25 @@ type Extractor interface { // DefaultExtractors returns the standard extractors in priority order: // pdftotext, plaintext, PDF OCR, image OCR. maxPages of 0 means no limit // (all pages). Zero timeout causes the concrete extractor to use its default. -func DefaultExtractors(maxPages int, timeout time.Duration) []Extractor { - return []Extractor{ +// ocrEnabled controls whether OCR extractors are included (default true). +// confidenceThreshold filters OCR words below this confidence (0 = no filter). +func DefaultExtractors( + maxPages int, + timeout time.Duration, + ocrEnabled bool, + confidenceThreshold int, +) []Extractor { + ext := []Extractor{ &PDFTextExtractor{Timeout: timeout}, &PlainTextExtractor{}, - &PDFOCRExtractor{MaxPages: maxPages}, - &ImageOCRExtractor{}, } + if ocrEnabled { + ext = append(ext, + &PDFOCRExtractor{MaxPages: maxPages, ConfidenceThreshold: confidenceThreshold}, + &ImageOCRExtractor{ConfidenceThreshold: confidenceThreshold}, + ) + } + return ext } // HasMatchingExtractor reports whether any extractor in the list with @@ -141,7 +153,8 @@ func (e *PlainTextExtractor) Extract(_ context.Context, data []byte) (TextSource // PDFOCRExtractor wraps ocrPDF for scanned PDF pages. type PDFOCRExtractor struct { - MaxPages int + MaxPages int + ConfidenceThreshold int } func (e *PDFOCRExtractor) Tool() string { return "tesseract" } @@ -156,6 +169,10 @@ func (e *PDFOCRExtractor) Extract(ctx context.Context, data []byte) (TextSource, if err != nil { return TextSource{}, err } + if e.ConfidenceThreshold > 0 { + tsv = filterTSVByConfidence(tsv, e.ConfidenceThreshold) + text = textFromTSV(tsv) + } return TextSource{ Tool: "tesseract", Desc: "Text recognized from rasterized page images. Covers scanned pages that pdftotext misses, but may contain OCR errors.", @@ -165,7 +182,9 @@ func (e *PDFOCRExtractor) Extract(ctx context.Context, data []byte) (TextSource, } // ImageOCRExtractor wraps ocrImage for direct image OCR. -type ImageOCRExtractor struct{} +type ImageOCRExtractor struct { + ConfidenceThreshold int +} func (e *ImageOCRExtractor) Tool() string { return "tesseract" } func (e *ImageOCRExtractor) Matches(mime string) bool { return IsImageMIME(mime) } @@ -179,6 +198,10 @@ func (e *ImageOCRExtractor) Extract(ctx context.Context, data []byte) (TextSourc if err != nil { return TextSource{}, err } + if e.ConfidenceThreshold > 0 { + tsv = filterTSVByConfidence(tsv, e.ConfidenceThreshold) + text = textFromTSV(tsv) + } return TextSource{ Tool: "tesseract", Desc: "Text recognized from the image. May contain OCR errors.", diff --git a/internal/extract/extractor_test.go b/internal/extract/extractor_test.go index c9aa0232..247a8299 100644 --- a/internal/extract/extractor_test.go +++ b/internal/extract/extractor_test.go @@ -150,7 +150,7 @@ func TestImageOCRExtractor_Available(t *testing.T) { func TestDefaultExtractors_Order(t *testing.T) { t.Parallel() - extractors := DefaultExtractors(0, 0) + extractors := DefaultExtractors(0, 0, true, 0) require.Len(t, extractors, 4) assert.Equal(t, "pdftotext", extractors[0].Tool()) assert.Equal(t, "plaintext", extractors[1].Tool()) @@ -166,7 +166,7 @@ func TestDefaultExtractors_Order(t *testing.T) { func TestDefaultExtractors_Passthrough(t *testing.T) { t.Parallel() - extractors := DefaultExtractors(42, 99) + extractors := DefaultExtractors(42, 99, true, 0) pdfExt, ok := extractors[0].(*PDFTextExtractor) require.True(t, ok) assert.Equal(t, 99, int(pdfExt.Timeout)) @@ -176,32 +176,52 @@ func TestDefaultExtractors_Passthrough(t *testing.T) { assert.Equal(t, 42, ocrExt.MaxPages) } +func TestDefaultExtractors_OCRDisabled(t *testing.T) { + t.Parallel() + extractors := DefaultExtractors(0, 0, false, 0) + require.Len(t, extractors, 2) + assert.Equal(t, "pdftotext", extractors[0].Tool()) + assert.Equal(t, "plaintext", extractors[1].Tool()) +} + +func TestDefaultExtractors_ConfidencePassthrough(t *testing.T) { + t.Parallel() + extractors := DefaultExtractors(0, 0, true, 70) + pdfOCR, ok := extractors[2].(*PDFOCRExtractor) + require.True(t, ok) + assert.Equal(t, 70, pdfOCR.ConfidenceThreshold) + + imgOCR, ok := extractors[3].(*ImageOCRExtractor) + require.True(t, ok) + assert.Equal(t, 70, imgOCR.ConfidenceThreshold) +} + // --- HasMatchingExtractor --- func TestHasMatchingExtractor_Tesseract_PDF(t *testing.T) { t.Parallel() - extractors := DefaultExtractors(0, 0) + extractors := DefaultExtractors(0, 0, true, 0) got := HasMatchingExtractor(extractors, "tesseract", "application/pdf") assert.Equal(t, OCRAvailable(), got) } func TestHasMatchingExtractor_Tesseract_Image(t *testing.T) { t.Parallel() - extractors := DefaultExtractors(0, 0) + extractors := DefaultExtractors(0, 0, true, 0) got := HasMatchingExtractor(extractors, "tesseract", "image/png") assert.Equal(t, ImageOCRAvailable(), got) } func TestHasMatchingExtractor_Pdftotext(t *testing.T) { t.Parallel() - extractors := DefaultExtractors(0, 0) + extractors := DefaultExtractors(0, 0, true, 0) got := HasMatchingExtractor(extractors, "pdftotext", "application/pdf") assert.Equal(t, HasPDFToText(), got) } func TestHasMatchingExtractor_NoMatch(t *testing.T) { t.Parallel() - extractors := DefaultExtractors(0, 0) + extractors := DefaultExtractors(0, 0, true, 0) assert.False(t, HasMatchingExtractor(extractors, "tesseract", "text/plain")) assert.False(t, HasMatchingExtractor(extractors, "pdftotext", "image/png")) assert.False(t, HasMatchingExtractor(extractors, "nonexistent", "application/pdf")) @@ -211,21 +231,21 @@ func TestHasMatchingExtractor_NoMatch(t *testing.T) { func TestNeedsOCR_PDF(t *testing.T) { t.Parallel() - extractors := DefaultExtractors(0, 0) + extractors := DefaultExtractors(0, 0, true, 0) got := NeedsOCR(extractors, "application/pdf") assert.Equal(t, OCRAvailable(), got) } func TestNeedsOCR_Image(t *testing.T) { t.Parallel() - extractors := DefaultExtractors(0, 0) + extractors := DefaultExtractors(0, 0, true, 0) got := NeedsOCR(extractors, "image/png") assert.Equal(t, ImageOCRAvailable(), got) } func TestNeedsOCR_PlainText(t *testing.T) { t.Parallel() - extractors := DefaultExtractors(0, 0) + extractors := DefaultExtractors(0, 0, true, 0) assert.False(t, NeedsOCR(extractors, "text/plain")) } @@ -240,7 +260,7 @@ func TestNeedsOCR_NoOCRExtractors(t *testing.T) { func TestExtractorTimeout(t *testing.T) { t.Parallel() - extractors := DefaultExtractors(0, 42) + extractors := DefaultExtractors(0, 42, true, 0) assert.Equal(t, time.Duration(42), ExtractorTimeout(extractors)) } @@ -252,7 +272,7 @@ func TestExtractorTimeout_NoPDFText(t *testing.T) { func TestExtractorMaxPages(t *testing.T) { t.Parallel() - extractors := DefaultExtractors(15, 0) + extractors := DefaultExtractors(15, 0, true, 0) assert.Equal(t, 15, ExtractorMaxPages(extractors)) } diff --git a/internal/extract/ocr.go b/internal/extract/ocr.go index 26b2b7c7..e01d3055 100644 --- a/internal/extract/ocr.go +++ b/internal/extract/ocr.go @@ -367,6 +367,33 @@ func atoi(b []byte) int { return n } +// filterTSVByConfidence removes rows from tesseract TSV output whose +// confidence (column 10) is below the threshold. +func filterTSVByConfidence(tsv []byte, threshold int) []byte { + lines := bytes.Split(tsv, []byte("\n")) + if len(lines) < 2 { + return tsv + } + var out bytes.Buffer + out.Write(lines[0]) // header + out.WriteByte('\n') + for _, line := range lines[1:] { + if len(line) == 0 { + continue + } + fields := bytes.Split(line, []byte("\t")) + if len(fields) < 12 { + continue + } + conf := atoi(fields[10]) + if conf >= threshold { + out.Write(line) + out.WriteByte('\n') + } + } + return out.Bytes() +} + // IsImageMIME reports whether the MIME type is an image format that // tesseract can process. func IsImageMIME(mime string) bool { diff --git a/internal/extract/ocr_coverage_test.go b/internal/extract/ocr_coverage_test.go index 9041f5f4..a57170e3 100644 --- a/internal/extract/ocr_coverage_test.go +++ b/internal/extract/ocr_coverage_test.go @@ -807,7 +807,7 @@ func TestExtractWithProgress_PDF_InvalidData(t *testing.T) { context.Background(), []byte("not a pdf"), "application/pdf", - DefaultExtractors(5, 0), + DefaultExtractors(5, 0, true, 0), ) var gotErr bool @@ -1018,7 +1018,7 @@ func TestExtractWithProgress_Image_InvalidData(t *testing.T) { context.Background(), []byte("not an image"), "image/png", - DefaultExtractors(5, 0), + DefaultExtractors(5, 0, true, 0), ) var gotErr bool diff --git a/internal/extract/ocr_progress_test.go b/internal/extract/ocr_progress_test.go index b46cc7e4..b7674bcd 100644 --- a/internal/extract/ocr_progress_test.go +++ b/internal/extract/ocr_progress_test.go @@ -22,7 +22,7 @@ func TestExtractWithProgress_EmptyData(t *testing.T) { context.Background(), nil, "application/pdf", - DefaultExtractors(20, 0), + DefaultExtractors(20, 0, true, 0), ) msg := <-ch assert.True(t, msg.Done) @@ -37,7 +37,12 @@ func TestExtractWithProgress_EmptyData(t *testing.T) { // TestExtractWithProgress_EmptyImage verifies the image path with empty data. func TestExtractWithProgress_EmptyImage(t *testing.T) { t.Parallel() - ch := ExtractWithProgress(context.Background(), nil, "image/png", DefaultExtractors(20, 0)) + ch := ExtractWithProgress( + context.Background(), + nil, + "image/png", + DefaultExtractors(20, 0, true, 0), + ) msg := <-ch assert.True(t, msg.Done) assert.Empty(t, msg.Text) @@ -52,7 +57,12 @@ func TestExtractWithProgress_ContextCancelled(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() // cancel immediately - ch := ExtractWithProgress(ctx, []byte("fake image data"), "image/png", DefaultExtractors(20, 0)) + ch := ExtractWithProgress( + ctx, + []byte("fake image data"), + "image/png", + DefaultExtractors(20, 0, true, 0), + ) var gotErr bool for msg := range ch { @@ -78,7 +88,12 @@ func TestExtractWithProgress_Image_Integration(t *testing.T) { skipOrFatalCI(t, "test fixture not found: "+imgPath) } - ch := ExtractWithProgress(context.Background(), data, "image/png", DefaultExtractors(20, 0)) + ch := ExtractWithProgress( + context.Background(), + data, + "image/png", + DefaultExtractors(20, 0, true, 0), + ) var progressCount int var finalText string @@ -117,7 +132,7 @@ func TestExtractWithProgress_PDF_Integration(t *testing.T) { context.Background(), data, "application/pdf", - DefaultExtractors(5, 0), + DefaultExtractors(5, 0, true, 0), ) var phases []string diff --git a/internal/extract/ocr_test.go b/internal/extract/ocr_test.go index 24e47604..ae1f2ca9 100644 --- a/internal/extract/ocr_test.go +++ b/internal/extract/ocr_test.go @@ -78,6 +78,28 @@ func TestAtoi(t *testing.T) { } } +func TestFilterTSVByConfidence(t *testing.T) { + t.Parallel() + tsv := []byte( + "level\tpage_num\tblock_num\tpar_num\tline_num\tword_num\tleft\ttop\twidth\theight\tconf\ttext\n" + + "5\t1\t1\t1\t1\t1\t100\t200\t50\t12\t96\tHigh\n" + + "5\t1\t1\t1\t1\t2\t160\t200\t50\t12\t30\tLow\n" + + "5\t1\t1\t1\t2\t1\t100\t220\t50\t12\t70\tBorder\n", + ) + + filtered := filterTSVByConfidence(tsv, 70) + text := textFromTSV(filtered) + assert.Contains(t, text, "High") + assert.NotContains(t, text, "Low") + assert.Contains(t, text, "Border") +} + +func TestFilterTSVByConfidence_Empty(t *testing.T) { + t.Parallel() + assert.Equal(t, []byte(nil), filterTSVByConfidence(nil, 70)) + assert.Equal(t, []byte(""), filterTSVByConfidence([]byte(""), 70)) +} + func TestIsImageMIME(t *testing.T) { t.Parallel() assert.True(t, IsImageMIME("image/png")) diff --git a/internal/extract/pipeline.go b/internal/extract/pipeline.go index eb727d53..8420be03 100644 --- a/internal/extract/pipeline.go +++ b/internal/extract/pipeline.go @@ -16,7 +16,7 @@ import ( // and gracefully degrades when its dependencies are unavailable. type Pipeline struct { LLMClient *llm.Client // nil = skip LLM extraction - Extractors []Extractor // nil = DefaultExtractors(0, 0) + Extractors []Extractor // nil = DefaultExtractors(0, 0, true, 0) Schema SchemaContext // DDL + entity rows for prompt DocID uint // document ID for UPDATE operations } @@ -72,7 +72,7 @@ func (p *Pipeline) Run( extractors := p.Extractors if extractors == nil { - extractors = DefaultExtractors(0, 0) + extractors = DefaultExtractors(0, 0, true, 0) } // Run all matching, available extractors. diff --git a/internal/extract/pipeline_test.go b/internal/extract/pipeline_test.go index b1dfbfc7..4f17eb17 100644 --- a/internal/extract/pipeline_test.go +++ b/internal/extract/pipeline_test.go @@ -112,7 +112,7 @@ func TestPipeline_OCRIntegration(t *testing.T) { } // Both pdftotext and OCR should run for PDFs. - p := &Pipeline{Extractors: DefaultExtractors(5, 0)} + p := &Pipeline{Extractors: DefaultExtractors(5, 0, true, 0)} r := p.Run(context.Background(), data, "sample.pdf", "application/pdf") require.NoError(t, r.Err) assert.True(t, r.HasSource("tesseract"), "OCR always runs for PDFs") @@ -141,7 +141,7 @@ func TestPipeline_MixedPDF(t *testing.T) { t.Skipf("test fixture not found (pdfunite unavailable?): %s", pdfPath) } - p := &Pipeline{Extractors: DefaultExtractors(5, 0)} + p := &Pipeline{Extractors: DefaultExtractors(5, 0, true, 0)} r := p.Run(context.Background(), data, "mixed-inspection.pdf", "application/pdf") require.NoError(t, r.Err)