feat(languages): real Czech & Slovenian stemmers + language-support consistency (#1033)

Author: thatjuan

packages/orama/src/components/tokenizer/languages.ts

@@ -1,35 +1,42 @@
-export const STEMMERS: Record<string, string> = {
+// Maps each supported language to its BCP-47 locale tag.
+// The keys are the canonical language names accepted by Orama (they define
+// `SUPPORTED_LANGUAGES` and the `Language` type); the values are passed to
+// `String.prototype.localeCompare` for locale-aware string sorting (see `getLocale`).
+// Note: these locale tags are intentionally decoupled from the file-id codes used by
+// `@orama/stemmers` / `@orama/stopwords` (e.g. Danish stems live in `dk.js` but the
+// locale is `da`), so keep them as valid BCP-47 primary subtags.
+export const SUPPORTED_LANGUAGE_LOCALES: Record<string, string> = {
   arabic: 'ar',
-  armenian: 'am',
+  armenian: 'hy',
   bulgarian: 'bg',
-  czech: 'cz',
-  danish: 'dk',
+  czech: 'cs',
+  danish: 'da',
   dutch: 'nl',
   english: 'en',
   finnish: 'fi',
   french: 'fr',
   german: 'de',
-  greek: 'gr',
+  greek: 'el',
   hungarian: 'hu',
-  indian: 'in',
+  indian: 'hi',
   indonesian: 'id',
-  irish: 'ie',
+  irish: 'ga',
   italian: 'it',
   lithuanian: 'lt',
-  nepali: 'np',
+  nepali: 'ne',
   norwegian: 'no',
   portuguese: 'pt',
   romanian: 'ro',
   russian: 'ru',
-  serbian: 'rs',
-  slovenian: 'ru',
+  serbian: 'sr',
+  slovenian: 'sl',
   spanish: 'es',
-  swedish: 'se',
+  swedish: 'sv',
   tamil: 'ta',
   turkish: 'tr',
   ukrainian: 'uk',
   vietnamese: 'vi',
-  sanskrit: 'sk'
+  sanskrit: 'sa'
 }
 
 export const SPLITTERS: Record<Language, RegExp> = {
@@ -66,10 +73,12 @@ export const SPLITTERS: Record<Language, RegExp> = {
   czech: /[^A-Z0-9a-zěščřžýáíéúůóťďĚŠČŘŽÝÁÍÉÓÚŮŤĎ-]+/gim
 }
 
-export const SUPPORTED_LANGUAGES = Object.keys(STEMMERS)
+export const SUPPORTED_LANGUAGES = Object.keys(SUPPORTED_LANGUAGE_LOCALES)
 
 export function getLocale(language: string | undefined) {
-  return language !== undefined && SUPPORTED_LANGUAGES.includes(language) ? STEMMERS[language] : undefined
+  return language !== undefined && SUPPORTED_LANGUAGES.includes(language)
+    ? SUPPORTED_LANGUAGE_LOCALES[language]
+    : undefined
 }
 
 export type Language = (typeof SUPPORTED_LANGUAGES)[number]

packages/orama/tests/languages.test.ts

@@ -0,0 +1,50 @@
+import t from 'tap'
+
+import { getLocale, SUPPORTED_LANGUAGES, SUPPORTED_LANGUAGE_LOCALES } from '../src/components/tokenizer/languages.js'
+
+t.test('language locales', async (t) => {
+  t.test('every supported language maps to a canonical, well-formed BCP-47 locale', async (t) => {
+    for (const language of SUPPORTED_LANGUAGES) {
+      const locale = getLocale(language)
+
+      t.type(locale, 'string', `getLocale('${language}') returns a locale`)
+
+      let canonical: string[] = []
+      t.doesNotThrow(() => {
+        canonical = Intl.getCanonicalLocales(locale)
+      }, `'${locale}' (${language}) is a well-formed BCP-47 tag`)
+      t.strictSame(canonical, [locale], `'${locale}' (${language}) is already in canonical form`)
+    }
+  })
+
+  t.test('specific language to locale mappings', async (t) => {
+    const expectedLocales = {
+      czech: 'cs',
+      slovenian: 'sl',
+      danish: 'da',
+      greek: 'el',
+      swedish: 'sv',
+      serbian: 'sr',
+      armenian: 'hy',
+      sanskrit: 'sa',
+      indian: 'hi',
+      irish: 'ga',
+      nepali: 'ne'
+    }
+
+    for (const [language, locale] of Object.entries(expectedLocales)) {
+      t.equal(getLocale(language), locale, `getLocale('${language}') returns '${locale}'`)
+    }
+  })
+
+  t.test('supported languages include czech and slovenian', async (t) => {
+    t.ok(SUPPORTED_LANGUAGES.includes('czech'))
+    t.ok(SUPPORTED_LANGUAGES.includes('slovenian'))
+    t.strictSame(SUPPORTED_LANGUAGES, Object.keys(SUPPORTED_LANGUAGE_LOCALES))
+  })
+
+  t.test('unknown or missing languages return undefined', async (t) => {
+    t.equal(getLocale(undefined), undefined)
+    t.equal(getLocale('klingon'), undefined)
+  })
+})

packages/orama/tests/tokenizer.test.ts

@@ -1,6 +1,7 @@
 import t from 'tap'
 
 import { stemmer as bulgarianStemmer, language as bulgarianLanguage } from '@orama/stemmers/bulgarian'
+import { stemmer as czechStemmer, language as czechLanguage } from '@orama/stemmers/czech'
 import { stemmer as danishStemmer, language as danishLanguage } from '@orama/stemmers/danish'
 import { stemmer as dutchStemmer, language as dutchLanguage } from '@orama/stemmers/dutch'
 import { stemmer as finnishStemmer, language as finnishLanguage } from '@orama/stemmers/finnish'
@@ -10,12 +11,14 @@ import { stemmer as italianStemmer, language as italianLanguage } from '@orama/s
 import { stemmer as norwegianStemmer, language as norwegianLanguage } from '@orama/stemmers/norwegian'
 import { stemmer as portugueseStemmer, language as portugueseLanguage } from '@orama/stemmers/portuguese'
 import { stemmer as russianStemmer, language as russianLanguage } from '@orama/stemmers/russian'
+import { stemmer as slovenianStemmer, language as slovenianLanguage } from '@orama/stemmers/slovenian'
 import { stemmer as spanishStemmer, language as spanishLanguage } from '@orama/stemmers/spanish'
 import { stemmer as swedishStemmer, language as swedishLanguage } from '@orama/stemmers/swedish'
 import { stemmer as ukrainianStemmer, language as ukrainianLanguage } from '@orama/stemmers/ukrainian'
 import { stemmer as tamilStemmer, language as tamilLanguage } from '@orama/stemmers/tamil'
 import { stemmer as vietnameseStemmer, language as vietnameseLanguage } from '@orama/stemmers/vietnamese'
 
+import { stopwords as czechStopwords } from '@orama/stopwords/czech'
 import { stopwords as danishStopwords } from '@orama/stopwords/danish'
 import { stopwords as dutchStopwords } from '@orama/stopwords/dutch'
 import { stopwords as englishStopwords } from '@orama/stopwords/english'
@@ -26,6 +29,7 @@ import { stopwords as italianStopwords } from '@orama/stopwords/italian'
 import { stopwords as norwegianStopwords } from '@orama/stopwords/norwegian'
 import { stopwords as portugueseStopwords } from '@orama/stopwords/portuguese'
 import { stopwords as russianStopwords } from '@orama/stopwords/russian'
+import { stopwords as slovenianStopwords } from '@orama/stopwords/slovenian'
 import { stopwords as spanishStopwords } from '@orama/stopwords/spanish'
 import { stopwords as swedishStopwords } from '@orama/stopwords/swedish'
 import { stopwords as ukrainianStopwords } from '@orama/stopwords/ukrainian'
@@ -368,6 +372,40 @@ t.test('Tokenizer', async (t) => {
     t.strictSame(O2, ['има', 'първ', 'вероятност', 'да', 'се', 'случ', 'нещ', 'неочакван', 'док', 'изпълняват', 'тест'])
   })
 
+  t.test('should tokenize and stem correctly in czech', async (t) => {
+    const tokenizer = await createTokenizer({
+      language: czechLanguage,
+      stemmer: czechStemmer,
+      stopWords: czechStopwords
+    })
+
+    const I1 = 'Upekla jsem nějaké koláče'
+    const I2 = 'žáci četli knihy ve škole'
+
+    const O1 = tokenizer.tokenize(I1)
+    const O2 = tokenizer.tokenize(I2)
+
+    t.strictSame(O1, ['upekl', 'nejak', 'kolak'])
+    t.strictSame(O2, ['zak', 'cetl', 'knih', 'skol'])
+  })
+
+  t.test('should tokenize and stem correctly in slovenian', async (t) => {
+    const tokenizer = await createTokenizer({
+      language: slovenianLanguage,
+      stemmer: slovenianStemmer,
+      stopWords: slovenianStopwords
+    })
+
+    const I1 = 'Spekla sem nekaj tort'
+    const I2 = 'otroci berejo knjige v mestih'
+
+    const O1 = tokenizer.tokenize(I1)
+    const O2 = tokenizer.tokenize(I2)
+
+    t.strictSame(O1, ['spekl', 'tort'])
+    t.strictSame(O2, ['otroc', 'ber', 'knjig', 'mest'])
+  })
+
   t.test('disable stemming', async (t) => {
     const tokenizer = await createTokenizer({ language: 'english', stemming: false, stopWords: englishStopwords })
 
@@ -393,6 +431,50 @@ t.test('Tokenizer', async (t) => {
   })
 })
 
+t.test('Czech and Slovenian stemming', async (t) => {
+  t.test('czech inflected forms collapse to a single stem', async (t) => {
+    for (const word of ['žák', 'žáci', 'žáky', 'žákům', 'žácích']) {
+      t.equal(czechStemmer(word), 'žák', `${word} stems to žák`)
+    }
+
+    for (const word of ['kniha', 'knihy', 'knihám', 'knihách']) {
+      t.equal(czechStemmer(word), 'knih', `${word} stems to knih`)
+    }
+
+    for (const word of ['malý', 'malá', 'malé', 'malému', 'malých']) {
+      t.equal(czechStemmer(word), 'mal', `${word} stems to mal`)
+    }
+  })
+
+  t.test('czech short words are left unchanged', async (t) => {
+    t.equal(czechStemmer('e'), 'e')
+    t.equal(czechStemmer('zi'), 'zi')
+  })
+
+  t.test('slovenian inflected forms collapse to a single stem', async (t) => {
+    for (const word of ['mesto', 'mesta', 'mestu', 'mestom', 'mest', 'mestih']) {
+      t.equal(slovenianStemmer(word), 'mest', `${word} stems to mest`)
+    }
+
+    for (const word of ['hiša', 'hiše', 'hiši', 'hišo']) {
+      t.equal(slovenianStemmer(word), 'hiš', `${word} stems to hiš`)
+    }
+
+    for (const word of ['velik', 'velika', 'veliko', 'velikega', 'velikih']) {
+      t.equal(slovenianStemmer(word), 'velik', `${word} stems to velik`)
+    }
+
+    for (const word of ['delati', 'delam', 'delaš', 'dela', 'delamo', 'delajo']) {
+      t.equal(slovenianStemmer(word), 'del', `${word} stems to del`)
+    }
+  })
+
+  t.test('slovenian stemming keeps distinct words distinct', async (t) => {
+    t.not(slovenianStemmer('mesto'), slovenianStemmer('meso'), 'mesto and meso must not collapse')
+    t.not(slovenianStemmer('letalo'), slovenianStemmer('leto'), 'letalo and leto must not collapse')
+  })
+})
+
 t.test('Custom stop-words rules', async (t) => {
   t.test('custom array of stop-words', async (t) => {
     const tokenizer = await createTokenizer({

packages/stemmers/README.md

@@ -2,12 +2,13 @@
 
 Orama can analyze the input and perform a `stemming` operation, which allows the engine to perform more optimized queries, as well as save indexing space.
 
-Right now, Orama supports 30 languages and stemmers out of the box:
+<!-- LANGUAGES:START -->
+Right now, Orama supports 31 languages and stemmers out of the box:
 
 - Arabic
 - Armenian
 - Bulgarian
-- Chinese (Mandarin - stemmer not supported)
+- Czech
 - Danish
 - Dutch
 - English
@@ -20,7 +21,7 @@ Right now, Orama supports 30 languages and stemmers out of the box:
 - Indonesian
 - Irish
 - Italian
-- Japanese
+- Lithuanian
 - Nepali
 - Norwegian
 - Portuguese
@@ -34,6 +35,10 @@ Right now, Orama supports 30 languages and stemmers out of the box:
 - Tamil
 - Turkish
 - Ukrainian
+- Vietnamese
+<!-- LANGUAGES:END -->
+
+Chinese (Mandarin) and Japanese are supported through dedicated tokenizers (`@orama/tokenizers`) and stop-word removal (`@orama/stopwords`), not through stemming.
 
 ```js
 import { create } from '@orama/orama'