Skip to content

Commit 649868d

Browse files
eddy-geekeddy-geek
authored
Merge pull request #1796 from c2corg/elasticsearch-search-indice-update
Elasticsearch search indice update
2 parents ba24abe + 68b8522 commit 649868d

17 files changed

Lines changed: 6062 additions & 36 deletions

File tree

‎.github/workflows/ci.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,7 @@ jobs:
7979
run: |
8080
mkdir /tmp/elasticsearch
8181
wget -O - https://download.elasticsearch.org/elasticsearch/release/org/elasticsearch/distribution/tar/elasticsearch/2.4.6/elasticsearch-2.4.6.tar.gz | tar xz --directory=/tmp/elasticsearch --strip-components=1
82+
/tmp/elasticsearch/bin/plugin install analysis-icu
8283
/tmp/elasticsearch/bin/elasticsearch --daemonize --path.data /tmp
8384
sleep 30 # ElasticSearch takes few seconds to start, make sure it is available when the build script runs
8485
- run: curl -v http://localhost:9200

‎c2corg_api/search/__init__.py‎

Lines changed: 36 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -84,26 +84,46 @@ def create_search(document_type):
8484

8585
def get_text_query_on_title(search_term, search_lang=None):
8686
fields = []
87-
# search in all title* (title_en, title_fr, ...) fields.
87+
88+
if search_term.count(' ') == 0:
89+
mots = False
90+
else:
91+
mots = True
92+
8893
if not search_lang:
89-
fields.append('title_*.ngram')
90-
fields.append('title_*.raw^2')
94+
if not mots:
95+
fields.append('title_*.ngram')
96+
else:
97+
fields.append('title_*.contentheavy')
98+
9199
else:
92-
# if a language is given, boost the fields for the language
93100
for lang in default_langs:
94101
if lang == search_lang:
95-
fields.append('title_{0}.ngram^2'.format(lang))
96-
fields.append('title_{0}.raw^3'.format(lang))
97-
else:
98-
fields.append('title_{0}.ngram'.format(lang))
99-
fields.append('title_{0}.raw^2'.format(lang))
100-
101-
return MultiMatch(
102-
query=search_term,
103-
fuzziness='auto',
104-
operator='and',
105-
fields=fields
106-
)
102+
if not mots:
103+
fields.append('title_{0}.ngram'.format(lang))
104+
else:
105+
fields.append('title_{0}.contentheavy'.format(lang))
106+
107+
if not mots:
108+
return MultiMatch(
109+
query=search_term,
110+
fields=fields,
111+
type='best_fields',
112+
fuzziness=1,
113+
max_expansions=2,
114+
zero_terms_query="none",
115+
slop=4,
116+
)
117+
else:
118+
return MultiMatch(
119+
query=search_term,
120+
fields=fields,
121+
type='phrase',
122+
fuzziness=2,
123+
max_expansions=3,
124+
zero_terms_query="none",
125+
slop=4,
126+
)
107127

108128

109129
search_documents = {

‎c2corg_api/search/mapping.py‎

Lines changed: 161 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -19,15 +19,29 @@ class BaseMeta:
1919
# the configuration is based on the one by Photon:
2020
# https://github.com/komoot/photon/blob/master/es/mappings.json
2121
# https://github.com/komoot/photon/blob/master/es/index_settings.json
22-
def default_title_field():
23-
return String(
24-
index='not_analyzed',
25-
similarity='c2corgsimilarity',
26-
fields={
27-
'ngram': String(
28-
analyzer='index_ngram', search_analyzer='search_ngram'),
29-
'raw': String(
30-
analyzer='index_raw', search_analyzer='search_raw')})
22+
def default_title_field(lang: None):
23+
if lang is None:
24+
return String(
25+
index='not_analyzed',
26+
similarity='c2corgsimilarity',
27+
fields={
28+
'ngram': String(
29+
analyzer='index_ngram', search_analyzer='search_ngram'),
30+
'raw': String(
31+
analyzer='index_raw', search_analyzer='search_raw')
32+
})
33+
else:
34+
return String(
35+
index='not_analyzed',
36+
similarity='c2corgsimilarity',
37+
fields={
38+
'ngram': String(
39+
analyzer='index_ngram', search_analyzer='search_ngram'),
40+
'raw': String(
41+
analyzer='index_raw', search_analyzer='search_raw'),
42+
'contentheavy': String(
43+
analyzer='{0}_heavy'.format(lang))
44+
})
3145

3246

3347
class SearchDocument(DocType):
@@ -59,54 +73,68 @@ class Meta(BaseMeta):
5973
areas = QLong('a', is_id=True)
6074

6175
# fr
62-
title_fr = default_title_field()
76+
title_fr = default_title_field("french")
6377
summary_fr = String(
6478
analyzer='index_french', search_analyzer='search_french')
6579
description_fr = String(
6680
analyzer='index_french', search_analyzer='search_french')
6781

6882
# it
69-
title_it = default_title_field()
83+
title_it = default_title_field("italian")
7084
summary_it = String(
7185
analyzer='index_italian', search_analyzer='search_italian')
7286
description_it = String(
7387
analyzer='index_italian', search_analyzer='search_italian')
7488

7589
# de
76-
title_de = default_title_field()
90+
title_de = default_title_field("german")
7791
summary_de = String(
7892
analyzer='index_german', search_analyzer='search_german')
7993
description_de = String(
8094
analyzer='index_german', search_analyzer='search_german')
8195

8296
# en
83-
title_en = default_title_field()
97+
title_en = default_title_field("english")
8498
summary_en = String(
8599
analyzer='index_english', search_analyzer='search_english')
86100
description_en = String(
87101
analyzer='index_english', search_analyzer='search_english')
88102

89103
# es
90-
title_es = default_title_field()
104+
title_es = default_title_field("spanish")
91105
summary_es = String(
92106
analyzer='index_spanish', search_analyzer='search_spanish')
93107
description_es = String(
94108
analyzer='index_spanish', search_analyzer='search_spanish')
95109

96110
# ca
97-
title_ca = default_title_field()
111+
title_ca = default_title_field("catalan")
98112
summary_ca = String(
99113
analyzer='index_catalan', search_analyzer='search_catalan')
100114
description_ca = String(
101115
analyzer='index_catalan', search_analyzer='search_catalan')
102116

103117
# eu
104-
title_eu = default_title_field()
118+
title_eu = default_title_field("basque")
105119
summary_eu = String(
106120
analyzer='index_basque', search_analyzer='search_basque')
107121
description_eu = String(
108122
analyzer='index_basque', search_analyzer='search_basque')
109123

124+
# sl
125+
title_sl = default_title_field("slovene")
126+
summary_sl = String(
127+
analyzer='index_slovene', search_analyzer='search_slovene')
128+
description_sl = String(
129+
analyzer='index_slovene', search_analyzer='search_slovene')
130+
131+
# zh
132+
title_zh = default_title_field("chinois")
133+
summary_zh = String(
134+
analyzer='index_chinois', search_analyzer='search_chinois')
135+
description_zh = String(
136+
analyzer='index_chinois', search_analyzer='search_chinois')
137+
110138
@staticmethod
111139
def to_search_document(document, index, include_areas=True):
112140
search_document = {
@@ -190,8 +218,12 @@ def copy_enum_range_fields(
190218
"filter": {
191219
"autocomplete_filter": {
192220
"type": "edge_ngram",
193-
"min_gram": 2,
194-
"max_gram": 20
221+
"min_gram": "2",
222+
"max_gram": "15",
223+
"token_chars": [
224+
"letter",
225+
"digit"
226+
]
195227
},
196228
# filters for the english analyzers
197229
"english_stop": {
@@ -300,9 +332,9 @@ def copy_enum_range_fields(
300332
"index_ngram": {
301333
"char_filter": ["punctuationgreedy"],
302334
"filter": [
303-
"word_delimiter", "lowercase", "asciifolding", "unique",
335+
"word_delimiter", "lowercase", "icu_folding",
304336
"autocomplete_filter"],
305-
"tokenizer": "standard"
337+
"tokenizer": "icu_tokenizer"
306338
},
307339
"search_ngram": {
308340
"char_filter": ["punctuationgreedy"],
@@ -453,6 +485,115 @@ def copy_enum_range_fields(
453485
"basque_stop",
454486
"basque_stemmer"
455487
]
488+
},
489+
"french_heavy": {
490+
"tokenizer": "icu_tokenizer",
491+
"filter": [
492+
"french_elision",
493+
"french_stop",
494+
"icu_folding",
495+
"lowercase",
496+
"french_stemmer"
497+
]
498+
},
499+
"german_heavy": {
500+
"tokenizer": "icu_tokenizer",
501+
"filter": [
502+
"german_stop",
503+
"german_stemmer",
504+
"lowercase",
505+
"icu_folding"
506+
]
507+
},
508+
"english_heavy": {
509+
"tokenizer": "icu_tokenizer",
510+
"filter": [
511+
"english_possessive_stemmer",
512+
"english_stop",
513+
"lowercase",
514+
"icu_folding"
515+
]
516+
},
517+
"italian_heavy": {
518+
"tokenizer": "icu_tokenizer",
519+
"filter": [
520+
"italian_elision",
521+
"italian_stop",
522+
"lowercase",
523+
"icu_folding",
524+
"italian_stemmer"
525+
]
526+
},
527+
"spanish_heavy": {
528+
"tokenizer": "icu_tokenizer",
529+
"filter": [
530+
"lowercase",
531+
"spanish_stop",
532+
"spanish_stemmer",
533+
"icu_folding"
534+
]
535+
},
536+
"catalan_heavy": {
537+
"tokenizer": "icu_tokenizer",
538+
"filter": [
539+
"catalan_elision",
540+
"lowercase",
541+
"catalan_stop",
542+
"catalan_stemmer",
543+
"icu_folding"
544+
]
545+
},
546+
"basque_heavy": {
547+
"tokenizer": "icu_tokenizer",
548+
"filter": [
549+
"lowercase",
550+
"basque_stop",
551+
"basque_stemmer",
552+
"icu_folding"
553+
]
554+
},
555+
"index_slovene": {
556+
"type": "custom",
557+
"tokenizer": "standard",
558+
"filter": [
559+
"autocomplete_filter",
560+
"lowercase"
561+
]
562+
},
563+
"search_slovene": {
564+
"type": "custom",
565+
"tokenizer": "standard",
566+
"filter": [
567+
"lowercase"
568+
]
569+
},
570+
"slovene_heavy": {
571+
"tokenizer": "icu_tokenizer",
572+
"filter": [
573+
"lowercase",
574+
"icu_folding"
575+
]
576+
},
577+
"index_chinois": {
578+
"type": "custom",
579+
"tokenizer": "standard",
580+
"filter": [
581+
"autocomplete_filter"
582+
]
583+
},
584+
"search_chinois": {
585+
"type": "custom",
586+
"tokenizer": "standard",
587+
"filter": [
588+
"lowercase"
589+
]
590+
},
591+
"chinois_heavy": {
592+
"tokenizer": "icu_tokenizer",
593+
"filter": [
594+
"lowercase",
595+
"icu_folding"
596+
]
456597
}
457598
}
458599
}

‎scripts/README.md‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
# ElasticSearch 2.4 update for search improvement
2+
3+
Script has to be execute one time in order to be relevant with the code update.
4+
5+
## Script has to be execute from the host
6+
7+
### script requirement :
8+
- it requires curl from the execution machine
9+
10+
- it will update elasticsearch on localhost:9200, if elasticsearch is on other server, please update script changing localhost by the correct ES url.
11+
- an elasticsearch plugin has to be install, in case of elasticsearch cluster - the plugin has to be install on each node
12+
13+
14+
To install manually the the plugin analysis-icu from elasticsearch server :
15+
16+
```
17+
./bin/plugin install analysis-icu analysis-icu
18+
```
19+
20+
21+
The installation process is :
22+
- Install the plugin,
23+
- restart ElasticSearch instance,
24+
- update analyser (_settings)
25+
- update document mapping (_mappings /espon)
26+
- start a full indexation.
27+
28+
To launch the script from the host side :
29+
```
30+
cd scripts/
31+
./esUpdateMapSettings.sh
32+
```
33+
34+

0 commit comments

Comments
 (0)