Skip to content

Commit e4a1cbe

Browse files
committed
Expand configuration management system
This commit copies some of the improvements from the v2 settings system to the master branch. The most important change is allowing settings to be configured from a JSON file.
1 parent 6b552be commit e4a1cbe

File tree

1 file changed

+157
-32
lines changed

1 file changed

+157
-32
lines changed

argostranslate/settings.py

+157-32
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,42 @@
1+
import json
12
import os
23
from enum import Enum
34
from pathlib import Path
5+
from typing import Any, Dict
46

5-
TRUE_VALUES = ["1", "TRUE", "True", "true"]
7+
"""
8+
Argos Translate can be configured using either environment variables or json file
69
7-
debug = os.getenv("ARGOS_DEBUG") in TRUE_VALUES
10+
### Environment variables
11+
```
12+
export ARGOS_DEBUG="0"
13+
export ARGOS_PACKAGE_INDEX="https://raw.githubusercontent.com/argosopentech/argospm-index/main/"
14+
export ARGOS_PACKAGES_DIR="/home/<username>/.local/share/argos-translate/packages/"
15+
export ARGOS_DEVICE_TYPE="cpu"
816
9-
dev_mode = os.getenv("ARGOS_DEV_MODE") in TRUE_VALUES
17+
```
18+
19+
### JSON
20+
21+
# $HOME/.config/argos-translate/settings.json
22+
```
23+
{
24+
"ARGOS_DEBUG": "0",
25+
"ARGOS_PACKAGES_INDEX": "https://raw.githubusercontent.com/argosopentech/argospm-index/main/",
26+
"ARGOS_PACKAGE_DIR": "/home/<username>/.local/share/argos-translate/packages/",
27+
"ARGOS_DEVICE_TYPE": "cpu"
28+
}
29+
```
30+
"""
31+
32+
"""
33+
Importing argostranslate.settings will create the Argos Translate data directory (~/.local/share/argos-translate),
34+
the Argos Translate config directory (~/.config/argos-translate),
35+
and the Argos Translate cache directory (~/.local/cache/argos-translate) if they do not already exist.
36+
"""
1037

1138
home_dir = Path.home()
39+
1240
if "SNAP" in os.environ:
1341
home_dir = Path(os.environ["SNAP_USER_DATA"])
1442

@@ -22,15 +50,97 @@
2250
legacy_package_data_dir = Path(
2351
os.getenv("ARGOS_TRANSLATE_PACKAGES_DIR", default=data_dir / "packages")
2452
)
25-
package_data_dir = Path(os.getenv("ARGOS_PACKAGES_DIR", legacy_package_data_dir))
26-
os.makedirs(package_data_dir, exist_ok=True)
53+
54+
config_dir = (
55+
Path(os.getenv("XDG_CONFIG_HOME", default=home_dir / ".config")) / "argos-translate"
56+
)
57+
os.makedirs(config_dir, exist_ok=True)
2758

2859
cache_dir = (
2960
Path(os.getenv("XDG_CACHE_HOME", default=home_dir / ".local" / "cache"))
3061
/ "argos-translate"
3162
)
3263
os.makedirs(cache_dir, exist_ok=True)
3364

65+
66+
downloads_dir = cache_dir / "downloads"
67+
os.makedirs(downloads_dir, exist_ok=True)
68+
69+
settings_file = config_dir / "settings.json"
70+
71+
72+
def load_settings_dict() -> Dict[str, Any]:
73+
settings_dict = dict()
74+
if settings_file.exists():
75+
try:
76+
with open(settings_file, "r") as settings_file_data:
77+
settings_dict = json.load(settings_file_data)
78+
assert isinstance(
79+
settings_dict, dict
80+
), "settings.json should contain a dictionary"
81+
except FileNotFoundError as e:
82+
print(f"{settings_file} not found : FileNotFoundError {e}")
83+
except json.JSONDecodeError as e:
84+
print(f"Error decoding {settings_file}: JSONDecodeError {e}")
85+
return settings_dict
86+
87+
88+
def get_setting(key: str, default=None):
89+
"""Gets a setting from either environment variables or settings.json
90+
91+
Settings from environment variables take precedence over settings.json
92+
93+
Args:
94+
key (str): Key value
95+
default: The default setting value. Defaults to None.
96+
97+
Returns:
98+
The setting value
99+
"""
100+
value_from_environment = os.getenv(key)
101+
value_from_file = load_settings_dict().get(key)
102+
if value_from_environment is not None:
103+
return value_from_environment
104+
else:
105+
if value_from_file is not None:
106+
return value_from_file
107+
return default
108+
109+
110+
def set_setting(key: str, value):
111+
"""Sets a setting in the settings.json file.
112+
113+
Args:
114+
key (str): The key to set.
115+
value: The value to set.
116+
"""
117+
settings = load_settings_dict()
118+
settings[key] = value
119+
with open(settings_file, "w") as settings_file_data:
120+
json.dump(settings, settings_file_data, indent=4)
121+
122+
123+
TRUE_VALUES = ["1", "TRUE", "True", "true", 1, True]
124+
125+
126+
debug = get_setting("ARGOS_DEBUG") in TRUE_VALUES
127+
128+
dev_mode = get_setting("ARGOS_DEV_MODE") in TRUE_VALUES
129+
130+
package_index = get_setting(
131+
"ARGOS_PACKAGE_INDEX",
132+
default="https://raw.githubusercontent.com/argosopentech/argospm-index/main/",
133+
)
134+
135+
package_data_dir = Path(
136+
get_setting("ARGOS_PACKAGES_DIR", default=data_dir / "packages")
137+
)
138+
os.makedirs(package_data_dir, exist_ok=True)
139+
140+
141+
downloads_dir = cache_dir / "downloads"
142+
os.makedirs(downloads_dir, exist_ok=True)
143+
34144
if not dev_mode:
35145
remote_repo = os.getenv(
36146
"ARGOS_PACKAGE_INDEX",
@@ -41,12 +151,22 @@
41151
"ARGOS_PACKAGE_INDEX",
42152
default="https://raw.githubusercontent.com/argosopentech/argospm-index-dev/main",
43153
)
44-
remote_package_index = remote_repo + "/index.json"
154+
155+
remote_package_index = package_index + "index.json"
156+
157+
local_package_index = data_dir / "index.json"
45158

46159
experimental_enabled = os.getenv("ARGOS_EXPERIMENTAL_ENABLED") in TRUE_VALUES
47160

48161
stanza_available = os.getenv("ARGOS_STANZA_AVAILABLE") in (TRUE_VALUES + [None])
49162

163+
# Supported values: "cpu" and "cuda"
164+
device = get_setting("ARGOS_DEVICE_TYPE", "cpu")
165+
166+
# https://opennmt.net/CTranslate2/python/ctranslate2.Translator.html
167+
inter_threads = int(get_setting("ARGOS_INTER_THREADS", "1"))
168+
intra_threads = int(get_setting("ARGOS_INTRA_THREADS", "0"))
169+
50170

51171
class ModelProvider(Enum):
52172
OPENNMT = 0
@@ -59,15 +179,39 @@ class ModelProvider(Enum):
59179
"LIBRETRANSLATE": ModelProvider.LIBRETRANSLATE,
60180
"OPENAI": ModelProvider.OPENAI,
61181
}
62-
model_provider = os.getenv("ARGOS_MODEL_PROVIDER", "OPENNMT")
63-
assert model_provider in model_mapping.keys()
64-
model_provider = model_mapping[model_provider]
182+
model_provider = model_mapping[get_setting("ARGOS_MODEL_PROVIDER", default="OPENNMT")]
65183

66-
libretranslate_api_key = os.getenv("LIBRETRANSLATE_API_KEY", None)
67-
openai_api_key = os.getenv("OPENAI_API_KEY", None)
68184

69-
downloads_dir = cache_dir / "downloads"
70-
os.makedirs(downloads_dir, exist_ok=True)
185+
# Sentence boundary detection
186+
class ChunkType(Enum):
187+
DEFAULT = 0
188+
ARGOSTRANSLATE = 1
189+
NONE = 2 # No sentence splitting
190+
191+
192+
chunk_type_mapping = {
193+
"DEFAULT": ChunkType.DEFAULT,
194+
"ARGOSTRANSLATE": ChunkType.ARGOSTRANSLATE,
195+
"NONE": ChunkType.NONE,
196+
}
197+
chunk_type = chunk_type_mapping[get_setting("ARGOS_CHUNK_TYPE", default="DEFAULT")]
198+
if chunk_type == ChunkType.DEFAULT:
199+
chunk_type = ChunkType.ARGOSTRANSLATE
200+
201+
202+
libretranslate_api_key = get_setting("LIBRETRANSLATE_API_KEY", None)
203+
openai_api_key = get_setting("OPENAI_API_KEY", None)
204+
205+
206+
argos_translate_about_text = (
207+
"Argos Translate is an open source neural machine "
208+
+ "translation application created by Argos Open "
209+
+ "Technologies, LLC (www.argosopentech.com). "
210+
)
211+
212+
# Fix Intel bug
213+
# https://github.com/argosopentech/argos-translate/issues/40
214+
os.environ["KMP_DUPLICATE_LIB_OK"] = "True"
71215

72216
# Will search all of these directories for packages
73217
package_dirs = [package_data_dir]
@@ -85,22 +229,3 @@ class ModelProvider(Enum):
85229
for package_dir in content_snap_packages.iterdir():
86230
if package_dir.is_dir():
87231
package_dirs.append(package_dir)
88-
89-
local_package_index = cache_dir / "index.json"
90-
91-
about_text = (
92-
"Argos Translate is an open source neural machine "
93-
"translation application created by Argos Open "
94-
"Technologies, LLC (www.argosopentech.com). "
95-
)
96-
97-
# Fix Intel bug
98-
# https://github.com/argosopentech/argos-translate/issues/40
99-
os.environ["KMP_DUPLICATE_LIB_OK"] = "True"
100-
101-
# Supported values: cpu and cuda
102-
device = os.environ.get("ARGOS_DEVICE_TYPE", "cpu")
103-
104-
# https://opennmt.net/CTranslate2/python/ctranslate2.Translator.html
105-
inter_threads = int(os.environ.get("ARGOS_INTER_THREADS", "1"))
106-
intra_threads = int(os.environ.get("ARGOS_INTRA_THREADS", "0"))

0 commit comments

Comments
 (0)