Migrate from ENV to YAML config (#59)

Author: No767

.gitignore

@@ -129,6 +129,13 @@ ENV/
 env.bak/
 venv.bak/
 
+# Docker Compose ENV
+docker-compose.env
+
+# Rodhaj configurations
+config.yml
+config.yaml
+
 # Spyder project settings
 .spyderproject
 .spyproject
@@ -176,4 +183,4 @@ inv.yml
 test-scripts/
 
 # Ruff cache
-.ruff_cache/
+.ruff_cache/

bot/cogs/utilities.py

@@ -9,6 +9,7 @@
 from discord.ext import commands
 from discord.utils import format_dt
 from libs.utils import Embed, RoboContext, human_timedelta, is_docker
+from pygit2.enums import SortMode
 from rodhaj import Rodhaj
 
 
@@ -46,9 +47,7 @@ def format_commit(self, commit: pygit2.Commit) -> str:
     def get_last_commits(self, count: int = 5):
         repo = pygit2.Repository(".git")
         commits = list(
-            itertools.islice(
-                repo.walk(repo.head.target, pygit2.GIT_SORT_TOPOLOGICAL), count
-            )
+            itertools.islice(repo.walk(repo.head.target, SortMode.TOPOLOGICAL), count)
         )
         return "\n".join(self.format_commit(c) for c in commits)
 

bot/launcher.py

@@ -1,25 +1,24 @@
 import os
 import signal
+from pathlib import Path
 
 import asyncpg
 import discord
 from aiohttp import ClientSession
-from environs import Env
 from libs.utils import KeyboardInterruptHandler, RodhajLogger
+from libs.utils.config import RodhajConfig
 from rodhaj import Rodhaj
 
 if os.name == "nt":
     from winloop import run
 else:
     from uvloop import run
 
-# Hope not to trip pyright
-env = Env()
-env.read_env()
+config_path = Path(__file__).parent / "config.yml"
+config = RodhajConfig(config_path)
 
-TOKEN = env("TOKEN")
-DEV_MODE = env.bool("DEV_MODE", False)
-POSTGRES_URI = env("POSTGRES_URI")
+TOKEN = config["rodhaj"]["token"]
+POSTGRES_URI = config["postgres_uri"]
 
 intents = discord.Intents.default()
 intents.message_content = True
@@ -31,7 +30,7 @@ async def main() -> None:
         dsn=POSTGRES_URI, min_size=25, max_size=25, command_timeout=30
     ) as pool:
         async with Rodhaj(
-            intents=intents, session=session, pool=pool, dev_mode=DEV_MODE
+            config=config, intents=intents, session=session, pool=pool
         ) as bot:
             bot.loop.add_signal_handler(signal.SIGTERM, KeyboardInterruptHandler(bot))
             bot.loop.add_signal_handler(signal.SIGINT, KeyboardInterruptHandler(bot))

bot/libs/utils/__init__.py

@@ -4,6 +4,7 @@
     is_manager as is_manager,
     is_mod as is_mod,
 )
+from .config import RodhajConfig as RodhajConfig
 from .context import GuildContext as GuildContext, RoboContext as RoboContext
 from .embeds import (
     Embed as Embed,

bot/libs/utils/config.py

@@ -0,0 +1,48 @@
+from pathlib import Path
+from typing import Any, Generic, Optional, TypeVar, Union, overload
+
+import yaml
+
+_T = TypeVar("_T")
+
+
+class RodhajConfig(Generic[_T]):
+    def __init__(self, path: Path):
+        self.path = path
+        self._config: dict[str, Union[_T, Any]] = {}
+        self.load_from_file()
+
+    def load_from_file(self) -> None:
+        try:
+            with open(self.path, "r") as f:
+                self._config: dict[str, Union[_T, Any]] = yaml.safe_load(f.read())
+        except FileNotFoundError:
+            self._config = {}
+
+    @property
+    def rodhaj(self) -> _T:
+        return self._config.get("rodhaj", {})
+
+    @overload
+    def get(self, key: Any) -> Optional[Union[_T, Any]]:
+        ...
+
+    @overload
+    def get(self, key: Any, default: Any) -> Union[_T, Any]:
+        ...
+
+    def get(self, key: Any, default: Any = None) -> Optional[Union[_T, Any]]:
+        """Retrieves a config entry."""
+        return self._config.get(str(key), default)
+
+    def __contains__(self, item: Any) -> bool:
+        return str(item) in self._config
+
+    def __getitem__(self, item: Any) -> Union[_T, Any]:
+        return self._config[str(item)]
+
+    def __len__(self) -> int:
+        return len(self._config)
+
+    def all(self) -> dict[str, Union[_T, Any]]:
+        return self._config

bot/migrations.py

@@ -1,6 +1,5 @@
 import asyncio
 import datetime
-import os
 import re
 import traceback
 from functools import wraps
@@ -10,15 +9,16 @@
 
 import asyncpg
 import click
-from dotenv import load_dotenv
+from libs.utils.config import RodhajConfig
 from typing_extensions import Self
 
-load_dotenv()
+path = Path(__file__).parent / "config.yml"
+config = RodhajConfig(path)
 
 BE = TypeVar("BE", bound=BaseException)
 
 REVISION_FILE = re.compile(r"(?P<kind>V)(?P<version>[0-9]+)__(?P<description>.+).sql")
-POSTGRES_URI = os.environ["POSTGRES_URI"]
+POSTGRES_URI = config["postgres_uri"]
 
 CREATE_MIGRATIONS_TABLE = """
 CREATE TABLE IF NOT EXISTS migrations (

bot/rodhaj.py

@@ -19,23 +19,22 @@
     RodhajHelp,
     send_error_embed,
 )
+from libs.utils.config import RodhajConfig
 from libs.utils.reloader import Reloader
 
 if TYPE_CHECKING:
     from cogs.tickets import Tickets
 
-TRANSPROGRAMMER_GUILD_ID = 1183302385020436480
-
 
 class Rodhaj(commands.Bot):
     """Main bot for Rodhaj"""
 
     def __init__(
         self,
+        config: RodhajConfig,
         intents: discord.Intents,
         session: ClientSession,
         pool: asyncpg.Pool,
-        dev_mode: bool = False,
         *args,
         **kwargs,
     ):
@@ -57,9 +56,11 @@ def __init__(
         self.session = session
         self.partial_config: Optional[PartialConfig] = None
         self.pool = pool
-        self.transprogrammer_guild_id = TRANSPROGRAMMER_GUILD_ID
         self.version = str(VERSION)
-        self._dev_mode = dev_mode
+        self.transprogrammer_guild_id = config.rodhaj.get(
+            "guild_id", 1183302385020436480
+        )
+        self._dev_mode = config.rodhaj.get("dev_mode", False)
         self._reloader = Reloader(self, Path(__file__).parent)
 
     ### Ticket related utils

config-example.yml

@@ -0,0 +1,27 @@
+# --------------------------------- #
+#    Rodhaj's Configuration file    #
+# --------------------------------- #
+# This holds the configuration for Rodhaj. This file is not settable during runtime.
+# If you wish to change the values, change and save, and restart your bot
+
+# Entries pertaining to Rodhaj are located here
+rodhaj:
+
+  # The token that Rodhaj will use.
+  # Ensure that this is kept secret, as these are just like the password to your bot.
+  token: ""
+
+  # The Guild ID that is associated with Rodhaj. This will be the guild that Rodhaj
+  # will operate in, and will be the guild that Rodhaj will use to associate tickets and co.
+  guild_id: 12345
+
+  # The dev mode for Rodhaj. This will enable certain features
+  # such as an extension and library hot reloader in order to
+  # make development easier
+  # Note: Set this to false or remove this entry when running Rodhaj in production
+  dev_mode: False
+
+# The PostgreSQL connection URI that is used to connect to the database
+# The URI must be valid, and components will need to be quoted.
+# See https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNSTRING
+postgres_uri: "postgresql://user:password@localhost:5432/user"

docker-compose-dev.yml

@@ -3,9 +3,12 @@ name: "rodhaj-dev"
 services:
   postgres:
     container_name: Rodhaj-Postgres
-    image: postgres:15 
+    image: rodhaj-pg:dev-latest
+    build:
+      context: ./docker/pg
+      dockerfile: Dockerfile
     env_file:
-      - bot/.env
+      - docker-compose.env
     volumes:
       - postgres_volume:/var/lib/postgresql/data
     ports:

docker/pg/Dockerfile

@@ -0,0 +1,2 @@
+FROM postgres:15
+COPY /init.sh /docker-entrypoint-initdb.d/

docker/pg/init.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+set -e
+
+psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL
+  CREATE ROLE rodhaj WITH LOGIN PASSWORD "$RODHAJ_PASSWORD";
+  CREATE DATABASE rodhaj OWNER rodhaj;
+EOSQL

docs/dev-guide/index.rst

@@ -10,3 +10,4 @@ Rodhaj offers a developer guide so future developers can easily get started with
    intro
    contributing
    faq
+   migrating-to-yaml

docs/dev-guide/intro.rst

@@ -53,11 +53,11 @@ Setup
     poetry install \
     && poetry run pre-commit install
 
-3. Copy over the ``.env`` template over to the ``bot`` directory. Modify the values as appropriate.
+3. Copy over the ``config-example.yml`` template over to the ``bot`` directory. Modify the values as appropriate.
 
 .. code-block:: bash
 
-    cp envs/dev.env bot/.env
+    cp config-example.yml bot/config.yml
 
 4. Run the SQL migrations
 
@@ -91,7 +91,15 @@ Using Docker
 ^^^^^^^^^^^^
 
 If you decide to use Docker to run the local PostgreSQL server, then a
-pre-built Docker Compose file is provided. Simply run the following:
+pre-built Docker Compose file is provided. Setup instructions are as follows:
+
+1. Copy ``envs/docker.env`` to ``docker-compose.env`` within the root of the repo. Modify as appropriate.
+
+.. code-block:: bash
+
+    cp envs/docker.env docker-compose.env
+
+2. Run the following command to start the PostgreSQL server
 
 .. code-block:: bash
 

docs/dev-guide/migrating-to-yaml.rst

@@ -0,0 +1,59 @@
+=================
+Migrating to YAML
+=================
+
+Rodhaj now uses an new config format, YAML.
+Previously, the project utilized an ENV file for configuration, but
+due to issues such as the inability for better structured data and others,
+the project has migrated to YAML.
+
+The details of the motivation can be found in
+`PR #59 <https://github.com/transprogrammer/rodhaj/pull/59>`_.
+
+.. note::
+
+    - ``*.yaml`` files are ignored and not read by the config loader. Ensure that your YAML file uses the ``*.yml`` file extension.
+    - The config file is stored in the same location as the ``.env`` file. It will always be read under ``bot/config.yml``.
+    
+Changes
+=======
+
+Grouped entries
+---------------
+
+The bot token, and others are merged into one mapping. This mapping key is denoted
+by ``rodhaj``. Any and all configuration options in relation to the bot can be found
+under that mapping.
+
+To illustrate, an example comparison of the old and new config format is shown below:
+
+``.env``
+
+.. code-block:: bash
+
+    TOKEN=...
+    DEV_MODE=False
+
+``config.yml``
+
+.. code-block:: yaml
+
+    rodhaj:
+        token: ...
+        dev_mode: False # This key-value pair can also be removed to disable the dev mode feature
+
+Rodhaj's Guild ID can now be set via the config
+-----------------------------------------------
+
+Instead of hardcoding the Guild ID into the code, to allow for flexibity in development,
+the Guild ID can now be set via the config. This is acheived through the ``guild_id`` entry as illustrated.
+
+.. code-block:: yaml
+
+    rodhaj:
+        guild_id: 1234567890 # This ID is the guild that is associated with Rodhaj for operations
+
+PostgreSQL Connection URI
+-------------------------
+
+The PostgreSQL URI is mostly the same, but is now under the ``postgres_uri`` entry.

envs/dev.env

@@ -0,0 +1,12 @@
+# Docker Compose ENV file
+# This is used for setting up the PostgreSQL server
+# found within the docker compose file
+
+# Docker PostgreSQL root credientials
+# This account has root access, so please ensure that this is a secure password
+POSTGRES_USER=postgres
+POSTGRES_PASSWORD=password
+
+# Account for the rodhaj user on PostgreSQL
+# This is used to create the internal user, rodhaj
+RODHAJ_PASSWORD=password