CU-869aa22g2 Add ElasticSearch bits from working_with_cogstack

.github/workflows/cogstack-es_main.yml

@@ -0,0 +1,107 @@
+name: cogstack-es - Test
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    paths:
+      - 'cogstack-es/**'
+      - '.github/workflows/cogstack-es**'
+
+defaults:
+  run:
+    working-directory: ./cogstack-es
+
+permissions:
+  id-token: write
+
+jobs:
+  types-only-with-ES-and-OS:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    strategy:
+      matrix:
+        python-version: [ '3.10', '3.11', '3.12', '3.13' ]
+      max-parallel: 4
+
+    steps:
+      - uses: actions/checkout@v5
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install ".[dev,OS,ES9]"
+      - name: Check types
+        run: |
+          python -m mypy --follow-imports=normal src/cogstack
+
+  types-lint-tests:
+    runs-on: ubuntu-latest
+    needs: types-only-with-ES-and-OS
+    permissions:
+      contents: read
+    strategy:
+      matrix:
+        python-version: [ '3.10', '3.11', '3.12', '3.13' ]
+        install-target: [ "ES9", "ES8", "OS", ]
+      max-parallel: 4
+
+    steps:
+      - uses: actions/checkout@v5
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install ".[dev,${{ matrix.install-target }}]"
+      - name: Lint
+        run: |
+          ruff check src/cogstack
+      - name: Test
+        run: |
+          pytest tests
+
+  publish-to-test-PyPI:
+    runs-on: ubuntu-latest
+    needs: types-lint-tests
+    steps:
+      - name: Checkout main
+        uses: actions/checkout@v5
+        with:
+          fetch-depth: 0        # fetch all history
+          fetch-tags: true      # fetch tags explicitly
+
+      - name: Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: '3.10'
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install --upgrade build
+
+      - name: Set timestamp-based dev version
+        run: |
+          TS=$(date -u +"%Y%m%d%H%M%S")
+          echo "SETUPTOOLS_SCM_PRETEND_VERSION_FOR_COGSTACK_ES=0.1.1.dev${TS}" >> $GITHUB_ENV
+
+      - name: Install package in development mode
+        run: |
+          pip install -e ".[dev,ES9,OS]"
+
+      - name: Build package
+        run: |
+          python -m build
+
+      - name: Publish distribution to TestPyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          repository_url: https://test.pypi.org/legacy/
+          packages_dir: cogstack-es/dist

.github/workflows/cogstack-es_release.yml

@@ -0,0 +1,51 @@
+name: cogstack-es release-build
+
+on:
+  push:
+    tags:
+      - 'cogstack-es/v*.*.*'  
+
+permissions:
+  id-token: write
+
+defaults:
+  run:
+    working-directory: ./cogstack-es
+
+jobs:
+  test-and-publish-to-PyPI:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout main
+        uses: actions/checkout@v5
+
+      - name: Release Tag
+        # If GITHUB_REF=refs/tags/cogstack-es/v0.1.2, this returns v0.1.2. Note it's including the "v" though it probably shouldnt
+        run: echo "RELEASE_VERSION=${GITHUB_REF##refs/*/}" >> $GITHUB_ENV
+
+      - name: Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: '3.10'
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install --upgrade build
+
+      - name: Install client package in development mode
+        run: |
+          pip install -e ".[dev,ES9,OS]"
+
+      - name: Test
+        run: |
+          pytest tests
+
+      - name: Build client package
+        run: |
+          python -m build
+
+      - name: Publish production distribution to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          packages_dir: cogstack-es/dist

cogstack-es/ReadMe.md

@@ -0,0 +1,50 @@
+
+# Login and search
+This project is responsible for logging in and performing a search for Elasticsearch or Opensearch.
+
+# Installation
+
+This package is distributed through PyPI and can be installed using one of:
+```
+pip install cogstack-es[ES9]  # For Elasticsearch 9
+pip install cogstack-es[ES8]  # For Elasticsearch 8
+pip install cogstack-es[OS]  # For Opensearch
+```
+
+PS:
+After installation, the import still remains `import cogstack` even though the installed package is called `cogstack-es`.
+
+## Login details
+You need to get your login details and host from your administrator.
+This is usually an API key.
+There is also a mechanism for reading hosts and credentials from environmental variables:
+```python
+from cogstack import read_from_env, CogStack
+hosts, api_key, (username, password) = read_from_env()
+# subsequently use one of
+cs = CogStack.with_api_key_auth(hosts=hosts, api_key=api_key)
+#cs = CogStack.with_basic_auth(hosts=hosts, username=username, password=password)
+```
+The `read_from_env` method will read the data from the following environmental variables:
+
+| Environmetnal variable name    |  Description                        | Example value |
+| ------------------------------ | ----------------------------------- | ------------- |
+| `COGSTACK_HOSTS`               | The host addresses, comma separated | `http://localhost:9200,http://localhost:9201` |
+| `COGSTACK_USERNAME`            | The username for basic auth         | `user123`     |
+| `COGSTACK_PASSWORD`            | The password for basic auth         | `sup3rsecur3-pw#946` |
+| `COGSTACK_API_KEY_ID`          | The API key ID for authentiaction   | `l0cGtvtlw1lbsyClOm6w` |
+| `COGSTACK_API_KEY`             | The unencoded API key for authentiaction with the ID | `I01NJf4Z6yvXyXThh1676g` |
+| `COGSTACK_API_KEY_ENCODED`     | The encoded API key for authentiaction with just the API key | `ZZpwMtW3ky6Tw9KEtfavVzTP0JcrC7iLnVf7zXbqAh70A15VKJwHd5YX3J==` |
+
+
+__Note__: If these fields are left blank then the user will be prompted to enter the details themselves.
+
+If you are unsure about the above information please contact your CogStack system administrator.
+
+## How to build a Search query
+
+A core component of cogstack is Elasticsearch which is a search engine built on top of Apache Lucene.
+
+Lucene has a custom query syntax for querying its indexes (Lucene Query Syntax). This query syntax allows for features such as Keyword matching, Wildcard matching, Regular expression, Proximity matching, Range searches.
+
+Full documentation for this syntax is available as part of Elasticsearch [query string syntax](https://www.elastic.co/guide/en/elasticsearch/reference/8.5/query-dsl-query-string-query.html#query-string-syntax).

cogstack-es/pyproject.toml

@@ -0,0 +1,61 @@
+[build-system]
+requires = ["setuptools>=61.0", "wheel",  "setuptools_scm>=8"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "cogstack-es"
+dynamic = ["version"]
+description = "ElasticSearch or OpenSearch wrapper for CogStack deployments"
+readme = "ReadMe.md"
+authors = [{ name = "Mart Ratas", email = "[email protected]" }]
+license = { text = "Apache-2.0" }
+requires-python = ">=3.10"
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Science/Research",
+    "Topic :: Scientific/Engineering :: Artificial Intelligence",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "License :: OSI Approved :: Apache Software License"
+]
+
+dependencies = [
+    "tqdm>=4.64,<5.0",
+    "pandas>=2.2,<3.0",
+    "ipython",
+]
+
+[project.optional-dependencies]
+dev = [
+    "mypy",
+    "pandas-stubs",
+    "types-tqdm",
+    "pytest",
+    "ruff",
+    "nbconvert",
+]
+ES8 = [
+    "elasticsearch>=8.0.0,<9.0",
+]
+ES9 = [
+    "elasticsearch>=9.0.0,<10.0",
+]
+OS = [
+    "opensearch-py>=2.0.0,<3.0",
+]
+
+[tool.setuptools]
+package-dir = {"" = "src"}
+
+[project.urls]
+Homepage = "https://github.com/CogStack/cogstack-nlp/tree/main/cogstack-es"
+Repository = "https://github.com/CogStack/cogstack-nlp/tree/main/cogstack-es"
+Issues = "https://github.com/CogStack/cogstack-nlp/issues"
+
+[tool.setuptools_scm]
+root = ".."
+tag_regex = "^cogstack-es/v(?P<version>[0-9]+(?:\\.[0-9]+)*)$"
+fallback_version = "0.1.0.dev0"