Merge pull request #1971 from dlt-hub/devel

master merge for 1.3.0 release

.github/workflows/test_destination_athena.yml

@@ -17,7 +17,7 @@ concurrency:
 env:
   DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }}
 
-  RUNTIME__SENTRY_DSN: https://[email protected]/4504819859914752
+  # RUNTIME__SENTRY_DSN: https://[email protected]/4504819859914752
   RUNTIME__LOG_LEVEL: ERROR
   RUNTIME__DLTHUB_TELEMETRY_ENDPOINT: ${{ secrets.RUNTIME__DLTHUB_TELEMETRY_ENDPOINT }}
   ACTIVE_DESTINATIONS: "[\"athena\"]"

.github/workflows/test_destination_athena_iceberg.yml

@@ -17,7 +17,7 @@ concurrency:
 env:
   DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }}
 
-  RUNTIME__SENTRY_DSN: https://[email protected]/4504819859914752
+  # RUNTIME__SENTRY_DSN: https://[email protected]/4504819859914752
   RUNTIME__LOG_LEVEL: ERROR
   RUNTIME__DLTHUB_TELEMETRY_ENDPOINT: ${{ secrets.RUNTIME__DLTHUB_TELEMETRY_ENDPOINT }}
   ACTIVE_DESTINATIONS: "[\"athena\"]"

.github/workflows/test_destination_bigquery.yml

@@ -17,7 +17,7 @@ concurrency:
 env:
   DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }}
 
-  RUNTIME__SENTRY_DSN: https://[email protected]/4504819859914752
+  # RUNTIME__SENTRY_DSN: https://[email protected]/4504819859914752
   RUNTIME__LOG_LEVEL: ERROR
   RUNTIME__DLTHUB_TELEMETRY_ENDPOINT: ${{ secrets.RUNTIME__DLTHUB_TELEMETRY_ENDPOINT }}
 

.github/workflows/test_destination_clickhouse.yml

@@ -14,7 +14,7 @@ concurrency:
   cancel-in-progress: true
 
 env:
-  RUNTIME__SENTRY_DSN: https://[email protected]/4504819859914752
+  # RUNTIME__SENTRY_DSN: https://[email protected]/4504819859914752
   RUNTIME__LOG_LEVEL: ERROR
   DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }}
 

.github/workflows/test_destination_databricks.yml

@@ -17,7 +17,7 @@ concurrency:
 env:
   DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }}
 
-  RUNTIME__SENTRY_DSN: https://[email protected]/4504819859914752
+  # RUNTIME__SENTRY_DSN: https://[email protected]/4504819859914752
   RUNTIME__LOG_LEVEL: ERROR
   RUNTIME__DLTHUB_TELEMETRY_ENDPOINT: ${{ secrets.RUNTIME__DLTHUB_TELEMETRY_ENDPOINT }}
 

.github/workflows/test_destination_dremio.yml

@@ -15,7 +15,7 @@ concurrency:
   cancel-in-progress: true
 
 env:
-  RUNTIME__SENTRY_DSN: https://[email protected]/4504819859914752
+  # RUNTIME__SENTRY_DSN: https://[email protected]/4504819859914752
   RUNTIME__LOG_LEVEL: ERROR
   RUNTIME__DLTHUB_TELEMETRY_ENDPOINT: ${{ secrets.RUNTIME__DLTHUB_TELEMETRY_ENDPOINT }}
 

.github/workflows/test_destination_lancedb.yml

@@ -16,7 +16,7 @@ concurrency:
 env:
   DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }}
 
-  RUNTIME__SENTRY_DSN: https://[email protected]/4504819859914752
+  # RUNTIME__SENTRY_DSN: https://[email protected]/4504819859914752
   RUNTIME__LOG_LEVEL: ERROR
   RUNTIME__DLTHUB_TELEMETRY_ENDPOINT: ${{ secrets.RUNTIME__DLTHUB_TELEMETRY_ENDPOINT }}
 

.github/workflows/test_destination_motherduck.yml

@@ -17,7 +17,7 @@ concurrency:
 env:
   DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }}
 
-  RUNTIME__SENTRY_DSN: https://[email protected]/4504819859914752
+  # RUNTIME__SENTRY_DSN: https://[email protected]/4504819859914752
   RUNTIME__LOG_LEVEL: ERROR
   RUNTIME__DLTHUB_TELEMETRY_ENDPOINT: ${{ secrets.RUNTIME__DLTHUB_TELEMETRY_ENDPOINT }}
 

.github/workflows/test_destination_mssql.yml

@@ -18,7 +18,7 @@ env:
 
   DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }}
 
-  RUNTIME__SENTRY_DSN: https://[email protected]/4504819859914752
+  # RUNTIME__SENTRY_DSN: https://[email protected]/4504819859914752
   RUNTIME__LOG_LEVEL: ERROR
   RUNTIME__DLTHUB_TELEMETRY_ENDPOINT: ${{ secrets.RUNTIME__DLTHUB_TELEMETRY_ENDPOINT }}
 

.github/workflows/test_destination_qdrant.yml

@@ -16,7 +16,7 @@ concurrency:
 env:
   DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }}
 
-  RUNTIME__SENTRY_DSN: https://[email protected]/4504819859914752
+  # RUNTIME__SENTRY_DSN: https://[email protected]/4504819859914752
   RUNTIME__LOG_LEVEL: ERROR
   RUNTIME__DLTHUB_TELEMETRY_ENDPOINT: ${{ secrets.RUNTIME__DLTHUB_TELEMETRY_ENDPOINT }}
 

.github/workflows/test_destination_snowflake.yml

@@ -17,7 +17,7 @@ concurrency:
 env:
   DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }}
 
-  RUNTIME__SENTRY_DSN: https://[email protected]/4504819859914752
+  # RUNTIME__SENTRY_DSN: https://[email protected]/4504819859914752
   RUNTIME__LOG_LEVEL: ERROR
   RUNTIME__DLTHUB_TELEMETRY_ENDPOINT: ${{ secrets.RUNTIME__DLTHUB_TELEMETRY_ENDPOINT }}
 

.github/workflows/test_destinations.yml

@@ -23,7 +23,7 @@ env:
   TESTS__R2_AWS_SECRET_ACCESS_KEY: ${{ secrets.CLOUDFLARE_R2_SECRET_ACCESS_KEY }}
   TESTS__R2_ENDPOINT_URL: https://9830548e4e4b582989be0811f2a0a97f.r2.cloudflarestorage.com
 
-  RUNTIME__SENTRY_DSN: https://[email protected]/4504819859914752
+  # RUNTIME__SENTRY_DSN: https://[email protected]/4504819859914752
   RUNTIME__LOG_LEVEL: ERROR
   RUNTIME__DLTHUB_TELEMETRY_ENDPOINT: ${{ secrets.RUNTIME__DLTHUB_TELEMETRY_ENDPOINT }}
   # Test redshift and filesystem with all buckets
@@ -77,11 +77,14 @@ jobs:
 
       - name: Install dependencies
         # if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
-        run: poetry install --no-interaction -E redshift -E gs -E s3 -E az -E parquet -E duckdb -E cli --with sentry-sdk --with pipeline -E deltalake
+        run: poetry install --no-interaction -E redshift -E gs -E s3 -E az -E parquet -E duckdb -E cli -E filesystem --with sentry-sdk --with pipeline -E deltalake
 
       - name: create secrets.toml
         run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml
 
+      - name: clear duckdb secrets and cache
+        run: rm -rf ~/.duckdb
+
       - run: |
           poetry run pytest tests/load --ignore tests/load/sources -m "essential"
         name: Run essential tests Linux

.github/workflows/test_doc_snippets.yml

@@ -15,7 +15,7 @@ concurrency:
 env:
   DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }}
 
-  RUNTIME__SENTRY_DSN: https://[email protected]/4504819859914752
+  # RUNTIME__SENTRY_DSN: https://[email protected]/4504819859914752
   RUNTIME__LOG_LEVEL: ERROR
   RUNTIME__DLTHUB_TELEMETRY_ENDPOINT: ${{ secrets.RUNTIME__DLTHUB_TELEMETRY_ENDPOINT }}
 
@@ -60,7 +60,7 @@ jobs:
         uses: actions/checkout@master
 
       - name: Start weaviate
-        run: docker compose -f ".github/weaviate-compose.yml" up -d
+        run:  docker compose -f "tests/load/weaviate/docker-compose.yml" up -d
 
       - name: Setup Python
         uses: actions/setup-python@v4

.github/workflows/test_local_destinations.yml

@@ -18,7 +18,7 @@ env:
   # NOTE: this workflow can't use github secrets!
   # DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }}
 
-  RUNTIME__SENTRY_DSN: https://[email protected]/4504819859914752
+  # RUNTIME__SENTRY_DSN: https://[email protected]/4504819859914752
   RUNTIME__LOG_LEVEL: ERROR
   RUNTIME__DLTHUB_TELEMETRY_ENDPOINT: ${{ secrets.RUNTIME__DLTHUB_TELEMETRY_ENDPOINT }}
   ACTIVE_DESTINATIONS: "[\"duckdb\", \"postgres\", \"filesystem\", \"weaviate\", \"qdrant\"]"
@@ -73,7 +73,7 @@ jobs:
         uses: actions/checkout@master
 
       - name: Start weaviate
-        run: docker compose -f ".github/weaviate-compose.yml" up -d
+        run:  docker compose -f "tests/load/weaviate/docker-compose.yml" up -d
 
       - name: Setup Python
         uses: actions/setup-python@v4
@@ -122,7 +122,7 @@ jobs:
 
       - name: Stop weaviate
         if: always()
-        run: docker compose -f ".github/weaviate-compose.yml" down -v
+        run:  docker compose -f "tests/load/weaviate/docker-compose.yml" down -v
 
       - name: Stop SFTP server
         if: always()

.github/workflows/test_local_sources.yml

@@ -15,7 +15,7 @@ concurrency:
 
 env:
 
-  RUNTIME__SENTRY_DSN: https://[email protected]/4504819859914752
+  # RUNTIME__SENTRY_DSN: https://[email protected]/4504819859914752
   RUNTIME__LOG_LEVEL: ERROR
   RUNTIME__DLTHUB_TELEMETRY_ENDPOINT: ${{ secrets.RUNTIME__DLTHUB_TELEMETRY_ENDPOINT }}
 

.github/workflows/test_pyarrow17.yml

@@ -18,7 +18,7 @@ env:
 
   DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }}
 
-  RUNTIME__SENTRY_DSN: https://[email protected]/4504819859914752
+  # RUNTIME__SENTRY_DSN: https://[email protected]/4504819859914752
   RUNTIME__LOG_LEVEL: ERROR
   RUNTIME__DLTHUB_TELEMETRY_ENDPOINT: ${{ secrets.RUNTIME__DLTHUB_TELEMETRY_ENDPOINT }}
 
@@ -65,14 +65,18 @@ jobs:
           key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}-pyarrow17
 
       - name: Install dependencies
-        run: poetry install --no-interaction --with sentry-sdk --with pipeline -E deltalake -E gs -E s3 -E az
+        run: poetry install --no-interaction --with sentry-sdk --with pipeline -E deltalake -E duckdb -E filesystem -E gs -E s3 -E az
+
 
       - name: Upgrade pyarrow
         run: poetry run pip install pyarrow==17.0.0
 
       - name: create secrets.toml
         run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml
 
+      - name: clear duckdb secrets and cache
+        run: rm -rf ~/.duckdb
+
       - name: Run needspyarrow17 tests Linux
         run: |
           poetry run pytest tests/libs -m "needspyarrow17"

.github/workflows/test_sqlalchemy_destinations.yml

@@ -18,7 +18,7 @@ env:
   # NOTE: this workflow can't use github secrets!
   # DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }}
 
-  RUNTIME__SENTRY_DSN: https://[email protected]/4504819859914752
+  # RUNTIME__SENTRY_DSN: https://[email protected]/4504819859914752
   RUNTIME__LOG_LEVEL: ERROR
   RUNTIME__DLTHUB_TELEMETRY_ENDPOINT: ${{ secrets.RUNTIME__DLTHUB_TELEMETRY_ENDPOINT }}
   ACTIVE_DESTINATIONS: "[\"sqlalchemy\"]"
@@ -94,6 +94,3 @@ jobs:
       # always run full suite, also on branches
       - run: poetry run pytest tests/load -x --ignore tests/load/sources
         name: Run tests Linux
-        env:
-          DESTINATION__SQLALCHEMY_MYSQL__CREDENTIALS: mysql://root:[email protected]:3306/dlt_data  # Use root cause we need to create databases
-          DESTINATION__SQLALCHEMY_SQLITE__CREDENTIALS: sqlite:///_storage/dl_data.sqlite

CONTRIBUTING.md

@@ -68,12 +68,11 @@ feat/4922-add-avro-support
 * **fix** - a change that fixes a bug (ticket required)
 * **exp** - an experiment where we are testing a new  idea or want to demonstrate something to the team, might turn into a `feat` later (ticket encouraged)
 * **test** - anything related to the tests (ticket encouraged)
-* **blogs** - a new entry to our blog (ticket optional)
 * **docs** - a change to our docs (ticket optional)
 
 #### Ticket Numbers
 
-We encourage you to attach your branches to a ticket, if none exists, create one and explain what you are doing. For `feat` and `fix` branches, tickets are mandatory, for `exp` and `test` branches encouraged and for `blogs` and `docs` branches optional.
+We encourage you to attach your branches to a ticket, if none exists, create one and explain what you are doing. For `feat` and `fix` branches, tickets are mandatory, for `exp` and `test` branches encouraged and for `docs` branches optional.
 
 ### Submitting a hotfix
 We'll fix critical bugs and release `dlt` out of the schedule. Follow the regular procedure, but make your PR against **master** branch. Please ping us on Slack if you do it.

Makefile

@@ -107,6 +107,13 @@ test-build-images: build-library
 	docker build -f deploy/dlt/Dockerfile.airflow --build-arg=COMMIT_SHA="$(shell git log -1 --pretty=%h)" --build-arg=IMAGE_VERSION="$(shell poetry version -s)" .
 	# docker build -f deploy/dlt/Dockerfile --build-arg=COMMIT_SHA="$(shell git log -1 --pretty=%h)" --build-arg=IMAGE_VERSION="$(shell poetry version -s)" .
 
-preprocess-docs: 
+preprocess-docs:
 	# run docs preprocessing to run a few checks and ensure examples can be parsed
-	cd docs/website && npm i && npm run preprocess-docs
+	cd docs/website && npm i && npm run preprocess-docs
+
+start-test-containers:
+	docker compose -f "tests/load/dremio/docker-compose.yml" up -d
+	docker compose -f "tests/load/postgres/docker-compose.yml" up -d
+	docker compose -f "tests/load/weaviate/docker-compose.yml" up -d
+	docker compose -f "tests/load/filesystem_sftp/docker-compose.yml" up -d
+	docker compose -f "tests/load/sqlalchemy/docker-compose.yml" up -d

dlt/__init__.py

@@ -22,7 +22,7 @@
 
 from dlt.version import __version__
 from dlt.common.configuration.accessors import config, secrets
-from dlt.common.typing import TSecretValue as _TSecretValue
+from dlt.common.typing import TSecretValue as _TSecretValue, TSecretStrValue as _TSecretStrValue
 from dlt.common.configuration.specs import CredentialsConfiguration as _CredentialsConfiguration
 from dlt.common.pipeline import source_state as state
 from dlt.common.schema import Schema
@@ -42,6 +42,7 @@
 )
 from dlt.pipeline import progress
 from dlt import destinations
+from dlt.destinations.dataset import dataset as _dataset
 
 pipeline = _pipeline
 current = _current
@@ -50,10 +51,12 @@
 TSecretValue = _TSecretValue
 "When typing source/resource function arguments it indicates that a given argument is a secret and should be taken from dlt.secrets."
 
+TSecretStrValue = _TSecretStrValue
+"When typing source/resource function arguments it indicates that a given argument is a secret STRING and should be taken from dlt.secrets."
+
 TCredentials = _CredentialsConfiguration
 "When typing source/resource function arguments it indicates that a given argument represents credentials and should be taken from dlt.secrets. Credentials may be a string, dictionary or any other type."
 
-
 __all__ = [
     "__version__",
     "config",
@@ -77,4 +80,14 @@
     "TCredentials",
     "sources",
     "destinations",
+    "_dataset",
 ]
+
+# verify that no injection context was created
+from dlt.common.configuration.container import Container as _Container
+
+assert (
+    _Container._INSTANCE is None
+), "Injection container should not be initialized during initial import"
+# create injection container
+_Container()

dlt/cli/__init__.py

@@ -0,0 +1 @@
+from dlt.cli.reference import SupportsCliCommand