Merge branch 'main' into release-3.0.0

Author: jaidisido

.github/workflows/cfn-nag.yml

@@ -62,7 +62,7 @@ jobs:
           }
           EOT
           python -m pip install --upgrade pip
-          python -m pip install poetry==1.1.15 # 1.2.0 breaking resolution of packages
+          python -m pip install poetry==1.2.2
           poetry config virtualenvs.create false --local
           poetry install -vvv
       - name: CDK Synth

.github/workflows/minimal-tests.yml

@@ -32,7 +32,7 @@ jobs:
       - name: Install Requirements
         run: |
           python -m pip install --upgrade pip
-          python -m pip install poetry==1.1.15
+          python -m pip install poetry==1.2.2
           poetry config virtualenvs.create false --local
           poetry install --extras "sqlserver" -vvv
       - name: Test Metadata

.github/workflows/static-checking.yml

@@ -31,7 +31,7 @@ jobs:
       - name: Install Requirements
         run: |
           python -m pip install --upgrade pip
-          python -m pip install poetry==1.1.15 # 1.2.0 breaking resolution of packages
+          python -m pip install poetry==1.2.2
           poetry config virtualenvs.create false --local
           poetry install --extras "sqlserver sparql modin ray" -vvv
       - name: mypy check

awswrangler/s3/_write_dataset.py

@@ -16,9 +16,13 @@
 
 
 def _get_bucketing_series(df: pd.DataFrame, bucketing_info: Tuple[List[str], int]) -> pd.Series:
-    bucket_number_series = df.astype("O").apply(
-        lambda row: _get_bucket_number(bucketing_info[1], [row[col_name] for col_name in bucketing_info[0]]),
-        axis="columns",
+    bucket_number_series = (
+        df[bucketing_info[0]]
+        # Prevent "upcasting" mixed types by casting to object
+        .astype("O").apply(
+            lambda row: _get_bucket_number(bucketing_info[1], [row[col_name] for col_name in bucketing_info[0]]),
+            axis="columns",
+        )
     )
     return bucket_number_series.astype(pd.CategoricalDtype(range(bucketing_info[1])))
 
@@ -200,8 +204,7 @@ def _to_buckets(
     **func_kwargs: Any,
 ) -> List[str]:
     _proxy: _WriteProxy = proxy if proxy else _WriteProxy(use_threads=False)
-    df_groups = df.groupby(by=_get_bucketing_series(df=df, bucketing_info=bucketing_info))
-    for bucket_number, subgroup in df_groups:
+    for bucket_number, subgroup in df.groupby(by=_get_bucketing_series(df=df, bucketing_info=bucketing_info)):
         _proxy.write(
             func,
             boto3_session,

pyproject.toml

@@ -67,7 +67,7 @@ pydocstyle = "^6.1.1"
 doc8 = "^0.11.1"
 tox = "^3.25.1"
 pytest = "^7.1.2"
-pytest-cov = "^3.0.0"
+pytest-cov = "^4.0.0"
 pytest-rerunfailures = "^10.2"
 pytest-xdist = "^2.5.0"
 pytest-timeout = "^2.1.0"

test_infra/stacks/databases_stack.py

@@ -232,6 +232,7 @@ def _setup_redshift(self) -> None:
         redshift_cluster = redshift.Cluster(
             self,
             "aws-sdk-pandas-redshift-cluster",
+            removal_policy=RemovalPolicy.DESTROY,
             default_database_name=database,
             master_user=redshift.Login(
                 master_username=self.db_username,
@@ -676,6 +677,7 @@ def _setup_neptune(self, iam_enabled: bool = False, port: int = 8182) -> None:
         cluster = neptune.DatabaseCluster(
             self,
             "aws-sdk-pandas-neptune-cluster",
+            removal_policy=RemovalPolicy.DESTROY,
             vpc=self.vpc,
             instance_type=neptune.InstanceType.R5_LARGE,
             iam_authentication=iam_enabled,

tox.ini

@@ -31,7 +31,7 @@ commands =
 passenv = AWS_PROFILE AWS_DEFAULT_REGION AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_SESSION_TOKEN
 deps =
        {[testenv]deps}
-       pytest-cov==3.0.0
+       pytest-cov==4.0.0
 commands =
        pytest -n {posargs} -s -v --timeout=300 --reruns=2 --reruns-delay=60 --cov=awswrangler \
               --junitxml=test-reports/junit.xml --log-file=test-reports/logs.txt tests/unit

tutorials/001 - Introduction.ipynb

@@ -108,9 +108,9 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "conda_python3",
+   "display_name": "Python 3.9.14",
    "language": "python",
-   "name": "conda_python3"
+   "name": "python3"
   },
   "language_info": {
    "codemirror_mode": {
@@ -122,9 +122,9 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.6.10"
+   "version": "3.9.14"
   }
  },
  "nbformat": 4,
  "nbformat_minor": 4
-}
+}

tutorials/002 - Sessions.ipynb

@@ -128,7 +128,7 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "Python 3.9.13 ('awswrangler-mo8sEp3D-py3.9')",
+   "display_name": "Python 3.9.14",
    "language": "python",
    "name": "python3"
   },
@@ -142,7 +142,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.9.13"
+   "version": "3.9.14"
   },
   "vscode": {
    "interpreter": {

tutorials/003 - Amazon S3.ipynb

@@ -122,7 +122,7 @@
     "path2 = f\"s3://{bucket}/csv/file2.csv\"\n",
     "\n",
     "wr.s3.to_csv(df1, path1, index=False)\n",
-    "wr.s3.to_csv(df2, path2, index=False);"
+    "wr.s3.to_csv(df2, path2, index=False)"
    ]
   },
   {
@@ -362,7 +362,7 @@
     "path2 = f\"s3://{bucket}/parquet/file2.parquet\"\n",
     "\n",
     "wr.s3.to_parquet(df1, path1)\n",
-    "wr.s3.to_parquet(df2, path2);"
+    "wr.s3.to_parquet(df2, path2)"
    ]
   },
   {
@@ -708,7 +708,7 @@
     "wr.s3.read_fwf(f\"s3://{bucket}/fwf/\", names=[\"id\", \"name\", \"date\"], last_modified_begin=begin_utc, last_modified_end=end_utc)\n",
     "wr.s3.read_json(f\"s3://{bucket}/json/\", last_modified_begin=begin_utc, last_modified_end=end_utc)\n",
     "wr.s3.read_csv(f\"s3://{bucket}/csv/\", last_modified_begin=begin_utc, last_modified_end=end_utc)\n",
-    "wr.s3.read_parquet(f\"s3://{bucket}/parquet/\", last_modified_begin=begin_utc, last_modified_end=end_utc);"
+    "wr.s3.read_parquet(f\"s3://{bucket}/parquet/\", last_modified_begin=begin_utc, last_modified_end=end_utc)"
    ]
   },
   {
@@ -965,7 +965,7 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "Python 3",
+   "display_name": "Python 3.9.14",
    "language": "python",
    "name": "python3"
   },
@@ -979,9 +979,9 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.8.5"
+   "version": "3.9.14"
   }
  },
  "nbformat": 4,
  "nbformat_minor": 4
-}
+}