fix: windows compatibility

lihsai0 · lihsai0 · commit b4c7d4131e3a · 2024-10-11T18:14:19.000+08:00
- fix file write error
- fix datetime to timestamp error
- fix read file line by line then json.parse error
- test, add windows test actions
- build, remove useless test packages
- test, fix windows platform not work with unicode characters in codecov config
diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml
@@ -60,14 +60,106 @@ jobs:
           QINIU_UPLOAD_CALLBACK_URL: ${{secrets.QINIU_UPLOAD_CALLBACK_URL}}
           QINIU_TEST_ENV: "travis"
           MOCK_SERVER_ADDRESS: "http://127.0.0.1:9000"
-          PYTHONPATH: "$PYTHONPATH:."
         run: |
           flake8 --show-source --max-line-length=160 ./qiniu
-          coverage run -m pytest ./test_qiniu.py ./tests/cases
-          ocular --data-file .coverage
-          codecov
+          python -m pytest ./test_qiniu.py tests --cov qiniu --cov-report=xml
+      - name: Post Setup mock server
+        if: ${{ always() }}
+        shell: bash
+        run: |
+          set +e
           cat mock-server.pid | xargs kill
+          rm mock-server.pid
       - name: Print mock server log
         if: ${{ failure() }}
         run: |
           cat py-mock-server.log
+      - name: Upload results to Codecov
+        uses: codecov/codecov-action@v4
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+  test-win:
+    strategy:
+      fail-fast: false
+      max-parallel: 1
+      matrix:
+        python_version: ['2.7', '3.5', '3.9']
+    runs-on: windows-2019
+    # make sure only one test running,
+    # remove this when cases could run in parallel.
+    needs: test
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v2
+        with:
+          ref: ${{ github.ref }}
+      - name: Setup miniconda
+        uses: conda-incubator/setup-miniconda@v2
+        with:
+          auto-update-conda: true
+          channels: conda-forge
+          python-version: ${{ matrix.python_version }}
+          activate-environment: qiniu-sdk
+          auto-activate-base: false
+      - name: Setup pip
+        env:
+          PYTHON_VERSION: ${{ matrix.python_version }}
+          PIP_BOOTSTRAP_SCRIPT_PREFIX: https://bootstrap.pypa.io/pip
+        run: |
+          # reinstall pip by some python(<3.7) not compatible
+          $pyversion = [Version]"$ENV:PYTHON_VERSION"
+          if ($pyversion -lt [Version]"3.7") {
+            Invoke-WebRequest "$ENV:PIP_BOOTSTRAP_SCRIPT_PREFIX/$($pyversion.Major).$($pyversion.Minor)/get-pip.py" -OutFile "$ENV:TEMP\get-pip.py"
+            python $ENV:TEMP\get-pip.py --user
+            Remove-Item -Path "$ENV:TEMP\get-pip.py"
+          }
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install -I -e ".[dev]"
+      - name: Run cases
+        env:
+          QINIU_ACCESS_KEY: ${{ secrets.QINIU_ACCESS_KEY }}
+          QINIU_SECRET_KEY: ${{ secrets.QINIU_SECRET_KEY }}
+          QINIU_TEST_BUCKET: ${{ secrets.QINIU_TEST_BUCKET }}
+          QINIU_TEST_NO_ACC_BUCKET: ${{ secrets.QINIU_TEST_NO_ACC_BUCKET }}
+          QINIU_TEST_DOMAIN: ${{ secrets.QINIU_TEST_DOMAIN }}
+          QINIU_UPLOAD_CALLBACK_URL: ${{secrets.QINIU_UPLOAD_CALLBACK_URL}}
+          QINIU_TEST_ENV: "github"
+          MOCK_SERVER_ADDRESS: "http://127.0.0.1:9000"
+          PYTHONPATH: "$PYTHONPATH:."
+        run: |
+          Write-Host "======== Setup Mock Server ========="
+          conda create -y -n mock-server python=3.10
+          conda activate mock-server
+          python --version
+          $processOptions = @{
+            FilePath="python"
+            ArgumentList="tests/mock_server/main.py", "--port", "9000"
+            PassThru=$true
+            RedirectStandardOutput="py-mock-server.log"
+          }
+          $mocksrvp = Start-Process @processOptions
+          $mocksrvp.Id | Out-File -FilePath "mock-server.pid"
+          conda deactivate
+          Write-Host "======== Running Test ========="
+          python --version
+          python -m pytest ./test_qiniu.py tests --cov qiniu --cov-report=xml
+      - name: Post Setup mock server
+        if: ${{ always() }}
+        run: |
+          Try {
+            $mocksrvpid = Get-Content -Path "mock-server.pid"
+            Stop-Process -Id $mocksrvpid
+            Remove-Item -Path "mock-server.pid"
+          } Catch {
+            Write-Host -Object $_
+          }
+      - name: Print mock server log
+        if: ${{ failure() }}
+        run: |
+          Get-Content -Path "py-mock-server.log"
+      - name: Upload results to Codecov
+        uses: codecov/codecov-action@v4
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
diff --git a/codecov.yml b/codecov.yml
@@ -1,28 +1,28 @@
 codecov:
   ci:
-    - prow.qiniu.io       # prow 里面运行需添加，其他 CI 不要
-  require_ci_to_pass: no  # 改为 no，否则 codecov 会等待其他 GitHub 上所有 CI 通过才会留言。
+    - prow.qiniu.io       # prow need this. seems useless
+  require_ci_to_pass: no  # `no` means the bot will comment on the PR even before all ci passed
 
-github_checks:              #关闭github checks
+github_checks:              # close github checks
   annotations: false
 
 comment:
   layout: "reach, diff, flags, files"
-  behavior: new           # 默认是更新旧留言，改为 new，删除旧的，增加新的。
+  behavior: new           # `new` means the bot will comment a new message instead of edit the old one
   require_changes: false  # if true: only post the comment if coverage changes
   require_base: no        # [yes :: must have a base report to post]
   require_head: yes       # [yes :: must have a head report to post]
   branches:               # branch names that can post comment
       - "master"
 
 coverage:
-  status:                                 # 评判 pr 通过的标准
+  status:                                 # check coverage status to pass or fail
       patch: off
-      project:                            # project 统计所有代码x
+      project:                            # project analyze all code in the project
           default:
             # basic
-            target: 73.5%                  # 总体通过标准
-            threshold: 3%                 # 允许单次下降的幅度
+            target: 73.5%                 # the minimum coverage ratio that the commit must meet
+            threshold: 3%                 # allow the coverage to drop
             base: auto
             if_not_found: success
             if_ci_failed: error
diff --git a/qiniu/http/regions_provider.py b/qiniu/http/regions_provider.py
@@ -5,9 +5,10 @@
 import logging
 import tempfile
 import os
+import shutil
 
 from qiniu.compat import json, b as to_bytes
-from qiniu.utils import io_md5
+from qiniu.utils import io_md5, dt2ts
 
 from .endpoint import Endpoint
 from .region import Region, ServiceName
@@ -264,7 +265,7 @@ def _persist_region(region):
         },
         ttl=region.ttl,
         # use datetime.datetime.timestamp() when min version of python >= 3
-        createTime=int(float(region.create_time.strftime('%s.%f')) * 1000)
+        createTime=dt2ts(region.create_time)
     )._asdict()
 
 
@@ -338,8 +339,10 @@ def _walk_persist_cache_file(persist_path, ignore_parse_error=False):
 
     with open(persist_path, 'r') as f:
         for line in f:
+            if not line.strip():
+                continue
             try:
-                cache_key, regions = _parse_persisted_regions(line)
+                cache_key, regions = _parse_persisted_regions(line.strip())
                 yield cache_key, regions
             except Exception as err:
                 if not ignore_parse_error:
@@ -655,7 +658,7 @@ def __shrink_cache(self):
                         )
 
                 # rename file
-                os.rename(shrink_file_path, self._cache_scope.persist_path)
+                shutil.move(shrink_file_path, self._cache_scope.persist_path)
         except FileAlreadyLocked:
             pass
         finally:
diff --git a/qiniu/region.py b/qiniu/region.py
@@ -6,7 +6,7 @@
 
 
 from .compat import json, s as str_from_bytes
-from .utils import urlsafe_base64_decode
+from .utils import urlsafe_base64_decode, dt2ts
 from .config import UC_HOST, is_customized_default, get_default
 from .http.endpoint import Endpoint as _HTTPEndpoint
 from .http.regions_provider import Region as _HTTPRegion, ServiceName, get_default_regions_provider
@@ -190,7 +190,7 @@ def get_bucket_hosts(self, ak, bucket, home_dir=None, force=False):
 
         ttl = region.ttl if region.ttl > 0 else 24 * 3600  # 1 day
         # use datetime.datetime.timestamp() when min version of python >= 3
-        create_time = int(float(region.create_time.strftime('%s.%f')) * 1000)
+        create_time = dt2ts(region.create_time)
         bucket_hosts['deadline'] = create_time + ttl
 
         return bucket_hosts
diff --git a/qiniu/utils.py b/qiniu/utils.py
@@ -1,7 +1,8 @@
 # -*- coding: utf-8 -*-
 from hashlib import sha1, new as hashlib_new
 from base64 import urlsafe_b64encode, urlsafe_b64decode
-from datetime import datetime
+from datetime import datetime, tzinfo, timedelta
+
 from .compat import b, s
 
 try:
@@ -236,3 +237,30 @@ def canonical_mime_header_key(field_name):
             result += ch
         upper = ch == "-"
     return result
+
+
+class _UTC_TZINFO(tzinfo):
+    def utcoffset(self, dt):
+        return timedelta(hours=0)
+
+    def tzname(self, dt):
+        return "UTC"
+
+    def dst(self, dt):
+        return timedelta(0)
+
+
+def dt2ts(dt):
+    """
+    converte datetime to timestamp
+
+    Parameters
+    ----------
+    dt: datetime.datetime
+    """
+    if not dt.tzinfo:
+        st = (dt - datetime(1970, 1, 1)).total_seconds()
+    else:
+        st = (dt - datetime(1970, 1, 1, tzinfo=_UTC_TZINFO())).total_seconds()
+
+    return int(st)
diff --git a/setup.py b/setup.py
@@ -42,10 +42,8 @@ def find_version(*file_paths):
         'Operating System :: OS Independent',
         'Programming Language :: Python',
         'Programming Language :: Python :: 2',
-        'Programming Language :: Python :: 2.6',
         'Programming Language :: Python :: 2.7',
         'Programming Language :: Python :: 3',
-        'Programming Language :: Python :: 3.3',
         'Programming Language :: Python :: 3.4',
         'Programming Language :: Python :: 3.5',
         'Programming Language :: Python :: 3.6',
@@ -66,8 +64,6 @@ def find_version(*file_paths):
             'pytest',
             'pytest-cov',
             'freezegun',
-            'scrutinizer-ocular',
-            'codecov'
         ]
     },
 
diff --git a/test_qiniu.py b/test_qiniu.py
@@ -65,129 +65,6 @@ def remove_temp_file(file):
     except OSError:
         pass
 
-
-class UtilsTest(unittest.TestCase):
-    def test_urlsafe(self):
-        a = 'hello\x96'
-        u = urlsafe_base64_encode(a)
-        assert b(a) == urlsafe_base64_decode(u)
-
-    def test_canonical_mime_header_key(self):
-        field_names = [
-            ":status",
-            ":x-test-1",
-            ":x-Test-2",
-            "content-type",
-            "CONTENT-LENGTH",
-            "oRiGin",
-            "ReFer",
-            "Last-Modified",
-            "acCePt-ChArsEt",
-            "x-test-3",
-            "cache-control",
-        ]
-        expect_canonical_field_names = [
-            ":status",
-            ":x-test-1",
-            ":x-Test-2",
-            "Content-Type",
-            "Content-Length",
-            "Origin",
-            "Refer",
-            "Last-Modified",
-            "Accept-Charset",
-            "X-Test-3",
-            "Cache-Control",
-        ]
-        assert len(field_names) == len(expect_canonical_field_names)
-        for i in range(len(field_names)):
-            assert canonical_mime_header_key(field_names[i]) == expect_canonical_field_names[i]
-
-    def test_entry(self):
-        case_list = [
-            {
-                'msg': 'normal',
-                'bucket': 'qiniuphotos',
-                'key': 'gogopher.jpg',
-                'expect': 'cWluaXVwaG90b3M6Z29nb3BoZXIuanBn'
-            },
-            {
-                'msg': 'key empty',
-                'bucket': 'qiniuphotos',
-                'key': '',
-                'expect': 'cWluaXVwaG90b3M6'
-            },
-            {
-                'msg': 'key undefined',
-                'bucket': 'qiniuphotos',
-                'key': None,
-                'expect': 'cWluaXVwaG90b3M='
-            },
-            {
-                'msg': 'key need replace plus symbol',
-                'bucket': 'qiniuphotos',
-                'key': '012ts>a',
-                'expect': 'cWluaXVwaG90b3M6MDEydHM-YQ=='
-            },
-            {
-                'msg': 'key need replace slash symbol',
-                'bucket': 'qiniuphotos',
-                'key': '012ts?a',
-                'expect': 'cWluaXVwaG90b3M6MDEydHM_YQ=='
-            }
-        ]
-        for c in case_list:
-            assert c.get('expect') == entry(c.get('bucket'), c.get('key')), c.get('msg')
-
-    def test_decode_entry(self):
-        case_list = [
-            {
-                'msg': 'normal',
-                'expect': {
-                    'bucket': 'qiniuphotos',
-                    'key': 'gogopher.jpg'
-                },
-                'entry': 'cWluaXVwaG90b3M6Z29nb3BoZXIuanBn'
-            },
-            {
-                'msg': 'key empty',
-                'expect': {
-                    'bucket': 'qiniuphotos',
-                    'key': ''
-                },
-                'entry': 'cWluaXVwaG90b3M6'
-            },
-            {
-                'msg': 'key undefined',
-                'expect': {
-                    'bucket': 'qiniuphotos',
-                    'key': None
-                },
-                'entry': 'cWluaXVwaG90b3M='
-            },
-            {
-                'msg': 'key need replace plus symbol',
-                'expect': {
-                    'bucket': 'qiniuphotos',
-                    'key': '012ts>a'
-                },
-                'entry': 'cWluaXVwaG90b3M6MDEydHM-YQ=='
-            },
-            {
-                'msg': 'key need replace slash symbol',
-                'expect': {
-                    'bucket': 'qiniuphotos',
-                    'key': '012ts?a'
-                },
-                'entry': 'cWluaXVwaG90b3M6MDEydHM_YQ=='
-            }
-        ]
-        for c in case_list:
-            bucket, key = decode_entry(c.get('entry'))
-            assert bucket == c.get('expect').get('bucket'), c.get('msg')
-            assert key == c.get('expect').get('key'), c.get('msg')
-
-
 class BucketTestCase(unittest.TestCase):
     q = Auth(access_key, secret_key)
     bucket = BucketManager(q)
diff --git a/tests/cases/test_utils.py b/tests/cases/test_utils.py
