Fetch and Merge Crypto Data #43
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Fetch and Merge Crypto Data | |
| on: | |
| schedule: | |
| - cron: "0 0 * * *" # Runs daily at 00:00 UTC | |
| workflow_dispatch: # Allows manual triggering from the GitHub UI | |
| jobs: | |
| fetch-and-merge: | |
| runs-on: ubuntu-latest | |
| steps: | |
| # Step 1: Checkout the repository | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| with: | |
| lfs: false # No need to fetch the bulk data for this workflow | |
| fetch-depth: 2 # Fetch enough history to reference HEAD~ | |
| # Step 2: Set up Python 3.12 | |
| - name: Set up Python 3.12 | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.12" | |
| # Step 3: Install Poetry | |
| - name: Install Poetry | |
| shell: bash | |
| run: | | |
| pipx install poetry==2.1.1 --python python3.12 | |
| echo $PIPX_BIN_DIR >> $GITHUB_PATH | |
| # Step 4: Install project dependencies using Poetry | |
| - name: Install dependencies | |
| run: | | |
| poetry install | |
| # Step 5: Run the Python script to fetch and process data | |
| - name: Run data processing script | |
| run: | | |
| set -o pipefail | |
| poetry run python scripts/update_data.py 2>&1 | tee script_output.log | |
| if grep -q "Successfully saved" script_output.log; then | |
| echo "SCRIPT_SUCCESS=true" >> $GITHUB_ENV | |
| else | |
| echo "SCRIPT_SUCCESS=false" >> $GITHUB_ENV | |
| fi | |
| # Step 6: Check previous commit message | |
| - name: Check previous commit message | |
| if: env.SCRIPT_SUCCESS == 'true' | |
| run: | | |
| LAST_COMMIT_MSG=$(git log -1 --pretty=%B) | |
| if [[ "$LAST_COMMIT_MSG" == ":package: Daily update"* ]]; then | |
| echo "Previous commit is a daily update. Preparing to overwrite." | |
| echo "RESET_PREVIOUS_COMMIT=true" >> $GITHUB_ENV | |
| else | |
| echo "RESET_PREVIOUS_COMMIT=false" >> $GITHUB_ENV | |
| fi | |
| # Step 7: Commit and force push if previous commit was a daily update | |
| - name: Commit and Push Updates | |
| if: env.SCRIPT_SUCCESS == 'true' | |
| run: | | |
| git config --local user.name "github-actions[bot]" | |
| git config --local user.email "github-actions[bot]@users.noreply.github.com" | |
| if [ "$RESET_PREVIOUS_COMMIT" == "true" ]; then | |
| git reset HEAD~ # Drop the previous commit | |
| fi | |
| git add data/updates/btcusd_bitstamp_1min_latest.csv | |
| git commit -m ":package: Daily update: $(date -u +'%Y-%m-%d')" | |
| if [ "$RESET_PREVIOUS_COMMIT" == "true" ]; then | |
| git push -f origin main # Force push to overwrite the previous commit | |
| else | |
| git push origin main | |
| fi |